qjyu
/
OrangePi_CM5
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'android12-5.10' into branch 'android12-5.10-lts' 

Sync up with the android12-5.10 branch for the following commit:

c4bd1c03ba ANDROID: qcom: Add is_dma_buf_file to ABI
268088c42e ANDROID: GKI: update .xml file
a7f3a36529 ANDROID: GKI: enable KFENCE by setting the sample interval to 500ms
9d1c78d623 ANDROID: abi_gki_aarch64_qcom: Add xhci symbols to list
5896d00f9c ANDROID: vmlinux.lds.h: Define SANITIZER_DISCARDS with CONFIG_CFI_CLANG
ffacd097fc ANDROID: usb: typec: tcpci: Add vendor hook to mask vbus present
1a93d556d7 ANDROID: usb: typce: tcpci: Add vendor hook for chip specific features
08879ea0d6 ANDROID: usb: typec: tcpci: Add vendor hooks for tcpci interface
892e618852 FROMGIT: f2fs: add sysfs nodes to get runtime compression stat
58f0e25d0d ANDROID: dma-buf: Fix error path on system heaps use of the page pool
958c19b19b ANDROID: usb: typec: tcpm: Fix event storm caused by error in backport
0b6fadd02e ANDROID: GKI: USB: XHCI: add Android ABI padding to lots of xhci structures
6f00446c76 FROMGIT: KVM: arm64: Fix host's ZCR_EL2 restore on nVHE
b0671cd753 FROMGIT: KVM: arm64: Force SCTLR_EL2.WXN when running nVHE
8f25c7432f FROMGIT: KVM: arm64: Turn SCTLR_ELx_FLAGS into INIT_SCTLR_EL2_MMU_ON
f47a4d91e5 FROMGIT: KVM: arm64: Use INIT_SCTLR_EL2_MMU_OFF to disable the MMU on KVM teardown
3ff7250562 FROMGIT: arm64: Use INIT_SCTLR_EL1_MMU_OFF to disable the MMU on CPU restart
a8a9fadaf8 FROMGIT: KVM: arm64: Enable SVE support for nVHE
b0e15c8c44 FROMGIT: KVM: arm64: Save/restore SVE state for nVHE
194fd166b5 BACKPORT: FROMGIT: KVM: arm64: Trap host SVE accesses when the FPSIMD state is dirty
507e10616c FROMGIT: KVM: arm64: Save guest's ZCR_EL1 before saving the FPSIMD state
4cb78b26ea FROMGIT: KVM: arm64: Map SVE context at EL2 when available
1105b4d1ce BACKPORT: FROMGIT: KVM: arm64: Rework SVE host-save/guest-restore
57171b3636 FROMGIT: arm64: sve: Provide a conditional update accessor for ZCR_ELx
ac5f5708bb FROMGIT: KVM: arm64: Introduce vcpu_sve_vq() helper
f9bab75599 FROMGIT: KVM: arm64: Let vcpu_sve_pffr() handle HYP VAs
c2469fa4b4 FROMGIT: KVM: arm64: Use {read,write}_sysreg_el1 to access ZCR_EL1
f5e060d65e FROMGIT: KVM: arm64: Provide KVM's own save/restore SVE primitives
af855ca1c9 ANDROID: GKI: USB: Gadget: add Android ABI padding to struct usb_gadget
24149445ad ANDROID: vendor_hooks: Add hooks for memory when debug
00185a421a ANDROID: vendor_hooks: Add hooks for ufs scheduler
b8981993c2 ANDROID: GKI: sound/usb/card.h: add Android ABI padding to struct snd_usb_endpoint
5a56f5119e ANDROID: GKI: user_namespace.h: add Android ABI padding to a structure
599e75fcfd ANDROID: GKI: timer.h: add Android ABI padding to a structure
303a28f485 ANDROID: GKI: quota.h: add Android ABI padding to some structures
a06d91262a ANDROID: GKI: mmu_notifier.h: add Android ABI padding to some structures
8faaa07702 ANDROID: GKI: mm.h: add Android ABI padding to a structure
f21777c57f ANDROID: GKI: kobject.h: add Android ABI padding to some structures
31f6b7a1ae ANDROID: GKI: kernfs.h: add Android ABI padding to some structures
1a517d8668 ANDROID: GKI: irqdomain.h: add Android ABI padding to a structure
c6adf0c7b9 ANDROID: GKI: ioport.h: add Android ABI padding to a structure
c469de24a1 ANDROID: GKI: iomap.h: add Android ABI padding to a structure
3cb5c28964 ANDROID: GKI: hrtimer.h: add Android ABI padding to a structure
eff6517b1e ANDROID: GKI: genhd.h: add Android ABI padding to some structures
9a92773dc8 ANDROID: GKI: ethtool.h: add Android ABI padding to a structure
774168de67 ANDROID: GKI: dma-mapping.h: add Android ABI padding to a structure
73ff58b01f ANDROID: GKI: networking: add Android ABI padding to a lot of networking structures
f0688df6b5 ANDROID: GKI: blk_types.h: add Android ABI padding to a structure
4a895f1f2b ANDROID: GKI: scsi.h: add Android ABI padding to a structure
48691266ca ANDROID: GKI: pci: add Android ABI padding to some structures
4c95a92b84 ANDROID: GKI: add Android ABI padding to struct nf_conn
e6f5502bc3 UPSTREAM: usb: typec: tcpm: Skip sink_cap query only when VDM sm is busy
36547f2f6d FROMGIT: usb: typec: tcpm: PD3.0 sinks can send Discover Identity even in device mode
3f7ada406d UPSTREAM: usb: gadget: configfs: Fix KASAN use-after-free
9bd96584f4 UPSTREAM: usb: typec: Remove vdo[3] part of tps6598x_rx_identity_reg struct
f2e7bdcdfa UPSTREAM: usb: dwc3: gadget: Prevent EP queuing while stopping transfers
462c92e8f2 FROMGIT: usb: typec: tcpci: Added few missing TCPCI register definitions
6f85b7e0f8 UPSTREAM: usb: typec: tcpm: Invoke power_supply_changed for tcpm-source-psy-
8cf94785bb Revert "ANDROID: GKI: Enable bounds sanitizer"
0eecc2aa30 ANDROID: GKI: temporarily disable LTO/CFI
3520187422 ANDROID: Revert "f2fs: fix to tag FIEMAP_EXTENT_MERGED in f2fs_fiemap()"
16982b6a18 FROMLIST: KVM: arm64: Disable FWB in host stage-2
3a16a9e7be FROMLIST: KVM: arm64: Introduce KVM_PGTABLE_S2_NOFWB Stage-2 flag
bcdcab0253 FROMLIST: KVM: arm64: Protect the .hyp sections from the host
317d5a5030 FROMLIST: KVM: arm64: Disable PMU support in protected mode
6697310c08 FROMLIST: KVM: arm64: Page-align the .hyp sections
68bd98a28a FROMLIST: KVM: arm64: Wrap the host with a stage 2
5a128f0626 FROMLIST: KVM: arm64: Provide sanitized mmfr* registers at EL2
0af83133e3 FROMLIST: KVM: arm64: Add kvm_pgtable_stage2_find_range()
c3f49047ca FROMLIST: KVM: arm64: Refactor the *_map_set_prot_attr() helpers
9349978462 FROMLIST: KVM: arm64: Use page-table to track page ownership
845457f549 FROMLIST: KVM: arm64: Always zero invalid PTEs
1d349d64ba FROMLIST: KVM: arm64: Sort the hypervisor memblocks
1e798d4121 FROMLIST: KVM: arm64: Reserve memory for host stage 2
6fddc52e50 FROMLIST: KVM: arm64: Make memcache anonymous in pgtable allocator
a20c8e1c48 FROMLIST: KVM: arm64: Refactor __populate_fault_info()
21a1f89953 FROMLIST: KVM: arm64: Refactor __load_guest_stage2()
1a9bba9121 FROMLIST: KVM: arm64: Refactor kvm_arm_setup_stage2()
6d3f5c07aa FROMLIST: KVM: arm64: Set host stage 2 using kvm_nvhe_init_params
eef107d42a FROMLIST: KVM: arm64: Use kvm_arch in kvm_s2_mmu
1a76f891e1 FROMLIST: KVM: arm64: Use kvm_arch for stage 2 pgtable
d8eb170aa9 FROMLIST: KVM: arm64: Elevate hypervisor mappings creation at EL2
ae1ed8a984 FROMLIST: KVM: arm64: Prepare the creation of s1 mappings at EL2
364f0639db FROMLIST: arm64: asm: Provide set_sctlr_el2 macro
ca52b29511 FROMLIST: KVM: arm64: Factor out vector address calculation
a1180dd384 FROMLIST: KVM: arm64: Provide __flush_dcache_area at EL2
37b5552dfb FROMLIST: KVM: arm64: Enable access to sanitized CPU features at EL2
75fba96057 FROMLIST: KVM: arm64: Introduce a Hyp buddy page allocator
f6e834486a FROMLIST: KVM: arm64: Stub CONFIG_DEBUG_LIST at Hyp
01253f8e2c FROMLIST: KVM: arm64: Introduce an early Hyp page allocator
a3aad4cf4a FROMLIST: KVM: arm64: Allow using kvm_nvhe_sym() in hyp code
81181250ba FROMLIST: KVM: arm64: Make kvm_call_hyp() a function call at Hyp
542dace0bc FROMLIST: KVM: arm64: Introduce a BSS section for use at Hyp
4a4beb1005 FROMLIST: KVM: arm64: Factor memory allocation out of pgtable.c
620bb5f9c9 FROMLIST: KVM: arm64: Avoid free_page() in page-table allocator
c758e8e4b1 FROMLIST: KVM: arm64: Initialize kvm_nvhe_init_params early
ce5e848201 FROMLIST: arm64: kvm: Add standalone ticket spinlock implementation for use at hyp
9833905e2d FROMLIST: KVM: arm64: Link position-independent string routines into .hyp.text
0f63237470 FROMLIST: arm64: lib: Annotate {clear, copy}_page() as position-independent
edab6a7245 Revert "ANDROID: sched: cpufreq_schedutil: add sugov tracepoints"
187306ab1a ANDROID: Partial revert of 06881e01b5 ("ANDROID: sched: Add vendor hooks for override sugov behavior")
b542f4c389 Revert "ANDROID: sched: Add vendor hooks for skipping sugov update"
27d6dac0df Revert "ANDROID: sched: cpufreq_schedutil: move sugov traces to sched"
cc574f0d36 ANDROID: thermal: Add logic for filter on-die tz genl event.
6c6fac8f39 ANDROID: mmc: support hardware that takes key directly
e4636b47ef UPSTREAM: zram: fix broken page writeback
542d1937ba UPSTREAM: zram: fix return value on writeback_store
475aea007d ANDROID: sched: Add vendor hook for util_est_update
89ea2f1eca ANDROID: GKI: Enable DTPM framework
a3a580961b ANDROID: GKI: Add remoteproc framework symbols to symbol list
53e8099784 ANDROID: vendor_hooks: Add hooks for scheduler
7e2c33ac0e FROMGIT: configfs: fix a use-after-free in __configfs_open_file
7656aa0f48 ANDROID: lib/plist.c: Export plist-related APIs
4d63efb9ae ANDROID: module: Add vendor hook
2ff446fc4d ANDROID: bpf: Add vendor hook
dccee128b7 ANDROID: kernel: Add vendor hook in creds
0a3b407463 ANDROID: security: selinux: Add vendor hook in avc
a128ec2b2e UPSTREAM: KVM: arm64: Fix exclusive limit for IPA size
ac821b6257 UPSTREAM: KVM: arm64: Reject VM creation when the default IPA size is unsupported
d2eb8b0028 ANDROID: GKI: add android_kabi.h
45021f7679 UPSTREAM: MAINTAINERS: add entry for KFENCE
90d03f6fcf ANDROID: Clang LTO: Comment on symbol visibility workaround
169ddec367 FROMLIST: mm: fs: Invalidate BH LRU during page migration
3039d8580c BACKPORT: FROMLIST: mm: disable LRU pagevec during the migration temporarily
134ac2d4dc FROMLIST: mm: replace migrate_prep with lru_add_drain_all
a71a3e1ce6 UPSTREAM: mm: migrate: initialize err in do_migrate_pages
059eaf82c0 UPSTREAM: mm: migrate: clean up migrate_prep{_local}
be409db652 ANDROID: Clang LTO: Only set -fvisibility=hidden for x86
02efd18b77 UPSTREAM: usb: typec: tcpci: Refactor tcpc_presenting_cc1_rd macro
8d03e49505 ANDROID: mm: build alloc_contig_dump_pages in page_alloc.o
28f6641041 FROMLIST: mm: page_alloc: dump migrate-failed pages
cc1f93cb20 ANDROID: sched: Add vendor hook for uclamp_eff_get
5e1e3e9387 ANDROID: cpu/hotplug: failure to offline 2nd to last cpu
2c728929c1 ANDROID: qcom: Add devm_blk_ksm_init to ABI
9108e9ba66 ANDROID: arm64: add vendor hooks for unusal abort cases
5f1e1f42a5 UPSTREAM: powercap/drivers/dtpm: Fix size of object being allocated
fd2c659694 UPSTREAM: powercap/drivers/dtpm: Fix an IS_ERR() vs NULL check
2a1cd71862 UPSTREAM: powercap/drivers/dtpm: Fix some missing unlock bugs
7112d4320f UPSTREAM: powercap/drivers/dtpm: Fix a double shift bug
578572c48c UPSTREAM: powercap/drivers/dtpm: Fix __udivdi3 and __aeabi_uldivmod unresolved symbols
7f17016dc8 UPSTREAM: powercap/drivers/dtpm: Add CPU energy model based support
082e8dcea9 UPSTREAM: powercap/drivers/dtpm: Add API for dynamic thermal power management

Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
Change-Id: If05a0d9902893cd29e8bb5eb175bd4ce30250697
This commit is contained in:
Greg Kroah-Hartman 2021-03-21 15:01:22 +01:00
parent 061e6e09f0 c4bd1c03ba
commit e8f79d9cbb
158 changed files with 20945 additions and 14753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Documentation/ABI/testing/sysfs-fs-f2fs
View File

@ -409,3 +409,27 @@ Description: Give a way to change checkpoint merge daemon's io priority.
I/O priority "3". We can select the class between "rt" and "be",
and set the I/O priority within valid range of it. "," delimiter
is necessary in between I/O class and priority number.
What: /sys/fs/f2fs/<disk>/compr_written_block
Date: March 2021
Contact: "Daeho Jeong" <daehojeong@google.com>
Description: Show the block count written after compression since mount. Note
that when the compressed blocks are deleted, this count doesn't
decrease. If you write "0" here, you can initialize
compr_written_block and compr_saved_block to "0".
What: /sys/fs/f2fs/<disk>/compr_saved_block
Date: March 2021
Contact: "Daeho Jeong" <daehojeong@google.com>
Description: Show the saved block count with compression since mount. Note
that when the compressed blocks are deleted, this count doesn't
decrease. If you write "0" here, you can initialize
compr_written_block and compr_saved_block to "0".
What: /sys/fs/f2fs/<disk>/compr_new_inode
Date: March 2021
Contact: "Daeho Jeong" <daehojeong@google.com>
Description: Show the count of inode newly enabled for compression since mount.
Note that when the compression is disabled for the files, this count
doesn't decrease. If you write "0" here, you can initialize
compr_new_inode to "0".

12
MAINTAINERS
View File

@ -9757,6 +9757,18 @@ F: include/linux/keyctl.h
F: include/uapi/linux/keyctl.h
F: security/keys/
KFENCE
M: Alexander Potapenko <glider@google.com>
M: Marco Elver <elver@google.com>
R: Dmitry Vyukov <dvyukov@google.com>
L: kasan-dev@googlegroups.com
S: Maintained
F: Documentation/dev-tools/kfence.rst
F: arch/*/include/asm/kfence.h
F: include/linux/kfence.h
F: lib/Kconfig.kfence
F: mm/kfence/
KFIFO
M: Stefani Seibold <stefani@seibold.net>
S: Maintained

6
Makefile
View File

@ -906,7 +906,13 @@ KBUILD_LDFLAGS += --thinlto-cache-dir=$(extmod-prefix).thinlto-cache
else
CC_FLAGS_LTO += -flto
endif
ifeq ($(SRCARCH),x86)
# TODO(b/182572011): Revert workaround for compiler / linker bug.
CC_FLAGS_LTO += -fvisibility=hidden
else
CC_FLAGS_LTO += -fvisibility=default
endif
# Limit inlining across translation units to reduce binary size
KBUILD_LDFLAGS += -mllvm -import-instr-limit=5

30610
android/abi_gki_aarch64.xml
View File

File diff suppressed because it is too large Load Diff

14
android/abi_gki_aarch64_qcom
View File

@ -399,6 +399,7 @@
__dev_kfree_skb_any
devm_add_action
devm_backlight_device_register
devm_blk_ksm_init
devm_clk_bulk_get
devm_clk_bulk_get_all
devm_clk_get
@ -1170,6 +1171,7 @@
irq_work_queue
irq_work_queue_on
irq_work_sync
is_dma_buf_file
is_vmalloc_addr
jiffies
jiffies_to_msecs
@ -1921,8 +1923,10 @@
rproc_add_subdev
rproc_alloc
rproc_boot
rproc_coredump_add_custom_segment
rproc_coredump_add_segment
rproc_coredump_set_elf_info
rproc_coredump_using_sections
rproc_del
rproc_free
rproc_get_by_child
@ -2662,5 +2666,15 @@
xa_find_after
xa_load
xa_store
xhci_alloc_command
xhci_alloc_erst
xhci_free_command
xhci_free_erst
xhci_get_endpoint_index
xhci_queue_stop_endpoint
xhci_ring_alloc
xhci_ring_cmd_db
xhci_ring_free
xhci_trb_virt_to_dma
zone_watermark_ok
zone_watermark_ok_safe

7
arch/arm64/Kconfig
View File

@ -1712,7 +1712,6 @@ endmenu
config ARM64_SVE
bool "ARM Scalable Vector Extension support"
default y
depends on !KVM || ARM64_VHE
help
The Scalable Vector Extension (SVE) is an extension to the AArch64
execution state which complements and extends the SIMD functionality
@ -1741,12 +1740,6 @@ config ARM64_SVE
booting the kernel. If unsure and you are not observing these
symptoms, you should assume that it is safe to say Y.
CPUs that support SVE are architecturally required to support the
Virtualization Host Extensions (VHE), so the kernel makes no
provision for supporting SVE alongside KVM without VHE enabled.
Thus, you will need to enable CONFIG_ARM64_VHE if you want to support
KVM in the same kernel image.
config ARM64_MODULE_PLTS
bool "Use PLTs to allow module memory to spill over into vmalloc area"
depends on MODULES

11
arch/arm64/configs/gki_defconfig
View File

@ -86,8 +86,6 @@ CONFIG_CRYPTO_AES_ARM64_CE_BLK=y
CONFIG_KPROBES=y
CONFIG_JUMP_LABEL=y
CONFIG_SHADOW_CALL_STACK=y
CONFIG_LTO_CLANG_FULL=y
CONFIG_CFI_CLANG=y
CONFIG_MODULES=y
CONFIG_MODULE_UNLOAD=y
CONFIG_MODVERSIONS=y
@ -503,6 +501,9 @@ CONFIG_IIO_BUFFER=y
CONFIG_IIO_TRIGGER=y
CONFIG_PWM=y
CONFIG_GENERIC_PHY=y
CONFIG_POWERCAP=y
CONFIG_DTPM=y
CONFIG_DTPM_CPU=y
CONFIG_RAS=y
CONFIG_ANDROID=y
CONFIG_ANDROID_BINDER_IPC=y
@ -619,17 +620,13 @@ CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_HEADERS_INSTALL=y
# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
CONFIG_MAGIC_SYSRQ=y
CONFIG_UBSAN=y
CONFIG_UBSAN_TRAP=y
CONFIG_UBSAN_LOCAL_BOUNDS=y
# CONFIG_UBSAN_MISC is not set
CONFIG_PAGE_OWNER=y
CONFIG_DEBUG_STACK_USAGE=y
CONFIG_DEBUG_MEMORY_INIT=y
CONFIG_KASAN=y
CONFIG_KASAN_HW_TAGS=y
CONFIG_KFENCE=y
CONFIG_KFENCE_SAMPLE_INTERVAL=0
CONFIG_KFENCE_SAMPLE_INTERVAL=500
CONFIG_PANIC_ON_OOPS=y
CONFIG_PANIC_TIMEOUT=-1
CONFIG_WQ_WATCHDOG=y

14
arch/arm64/include/asm/assembler.h
View File

@ -676,11 +676,11 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
.endm
/*
* Set SCTLR_EL1 to the passed value, and invalidate the local icache
* Set SCTLR_ELx to the @reg value, and invalidate the local icache
* in the process. This is called when setting the MMU on.
*/
.macro set_sctlr_el1, reg
msr sctlr_el1, \reg
.macro set_sctlr, sreg, reg
msr \sreg, \reg
isb
/*
* Invalidate the local I-cache so that any instructions fetched
@ -692,6 +692,14 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU
isb
.endm
.macro set_sctlr_el1, reg
set_sctlr sctlr_el1, \reg
.endm
.macro set_sctlr_el2, reg
set_sctlr sctlr_el2, \reg
.endm
/*
* Check whether to yield to another runnable task from kernel mode NEON code
* (which runs with preemption disabled).

1
arch/arm64/include/asm/cpufeature.h
View File

@ -607,6 +607,7 @@ void check_local_cpu_capabilities(void);
u64 read_sanitised_ftr_reg(u32 id);
u64 __read_sysreg_by_encoding(u32 sys_id);
int copy_ftr_reg(u32 id, struct arm64_ftr_reg *dst);
static inline bool cpu_supports_mixed_endian_el0(void)
{

9
arch/arm64/include/asm/fpsimd.h
View File

@ -130,6 +130,15 @@ static inline void sve_user_enable(void)
sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
}
#define sve_cond_update_zcr_vq(val, reg) \
do { \
u64 __zcr = read_sysreg_s((reg)); \
u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
__new |= (val) & ZCR_ELx_LEN_MASK; \
if (__zcr != __new) \
write_sysreg_s(__new, (reg)); \
} while (0)
/*
* Probing and setup functions.
* Calls to these functions must be serialised with one another.

10
arch/arm64/include/asm/fpsimdmacros.h
View File

@ -6,6 +6,8 @@
* Author: Catalin Marinas <catalin.marinas@arm.com>
*/
#include <asm/assembler.h>
.macro fpsimd_save state, tmpnr
stp q0, q1, [\state, #16 * 0]
stp q2, q3, [\state, #16 * 2]
@ -230,8 +232,7 @@
str w\nxtmp, [\xpfpsr, #4]
.endm
.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
sve_load_vq \xvqminus1, x\nxtmp, \xtmp2
.macro __sve_load nxbase, xpfpsr, nxtmp
_for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
_sve_ldr_p 0, \nxbase
_sve_wrffr 0
@ -242,3 +243,8 @@
ldr w\nxtmp, [\xpfpsr, #4]
msr fpcr, x\nxtmp
.endm
.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
sve_load_vq \xvqminus1, x\nxtmp, \xtmp2
__sve_load \nxbase, \xpfpsr, \nxtmp
.endm

7
arch/arm64/include/asm/hyp_image.h
View File

@ -10,11 +10,15 @@
#define __HYP_CONCAT(a, b) a ## b
#define HYP_CONCAT(a, b) __HYP_CONCAT(a, b)
#ifndef __KVM_NVHE_HYPERVISOR__
/*
* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_,
* to separate it from the kernel proper.
*/
#define kvm_nvhe_sym(sym) __kvm_nvhe_##sym
#else
#define kvm_nvhe_sym(sym) sym
#endif
#ifdef LINKER_SCRIPT
@ -56,6 +60,9 @@
*/
#define KVM_NVHE_ALIAS(sym) kvm_nvhe_sym(sym) = sym;
/* Defines a linker script alias for KVM nVHE hyp symbols */
#define KVM_NVHE_ALIAS_HYP(first, sec) kvm_nvhe_sym(first) = kvm_nvhe_sym(sec);
#endif /* LINKER_SCRIPT */
#endif /* __ARM64_HYP_IMAGE_H__ */

9
arch/arm64/include/asm/kvm_asm.h
View File

@ -57,6 +57,12 @@
#define __KVM_HOST_SMCCC_FUNC___kvm_get_mdcr_el2 12
#define __KVM_HOST_SMCCC_FUNC___vgic_v3_save_aprs 13
#define __KVM_HOST_SMCCC_FUNC___vgic_v3_restore_aprs 14
#define __KVM_HOST_SMCCC_FUNC___pkvm_init 15
#define __KVM_HOST_SMCCC_FUNC___pkvm_create_mappings 16
#define __KVM_HOST_SMCCC_FUNC___pkvm_create_private_mapping 17
#define __KVM_HOST_SMCCC_FUNC___pkvm_cpu_set_vector 18
#define __KVM_HOST_SMCCC_FUNC___pkvm_prot_finalize 19
#define __KVM_HOST_SMCCC_FUNC___pkvm_mark_hyp 20
#ifndef __ASSEMBLY__
@ -154,6 +160,9 @@ struct kvm_nvhe_init_params {
unsigned long tpidr_el2;
unsigned long stack_hyp_va;
phys_addr_t pgd_pa;
unsigned long hcr_el2;
unsigned long vttbr;
unsigned long vtcr;
};
/* Translate a kernel address @ptr into its equivalent linear mapping */

19
arch/arm64/include/asm/kvm_cpufeature.h Normal file
View File

@ -0,0 +1,19 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020 - Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <asm/cpufeature.h>
#ifndef KVM_HYP_CPU_FTR_REG
#if defined(__KVM_NVHE_HYPERVISOR__)
#define KVM_HYP_CPU_FTR_REG(name) extern struct arm64_ftr_reg name
#else
#define KVM_HYP_CPU_FTR_REG(name) extern struct arm64_ftr_reg kvm_nvhe_sym(name)
#endif
#endif
KVM_HYP_CPU_FTR_REG(arm64_ftr_reg_ctrel0);
KVM_HYP_CPU_FTR_REG(arm64_ftr_reg_id_aa64mmfr0_el1);
KVM_HYP_CPU_FTR_REG(arm64_ftr_reg_id_aa64mmfr1_el1);

40
arch/arm64/include/asm/kvm_host.h
View File

@ -95,7 +95,7 @@ struct kvm_s2_mmu {
/* The last vcpu id that ran on each physical CPU */
int __percpu *last_vcpu_ran;
struct kvm *kvm;
struct kvm_arch *arch;
};
struct kvm_arch_memory_slot {
@ -375,8 +375,10 @@ struct kvm_vcpu_arch {
};
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
#define vcpu_sve_pffr(vcpu) ((void *)((char *)((vcpu)->arch.sve_state) + \
sve_ffr_offset((vcpu)->arch.sve_max_vl)))
#define vcpu_sve_pffr(vcpu) (kern_hyp_va((vcpu)->arch.sve_state) + \
sve_ffr_offset((vcpu)->arch.sve_max_vl))
#define vcpu_sve_max_vq(vcpu) sve_vq_from_vl((vcpu)->arch.sve_max_vl)
#define vcpu_sve_state_size(vcpu) ({ \
size_t __size_ret; \
@ -385,7 +387,7 @@ struct kvm_vcpu_arch {
if (WARN_ON(!sve_vl_valid((vcpu)->arch.sve_max_vl))) { \
__size_ret = 0; \
} else { \
__vcpu_vq = sve_vq_from_vl((vcpu)->arch.sve_max_vl); \
__vcpu_vq = vcpu_sve_max_vq(vcpu); \
__size_ret = SVE_SIG_REGS_SIZE(__vcpu_vq); \
} \
\
@ -594,6 +596,7 @@ int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
void kvm_arm_halt_guest(struct kvm *kvm);
void kvm_arm_resume_guest(struct kvm *kvm);
#ifndef __KVM_NVHE_HYPERVISOR__
#define kvm_call_hyp_nvhe(f, ...) \
({ \
struct arm_smccc_res res; \
@ -633,6 +636,11 @@ void kvm_arm_resume_guest(struct kvm *kvm);
\
ret; \
})
#else /* __KVM_NVHE_HYPERVISOR__ */
#define kvm_call_hyp(f, ...) f(__VA_ARGS__)
#define kvm_call_hyp_ret(f, ...) f(__VA_ARGS__)
#define kvm_call_hyp_nvhe(f, ...) f(__VA_ARGS__)
#endif /* __KVM_NVHE_HYPERVISOR__ */
void force_vm_exit(const cpumask_t *mask);
void kvm_mmu_wp_memory_region(struct kvm *kvm, int slot);
@ -695,19 +703,6 @@ static inline void kvm_init_host_cpu_context(struct kvm_cpu_context *cpu_ctxt)
ctxt_sys_reg(cpu_ctxt, MPIDR_EL1) = read_cpuid_mpidr();
}
static inline bool kvm_arch_requires_vhe(void)
{
/*
* The Arm architecture specifies that implementation of SVE
* requires VHE also to be implemented. The KVM code for arm64
* relies on this when SVE is present:
*/
if (system_supports_sve())
return true;
return false;
}
void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu);
static inline void kvm_arch_hardware_unsetup(void) {}
@ -748,9 +743,13 @@ void kvm_clr_pmu_events(u32 clr);
void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu);
void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu);
void setup_kvm_el2_caps(void);
#else
static inline void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr) {}
static inline void kvm_clr_pmu_events(u32 clr) {}
static inline void setup_kvm_el2_caps(void) {}
#endif
void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu);
@ -774,5 +773,12 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu);
(test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features))
int kvm_trng_call(struct kvm_vcpu *vcpu);
#ifdef CONFIG_KVM
extern phys_addr_t hyp_mem_base;
extern phys_addr_t hyp_mem_size;
void __init kvm_hyp_reserve(void);
#else
static inline void kvm_hyp_reserve(void) { }
#endif
#endif /* __ARM64_KVM_HOST_H__ */

10
arch/arm64/include/asm/kvm_hyp.h
View File

@ -90,6 +90,8 @@ void __debug_restore_host_buffers_nvhe(struct kvm_vcpu *vcpu);
void __fpsimd_save_state(struct user_fpsimd_state *fp_regs);
void __fpsimd_restore_state(struct user_fpsimd_state *fp_regs);
void __sve_save_state(void *sve_pffr, u32 *fpsr);
void __sve_restore_state(void *sve_pffr, u32 *fpsr);
#ifndef __KVM_NVHE_HYPERVISOR__
void activate_traps_vhe_load(struct kvm_vcpu *vcpu);
@ -106,4 +108,12 @@ void __noreturn __hyp_do_panic(struct kvm_cpu_context *host_ctxt, u64 spsr,
u64 elr, u64 par);
#endif
#ifdef __KVM_NVHE_HYPERVISOR__
void __pkvm_init_switch_pgd(phys_addr_t phys, unsigned long size,
phys_addr_t pgd, void *sp, void *cont_fn);
int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
unsigned long *per_cpu_base, u32 hyp_va_bits);
void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
#endif
#endif /* __ARM64_KVM_HYP_H__ */

23
arch/arm64/include/asm/kvm_mmu.h
View File

@ -166,7 +166,15 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu);
phys_addr_t kvm_mmu_get_httbr(void);
phys_addr_t kvm_get_idmap_vector(void);
int kvm_mmu_init(void);
int kvm_mmu_init(u32 *hyp_va_bits);
static inline void *__kvm_vector_slot2addr(void *base,
enum arm64_hyp_spectre_vector slot)
{
int idx = slot - (slot != HYP_VECTOR_DIRECT);
return base + (idx * SZ_2K);
}
struct kvm;
@ -262,9 +270,9 @@ static __always_inline u64 kvm_get_vttbr(struct kvm_s2_mmu *mmu)
* Must be called from hyp code running at EL2 with an updated VTTBR
* and interrupts disabled.
*/
static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu)
static __always_inline void __load_stage2(struct kvm_s2_mmu *mmu, unsigned long vtcr)
{
write_sysreg(kern_hyp_va(mmu->kvm)->arch.vtcr, vtcr_el2);
write_sysreg(vtcr, vtcr_el2);
write_sysreg(kvm_get_vttbr(mmu), vttbr_el2);
/*
@ -275,5 +283,14 @@ static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu)
asm(ALTERNATIVE("nop", "isb", ARM64_WORKAROUND_SPECULATIVE_AT));
}
static __always_inline void __load_guest_stage2(struct kvm_s2_mmu *mmu)
{
__load_stage2(mmu, kern_hyp_va(mmu->arch)->vtcr);
}
static inline struct kvm *kvm_s2_mmu_to_kvm(struct kvm_s2_mmu *mmu)
{
return container_of(mmu->arch, struct kvm, arch);
}
#endif /* __ASSEMBLY__ */
#endif /* __ARM64_KVM_MMU_H__ */

143
arch/arm64/include/asm/kvm_pgtable.h
View File

@ -11,22 +11,77 @@
#include <linux/kvm_host.h>
#include <linux/types.h>
#define KVM_PGTABLE_MAX_LEVELS 4U
static inline u64 kvm_get_parange(u64 mmfr0)
{
u64 parange = cpuid_feature_extract_unsigned_field(mmfr0,
ID_AA64MMFR0_PARANGE_SHIFT);
if (parange > ID_AA64MMFR0_PARANGE_MAX)
parange = ID_AA64MMFR0_PARANGE_MAX;
return parange;
}
typedef u64 kvm_pte_t;
/**
* struct kvm_pgtable_mm_ops - Memory management callbacks.
* @zalloc_page: Allocate a single zeroed memory page. The @arg parameter
* can be used by the walker to pass a memcache. The
* initial refcount of the page is 1.
* @zalloc_pages_exact: Allocate an exact number of zeroed memory pages. The
* @size parameter is in bytes, and is rounded-up to the
* next page boundary. The resulting allocation is
* physically contiguous.
* @free_pages_exact: Free an exact number of memory pages previously
* allocated by zalloc_pages_exact.
* @get_page: Increment the refcount on a page.
* @put_page: Decrement the refcount on a page. When the refcount
* reaches 0 the page is automatically freed.
* @page_count: Return the refcount of a page.
* @phys_to_virt: Convert a physical address into a virtual address mapped
* in the current context.
* @virt_to_phys: Convert a virtual address mapped in the current context
* into a physical address.
*/
struct kvm_pgtable_mm_ops {
void* (*zalloc_page)(void *arg);
void* (*zalloc_pages_exact)(size_t size);
void (*free_pages_exact)(void *addr, size_t size);
void (*get_page)(void *addr);
void (*put_page)(void *addr);
int (*page_count)(void *addr);
void* (*phys_to_virt)(phys_addr_t phys);
phys_addr_t (*virt_to_phys)(void *addr);
};
/**
* enum kvm_pgtable_stage2_flags - Stage-2 page-table flags.
* @KVM_PGTABLE_S2_NOFWB: Don't enforce Normal-WB even if the CPUs have
* ARM64_HAS_STAGE2_FWB.
*/
enum kvm_pgtable_stage2_flags {
KVM_PGTABLE_S2_NOFWB = BIT(0),
};
/**
* struct kvm_pgtable - KVM page-table.
* @ia_bits: Maximum input address size, in bits.
* @start_level: Level at which the page-table walk starts.
* @pgd: Pointer to the first top-level entry of the page-table.
* @mm_ops: Memory management callbacks.
* @mmu: Stage-2 KVM MMU struct. Unused for stage-1 page-tables.
*/
struct kvm_pgtable {
u32 ia_bits;
u32 start_level;
kvm_pte_t *pgd;
struct kvm_pgtable_mm_ops *mm_ops;
/* Stage-2 only */
struct kvm_s2_mmu *mmu;
enum kvm_pgtable_stage2_flags flags;
};
/**
@ -49,6 +104,16 @@ enum kvm_pgtable_prot {
#define PAGE_HYP_RO (KVM_PGTABLE_PROT_R)
#define PAGE_HYP_DEVICE (PAGE_HYP | KVM_PGTABLE_PROT_DEVICE)
/**
* struct kvm_mem_range - Range of Intermediate Physical Addresses
* @start: Start of the range.
* @end: End of the range.
*/
struct kvm_mem_range {
u64 start;
u64 end;
};
/**
* enum kvm_pgtable_walk_flags - Flags to control a depth-first page-table walk.
* @KVM_PGTABLE_WALK_LEAF: Visit leaf entries, including invalid
@ -86,10 +151,12 @@ struct kvm_pgtable_walker {
* kvm_pgtable_hyp_init() - Initialise a hypervisor stage-1 page-table.
* @pgt: Uninitialised page-table structure to initialise.
* @va_bits: Maximum virtual address bits.
* @mm_ops: Memory management callbacks.
*
* Return: 0 on success, negative error code on failure.
*/
int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits);
int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
struct kvm_pgtable_mm_ops *mm_ops);
/**
* kvm_pgtable_hyp_destroy() - Destroy an unused hypervisor stage-1 page-table.
@ -122,14 +189,38 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt);
int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
enum kvm_pgtable_prot prot);
/**
* kvm_get_vtcr() - Helper to construct VTCR_EL2
* @mmfr0: Sanitized value of SYS_ID_AA64MMFR0_EL1 register.
* @mmfr1: Sanitized value of SYS_ID_AA64MMFR1_EL1 register.
* @phys_shfit: Value to set in VTCR_EL2.T0SZ.
*
* The VTCR value is common across all the physical CPUs on the system.
* We use system wide sanitised values to fill in different fields,
* except for Hardware Management of Access Flags. HA Flag is set
* unconditionally on all CPUs, as it is safe to run with or without
* the feature and the bit is RES0 on CPUs that don't support it.
*
* Return: VTCR_EL2 value
*/
u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift);
/**
* kvm_pgtable_stage2_init() - Initialise a guest stage-2 page-table.
* @pgt: Uninitialised page-table structure to initialise.
* @kvm: KVM structure representing the guest virtual machine.
* @arch: Arch-specific KVM structure representing the guest virtual
* machine.
* @mm_ops: Memory management callbacks.
* @flags: Stage-2 configuration flags.
*
* Return: 0 on success, negative error code on failure.
*/
int kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm *kvm);
int kvm_pgtable_stage2_init_flags(struct kvm_pgtable *pgt, struct kvm_arch *arch,
struct kvm_pgtable_mm_ops *mm_ops,
enum kvm_pgtable_stage2_flags flags);
#define kvm_pgtable_stage2_init(pgt, arch, mm_ops) \
kvm_pgtable_stage2_init_flags(pgt, arch, mm_ops, 0)
/**
* kvm_pgtable_stage2_destroy() - Destroy an unused guest stage-2 page-table.
@ -147,8 +238,8 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
* @size: Size of the mapping.
* @phys: Physical address of the memory to map.
* @prot: Permissions and attributes for the mapping.
* @mc: Cache of pre-allocated GFP_PGTABLE_USER memory from which to
* allocate page-table pages.
* @mc: Cache of pre-allocated and zeroed memory from which to allocate
* page-table pages.
*
* The offset of @addr within a page is ignored, @size is rounded-up to
* the next page boundary and @phys is rounded-down to the previous page
@ -170,7 +261,28 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt);
*/
int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
u64 phys, enum kvm_pgtable_prot prot,
struct kvm_mmu_memory_cache *mc);
void *mc);
/**
* kvm_pgtable_stage2_set_owner() - Annotate invalid mappings with metadata
* encoding the ownership of a page in the
* IPA space.
* @pgt: Page-table structure initialised by kvm_pgtable_stage2_init().
* @addr: Base intermediate physical address to annotate.
* @size: Size of the annotated range.
* @mc: Cache of pre-allocated and zeroed memory from which to allocate
* page-table pages.
* @owner_id: Unique identifier for the owner of the page.
*
* By default, all page-tables are owned by identifier 0. This function can be
* used to mark portions of the IPA space as owned by other entities. When a
* stage 2 is used with identity-mappings, these annotations allow to use the
* page-table data structure as a simple rmap.
*
* Return: 0 on success, negative error code on failure.
*/
int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
void *mc, u8 owner_id);
/**
* kvm_pgtable_stage2_unmap() - Remove a mapping from a guest stage-2 page-table.
@ -311,4 +423,23 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size);
int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
struct kvm_pgtable_walker *walker);
/**
* kvm_pgtable_stage2_find_range() - Find a range of Intermediate Physical
* Addresses with compatible permission
* attributes.
* @pgt: Page-table structure initialised by kvm_pgtable_stage2_init().
* @addr: Address that must be covered by the range.
* @prot: Protection attributes that the range must be compatible with.
* @range: Range structure used to limit the search space at call time and
* that will hold the result.
*
* The offset of @addr within a page is ignored. An IPA is compatible with @prot
* iff its corresponding stage-2 page-table entry has default ownership and, if
* valid, is mapped with protection attributes identical to @prot.
*
* Return: 0 on success, negative error code on failure.
*/
int kvm_pgtable_stage2_find_range(struct kvm_pgtable *pgt, u64 addr,
enum kvm_pgtable_prot prot,
struct kvm_mem_range *range);
#endif /* __ARM64_KVM_PGTABLE_H__ */

4
arch/arm64/include/asm/pgtable-prot.h
View File

@ -71,10 +71,10 @@ extern bool arm64_use_ng_mappings;
#define PAGE_KERNEL_EXEC __pgprot(PROT_NORMAL & ~PTE_PXN)
#define PAGE_KERNEL_EXEC_CONT __pgprot((PROT_NORMAL & ~PTE_PXN) | PTE_CONT)
#define PAGE_S2_MEMATTR(attr) \
#define PAGE_S2_MEMATTR(attr, has_fwb) \
({ \
u64 __val; \
if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) \
if (has_fwb) \
__val = PTE_S2_MEMATTR(MT_S2_FWB_ ## attr); \
else \
__val = PTE_S2_MEMATTR(MT_S2_ ## attr); \

1
arch/arm64/include/asm/sections.h
View File

@ -13,6 +13,7 @@ extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
extern char __hyp_text_start[], __hyp_text_end[];
extern char __hyp_rodata_start[], __hyp_rodata_end[];
extern char __hyp_reloc_begin[], __hyp_reloc_end[];
extern char __hyp_bss_start[], __hyp_bss_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];

7
arch/arm64/include/asm/sysreg.h
View File

@ -624,9 +624,6 @@
#define SCTLR_ELx_A (BIT(1))
#define SCTLR_ELx_M (BIT(0))
#define SCTLR_ELx_FLAGS (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \
SCTLR_ELx_SA | SCTLR_ELx_I | SCTLR_ELx_IESB)
/* SCTLR_EL2 specific flags. */
#define SCTLR_EL2_RES1 ((BIT(4)) | (BIT(5)) | (BIT(11)) | (BIT(16)) | \
(BIT(18)) | (BIT(22)) | (BIT(23)) | (BIT(28)) | \
@ -638,6 +635,10 @@
#define ENDIAN_SET_EL2 0
#endif
#define INIT_SCTLR_EL2_MMU_ON \
(SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \
SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 | SCTLR_EL2_RES1)
#define INIT_SCTLR_EL2_MMU_OFF \
(SCTLR_EL2_RES1 | ENDIAN_SET_EL2)

3
arch/arm64/kernel/asm-offsets.c
View File

@ -121,6 +121,9 @@ int main(void)
DEFINE(NVHE_INIT_TPIDR_EL2, offsetof(struct kvm_nvhe_init_params, tpidr_el2));
DEFINE(NVHE_INIT_STACK_HYP_VA, offsetof(struct kvm_nvhe_init_params, stack_hyp_va));
DEFINE(NVHE_INIT_PGD_PA, offsetof(struct kvm_nvhe_init_params, pgd_pa));
DEFINE(NVHE_INIT_HCR_EL2, offsetof(struct kvm_nvhe_init_params, hcr_el2));
DEFINE(NVHE_INIT_VTTBR, offsetof(struct kvm_nvhe_init_params, vttbr));
DEFINE(NVHE_INIT_VTCR, offsetof(struct kvm_nvhe_init_params, vtcr));
#endif
#ifdef CONFIG_CPU_PM
DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));

5
arch/arm64/kernel/cpu-reset.S
View File

@ -30,10 +30,7 @@
* flat identity mapping.
*/
SYM_CODE_START(__cpu_soft_restart)
/* Clear sctlr_el1 flags. */
mrs x12, sctlr_el1
mov_q x13, SCTLR_ELx_FLAGS
bic x12, x12, x13
mov_q x12, INIT_SCTLR_EL1_MMU_OFF
pre_disable_mmu_workaround
/*
* either disable EL1&0 translation regime or disable EL2&0 translation

13
arch/arm64/kernel/cpufeature.c
View File

@ -1199,6 +1199,18 @@ u64 read_sanitised_ftr_reg(u32 id)
}
EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg);
int copy_ftr_reg(u32 id, struct arm64_ftr_reg *dst)
{
struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
if (!regp)
return -EINVAL;
*dst = *regp;
return 0;
}
#define read_sysreg_case(r) \
case r: val = read_sysreg_s(r); break;
@ -2930,6 +2942,7 @@ void __init setup_cpu_features(void)
setup_system_capabilities();
setup_elf_hwcaps(arm64_elf_hwcaps);
setup_kvm_el2_caps();
if (system_supports_32bit_el0())
setup_elf_hwcaps(compat_elf_hwcaps);

30
arch/arm64/kernel/image-vars.h
View File

@ -104,6 +104,36 @@ KVM_NVHE_ALIAS(kvm_arm_hyp_percpu_base);
/* PMU available static key */
KVM_NVHE_ALIAS(kvm_arm_pmu_available);
/* Position-independent library routines */
KVM_NVHE_ALIAS_HYP(clear_page, __pi_clear_page);
KVM_NVHE_ALIAS_HYP(copy_page, __pi_copy_page);
KVM_NVHE_ALIAS_HYP(memcpy, __pi_memcpy);
KVM_NVHE_ALIAS_HYP(memset, __pi_memset);
#ifdef CONFIG_KASAN
KVM_NVHE_ALIAS_HYP(__memcpy, __pi_memcpy);
KVM_NVHE_ALIAS_HYP(__memset, __pi_memset);
#endif
/* Kernel memory sections */
KVM_NVHE_ALIAS(__start_rodata);
KVM_NVHE_ALIAS(__end_rodata);
KVM_NVHE_ALIAS(__bss_start);
KVM_NVHE_ALIAS(__bss_stop);
/* Hyp memory sections */
KVM_NVHE_ALIAS(__hyp_idmap_text_start);
KVM_NVHE_ALIAS(__hyp_idmap_text_end);
KVM_NVHE_ALIAS(__hyp_text_start);
KVM_NVHE_ALIAS(__hyp_text_end);
KVM_NVHE_ALIAS(__hyp_bss_start);
KVM_NVHE_ALIAS(__hyp_bss_end);
KVM_NVHE_ALIAS(__hyp_rodata_start);
KVM_NVHE_ALIAS(__hyp_rodata_end);
/* pKVM static key */
KVM_NVHE_ALIAS(kvm_protected_mode_initialized);
#endif /* CONFIG_KVM */
#endif /* __ARM64_KERNEL_IMAGE_VARS_H */

6
arch/arm64/kernel/traps.c
View File

@ -46,6 +46,8 @@
#include <asm/system_misc.h>
#include <asm/sysreg.h>
#include <trace/hooks/traps.h>
static const char *handler[]= {
"Synchronous Abort",
"IRQ",
@ -404,6 +406,7 @@ void do_undefinstr(struct pt_regs *regs)
if (call_undef_hook(regs) == 0)
return;
trace_android_rvh_do_undefinstr(regs, user_mode(regs));
BUG_ON(!user_mode(regs));
force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0);
}
@ -765,6 +768,7 @@ asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int
handler[reason], smp_processor_id(), esr,
esr_get_class_string(esr));
trace_android_rvh_bad_mode(regs, esr, reason);
__show_regs(regs);
local_daif_mask();
panic("bad mode");
@ -830,6 +834,8 @@ void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
smp_processor_id(), esr, esr_get_class_string(esr));
trace_android_rvh_arm64_serror_panic(regs, esr);
if (regs)
__show_regs(regs);

74
arch/arm64/kernel/vmlinux.lds.S
View File

@ -5,24 +5,7 @@
* Written by Martin Mares <mj@atrey.karlin.mff.cuni.cz>
*/
#define RO_EXCEPTION_TABLE_ALIGN 8
#define RUNTIME_DISCARD_EXIT
#include <asm-generic/vmlinux.lds.h>
#include <asm/cache.h>
#include <asm/hyp_image.h>
#include <asm/kernel-pgtable.h>
#include <asm/memory.h>
#include <asm/page.h>
#include "image.h"
OUTPUT_ARCH(aarch64)
ENTRY(_text)
jiffies = jiffies_64;
#ifdef CONFIG_KVM
#define HYPERVISOR_EXTABLE \
. = ALIGN(SZ_8); \
@ -32,9 +15,11 @@ jiffies = jiffies_64;
#define HYPERVISOR_DATA_SECTIONS \
HYP_SECTION_NAME(.rodata) : { \
. = ALIGN(PAGE_SIZE); \
__hyp_rodata_start = .; \
*(HYP_SECTION_NAME(.data..ro_after_init)) \
*(HYP_SECTION_NAME(.rodata)) \
. = ALIGN(PAGE_SIZE); \
__hyp_rodata_end = .; \
}
@ -51,29 +36,52 @@ jiffies = jiffies_64;
__hyp_reloc_end = .; \
}
#define BSS_FIRST_SECTIONS \
__hyp_bss_start = .; \
*(HYP_SECTION_NAME(.bss)) \
. = ALIGN(PAGE_SIZE); \
__hyp_bss_end = .;
/*
* We require that __hyp_bss_start and __bss_start are aligned, and enforce it
* with an assertion. But the BSS_SECTION macro places an empty .sbss section
* between them, which can in some cases cause the linker to misalign them. To
* work around the issue, force a page alignment for __bss_start.
*/
#define SBSS_ALIGN PAGE_SIZE
#else /* CONFIG_KVM */
#define HYPERVISOR_EXTABLE
#define HYPERVISOR_DATA_SECTIONS
#define HYPERVISOR_PERCPU_SECTION
#define HYPERVISOR_RELOC_SECTION
#define SBSS_ALIGN 0
#endif
#define RO_EXCEPTION_TABLE_ALIGN 8
#define RUNTIME_DISCARD_EXIT
#include <asm-generic/vmlinux.lds.h>
#include <asm/cache.h>
#include <asm/kernel-pgtable.h>
#include <asm/memory.h>
#include <asm/page.h>
#include "image.h"
OUTPUT_ARCH(aarch64)
ENTRY(_text)
jiffies = jiffies_64;
#define HYPERVISOR_TEXT \
/* \
* Align to 4 KB so that \
* a) the HYP vector table is at its minimum \
* alignment of 2048 bytes \
* b) the HYP init code will not cross a page \
* boundary if its size does not exceed \
* 4 KB (see related ASSERT() below) \
*/ \
. = ALIGN(SZ_4K); \
. = ALIGN(PAGE_SIZE); \
__hyp_idmap_text_start = .; \
*(.hyp.idmap.text) \
__hyp_idmap_text_end = .; \
__hyp_text_start = .; \
*(.hyp.text) \
HYPERVISOR_EXTABLE \
. = ALIGN(PAGE_SIZE); \
__hyp_text_end = .;
#define IDMAP_TEXT \
@ -278,7 +286,7 @@ SECTIONS
__pecoff_data_rawsize = ABSOLUTE(. - __initdata_begin);
_edata = .;
BSS_SECTION(0, 0, 0)
BSS_SECTION(SBSS_ALIGN, 0, 0)
. = ALIGN(PAGE_SIZE);
init_pg_dir = .;
@ -311,11 +319,12 @@ SECTIONS
#include "image-vars.h"
/*
* The HYP init code and ID map text can't be longer than a page each,
* and should not cross a page boundary.
* The HYP init code and ID map text can't be longer than a page each. The
* former is page-aligned, but the latter may not be with 16K or 64K pages, so
* it should also not cross a page boundary.
*/
ASSERT(__hyp_idmap_text_end - (__hyp_idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
"HYP init code too big or misaligned")
ASSERT(__hyp_idmap_text_end - __hyp_idmap_text_start <= PAGE_SIZE,
"HYP init code too big")
ASSERT(__idmap_text_end - (__idmap_text_start & ~(SZ_4K - 1)) <= SZ_4K,
"ID map text too big or misaligned")
#ifdef CONFIG_HIBERNATION
@ -326,6 +335,9 @@ ASSERT(__hibernate_exit_text_end - (__hibernate_exit_text_start & ~(SZ_4K - 1))
ASSERT((__entry_tramp_text_end - __entry_tramp_text_start) == PAGE_SIZE,
"Entry trampoline text too big")
#endif
#ifdef CONFIG_KVM
ASSERT(__hyp_bss_start == __bss_start, "HYP and Host BSS are misaligned")
#endif
/*
* If padding is applied before .head.text, virt<->phys conversions will fail.
*/

204
arch/arm64/kvm/arm.c
View File

@ -1359,16 +1359,9 @@ static unsigned long nvhe_percpu_order(void)
/* A lookup table holding the hypervisor VA for each vector slot */
static void *hyp_spectre_vector_selector[BP_HARDEN_EL2_SLOTS];
static int __kvm_vector_slot2idx(enum arm64_hyp_spectre_vector slot)
{
return slot - (slot != HYP_VECTOR_DIRECT);
}
static void kvm_init_vector_slot(void *base, enum arm64_hyp_spectre_vector slot)
{
int idx = __kvm_vector_slot2idx(slot);
hyp_spectre_vector_selector[slot] = base + (idx * SZ_2K);
hyp_spectre_vector_selector[slot] = __kvm_vector_slot2addr(base, slot);
}
static int kvm_init_vector_slots(void)
@ -1397,22 +1390,18 @@ static int kvm_init_vector_slots(void)
return 0;
}
static void cpu_init_hyp_mode(void)
static void cpu_prepare_hyp_mode(int cpu)
{
struct kvm_nvhe_init_params *params = this_cpu_ptr_nvhe_sym(kvm_init_params);
struct arm_smccc_res res;
struct kvm_nvhe_init_params *params = per_cpu_ptr_nvhe_sym(kvm_init_params, cpu);
unsigned long tcr;
/* Switch from the HYP stub to our own HYP init vector */
__hyp_set_vectors(kvm_get_idmap_vector());
/*
* Calculate the raw per-cpu offset without a translation from the
* kernel's mapping to the linear mapping, and store it in tpidr_el2
* so that we can use adr_l to access per-cpu variables in EL2.
* Also drop the KASAN tag which gets in the way...
*/
params->tpidr_el2 = (unsigned long)kasan_reset_tag(this_cpu_ptr_nvhe_sym(__per_cpu_start)) -
params->tpidr_el2 = (unsigned long)kasan_reset_tag(per_cpu_ptr_nvhe_sym(__per_cpu_start, cpu)) -
(unsigned long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
params->mair_el2 = read_sysreg(mair_el1);
@ -1436,14 +1425,28 @@ static void cpu_init_hyp_mode(void)
tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
params->tcr_el2 = tcr;
params->stack_hyp_va = kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE);
params->stack_hyp_va = kern_hyp_va(per_cpu(kvm_arm_hyp_stack_page, cpu) + PAGE_SIZE);
params->pgd_pa = kvm_mmu_get_httbr();
if (is_protected_kvm_enabled())
params->hcr_el2 = HCR_HOST_NVHE_PROTECTED_FLAGS;
else
params->hcr_el2 = HCR_HOST_NVHE_FLAGS;
params->vttbr = params->vtcr = 0;
/*
* Flush the init params from the data cache because the struct will
* be read while the MMU is off.
*/
kvm_flush_dcache_to_poc(params, sizeof(*params));
}
static void hyp_install_host_vector(void)
{
struct kvm_nvhe_init_params *params;
struct arm_smccc_res res;
/* Switch from the HYP stub to our own HYP init vector */
__hyp_set_vectors(kvm_get_idmap_vector());
/*
* Call initialization code, and switch to the full blown HYP code.
@ -1452,8 +1455,14 @@ static void cpu_init_hyp_mode(void)
* cpus_have_const_cap() wrapper.
*/
BUG_ON(!system_capabilities_finalized());
params = this_cpu_ptr_nvhe_sym(kvm_init_params);
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init), virt_to_phys(params), &res);
WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
}
static void cpu_init_hyp_mode(void)
{
hyp_install_host_vector();
/*
* Disabling SSBD on a non-VHE system requires us to enable SSBS
@ -1496,7 +1505,10 @@ static void cpu_set_hyp_vector(void)
struct bp_hardening_data *data = this_cpu_ptr(&bp_hardening_data);
void *vector = hyp_spectre_vector_selector[data->slot];
*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
if (!is_protected_kvm_enabled())
*this_cpu_ptr_hyp_sym(kvm_hyp_vector) = (unsigned long)vector;
else
kvm_call_hyp_nvhe(__pkvm_cpu_set_vector, data->slot);
}
static void cpu_hyp_reinit(void)
@ -1504,13 +1516,14 @@ static void cpu_hyp_reinit(void)
kvm_init_host_cpu_context(&this_cpu_ptr_hyp_sym(kvm_host_data)->host_ctxt);
cpu_hyp_reset();
cpu_set_hyp_vector();
if (is_kernel_in_hyp_mode())
kvm_timer_init_vhe();
else
cpu_init_hyp_mode();
cpu_set_hyp_vector();
kvm_arm_init_debug();
if (vgic_present)
@ -1706,18 +1719,59 @@ static void teardown_hyp_mode(void)
}
}
static int do_pkvm_init(u32 hyp_va_bits)
{
void *per_cpu_base = kvm_ksym_ref(kvm_arm_hyp_percpu_base);
int ret;
preempt_disable();
hyp_install_host_vector();
ret = kvm_call_hyp_nvhe(__pkvm_init, hyp_mem_base, hyp_mem_size,
num_possible_cpus(), kern_hyp_va(per_cpu_base),
hyp_va_bits);
preempt_enable();
return ret;
}
static int kvm_hyp_init_protection(u32 hyp_va_bits)
{
void *addr = phys_to_virt(hyp_mem_base);
int ret;
ret = create_hyp_mappings(addr, addr + hyp_mem_size, PAGE_HYP);
if (ret)
return ret;
ret = do_pkvm_init(hyp_va_bits);
if (ret)
return ret;
free_hyp_pgds();
return 0;
}
/**
* Inits Hyp-mode on all online CPUs
*/
static int init_hyp_mode(void)
{
u32 hyp_va_bits;
int cpu;
int err = 0;
int err = -ENOMEM;
/*
* The protected Hyp-mode cannot be initialized if the memory pool
* allocation has failed.
*/
if (is_protected_kvm_enabled() && !hyp_mem_base)
goto out_err;
/*
* Allocate Hyp PGD and setup Hyp identity mapping
*/
err = kvm_mmu_init();
err = kvm_mmu_init(&hyp_va_bits);
if (err)
goto out_err;
@ -1778,7 +1832,19 @@ static int init_hyp_mode(void)
goto out_err;
}
err = create_hyp_mappings(kvm_ksym_ref(__bss_start),
/*
* .hyp.bss is guaranteed to be placed at the beginning of the .bss
* section thanks to an assertion in the linker script. Map it RW and
* the rest of .bss RO.
*/
err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_start),
kvm_ksym_ref(__hyp_bss_end), PAGE_HYP);
if (err) {
kvm_err("Cannot map hyp bss section: %d\n", err);
goto out_err;
}
err = create_hyp_mappings(kvm_ksym_ref(__hyp_bss_end),
kvm_ksym_ref(__bss_stop), PAGE_HYP_RO);
if (err) {
kvm_err("Cannot map bss section\n");
@ -1799,19 +1865,19 @@ static int init_hyp_mode(void)
}
}
/*
* Map Hyp percpu pages
*/
for_each_possible_cpu(cpu) {
char *percpu_begin = (char *)kvm_arm_hyp_percpu_base[cpu];
char *percpu_end = percpu_begin + nvhe_percpu_size();
/* Map Hyp percpu pages */
err = create_hyp_mappings(percpu_begin, percpu_end, PAGE_HYP);
if (err) {
kvm_err("Cannot map hyp percpu region\n");
goto out_err;
}
/* Prepare the CPU initialization parameters */
cpu_prepare_hyp_mode(cpu);
}
if (is_protected_kvm_enabled()) {
@ -1821,6 +1887,14 @@ static int init_hyp_mode(void)
goto out_err;
}
if (is_protected_kvm_enabled()) {
err = kvm_hyp_init_protection(hyp_va_bits);
if (err) {
kvm_err("Failed to init hyp memory protection\n");
goto out_err;
}
}
return 0;
out_err:
@ -1829,6 +1903,72 @@ out_err:
return err;
}
void _kvm_host_prot_finalize(void *discard)
{
WARN_ON(kvm_call_hyp_nvhe(__pkvm_prot_finalize));
}
static inline int pkvm_mark_hyp(phys_addr_t start, phys_addr_t end)
{
return kvm_call_hyp_nvhe(__pkvm_mark_hyp, start, end);
}
#define pkvm_mark_hyp_section(__section) \
pkvm_mark_hyp(__pa_symbol(__section##_start), \
__pa_symbol(__section##_end))
static int finalize_hyp_mode(void)
{
int cpu, ret;
if (!is_protected_kvm_enabled())
return 0;
ret = pkvm_mark_hyp_section(__hyp_idmap_text);
if (ret)
return ret;
ret = pkvm_mark_hyp_section(__hyp_text);
if (ret)
return ret;
ret = pkvm_mark_hyp_section(__hyp_rodata);
if (ret)
return ret;
ret = pkvm_mark_hyp_section(__hyp_bss);
if (ret)
return ret;
ret = pkvm_mark_hyp(hyp_mem_base, hyp_mem_base + hyp_mem_size);
if (ret)
return ret;
for_each_possible_cpu(cpu) {
phys_addr_t start = virt_to_phys((void *)kvm_arm_hyp_percpu_base[cpu]);
phys_addr_t end = start + (PAGE_SIZE << nvhe_percpu_order());
ret = pkvm_mark_hyp(start, end);
if (ret)
return ret;
start = virt_to_phys((void *)per_cpu(kvm_arm_hyp_stack_page, cpu));
end = start + PAGE_SIZE;
ret = pkvm_mark_hyp(start, end);
if (ret)
return ret;
}
/*
* Flip the static key upfront as that may no longer be possible
* once the host stage 2 is installed.
*/
static_branch_enable(&kvm_protected_mode_initialized);
on_each_cpu(_kvm_host_prot_finalize, NULL, 1);
return 0;
}
static void check_kvm_target_cpu(void *ret)
{
*(int *)ret = kvm_target_cpu();
@ -1903,11 +2043,6 @@ int kvm_arch_init(void *opaque)
in_hyp_mode = is_kernel_in_hyp_mode();
if (!in_hyp_mode && kvm_arch_requires_vhe()) {
kvm_pr_unimpl("CPU unsupported in non-VHE mode, not initializing\n");
return -ENODEV;
}
if (cpus_have_final_cap(ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) ||
cpus_have_final_cap(ARM64_WORKAROUND_1508412))
kvm_info("Guests without required CPU erratum workarounds can deadlock system!\n" \
@ -1945,8 +2080,15 @@ int kvm_arch_init(void *opaque)
if (err)
goto out_hyp;
if (!in_hyp_mode) {
err = finalize_hyp_mode();
if (err) {
kvm_err("Failed to finalize Hyp protection\n");
goto out_hyp;
}
}
if (is_protected_kvm_enabled()) {
static_branch_enable(&kvm_protected_mode_initialized);
kvm_info("Protected nVHE mode initialized successfully\n");
} else if (in_hyp_mode) {
kvm_info("VHE mode initialized successfully\n");

26
arch/arm64/kvm/fpsimd.c
View File

@ -11,6 +11,7 @@
#include <linux/kvm_host.h>
#include <asm/fpsimd.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/sysreg.h>
@ -42,6 +43,17 @@ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu)
if (ret)
goto error;
if (vcpu->arch.sve_state) {
void *sve_end;
sve_end = vcpu->arch.sve_state + vcpu_sve_state_size(vcpu);
ret = create_hyp_mappings(vcpu->arch.sve_state, sve_end,
PAGE_HYP);
if (ret)
goto error;
}
vcpu->arch.host_thread_info = kern_hyp_va(ti);
vcpu->arch.host_fpsimd_state = kern_hyp_va(fpsimd);
error:
@ -109,11 +121,17 @@ void kvm_arch_vcpu_put_fp(struct kvm_vcpu *vcpu)
local_irq_save(flags);
if (vcpu->arch.flags & KVM_ARM64_FP_ENABLED) {
fpsimd_save_and_flush_cpu_state();
if (guest_has_sve) {
__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_el1(SYS_ZCR);
if (guest_has_sve)
__vcpu_sys_reg(vcpu, ZCR_EL1) = read_sysreg_s(SYS_ZCR_EL12);
} else if (host_has_sve) {
/* Restore the VL that was saved when bound to the CPU */
if (!has_vhe())
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1,
SYS_ZCR_EL1);
}
fpsimd_save_and_flush_cpu_state();
} else if (has_vhe() && host_has_sve) {
/*
* The FPSIMD/SVE state in the CPU has not been touched, and we
* have SVE (and VHE): CPACR_EL1 (alias CPTR_EL2) has been

6
arch/arm64/kvm/guest.c
View File

@ -299,7 +299,7 @@ static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
memset(vqs, 0, sizeof(vqs));
max_vq = sve_vq_from_vl(vcpu->arch.sve_max_vl);
max_vq = vcpu_sve_max_vq(vcpu);
for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
if (sve_vq_available(vq))
vqs[vq_word(vq)] |= vq_mask(vq);
@ -427,7 +427,7 @@ static int sve_reg_to_region(struct sve_state_reg_region *region,
if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
return -ENOENT;
vq = sve_vq_from_vl(vcpu->arch.sve_max_vl);
vq = vcpu_sve_max_vq(vcpu);
reqoffset = SVE_SIG_ZREG_OFFSET(vq, reg_num) -
SVE_SIG_REGS_OFFSET;
@ -437,7 +437,7 @@ static int sve_reg_to_region(struct sve_state_reg_region *region,
if (!vcpu_has_sve(vcpu) || (reg->id & SVE_REG_SLICE_MASK) > 0)
return -ENOENT;
vq = sve_vq_from_vl(vcpu->arch.sve_max_vl);
vq = vcpu_sve_max_vq(vcpu);
reqoffset = SVE_SIG_PREG_OFFSET(vq, reg_num) -
SVE_SIG_REGS_OFFSET;

2
arch/arm64/kvm/hyp/Makefile
View File

@ -10,4 +10,4 @@ subdir-ccflags-y := -I$(incdir) \
-DDISABLE_BRANCH_PROFILING \
$(DISABLE_STACKLEAK_PLUGIN)
obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o
obj-$(CONFIG_KVM) += vhe/ nvhe/ pgtable.o reserved_mem.o

10
arch/arm64/kvm/hyp/fpsimd.S
View File

@ -19,3 +19,13 @@ SYM_FUNC_START(__fpsimd_restore_state)
fpsimd_restore x0, 1
ret
SYM_FUNC_END(__fpsimd_restore_state)
SYM_FUNC_START(__sve_restore_state)
__sve_load 0, x1, 2
ret
SYM_FUNC_END(__sve_restore_state)
SYM_FUNC_START(__sve_save_state)
sve_save 0, x1, 2
ret
SYM_FUNC_END(__sve_save_state)

111
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -160,19 +160,9 @@ static inline bool __translate_far_to_hpfar(u64 far, u64 *hpfar)
return true;
}
static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
static inline bool __get_fault_info(u64 esr, struct kvm_vcpu_fault_info *fault)
{
u8 ec;
u64 esr;
u64 hpfar, far;
esr = vcpu->arch.fault.esr_el2;
ec = ESR_ELx_EC(esr);
if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
return true;
far = read_sysreg_el2(SYS_FAR);
fault->far_el2 = read_sysreg_el2(SYS_FAR);
/*
* The HPFAR can be invalid if the stage 2 fault did not
@ -188,39 +178,63 @@ static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
if (!(esr & ESR_ELx_S1PTW) &&
(cpus_have_final_cap(ARM64_WORKAROUND_834220) ||
(esr & ESR_ELx_FSC_TYPE) == FSC_PERM)) {
if (!__translate_far_to_hpfar(far, &hpfar))
if (!__translate_far_to_hpfar(fault->far_el2, &fault->hpfar_el2))
return false;
} else {
hpfar = read_sysreg(hpfar_el2);
fault->hpfar_el2 = read_sysreg(hpfar_el2);
}
vcpu->arch.fault.far_el2 = far;
vcpu->arch.fault.hpfar_el2 = hpfar;
return true;
}
static inline bool __populate_fault_info(struct kvm_vcpu *vcpu)
{
u8 ec;
u64 esr;
esr = vcpu->arch.fault.esr_el2;
ec = ESR_ELx_EC(esr);
if (ec != ESR_ELx_EC_DABT_LOW && ec != ESR_ELx_EC_IABT_LOW)
return true;
return __get_fault_info(esr, &vcpu->arch.fault);
}
static inline void __hyp_sve_save_host(struct kvm_vcpu *vcpu)
{
struct thread_struct *thread;
thread = container_of(vcpu->arch.host_fpsimd_state, struct thread_struct,
uw.fpsimd_state);
__sve_save_state(sve_pffr(thread), &vcpu->arch.host_fpsimd_state->fpsr);
}
static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
{
sve_cond_update_zcr_vq(vcpu_sve_max_vq(vcpu) - 1, SYS_ZCR_EL2);
__sve_restore_state(vcpu_sve_pffr(vcpu),
&vcpu->arch.ctxt.fp_regs.fpsr);
write_sysreg_el1(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR);
}
/* Check for an FPSIMD/SVE trap and handle as appropriate */
static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)
{
bool vhe, sve_guest, sve_host;
bool sve_guest, sve_host;
u8 esr_ec;
u64 reg;
if (!system_supports_fpsimd())
return false;
/*
* Currently system_supports_sve() currently implies has_vhe(),
* so the check is redundant. However, has_vhe() can be determined
* statically and helps the compiler remove dead code.
*/
if (has_vhe() && system_supports_sve()) {
if (system_supports_sve()) {
sve_guest = vcpu_has_sve(vcpu);
sve_host = vcpu->arch.flags & KVM_ARM64_HOST_SVE_IN_USE;
vhe = true;
} else {
sve_guest = false;
sve_host = false;
vhe = has_vhe();
}
esr_ec = kvm_vcpu_trap_get_class(vcpu);
@ -229,53 +243,38 @@ static inline bool __hyp_handle_fpsimd(struct kvm_vcpu *vcpu)
return false;
/* Don't handle SVE traps for non-SVE vcpus here: */
if (!sve_guest)
if (esr_ec != ESR_ELx_EC_FP_ASIMD)
return false;
if (!sve_guest && esr_ec != ESR_ELx_EC_FP_ASIMD)
return false;
/* Valid trap. Switch the context: */
if (vhe) {
u64 reg = read_sysreg(cpacr_el1) | CPACR_EL1_FPEN;
if (has_vhe()) {
reg = CPACR_EL1_FPEN;
if (sve_guest)
reg |= CPACR_EL1_ZEN;
write_sysreg(reg, cpacr_el1);
sysreg_clear_set(cpacr_el1, 0, reg);
} else {
write_sysreg(read_sysreg(cptr_el2) & ~(u64)CPTR_EL2_TFP,
cptr_el2);
}
reg = CPTR_EL2_TFP;
if (sve_guest)
reg |= CPTR_EL2_TZ;
sysreg_clear_set(cptr_el2, reg, 0);
}
isb();
if (vcpu->arch.flags & KVM_ARM64_FP_HOST) {
/*
* In the SVE case, VHE is assumed: it is enforced by
* Kconfig and kvm_arch_init().
*/
if (sve_host) {
struct thread_struct *thread = container_of(
vcpu->arch.host_fpsimd_state,
struct thread_struct, uw.fpsimd_state);
sve_save_state(sve_pffr(thread),
&vcpu->arch.host_fpsimd_state->fpsr);
} else {
if (sve_host)
__hyp_sve_save_host(vcpu);
else
__fpsimd_save_state(vcpu->arch.host_fpsimd_state);
}
vcpu->arch.flags &= ~KVM_ARM64_FP_HOST;
}
if (sve_guest) {
sve_load_state(vcpu_sve_pffr(vcpu),
&vcpu->arch.ctxt.fp_regs.fpsr,
sve_vq_from_vl(vcpu->arch.sve_max_vl) - 1);
write_sysreg_s(__vcpu_sys_reg(vcpu, ZCR_EL1), SYS_ZCR_EL12);
} else {
if (sve_guest)
__hyp_sve_restore_guest(vcpu);
else
__fpsimd_restore_state(&vcpu->arch.ctxt.fp_regs);
}
/* Skip restoring fpexc32 for AArch64 guests */
if (!(read_sysreg(hcr_el2) & HCR_RW))

14
arch/arm64/kvm/hyp/include/nvhe/early_alloc.h Normal file
View File

@ -0,0 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __KVM_HYP_EARLY_ALLOC_H
#define __KVM_HYP_EARLY_ALLOC_H
#include <asm/kvm_pgtable.h>
void hyp_early_alloc_init(void *virt, unsigned long size);
unsigned long hyp_early_alloc_nr_used_pages(void);
void *hyp_early_alloc_page(void *arg);
void *hyp_early_alloc_contig(unsigned int nr_pages);
extern struct kvm_pgtable_mm_ops hyp_early_alloc_mm_ops;
#endif /* __KVM_HYP_EARLY_ALLOC_H */

68
arch/arm64/kvm/hyp/include/nvhe/gfp.h Normal file
View File

@ -0,0 +1,68 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __KVM_HYP_GFP_H
#define __KVM_HYP_GFP_H
#include <linux/list.h>
#include <nvhe/memory.h>
#include <nvhe/spinlock.h>
#define HYP_NO_ORDER UINT_MAX
struct hyp_pool {
/*
* Spinlock protecting concurrent changes to the memory pool as well as
* the struct hyp_page of the pool's pages until we have a proper atomic
* API at EL2.
*/
hyp_spinlock_t lock;
struct list_head free_area[MAX_ORDER];
phys_addr_t range_start;
phys_addr_t range_end;
unsigned int max_order;
};
static inline void hyp_page_ref_inc(struct hyp_page *p)
{
struct hyp_pool *pool = hyp_page_to_pool(p);
hyp_spin_lock(&pool->lock);
p->refcount++;
hyp_spin_unlock(&pool->lock);
}
static inline int hyp_page_ref_dec_and_test(struct hyp_page *p)
{
struct hyp_pool *pool = hyp_page_to_pool(p);
int ret;
hyp_spin_lock(&pool->lock);
p->refcount--;
ret = (p->refcount == 0);
hyp_spin_unlock(&pool->lock);
return ret;
}
static inline void hyp_set_page_refcounted(struct hyp_page *p)
{
struct hyp_pool *pool = hyp_page_to_pool(p);
hyp_spin_lock(&pool->lock);
if (p->refcount) {
hyp_spin_unlock(&pool->lock);
hyp_panic();
}
p->refcount = 1;
hyp_spin_unlock(&pool->lock);
}
/* Allocation */
void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order);
void hyp_get_page(void *addr);
void hyp_put_page(void *addr);
/* Used pages cannot be freed */
int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages,
unsigned int reserved_pages);
#endif /* __KVM_HYP_GFP_H */

36
arch/arm64/kvm/hyp/include/nvhe/mem_protect.h Normal file
View File

@ -0,0 +1,36 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020 Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#ifndef __KVM_NVHE_MEM_PROTECT__
#define __KVM_NVHE_MEM_PROTECT__
#include <linux/kvm_host.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_pgtable.h>
#include <asm/virt.h>
#include <nvhe/spinlock.h>
struct host_kvm {
struct kvm_arch arch;
struct kvm_pgtable pgt;
struct kvm_pgtable_mm_ops mm_ops;
hyp_spinlock_t lock;
};
extern struct host_kvm host_kvm;
int __pkvm_prot_finalize(void);
int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end);
int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool);
void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt);
static __always_inline void __load_host_stage2(void)
{
if (static_branch_likely(&kvm_protected_mode_initialized))
__load_stage2(&host_kvm.arch.mmu, host_kvm.arch.vtcr);
else
write_sysreg(0, vttbr_el2);
}
#endif /* __KVM_NVHE_MEM_PROTECT__ */

52
arch/arm64/kvm/hyp/include/nvhe/memory.h Normal file
View File

@ -0,0 +1,52 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __KVM_HYP_MEMORY_H
#define __KVM_HYP_MEMORY_H
#include <asm/page.h>
#include <linux/types.h>
struct hyp_pool;
struct hyp_page {
unsigned int refcount;
unsigned int order;
struct hyp_pool *pool;
struct list_head node;
};
extern s64 hyp_physvirt_offset;
extern u64 __hyp_vmemmap;
#define hyp_vmemmap ((struct hyp_page *)__hyp_vmemmap)
#define __hyp_pa(virt) ((phys_addr_t)(virt) + hyp_physvirt_offset)
#define __hyp_va(phys) ((void *)((phys_addr_t)(phys) - hyp_physvirt_offset))
static inline void *hyp_phys_to_virt(phys_addr_t phys)
{
return __hyp_va(phys);
}
static inline phys_addr_t hyp_virt_to_phys(void *addr)
{
return __hyp_pa(addr);
}
#define hyp_phys_to_pfn(phys) ((phys) >> PAGE_SHIFT)
#define hyp_pfn_to_phys(pfn) ((phys_addr_t)((pfn) << PAGE_SHIFT))
#define hyp_phys_to_page(phys) (&hyp_vmemmap[hyp_phys_to_pfn(phys)])
#define hyp_virt_to_page(virt) hyp_phys_to_page(__hyp_pa(virt))
#define hyp_virt_to_pfn(virt) hyp_phys_to_pfn(__hyp_pa(virt))
#define hyp_page_to_pfn(page) ((struct hyp_page *)(page) - hyp_vmemmap)
#define hyp_page_to_phys(page) hyp_pfn_to_phys((hyp_page_to_pfn(page)))
#define hyp_page_to_virt(page) __hyp_va(hyp_page_to_phys(page))
#define hyp_page_to_pool(page) (((struct hyp_page *)page)->pool)
static inline int hyp_page_count(void *addr)
{
struct hyp_page *p = hyp_virt_to_page(addr);
return p->refcount;
}
#endif /* __KVM_HYP_MEMORY_H */

96
arch/arm64/kvm/hyp/include/nvhe/mm.h Normal file
View File

@ -0,0 +1,96 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef __KVM_HYP_MM_H
#define __KVM_HYP_MM_H
#include <asm/kvm_pgtable.h>
#include <asm/spectre.h>
#include <linux/memblock.h>
#include <linux/types.h>
#include <nvhe/memory.h>
#include <nvhe/spinlock.h>
#define HYP_MEMBLOCK_REGIONS 128
extern struct memblock_region kvm_nvhe_sym(hyp_memory)[];
extern unsigned int kvm_nvhe_sym(hyp_memblock_nr);
extern struct kvm_pgtable pkvm_pgtable;
extern hyp_spinlock_t pkvm_pgd_lock;
extern struct hyp_pool hpool;
extern u64 __io_map_base;
int hyp_create_idmap(u32 hyp_va_bits);
int hyp_map_vectors(void);
int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back);
int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot);
int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot);
int __pkvm_create_mappings(unsigned long start, unsigned long size,
unsigned long phys, enum kvm_pgtable_prot prot);
unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
enum kvm_pgtable_prot prot);
static inline void hyp_vmemmap_range(phys_addr_t phys, unsigned long size,
unsigned long *start, unsigned long *end)
{
unsigned long nr_pages = size >> PAGE_SHIFT;
struct hyp_page *p = hyp_phys_to_page(phys);
*start = (unsigned long)p;
*end = *start + nr_pages * sizeof(struct hyp_page);
*start = ALIGN_DOWN(*start, PAGE_SIZE);
*end = ALIGN(*end, PAGE_SIZE);
}
static inline unsigned long __hyp_pgtable_max_pages(unsigned long nr_pages)
{
unsigned long total = 0, i;
/* Provision the worst case scenario */
for (i = 0; i < KVM_PGTABLE_MAX_LEVELS; i++) {
nr_pages = DIV_ROUND_UP(nr_pages, PTRS_PER_PTE);
total += nr_pages;
}
return total;
}
static inline unsigned long __hyp_pgtable_total_pages(void)
{
unsigned long res = 0, i;
/* Cover all of memory with page-granularity */
for (i = 0; i < kvm_nvhe_sym(hyp_memblock_nr); i++) {
struct memblock_region *reg = &kvm_nvhe_sym(hyp_memory)[i];
res += __hyp_pgtable_max_pages(reg->size >> PAGE_SHIFT);
}
return res;
}
static inline unsigned long hyp_s1_pgtable_pages(void)
{
unsigned long res;
res = __hyp_pgtable_total_pages();
/* Allow 1 GiB for private mappings */
res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
return res;
}
static inline unsigned long host_s2_mem_pgtable_pages(void)
{
/*
* Include an extra 16 pages to safely upper-bound the worst case of
* concatenated pgds.
*/
return __hyp_pgtable_total_pages() + 16;
}
static inline unsigned long host_s2_dev_pgtable_pages(void)
{
/* Allow 1 GiB for MMIO mappings */
return __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT);
}
#endif /* __KVM_HYP_MM_H */

92
arch/arm64/kvm/hyp/include/nvhe/spinlock.h Normal file
View File

@ -0,0 +1,92 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* A stand-alone ticket spinlock implementation for use by the non-VHE
* KVM hypervisor code running at EL2.
*
* Copyright (C) 2020 Google LLC
* Author: Will Deacon <will@kernel.org>
*
* Heavily based on the implementation removed by c11090474d70 which was:
* Copyright (C) 2012 ARM Ltd.
*/
#ifndef __ARM64_KVM_NVHE_SPINLOCK_H__
#define __ARM64_KVM_NVHE_SPINLOCK_H__
#include <asm/alternative.h>
#include <asm/lse.h>
typedef union hyp_spinlock {
u32 __val;
struct {
#ifdef __AARCH64EB__
u16 next, owner;
#else
u16 owner, next;
#endif
};
} hyp_spinlock_t;
#define hyp_spin_lock_init(l) \
do { \
*(l) = (hyp_spinlock_t){ .__val = 0 }; \
} while (0)
static inline void hyp_spin_lock(hyp_spinlock_t *lock)
{
u32 tmp;
hyp_spinlock_t lockval, newval;
asm volatile(
/* Atomically increment the next ticket. */
ARM64_LSE_ATOMIC_INSN(
/* LL/SC */
" prfm pstl1strm, %3\n"
"1: ldaxr %w0, %3\n"
" add %w1, %w0, #(1 << 16)\n"
" stxr %w2, %w1, %3\n"
" cbnz %w2, 1b\n",
/* LSE atomics */
" mov %w2, #(1 << 16)\n"
" ldadda %w2, %w0, %3\n"
__nops(3))
/* Did we get the lock? */
" eor %w1, %w0, %w0, ror #16\n"
" cbz %w1, 3f\n"
/*
* No: spin on the owner. Send a local event to avoid missing an
* unlock before the exclusive load.
*/
" sevl\n"
"2: wfe\n"
" ldaxrh %w2, %4\n"
" eor %w1, %w2, %w0, lsr #16\n"
" cbnz %w1, 2b\n"
/* We got the lock. Critical section starts here. */
"3:"
: "=&r" (lockval), "=&r" (newval), "=&r" (tmp), "+Q" (*lock)
: "Q" (lock->owner)
: "memory");
}
static inline void hyp_spin_unlock(hyp_spinlock_t *lock)
{
u64 tmp;
asm volatile(
ARM64_LSE_ATOMIC_INSN(
/* LL/SC */
" ldrh %w1, %0\n"
" add %w1, %w1, #1\n"
" stlrh %w1, %0",
/* LSE atomics */
" mov %w1, #1\n"
" staddlh %w1, %0\n"
__nops(1))
: "=Q" (lock->owner), "=&r" (tmp)
:
: "memory");
}
#endif /* __ARM64_KVM_NVHE_SPINLOCK_H__ */

9
arch/arm64/kvm/hyp/nvhe/Makefile
View File

@ -9,10 +9,15 @@ ccflags-y := -D__KVM_NVHE_HYPERVISOR__ -D__DISABLE_EXPORTS
hostprogs := gen-hyprel
HOST_EXTRACFLAGS += -I$(objtree)/include
lib-objs := clear_page.o copy_page.o memcpy.o memset.o
lib-objs := $(addprefix ../../../lib/, $(lib-objs))
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
hyp-main.o hyp-smp.o psci-relay.o
hyp-main.o hyp-smp.o psci-relay.o early_alloc.o stub.o page_alloc.o \
cache.o setup.o mm.o mem_protect.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o
../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
obj-y += $(lib-objs)
##
## Build rules for compiling nVHE hyp code

13
arch/arm64/kvm/hyp/nvhe/cache.S Normal file
View File

@ -0,0 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Code copied from arch/arm64/mm/cache.S.
*/
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/alternative.h>
SYM_FUNC_START_PI(__flush_dcache_area)
dcache_by_line_op civac, sy, x0, x1, x2, x3
ret
SYM_FUNC_END_PI(__flush_dcache_area)

54
arch/arm64/kvm/hyp/nvhe/early_alloc.c Normal file
View File

@ -0,0 +1,54 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020 Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <asm/kvm_pgtable.h>
#include <nvhe/early_alloc.h>
#include <nvhe/memory.h>
struct kvm_pgtable_mm_ops hyp_early_alloc_mm_ops;
s64 __ro_after_init hyp_physvirt_offset;
static unsigned long base;
static unsigned long end;
static unsigned long cur;
unsigned long hyp_early_alloc_nr_used_pages(void)
{
return (cur - base) >> PAGE_SHIFT;
}
void *hyp_early_alloc_contig(unsigned int nr_pages)
{
unsigned long size = (nr_pages << PAGE_SHIFT);
void *ret = (void *)cur;
if (!nr_pages)
return NULL;
if (end - cur < size)
return NULL;
cur += size;
memset(ret, 0, size);
return ret;
}
void *hyp_early_alloc_page(void *arg)
{
return hyp_early_alloc_contig(1);
}
void hyp_early_alloc_init(void *virt, unsigned long size)
{
base = cur = (unsigned long)virt;
end = base + size;
hyp_early_alloc_mm_ops.zalloc_page = hyp_early_alloc_page;
hyp_early_alloc_mm_ops.phys_to_virt = hyp_phys_to_virt;
hyp_early_alloc_mm_ops.virt_to_phys = hyp_virt_to_phys;
}

54
arch/arm64/kvm/hyp/nvhe/hyp-init.S
View File

@ -83,11 +83,6 @@ SYM_CODE_END(__kvm_hyp_init)
* x0: struct kvm_nvhe_init_params PA
*/
SYM_CODE_START_LOCAL(___kvm_hyp_init)
alternative_if ARM64_KVM_PROTECTED_MODE
mov_q x1, HCR_HOST_NVHE_PROTECTED_FLAGS
msr hcr_el2, x1
alternative_else_nop_endif
ldr x1, [x0, #NVHE_INIT_TPIDR_EL2]
msr tpidr_el2, x1
@ -97,6 +92,15 @@ alternative_else_nop_endif
ldr x1, [x0, #NVHE_INIT_MAIR_EL2]
msr mair_el2, x1
ldr x1, [x0, #NVHE_INIT_HCR_EL2]
msr hcr_el2, x1
ldr x1, [x0, #NVHE_INIT_VTTBR]
msr vttbr_el2, x1
ldr x1, [x0, #NVHE_INIT_VTCR]
msr vtcr_el2, x1
ldr x1, [x0, #NVHE_INIT_PGD_PA]
phys_to_ttbr x2, x1
alternative_if ARM64_HAS_CNP
@ -115,15 +119,10 @@ alternative_else_nop_endif
/* Invalidate the stale TLBs from Bootloader */
tlbi alle2
tlbi vmalls12e1
dsb sy
/*
* Preserve all the RES1 bits while setting the default flags,
* as well as the EE bit on BE. Drop the A flag since the compiler
* is allowed to generate unaligned accesses.
*/
mov_q x0, (SCTLR_EL2_RES1 | (SCTLR_ELx_FLAGS & ~SCTLR_ELx_A))
CPU_BE( orr x0, x0, #SCTLR_ELx_EE)
mov_q x0, INIT_SCTLR_EL2_MMU_ON
alternative_if ARM64_HAS_ADDRESS_AUTH
mov_q x1, (SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
SCTLR_ELx_ENDA | SCTLR_ELx_ENDB)
@ -221,9 +220,7 @@ SYM_CODE_START(__kvm_handle_stub_hvc)
mov x0, xzr
reset:
/* Reset kvm back to the hyp stub. */
mrs x5, sctlr_el2
mov_q x6, SCTLR_ELx_FLAGS
bic x5, x5, x6 // Clear SCTL_M and etc
mov_q x5, INIT_SCTLR_EL2_MMU_OFF
pre_disable_mmu_workaround
msr sctlr_el2, x5
isb
@ -244,4 +241,31 @@ alternative_else_nop_endif
SYM_CODE_END(__kvm_handle_stub_hvc)
SYM_FUNC_START(__pkvm_init_switch_pgd)
/* Turn the MMU off */
pre_disable_mmu_workaround
mrs x2, sctlr_el2
bic x3, x2, #SCTLR_ELx_M
msr sctlr_el2, x3
isb
tlbi alle2
/* Install the new pgtables */
ldr x3, [x0, #NVHE_INIT_PGD_PA]
phys_to_ttbr x4, x3
alternative_if ARM64_HAS_CNP
orr x4, x4, #TTBR_CNP_BIT
alternative_else_nop_endif
msr ttbr0_el2, x4
/* Set the new stack pointer */
ldr x0, [x0, #NVHE_INIT_STACK_HYP_VA]
mov sp, x0
/* And turn the MMU back on! */
set_sctlr_el2 x2
ret x1
SYM_FUNC_END(__pkvm_init_switch_pgd)
.popsection

74
arch/arm64/kvm/hyp/nvhe/hyp-main.c
View File

@ -6,12 +6,15 @@
#include <hyp/switch.h>
#include <asm/pgtable-types.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_host.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <nvhe/mem_protect.h>
#include <nvhe/mm.h>
#include <nvhe/trap_handler.h>
DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
@ -106,6 +109,61 @@ static void handle___vgic_v3_restore_aprs(struct kvm_cpu_context *host_ctxt)
__vgic_v3_restore_aprs(kern_hyp_va(cpu_if));
}
static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
DECLARE_REG(unsigned long, size, host_ctxt, 2);
DECLARE_REG(unsigned long, nr_cpus, host_ctxt, 3);
DECLARE_REG(unsigned long *, per_cpu_base, host_ctxt, 4);
DECLARE_REG(u32, hyp_va_bits, host_ctxt, 5);
/*
* __pkvm_init() will return only if an error occurred, otherwise it
* will tail-call in __pkvm_init_finalise() which will have to deal
* with the host context directly.
*/
cpu_reg(host_ctxt, 1) = __pkvm_init(phys, size, nr_cpus, per_cpu_base,
hyp_va_bits);
}
static void handle___pkvm_cpu_set_vector(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(enum arm64_hyp_spectre_vector, slot, host_ctxt, 1);
cpu_reg(host_ctxt, 1) = pkvm_cpu_set_vector(slot);
}
static void handle___pkvm_create_mappings(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(unsigned long, start, host_ctxt, 1);
DECLARE_REG(unsigned long, size, host_ctxt, 2);
DECLARE_REG(unsigned long, phys, host_ctxt, 3);
DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 4);
cpu_reg(host_ctxt, 1) = __pkvm_create_mappings(start, size, phys, prot);
}
static void handle___pkvm_create_private_mapping(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(phys_addr_t, phys, host_ctxt, 1);
DECLARE_REG(size_t, size, host_ctxt, 2);
DECLARE_REG(enum kvm_pgtable_prot, prot, host_ctxt, 3);
cpu_reg(host_ctxt, 1) = __pkvm_create_private_mapping(phys, size, prot);
}
static void handle___pkvm_prot_finalize(struct kvm_cpu_context *host_ctxt)
{
cpu_reg(host_ctxt, 1) = __pkvm_prot_finalize();
}
static void handle___pkvm_mark_hyp(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(phys_addr_t, start, host_ctxt, 1);
DECLARE_REG(phys_addr_t, end, host_ctxt, 2);
cpu_reg(host_ctxt, 1) = __pkvm_mark_hyp(start, end);
}
typedef void (*hcall_t)(struct kvm_cpu_context *);
#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@ -125,6 +183,12 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__kvm_get_mdcr_el2),
HANDLE_FUNC(__vgic_v3_save_aprs),
HANDLE_FUNC(__vgic_v3_restore_aprs),
HANDLE_FUNC(__pkvm_init),
HANDLE_FUNC(__pkvm_cpu_set_vector),
HANDLE_FUNC(__pkvm_create_mappings),
HANDLE_FUNC(__pkvm_create_private_mapping),
HANDLE_FUNC(__pkvm_prot_finalize),
HANDLE_FUNC(__pkvm_mark_hyp),
};
static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
@ -177,6 +241,16 @@ void handle_trap(struct kvm_cpu_context *host_ctxt)
case ESR_ELx_EC_SMC64:
handle_host_smc(host_ctxt);
break;
case ESR_ELx_EC_SVE:
sysreg_clear_set(cptr_el2, CPTR_EL2_TZ, 0);
isb();
sve_cond_update_zcr_vq(ZCR_ELx_LEN_MASK, SYS_ZCR_EL2);
break;
case ESR_ELx_EC_IABT_LOW:
fallthrough;
case ESR_ELx_EC_DABT_LOW:
handle_host_mem_abort(host_ctxt);
break;
default:
hyp_panic();
}

7
arch/arm64/kvm/hyp/nvhe/hyp-smp.c
View File

@ -38,3 +38,10 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu)
elf_base = (unsigned long)&__per_cpu_start;
return this_cpu_base - elf_base;
}
/*
* Define the CPU feature registers variables that will hold the copies of
* the host's sanitized values.
*/
#define KVM_HYP_CPU_FTR_REG(name) struct arm64_ftr_reg name
#include <asm/kvm_cpufeature.h>

1
arch/arm64/kvm/hyp/nvhe/hyp.lds.S
View File

@ -25,4 +25,5 @@ SECTIONS {
BEGIN_HYP_SECTION(.data..percpu)
PERCPU_INPUT(L1_CACHE_BYTES)
END_HYP_SECTION
HYP_SECTION(.bss)
}

277
arch/arm64/kvm/hyp/nvhe/mem_protect.c Normal file
View File

@ -0,0 +1,277 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020 Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <linux/kvm_host.h>
#include <asm/kvm_cpufeature.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_pgtable.h>
#include <asm/stage2_pgtable.h>
#include <hyp/switch.h>
#include <nvhe/gfp.h>
#include <nvhe/memory.h>
#include <nvhe/mem_protect.h>
#include <nvhe/mm.h>
extern unsigned long hyp_nr_cpus;
struct host_kvm host_kvm;
struct hyp_pool host_s2_mem;
struct hyp_pool host_s2_dev;
static const u8 pkvm_hyp_id = 1;
static void *host_s2_zalloc_pages_exact(size_t size)
{
return hyp_alloc_pages(&host_s2_mem, get_order(size));
}
static void *host_s2_zalloc_page(void *pool)
{
return hyp_alloc_pages(pool, 0);
}
static int prepare_s2_pools(void *mem_pgt_pool, void *dev_pgt_pool)
{
unsigned long nr_pages, pfn;
int ret;
pfn = hyp_virt_to_pfn(mem_pgt_pool);
nr_pages = host_s2_mem_pgtable_pages();
ret = hyp_pool_init(&host_s2_mem, pfn, nr_pages, 0);
if (ret)
return ret;
pfn = hyp_virt_to_pfn(dev_pgt_pool);
nr_pages = host_s2_dev_pgtable_pages();
ret = hyp_pool_init(&host_s2_dev, pfn, nr_pages, 0);
if (ret)
return ret;
host_kvm.mm_ops = (struct kvm_pgtable_mm_ops) {
.zalloc_pages_exact = host_s2_zalloc_pages_exact,
.zalloc_page = host_s2_zalloc_page,
.phys_to_virt = hyp_phys_to_virt,
.virt_to_phys = hyp_virt_to_phys,
.page_count = hyp_page_count,
.get_page = hyp_get_page,
.put_page = hyp_put_page,
};
return 0;
}
static void prepare_host_vtcr(void)
{
u32 parange, phys_shift;
u64 mmfr0, mmfr1;
mmfr0 = arm64_ftr_reg_id_aa64mmfr0_el1.sys_val;
mmfr1 = arm64_ftr_reg_id_aa64mmfr1_el1.sys_val;
/* The host stage 2 is id-mapped, so use parange for T0SZ */
parange = kvm_get_parange(mmfr0);
phys_shift = id_aa64mmfr0_parange_to_phys_shift(parange);
host_kvm.arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
}
int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool)
{
struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu;
int ret;
prepare_host_vtcr();
hyp_spin_lock_init(&host_kvm.lock);
ret = prepare_s2_pools(mem_pgt_pool, dev_pgt_pool);
if (ret)
return ret;
ret = kvm_pgtable_stage2_init_flags(&host_kvm.pgt, &host_kvm.arch,
&host_kvm.mm_ops, KVM_PGTABLE_S2_NOFWB);
if (ret)
return ret;
mmu->pgd_phys = __hyp_pa(host_kvm.pgt.pgd);
mmu->arch = &host_kvm.arch;
mmu->pgt = &host_kvm.pgt;
mmu->vmid.vmid_gen = 0;
mmu->vmid.vmid = 0;
return 0;
}
int __pkvm_prot_finalize(void)
{
struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu;
struct kvm_nvhe_init_params *params = this_cpu_ptr(&kvm_init_params);
params->vttbr = kvm_get_vttbr(mmu);
params->vtcr = host_kvm.arch.vtcr;
params->hcr_el2 |= HCR_VM;
kvm_flush_dcache_to_poc(params, sizeof(*params));
write_sysreg(params->hcr_el2, hcr_el2);
__load_stage2(&host_kvm.arch.mmu, host_kvm.arch.vtcr);
/*
* Make sure to have an ISB before the TLB maintenance below but only
* when __load_stage2() doesn't include one already.
*/
asm(ALTERNATIVE("isb", "nop", ARM64_WORKAROUND_SPECULATIVE_AT));
/* Invalidate stale HCR bits that may be cached in TLBs */
__tlbi(vmalls12e1);
dsb(nsh);
isb();
return 0;
}
static int host_stage2_unmap_dev_all(void)
{
struct kvm_pgtable *pgt = &host_kvm.pgt;
struct memblock_region *reg;
u64 addr = 0;
int i, ret;
/* Unmap all non-memory regions to recycle the pages */
for (i = 0; i < hyp_memblock_nr; i++, addr = reg->base + reg->size) {
reg = &hyp_memory[i];
ret = kvm_pgtable_stage2_unmap(pgt, addr, reg->base - addr);
if (ret)
return ret;
}
return kvm_pgtable_stage2_unmap(pgt, addr, BIT(pgt->ia_bits) - addr);
}
static bool find_mem_range(phys_addr_t addr, struct kvm_mem_range *range)
{
int cur, left = 0, right = hyp_memblock_nr;
struct memblock_region *reg;
phys_addr_t end;
range->start = 0;
range->end = ULONG_MAX;
/* The list of memblock regions is sorted, binary search it */
while (left < right) {
cur = (left + right) >> 1;
reg = &hyp_memory[cur];
end = reg->base + reg->size;
if (addr < reg->base) {
right = cur;
range->end = reg->base;
} else if (addr >= end) {
left = cur + 1;
range->start = end;
} else {
range->start = reg->base;
range->end = end;
return true;
}
}
return false;
}
static bool range_is_memory(u64 start, u64 end)
{
struct kvm_mem_range r1, r2;
if (!find_mem_range(start, &r1) || !find_mem_range(end, &r2))
return false;
if (r1.start != r2.start)
return false;
return true;
}
static inline int __host_stage2_idmap(u64 start, u64 end,
enum kvm_pgtable_prot prot,
struct hyp_pool *pool)
{
return kvm_pgtable_stage2_map(&host_kvm.pgt, start, end - start, start,
prot, pool);
}
static int host_stage2_idmap(u64 addr)
{
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W;
struct kvm_mem_range range;
bool is_memory = find_mem_range(addr, &range);
struct hyp_pool *pool = is_memory ? &host_s2_mem : &host_s2_dev;
int ret;
if (is_memory)
prot |= KVM_PGTABLE_PROT_X;
hyp_spin_lock(&host_kvm.lock);
ret = kvm_pgtable_stage2_find_range(&host_kvm.pgt, addr, prot, &range);
if (ret)
goto unlock;
ret = __host_stage2_idmap(range.start, range.end, prot, pool);
if (is_memory || ret != -ENOMEM)
goto unlock;
/*
* host_s2_mem has been provided with enough pages to cover all of
* memory with page granularity, so we should never hit the ENOMEM case.
* However, it is difficult to know how much of the MMIO range we will
* need to cover upfront, so we may need to 'recycle' the pages if we
* run out.
*/
ret = host_stage2_unmap_dev_all();
if (ret)
goto unlock;
ret = __host_stage2_idmap(range.start, range.end, prot, pool);
unlock:
hyp_spin_unlock(&host_kvm.lock);
return ret;
}
int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end)
{
int ret;
/*
* host_stage2_unmap_dev_all() currently relies on MMIO mappings being
* non-persistent, so don't allow changing page ownership in MMIO range.
*/
if (!range_is_memory(start, end))
return -EINVAL;
hyp_spin_lock(&host_kvm.lock);
ret = kvm_pgtable_stage2_set_owner(&host_kvm.pgt, start, end - start,
&host_s2_mem, pkvm_hyp_id);
hyp_spin_unlock(&host_kvm.lock);
return ret != -EAGAIN ? ret : 0;
}
void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt)
{
struct kvm_vcpu_fault_info fault;
u64 esr, addr;
int ret = 0;
esr = read_sysreg_el2(SYS_ESR);
if (!__get_fault_info(esr, &fault))
hyp_panic();
addr = (fault.hpfar_el2 & HPFAR_MASK) << 8;
ret = host_stage2_idmap(addr);
if (ret && ret != -EAGAIN)
hyp_panic();
}

173
arch/arm64/kvm/hyp/nvhe/mm.c Normal file
View File

@ -0,0 +1,173 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020 Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <linux/kvm_host.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_pgtable.h>
#include <asm/spectre.h>
#include <nvhe/early_alloc.h>
#include <nvhe/gfp.h>
#include <nvhe/memory.h>
#include <nvhe/mm.h>
#include <nvhe/spinlock.h>
struct kvm_pgtable pkvm_pgtable;
hyp_spinlock_t pkvm_pgd_lock;
u64 __io_map_base;
struct memblock_region hyp_memory[HYP_MEMBLOCK_REGIONS];
unsigned int hyp_memblock_nr;
int __pkvm_create_mappings(unsigned long start, unsigned long size,
unsigned long phys, enum kvm_pgtable_prot prot)
{
int err;
hyp_spin_lock(&pkvm_pgd_lock);
err = kvm_pgtable_hyp_map(&pkvm_pgtable, start, size, phys, prot);
hyp_spin_unlock(&pkvm_pgd_lock);
return err;
}
unsigned long __pkvm_create_private_mapping(phys_addr_t phys, size_t size,
enum kvm_pgtable_prot prot)
{
unsigned long addr;
int err;
hyp_spin_lock(&pkvm_pgd_lock);
size = PAGE_ALIGN(size + offset_in_page(phys));
addr = __io_map_base;
__io_map_base += size;
/* Are we overflowing on the vmemmap ? */
if (__io_map_base > __hyp_vmemmap) {
__io_map_base -= size;
addr = (unsigned long)ERR_PTR(-ENOMEM);
goto out;
}
err = kvm_pgtable_hyp_map(&pkvm_pgtable, addr, size, phys, prot);
if (err) {
addr = (unsigned long)ERR_PTR(err);
goto out;
}
addr = addr + offset_in_page(phys);
out:
hyp_spin_unlock(&pkvm_pgd_lock);
return addr;
}
int pkvm_create_mappings(void *from, void *to, enum kvm_pgtable_prot prot)
{
unsigned long start = (unsigned long)from;
unsigned long end = (unsigned long)to;
unsigned long virt_addr;
phys_addr_t phys;
start = start & PAGE_MASK;
end = PAGE_ALIGN(end);
for (virt_addr = start; virt_addr < end; virt_addr += PAGE_SIZE) {
int err;
phys = hyp_virt_to_phys((void *)virt_addr);
err = __pkvm_create_mappings(virt_addr, PAGE_SIZE, phys, prot);
if (err)
return err;
}
return 0;
}
int hyp_back_vmemmap(phys_addr_t phys, unsigned long size, phys_addr_t back)
{
unsigned long start, end;
hyp_vmemmap_range(phys, size, &start, &end);
return __pkvm_create_mappings(start, end - start, back, PAGE_HYP);
}
static void *__hyp_bp_vect_base;
int pkvm_cpu_set_vector(enum arm64_hyp_spectre_vector slot)
{
void *vector;
switch (slot) {
case HYP_VECTOR_DIRECT: {
vector = __kvm_hyp_vector;
break;
}
case HYP_VECTOR_SPECTRE_DIRECT: {
vector = __bp_harden_hyp_vecs;
break;
}
case HYP_VECTOR_INDIRECT:
case HYP_VECTOR_SPECTRE_INDIRECT: {
vector = (void *)__hyp_bp_vect_base;
break;
}
default:
return -EINVAL;
}
vector = __kvm_vector_slot2addr(vector, slot);
*this_cpu_ptr(&kvm_hyp_vector) = (unsigned long)vector;
return 0;
}
int hyp_map_vectors(void)
{
phys_addr_t phys;
void *bp_base;
if (!cpus_have_const_cap(ARM64_SPECTRE_V3A))
return 0;
phys = __hyp_pa(__bp_harden_hyp_vecs);
bp_base = (void *)__pkvm_create_private_mapping(phys,
__BP_HARDEN_HYP_VECS_SZ,
PAGE_HYP_EXEC);
if (IS_ERR_OR_NULL(bp_base))
return PTR_ERR(bp_base);
__hyp_bp_vect_base = bp_base;
return 0;
}
int hyp_create_idmap(u32 hyp_va_bits)
{
unsigned long start, end;
start = hyp_virt_to_phys((void *)__hyp_idmap_text_start);
start = ALIGN_DOWN(start, PAGE_SIZE);
end = hyp_virt_to_phys((void *)__hyp_idmap_text_end);
end = ALIGN(end, PAGE_SIZE);
/*
* One half of the VA space is reserved to linearly map portions of
* memory -- see va_layout.c for more details. The other half of the VA
* space contains the trampoline page, and needs some care. Split that
* second half in two and find the quarter of VA space not conflicting
* with the idmap to place the IOs and the vmemmap. IOs use the lower
* half of the quarter and the vmemmap the upper half.
*/
__io_map_base = start & BIT(hyp_va_bits - 2);
__io_map_base ^= BIT(hyp_va_bits - 2);
__hyp_vmemmap = __io_map_base | BIT(hyp_va_bits - 3);
return __pkvm_create_mappings(start, end - start, start, PAGE_HYP_EXEC);
}

195
arch/arm64/kvm/hyp/nvhe/page_alloc.c Normal file
View File

@ -0,0 +1,195 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020 Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <asm/kvm_hyp.h>
#include <nvhe/gfp.h>
u64 __hyp_vmemmap;
/*
* Index the hyp_vmemmap to find a potential buddy page, but make no assumption
* about its current state.
*
* Example buddy-tree for a 4-pages physically contiguous pool:
*
* o : Page 3
* /
* o-o : Page 2
* /
* / o : Page 1
* / /
* o---o-o : Page 0
* Order 2 1 0
*
* Example of requests on this pool:
* __find_buddy_nocheck(pool, page 0, order 0) => page 1
* __find_buddy_nocheck(pool, page 0, order 1) => page 2
* __find_buddy_nocheck(pool, page 1, order 0) => page 0
* __find_buddy_nocheck(pool, page 2, order 0) => page 3
*/
static struct hyp_page *__find_buddy_nocheck(struct hyp_pool *pool,
struct hyp_page *p,
unsigned int order)
{
phys_addr_t addr = hyp_page_to_phys(p);
addr ^= (PAGE_SIZE << order);
/*
* Don't return a page outside the pool range -- it belongs to
* something else and may not be mapped in hyp_vmemmap.
*/
if (addr < pool->range_start || addr >= pool->range_end)
return NULL;
return hyp_phys_to_page(addr);
}
/* Find a buddy page currently available for allocation */
static struct hyp_page *__find_buddy_avail(struct hyp_pool *pool,
struct hyp_page *p,
unsigned int order)
{
struct hyp_page *buddy = __find_buddy_nocheck(pool, p, order);
if (!buddy || buddy->order != order || list_empty(&buddy->node))
return NULL;
return buddy;
}
static void __hyp_attach_page(struct hyp_pool *pool,
struct hyp_page *p)
{
unsigned int order = p->order;
struct hyp_page *buddy;
memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order);
/*
* Only the first struct hyp_page of a high-order page (otherwise known
* as the 'head') should have p->order set. The non-head pages should
* have p->order = HYP_NO_ORDER. Here @p may no longer be the head
* after coallescing, so make sure to mark it HYP_NO_ORDER proactively.
*/
p->order = HYP_NO_ORDER;
for (; (order + 1) < pool->max_order; order++) {
buddy = __find_buddy_avail(pool, p, order);
if (!buddy)
break;
/* Take the buddy out of its list, and coallesce with @p */
list_del_init(&buddy->node);
buddy->order = HYP_NO_ORDER;
p = min(p, buddy);
}
/* Mark the new head, and insert it */
p->order = order;
list_add_tail(&p->node, &pool->free_area[order]);
}
static void hyp_attach_page(struct hyp_page *p)
{
struct hyp_pool *pool = hyp_page_to_pool(p);
hyp_spin_lock(&pool->lock);
__hyp_attach_page(pool, p);
hyp_spin_unlock(&pool->lock);
}
static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool,
struct hyp_page *p,
unsigned int order)
{
struct hyp_page *buddy;
list_del_init(&p->node);
while (p->order > order) {
/*
* The buddy of order n - 1 currently has HYP_NO_ORDER as it
* is covered by a higher-level page (whose head is @p). Use
* __find_buddy_nocheck() to find it and inject it in the
* free_list[n - 1], effectively splitting @p in half.
*/
p->order--;
buddy = __find_buddy_nocheck(pool, p, p->order);
buddy->order = p->order;
list_add_tail(&buddy->node, &pool->free_area[buddy->order]);
}
return p;
}
void hyp_put_page(void *addr)
{
struct hyp_page *p = hyp_virt_to_page(addr);
if (hyp_page_ref_dec_and_test(p))
hyp_attach_page(p);
}
void hyp_get_page(void *addr)
{
struct hyp_page *p = hyp_virt_to_page(addr);
hyp_page_ref_inc(p);
}
void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order)
{
unsigned int i = order;
struct hyp_page *p;
hyp_spin_lock(&pool->lock);
/* Look for a high-enough-order page */
while (i < pool->max_order && list_empty(&pool->free_area[i]))
i++;
if (i >= pool->max_order) {
hyp_spin_unlock(&pool->lock);
return NULL;
}
/* Extract it from the tree at the right order */
p = list_first_entry(&pool->free_area[i], struct hyp_page, node);
p = __hyp_extract_page(pool, p, order);
hyp_spin_unlock(&pool->lock);
hyp_set_page_refcounted(p);
return hyp_page_to_virt(p);
}
int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages,
unsigned int reserved_pages)
{
phys_addr_t phys = hyp_pfn_to_phys(pfn);
struct hyp_page *p;
int i;
hyp_spin_lock_init(&pool->lock);
pool->max_order = min(MAX_ORDER, get_order(nr_pages << PAGE_SHIFT));
for (i = 0; i < pool->max_order; i++)
INIT_LIST_HEAD(&pool->free_area[i]);
pool->range_start = phys;
pool->range_end = phys + (nr_pages << PAGE_SHIFT);
/* Init the vmemmap portion */
p = hyp_phys_to_page(phys);
memset(p, 0, sizeof(*p) * nr_pages);
for (i = 0; i < nr_pages; i++) {
p[i].pool = pool;
INIT_LIST_HEAD(&p[i].node);
}
/* Attach the unused pages to the buddy tree */
for (i = reserved_pages; i < nr_pages; i++)
__hyp_attach_page(pool, &p[i]);
return 0;
}

4
arch/arm64/kvm/hyp/nvhe/psci-relay.c
View File

@ -11,6 +11,7 @@
#include <linux/kvm_host.h>
#include <uapi/linux/psci.h>
#include <nvhe/memory.h>
#include <nvhe/trap_handler.h>
void kvm_hyp_cpu_entry(unsigned long r0);
@ -20,9 +21,6 @@ void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
/* Config options set by the host. */
struct kvm_host_psci_config __ro_after_init kvm_host_psci_config;
s64 __ro_after_init hyp_physvirt_offset;
#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset)
#define INVALID_CPU_ID UINT_MAX

214
arch/arm64/kvm/hyp/nvhe/setup.c Normal file
View File

@ -0,0 +1,214 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2020 Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <linux/kvm_host.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
#include <asm/kvm_pgtable.h>
#include <nvhe/early_alloc.h>
#include <nvhe/gfp.h>
#include <nvhe/memory.h>
#include <nvhe/mem_protect.h>
#include <nvhe/mm.h>
#include <nvhe/trap_handler.h>
struct hyp_pool hpool;
struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops;
unsigned long hyp_nr_cpus;
#define hyp_percpu_size ((unsigned long)__per_cpu_end - \
(unsigned long)__per_cpu_start)
static void *vmemmap_base;
static void *hyp_pgt_base;
static void *host_s2_mem_pgt_base;
static void *host_s2_dev_pgt_base;
static int divide_memory_pool(void *virt, unsigned long size)
{
unsigned long vstart, vend, nr_pages;
hyp_early_alloc_init(virt, size);
hyp_vmemmap_range(__hyp_pa(virt), size, &vstart, &vend);
nr_pages = (vend - vstart) >> PAGE_SHIFT;
vmemmap_base = hyp_early_alloc_contig(nr_pages);
if (!vmemmap_base)
return -ENOMEM;
nr_pages = hyp_s1_pgtable_pages();
hyp_pgt_base = hyp_early_alloc_contig(nr_pages);
if (!hyp_pgt_base)
return -ENOMEM;
nr_pages = host_s2_mem_pgtable_pages();
host_s2_mem_pgt_base = hyp_early_alloc_contig(nr_pages);
if (!host_s2_mem_pgt_base)
return -ENOMEM;
nr_pages = host_s2_dev_pgtable_pages();
host_s2_dev_pgt_base = hyp_early_alloc_contig(nr_pages);
if (!host_s2_dev_pgt_base)
return -ENOMEM;
return 0;
}
static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
unsigned long *per_cpu_base,
u32 hyp_va_bits)
{
void *start, *end, *virt = hyp_phys_to_virt(phys);
unsigned long pgt_size = hyp_s1_pgtable_pages() << PAGE_SHIFT;
int ret, i;
/* Recreate the hyp page-table using the early page allocator */
hyp_early_alloc_init(hyp_pgt_base, pgt_size);
ret = kvm_pgtable_hyp_init(&pkvm_pgtable, hyp_va_bits,
&hyp_early_alloc_mm_ops);
if (ret)
return ret;
ret = hyp_create_idmap(hyp_va_bits);
if (ret)
return ret;
ret = hyp_map_vectors();
if (ret)
return ret;
ret = hyp_back_vmemmap(phys, size, hyp_virt_to_phys(vmemmap_base));
if (ret)
return ret;
ret = pkvm_create_mappings(__hyp_text_start, __hyp_text_end, PAGE_HYP_EXEC);
if (ret)
return ret;
ret = pkvm_create_mappings(__start_rodata, __end_rodata, PAGE_HYP_RO);
if (ret)
return ret;
ret = pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_HYP_RO);
if (ret)
return ret;
ret = pkvm_create_mappings(__hyp_bss_start, __hyp_bss_end, PAGE_HYP);
if (ret)
return ret;
ret = pkvm_create_mappings(__hyp_bss_end, __bss_stop, PAGE_HYP_RO);
if (ret)
return ret;
ret = pkvm_create_mappings(virt, virt + size, PAGE_HYP);
if (ret)
return ret;
for (i = 0; i < hyp_nr_cpus; i++) {
start = (void *)kern_hyp_va(per_cpu_base[i]);
end = start + PAGE_ALIGN(hyp_percpu_size);
ret = pkvm_create_mappings(start, end, PAGE_HYP);
if (ret)
return ret;
end = (void *)per_cpu_ptr(&kvm_init_params, i)->stack_hyp_va;
start = end - PAGE_SIZE;
ret = pkvm_create_mappings(start, end, PAGE_HYP);
if (ret)
return ret;
}
return 0;
}
static void update_nvhe_init_params(void)
{
struct kvm_nvhe_init_params *params;
unsigned long i;
for (i = 0; i < hyp_nr_cpus; i++) {
params = per_cpu_ptr(&kvm_init_params, i);
params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd);
__flush_dcache_area(params, sizeof(*params));
}
}
static void *hyp_zalloc_hyp_page(void *arg)
{
return hyp_alloc_pages(&hpool, 0);
}
void __noreturn __pkvm_init_finalise(void)
{
struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data);
struct kvm_cpu_context *host_ctxt = &host_data->host_ctxt;
unsigned long nr_pages, reserved_pages, pfn;
int ret;
/* Now that the vmemmap is backed, install the full-fledged allocator */
pfn = hyp_virt_to_pfn(hyp_pgt_base);
nr_pages = hyp_s1_pgtable_pages();
reserved_pages = hyp_early_alloc_nr_used_pages();
ret = hyp_pool_init(&hpool, pfn, nr_pages, reserved_pages);
if (ret)
goto out;
ret = kvm_host_prepare_stage2(host_s2_mem_pgt_base, host_s2_dev_pgt_base);
if (ret)
goto out;
pkvm_pgtable_mm_ops = (struct kvm_pgtable_mm_ops) {
.zalloc_page = hyp_zalloc_hyp_page,
.phys_to_virt = hyp_phys_to_virt,
.virt_to_phys = hyp_virt_to_phys,
.get_page = hyp_get_page,
.put_page = hyp_put_page,
};
pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops;
out:
/*
* We tail-called to here from handle___pkvm_init() and will not return,
* so make sure to propagate the return value to the host.
*/
cpu_reg(host_ctxt, 1) = ret;
__host_enter(host_ctxt);
}
int __pkvm_init(phys_addr_t phys, unsigned long size, unsigned long nr_cpus,
unsigned long *per_cpu_base, u32 hyp_va_bits)
{
struct kvm_nvhe_init_params *params;
void *virt = hyp_phys_to_virt(phys);
void (*fn)(phys_addr_t params_pa, void *finalize_fn_va);
int ret;
if (!PAGE_ALIGNED(phys) || !PAGE_ALIGNED(size))
return -EINVAL;
hyp_spin_lock_init(&pkvm_pgd_lock);
hyp_nr_cpus = nr_cpus;
ret = divide_memory_pool(virt, size);
if (ret)
return ret;
ret = recreate_hyp_mappings(phys, size, per_cpu_base, hyp_va_bits);
if (ret)
return ret;
update_nvhe_init_params();
/* Jump in the idmap page to switch to the new page-tables */
params = this_cpu_ptr(&kvm_init_params);
fn = (typeof(fn))__hyp_pa(__pkvm_init_switch_pgd);
fn(__hyp_pa(params), __pkvm_init_finalise);
unreachable();
}

22
arch/arm64/kvm/hyp/nvhe/stub.c Normal file
View File

@ -0,0 +1,22 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Stubs for out-of-line function calls caused by re-using kernel
* infrastructure at EL2.
*
* Copyright (C) 2020 - Google LLC
*/
#include <linux/list.h>
#ifdef CONFIG_DEBUG_LIST
bool __list_add_valid(struct list_head *new, struct list_head *prev,
struct list_head *next)
{
return true;
}
bool __list_del_entry_valid(struct list_head *entry)
{
return true;
}
#endif

25
arch/arm64/kvm/hyp/nvhe/switch.c
View File

@ -28,6 +28,8 @@
#include <asm/processor.h>
#include <asm/thread_info.h>
#include <nvhe/mem_protect.h>
/* Non-VHE specific context */
DEFINE_PER_CPU(struct kvm_host_data, kvm_host_data);
DEFINE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
@ -41,9 +43,9 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
__activate_traps_common(vcpu);
val = CPTR_EL2_DEFAULT;
val |= CPTR_EL2_TTA | CPTR_EL2_TZ | CPTR_EL2_TAM;
val |= CPTR_EL2_TTA | CPTR_EL2_TAM;
if (!update_fp_enabled(vcpu)) {
val |= CPTR_EL2_TFP;
val |= CPTR_EL2_TFP | CPTR_EL2_TZ;
__activate_traps_fpsimd32(vcpu);
}
@ -68,7 +70,7 @@ static void __activate_traps(struct kvm_vcpu *vcpu)
static void __deactivate_traps(struct kvm_vcpu *vcpu)
{
extern char __kvm_hyp_host_vector[];
u64 mdcr_el2;
u64 mdcr_el2, cptr;
___deactivate_traps(vcpu);
@ -98,17 +100,14 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
mdcr_el2 |= MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT;
write_sysreg(mdcr_el2, mdcr_el2);
if (is_protected_kvm_enabled())
write_sysreg(HCR_HOST_NVHE_PROTECTED_FLAGS, hcr_el2);
else
write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2);
write_sysreg(CPTR_EL2_DEFAULT, cptr_el2);
write_sysreg(__kvm_hyp_host_vector, vbar_el2);
}
write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2);
static void __load_host_stage2(void)
{
write_sysreg(0, vttbr_el2);
cptr = CPTR_EL2_DEFAULT;
if (vcpu_has_sve(vcpu) && (vcpu->arch.flags & KVM_ARM64_FP_ENABLED))
cptr |= CPTR_EL2_TZ;
write_sysreg(cptr, cptr_el2);
write_sysreg(__kvm_hyp_host_vector, vbar_el2);
}
/* Save VGICv3 state on non-VHE systems */

4
arch/arm64/kvm/hyp/nvhe/tlb.c
View File

@ -8,6 +8,8 @@
#include <asm/kvm_mmu.h>
#include <asm/tlbflush.h>
#include <nvhe/mem_protect.h>
struct tlb_inv_context {
u64 tcr;
};
@ -43,7 +45,7 @@ static void __tlb_switch_to_guest(struct kvm_s2_mmu *mmu,
static void __tlb_switch_to_host(struct tlb_inv_context *cxt)
{
write_sysreg(0, vttbr_el2);
__load_host_stage2();
if (cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT)) {
/* Ensure write of the host VMID */

415
arch/arm64/kvm/hyp/pgtable.c
View File

@ -9,8 +9,7 @@
#include <linux/bitfield.h>
#include <asm/kvm_pgtable.h>
#define KVM_PGTABLE_MAX_LEVELS 4U
#include <asm/stage2_pgtable.h>
#define KVM_PTE_VALID BIT(0)
@ -49,6 +48,11 @@
KVM_PTE_LEAF_ATTR_LO_S2_S2AP_W | \
KVM_PTE_LEAF_ATTR_HI_S2_XN)
#define KVM_PTE_LEAF_ATTR_S2_IGNORED GENMASK(58, 55)
#define KVM_INVALID_PTE_OWNER_MASK GENMASK(63, 56)
#define KVM_MAX_OWNER_ID 1
struct kvm_pgtable_walk_data {
struct kvm_pgtable *pgt;
struct kvm_pgtable_walker *walker;
@ -68,21 +72,36 @@ static u64 kvm_granule_size(u32 level)
return BIT(kvm_granule_shift(level));
}
static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)
{
u64 granule = kvm_granule_size(level);
#define KVM_PHYS_INVALID (-1ULL)
static bool kvm_phys_is_valid(u64 phys)
{
return phys < BIT(id_aa64mmfr0_parange_to_phys_shift(ID_AA64MMFR0_PARANGE_MAX));
}
static bool kvm_level_supports_block_mapping(u32 level)
{
/*
* Reject invalid block mappings and don't bother with 4TB mappings for
* 52-bit PAs.
*/
if (level == 0 || (PAGE_SIZE != SZ_4K && level == 1))
return !(level == 0 || (PAGE_SIZE != SZ_4K && level == 1));
}
static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level)
{
u64 granule = kvm_granule_size(level);
if (!kvm_level_supports_block_mapping(level))
return false;
if (granule > (end - addr))
return false;
return IS_ALIGNED(addr, granule) && IS_ALIGNED(phys, granule);
if (kvm_phys_is_valid(phys) && !IS_ALIGNED(phys, granule))
return false;
return IS_ALIGNED(addr, granule);
}
static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level)
@ -152,20 +171,20 @@ static kvm_pte_t kvm_phys_to_pte(u64 pa)
return pte;
}
static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte)
static kvm_pte_t *kvm_pte_follow(kvm_pte_t pte, struct kvm_pgtable_mm_ops *mm_ops)
{
return __va(kvm_pte_to_phys(pte));
return mm_ops->phys_to_virt(kvm_pte_to_phys(pte));
}
static void kvm_set_invalid_pte(kvm_pte_t *ptep)
static void kvm_clear_pte(kvm_pte_t *ptep)
{
kvm_pte_t pte = *ptep;
WRITE_ONCE(*ptep, pte & ~KVM_PTE_VALID);
WRITE_ONCE(*ptep, 0);
}
static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp)
static void kvm_set_table_pte(kvm_pte_t *ptep, kvm_pte_t *childp,
struct kvm_pgtable_mm_ops *mm_ops)
{
kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(__pa(childp));
kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(mm_ops->virt_to_phys(childp));
pte |= FIELD_PREP(KVM_PTE_TYPE, KVM_PTE_TYPE_TABLE);
pte |= KVM_PTE_VALID;
@ -187,6 +206,11 @@ static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32 level)
return pte;
}
static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id)
{
return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
}
static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr,
u32 level, kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag)
@ -228,7 +252,7 @@ static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data,
goto out;
}
childp = kvm_pte_follow(pte);
childp = kvm_pte_follow(pte, data->pgt->mm_ops);
ret = __kvm_pgtable_walk(data, childp, level + 1);
if (ret)
goto out;
@ -303,12 +327,12 @@ int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size,
}
struct hyp_map_data {
u64 phys;
kvm_pte_t attr;
u64 phys;
kvm_pte_t attr;
struct kvm_pgtable_mm_ops *mm_ops;
};
static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot,
struct hyp_map_data *data)
static int hyp_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep)
{
bool device = prot & KVM_PGTABLE_PROT_DEVICE;
u32 mtype = device ? MT_DEVICE_nGnRE : MT_NORMAL;
@ -333,7 +357,8 @@ static int hyp_map_set_prot_attr(enum kvm_pgtable_prot prot,
attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_AP, ap);
attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S1_SH, sh);
attr |= KVM_PTE_LEAF_ATTR_LO_S1_AF;
data->attr = attr;
*ptep = attr;
return 0;
}
@ -359,6 +384,8 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag, void * const arg)
{
kvm_pte_t *childp;
struct hyp_map_data *data = arg;
struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg))
return 0;
@ -366,11 +393,11 @@ static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1))
return -EINVAL;
childp = (kvm_pte_t *)get_zeroed_page(GFP_KERNEL);
childp = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
if (!childp)
return -ENOMEM;
kvm_set_table_pte(ptep, childp);
kvm_set_table_pte(ptep, childp, mm_ops);
return 0;
}
@ -380,6 +407,7 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
int ret;
struct hyp_map_data map_data = {
.phys = ALIGN_DOWN(phys, PAGE_SIZE),
.mm_ops = pgt->mm_ops,
};
struct kvm_pgtable_walker walker = {
.cb = hyp_map_walker,
@ -387,7 +415,7 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
.arg = &map_data,
};
ret = hyp_map_set_prot_attr(prot, &map_data);
ret = hyp_set_prot_attr(prot, &map_data.attr);
if (ret)
return ret;
@ -397,16 +425,18 @@ int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys,
return ret;
}
int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits)
int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits,
struct kvm_pgtable_mm_ops *mm_ops)
{
u64 levels = ARM64_HW_PGTABLE_LEVELS(va_bits);
pgt->pgd = (kvm_pte_t *)get_zeroed_page(GFP_KERNEL);
pgt->pgd = (kvm_pte_t *)mm_ops->zalloc_page(NULL);
if (!pgt->pgd)
return -ENOMEM;
pgt->ia_bits = va_bits;
pgt->start_level = KVM_PGTABLE_MAX_LEVELS - levels;
pgt->mm_ops = mm_ops;
pgt->mmu = NULL;
return 0;
}
@ -414,7 +444,9 @@ int kvm_pgtable_hyp_init(struct kvm_pgtable *pgt, u32 va_bits)
static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag, void * const arg)
{
free_page((unsigned long)kvm_pte_follow(*ptep));
struct kvm_pgtable_mm_ops *mm_ops = arg;
mm_ops->put_page((void *)kvm_pte_follow(*ptep, mm_ops));
return 0;
}
@ -423,30 +455,78 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)
struct kvm_pgtable_walker walker = {
.cb = hyp_free_walker,
.flags = KVM_PGTABLE_WALK_TABLE_POST,
.arg = pgt->mm_ops,
};
WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
free_page((unsigned long)pgt->pgd);
pgt->mm_ops->put_page(pgt->pgd);
pgt->pgd = NULL;
}
struct stage2_map_data {
u64 phys;
kvm_pte_t attr;
u8 owner_id;
kvm_pte_t *anchor;
kvm_pte_t *childp;
struct kvm_s2_mmu *mmu;
struct kvm_mmu_memory_cache *memcache;
void *memcache;
struct kvm_pgtable_mm_ops *mm_ops;
};
static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot,
struct stage2_map_data *data)
u64 kvm_get_vtcr(u64 mmfr0, u64 mmfr1, u32 phys_shift)
{
u64 vtcr = VTCR_EL2_FLAGS;
u8 lvls;
vtcr |= kvm_get_parange(mmfr0) << VTCR_EL2_PS_SHIFT;
vtcr |= VTCR_EL2_T0SZ(phys_shift);
/*
* Use a minimum 2 level page table to prevent splitting
* host PMD huge pages at stage2.
*/
lvls = stage2_pgtable_levels(phys_shift);
if (lvls < 2)
lvls = 2;
vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls);
/*
* Enable the Hardware Access Flag management, unconditionally
* on all CPUs. The features is RES0 on CPUs without the support
* and must be ignored by the CPUs.
*/
vtcr |= VTCR_EL2_HA;
/* Set the vmid bits */
vtcr |= (get_vmid_bits(mmfr1) == 16) ?
VTCR_EL2_VS_16BIT :
VTCR_EL2_VS_8BIT;
return vtcr;
}
static bool stage2_has_fwb(struct kvm_pgtable *pgt)
{
if (!cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
return false;
return !(pgt->flags & KVM_PGTABLE_S2_NOFWB);
}
static int stage2_set_prot_attr(enum kvm_pgtable_prot prot, kvm_pte_t *ptep,
struct kvm_pgtable *pgt)
{
bool device = prot & KVM_PGTABLE_PROT_DEVICE;
kvm_pte_t attr = device ? PAGE_S2_MEMATTR(DEVICE_nGnRE) :
PAGE_S2_MEMATTR(NORMAL);
u32 sh = KVM_PTE_LEAF_ATTR_LO_S2_SH_IS;
kvm_pte_t attr;
if (device)
attr = PAGE_S2_MEMATTR(DEVICE_nGnRE, stage2_has_fwb(pgt));
else
attr = PAGE_S2_MEMATTR(NORMAL, stage2_has_fwb(pgt));
if (!(prot & KVM_PGTABLE_PROT_X))
attr |= KVM_PTE_LEAF_ATTR_HI_S2_XN;
@ -461,44 +541,78 @@ static int stage2_map_set_prot_attr(enum kvm_pgtable_prot prot,
attr |= FIELD_PREP(KVM_PTE_LEAF_ATTR_LO_S2_SH, sh);
attr |= KVM_PTE_LEAF_ATTR_LO_S2_AF;
data->attr = attr;
*ptep = attr;
return 0;
}
static bool stage2_pte_needs_update(kvm_pte_t old, kvm_pte_t new)
{
if (!kvm_pte_valid(old) || !kvm_pte_valid(new))
return true;
return ((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS));
}
static bool stage2_pte_is_counted(kvm_pte_t pte)
{
/*
* The refcount tracks valid entries as well as invalid entries if they
* encode ownership of a page to another entity than the page-table
* owner, whose id is 0.
*/
return !!pte;
}
static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr,
u32 level, struct kvm_pgtable_mm_ops *mm_ops)
{
/*
* Clear the existing PTE, and perform break-before-make with
* TLB maintenance if it was valid.
*/
if (kvm_pte_valid(*ptep)) {
kvm_clear_pte(ptep);
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level);
}
mm_ops->put_page(ptep);
}
static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level,
kvm_pte_t *ptep,
struct stage2_map_data *data)
{
kvm_pte_t new, old = *ptep;
u64 granule = kvm_granule_size(level), phys = data->phys;
struct page *page = virt_to_page(ptep);
struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
if (!kvm_block_mapping_supported(addr, end, phys, level))
return -E2BIG;
new = kvm_init_valid_leaf_pte(phys, data->attr, level);
if (kvm_pte_valid(old)) {
if (kvm_phys_is_valid(phys))
new = kvm_init_valid_leaf_pte(phys, data->attr, level);
else
new = kvm_init_invalid_leaf_owner(data->owner_id);
if (stage2_pte_is_counted(old)) {
/*
* Skip updating the PTE if we are trying to recreate the exact
* same mapping or only change the access permissions. Instead,
* the vCPU will exit one more time from guest if still needed
* and then go through the path of relaxing permissions.
*/
if (!((old ^ new) & (~KVM_PTE_LEAF_ATTR_S2_PERMS)))
if (!stage2_pte_needs_update(old, new))
return -EAGAIN;
/*
* There's an existing different valid leaf entry, so perform
* break-before-make.
*/
kvm_set_invalid_pte(ptep);
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
put_page(page);
stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);
}
smp_store_release(ptep, new);
get_page(page);
data->phys += granule;
if (stage2_pte_is_counted(new))
mm_ops->get_page(ptep);
if (kvm_phys_is_valid(phys))
data->phys += granule;
return 0;
}
@ -512,7 +626,8 @@ static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,
if (!kvm_block_mapping_supported(addr, end, data->phys, level))
return 0;
kvm_set_invalid_pte(ptep);
data->childp = kvm_pte_follow(*ptep, data->mm_ops);
kvm_clear_pte(ptep);
/*
* Invalidate the whole stage-2, as we may have numerous leaf
@ -527,13 +642,13 @@ static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level,
static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
struct stage2_map_data *data)
{
int ret;
struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
kvm_pte_t *childp, pte = *ptep;
struct page *page = virt_to_page(ptep);
int ret;
if (data->anchor) {
if (kvm_pte_valid(pte))
put_page(page);
if (stage2_pte_is_counted(pte))
mm_ops->put_page(ptep);
return 0;
}
@ -548,7 +663,7 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
if (!data->memcache)
return -ENOMEM;
childp = kvm_mmu_memory_cache_alloc(data->memcache);
childp = mm_ops->zalloc_page(data->memcache);
if (!childp)
return -ENOMEM;
@ -557,14 +672,11 @@ static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
* a table. Accesses beyond 'end' that fall within the new table
* will be mapped lazily.
*/
if (kvm_pte_valid(pte)) {
kvm_set_invalid_pte(ptep);
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);
put_page(page);
}
if (stage2_pte_is_counted(pte))
stage2_put_pte(ptep, data->mmu, addr, level, mm_ops);
kvm_set_table_pte(ptep, childp);
get_page(page);
kvm_set_table_pte(ptep, childp, mm_ops);
mm_ops->get_page(ptep);
return 0;
}
@ -573,19 +685,25 @@ static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level,
kvm_pte_t *ptep,
struct stage2_map_data *data)
{
struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops;
kvm_pte_t *childp;
int ret = 0;
if (!data->anchor)
return 0;
free_page((unsigned long)kvm_pte_follow(*ptep));
put_page(virt_to_page(ptep));
if (data->anchor == ptep) {
childp = data->childp;
data->anchor = NULL;
data->childp = NULL;
ret = stage2_map_walk_leaf(addr, end, level, ptep, data);
} else {
childp = kvm_pte_follow(*ptep, mm_ops);
}
mm_ops->put_page(childp);
mm_ops->put_page(ptep);
return ret;
}
@ -627,13 +745,14 @@ static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
u64 phys, enum kvm_pgtable_prot prot,
struct kvm_mmu_memory_cache *mc)
void *mc)
{
int ret;
struct stage2_map_data map_data = {
.phys = ALIGN_DOWN(phys, PAGE_SIZE),
.mmu = pgt->mmu,
.memcache = mc,
.mm_ops = pgt->mm_ops,
};
struct kvm_pgtable_walker walker = {
.cb = stage2_map_walker,
@ -643,7 +762,7 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
.arg = &map_data,
};
ret = stage2_map_set_prot_attr(prot, &map_data);
ret = stage2_set_prot_attr(prot, &map_data.attr, pgt);
if (ret)
return ret;
@ -652,38 +771,69 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
return ret;
}
int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
void *mc, u8 owner_id)
{
int ret;
struct stage2_map_data map_data = {
.phys = KVM_PHYS_INVALID,
.mmu = pgt->mmu,
.memcache = mc,
.mm_ops = pgt->mm_ops,
.owner_id = owner_id,
};
struct kvm_pgtable_walker walker = {
.cb = stage2_map_walker,
.flags = KVM_PGTABLE_WALK_TABLE_PRE |
KVM_PGTABLE_WALK_LEAF |
KVM_PGTABLE_WALK_TABLE_POST,
.arg = &map_data,
};
if (owner_id > KVM_MAX_OWNER_ID)
return -EINVAL;
ret = kvm_pgtable_walk(pgt, addr, size, &walker);
dsb(ishst);
return ret;
}
static void stage2_flush_dcache(void *addr, u64 size)
{
if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
return;
__flush_dcache_area(addr, size);
}
static bool stage2_pte_cacheable(kvm_pte_t pte)
static bool stage2_pte_cacheable(kvm_pte_t pte, struct kvm_pgtable *pgt)
{
u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR;
return memattr == PAGE_S2_MEMATTR(NORMAL);
return memattr == PAGE_S2_MEMATTR(NORMAL, stage2_has_fwb(pgt));
}
static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag,
void * const arg)
{
struct kvm_s2_mmu *mmu = arg;
struct kvm_pgtable *pgt = arg;
struct kvm_s2_mmu *mmu = pgt->mmu;
struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
kvm_pte_t pte = *ptep, *childp = NULL;
bool need_flush = false;
if (!kvm_pte_valid(pte))
if (!kvm_pte_valid(pte)) {
if (stage2_pte_is_counted(pte)) {
kvm_clear_pte(ptep);
mm_ops->put_page(ptep);
}
return 0;
}
if (kvm_pte_table(pte, level)) {
childp = kvm_pte_follow(pte);
childp = kvm_pte_follow(pte, mm_ops);
if (page_count(virt_to_page(childp)) != 1)
if (mm_ops->page_count(childp) != 1)
return 0;
} else if (stage2_pte_cacheable(pte)) {
need_flush = true;
} else if (stage2_pte_cacheable(pte, pgt)) {
need_flush = !stage2_has_fwb(pgt);
}
/*
@ -691,17 +841,15 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
* block entry and rely on the remaining portions being faulted
* back lazily.
*/
kvm_set_invalid_pte(ptep);
kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level);
put_page(virt_to_page(ptep));
stage2_put_pte(ptep, mmu, addr, level, mm_ops);
if (need_flush) {
stage2_flush_dcache(kvm_pte_follow(pte),
stage2_flush_dcache(kvm_pte_follow(pte, mm_ops),
kvm_granule_size(level));
}
if (childp)
free_page((unsigned long)childp);
mm_ops->put_page(childp);
return 0;
}
@ -710,7 +858,7 @@ int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size)
{
struct kvm_pgtable_walker walker = {
.cb = stage2_unmap_walker,
.arg = pgt->mmu,
.arg = pgt,
.flags = KVM_PGTABLE_WALK_LEAF | KVM_PGTABLE_WALK_TABLE_POST,
};
@ -842,12 +990,14 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag,
void * const arg)
{
struct kvm_pgtable *pgt = arg;
struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops;
kvm_pte_t pte = *ptep;
if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pte))
if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pte, pgt))
return 0;
stage2_flush_dcache(kvm_pte_follow(pte), kvm_granule_size(level));
stage2_flush_dcache(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level));
return 0;
}
@ -856,30 +1006,35 @@ int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size)
struct kvm_pgtable_walker walker = {
.cb = stage2_flush_walker,
.flags = KVM_PGTABLE_WALK_LEAF,
.arg = pgt,
};
if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
if (stage2_has_fwb(pgt))
return 0;
return kvm_pgtable_walk(pgt, addr, size, &walker);
}
int kvm_pgtable_stage2_init(struct kvm_pgtable *pgt, struct kvm *kvm)
int kvm_pgtable_stage2_init_flags(struct kvm_pgtable *pgt, struct kvm_arch *arch,
struct kvm_pgtable_mm_ops *mm_ops,
enum kvm_pgtable_stage2_flags flags)
{
size_t pgd_sz;
u64 vtcr = kvm->arch.vtcr;
u64 vtcr = arch->vtcr;
u32 ia_bits = VTCR_EL2_IPA(vtcr);
u32 sl0 = FIELD_GET(VTCR_EL2_SL0_MASK, vtcr);
u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0;
pgd_sz = kvm_pgd_pages(ia_bits, start_level) * PAGE_SIZE;
pgt->pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
pgt->pgd = mm_ops->zalloc_pages_exact(pgd_sz);
if (!pgt->pgd)
return -ENOMEM;
pgt->flags = flags;
pgt->ia_bits = ia_bits;
pgt->start_level = start_level;
pgt->mmu = &kvm->arch.mmu;
pgt->mm_ops = mm_ops;
pgt->mmu = &arch->mmu;
/* Ensure zeroed PGD pages are visible to the hardware walker */
dsb(ishst);
@ -890,15 +1045,16 @@ static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag,
void * const arg)
{
struct kvm_pgtable_mm_ops *mm_ops = arg;
kvm_pte_t pte = *ptep;
if (!kvm_pte_valid(pte))
if (!stage2_pte_is_counted(pte))
return 0;
put_page(virt_to_page(ptep));
mm_ops->put_page(ptep);
if (kvm_pte_table(pte, level))
free_page((unsigned long)kvm_pte_follow(pte));
mm_ops->put_page(kvm_pte_follow(pte, mm_ops));
return 0;
}
@ -910,10 +1066,85 @@ void kvm_pgtable_stage2_destroy(struct kvm_pgtable *pgt)
.cb = stage2_free_walker,
.flags = KVM_PGTABLE_WALK_LEAF |
KVM_PGTABLE_WALK_TABLE_POST,
.arg = pgt->mm_ops,
};
WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker));
pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE;
free_pages_exact(pgt->pgd, pgd_sz);
pgt->mm_ops->free_pages_exact(pgt->pgd, pgd_sz);
pgt->pgd = NULL;
}
#define KVM_PTE_LEAF_S2_COMPAT_MASK (KVM_PTE_LEAF_ATTR_S2_PERMS | \
KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR | \
KVM_PTE_LEAF_ATTR_S2_IGNORED)
static int stage2_check_permission_walker(u64 addr, u64 end, u32 level,
kvm_pte_t *ptep,
enum kvm_pgtable_walk_flags flag,
void * const arg)
{
kvm_pte_t old_attr, pte = *ptep, *new_attr = arg;
/*
* Compatible mappings are either invalid and owned by the page-table
* owner (whose id is 0), or valid with matching permission attributes.
*/
if (kvm_pte_valid(pte)) {
old_attr = pte & KVM_PTE_LEAF_S2_COMPAT_MASK;
if (old_attr != *new_attr)
return -EEXIST;
} else if (pte) {
return -EEXIST;
}
return 0;
}
int kvm_pgtable_stage2_find_range(struct kvm_pgtable *pgt, u64 addr,
enum kvm_pgtable_prot prot,
struct kvm_mem_range *range)
{
kvm_pte_t attr;
struct kvm_pgtable_walker check_perm_walker = {
.cb = stage2_check_permission_walker,
.flags = KVM_PGTABLE_WALK_LEAF,
.arg = &attr,
};
u64 granule, start, end;
u32 level;
int ret;
ret = stage2_set_prot_attr(prot, &attr, pgt);
if (ret)
return ret;
attr &= KVM_PTE_LEAF_S2_COMPAT_MASK;
for (level = pgt->start_level; level < KVM_PGTABLE_MAX_LEVELS; level++) {
granule = kvm_granule_size(level);
start = ALIGN_DOWN(addr, granule);
end = start + granule;
if (!kvm_level_supports_block_mapping(level))
continue;
if (start < range->start || range->end < end)
continue;
/*
* Check the presence of existing mappings with incompatible
* permissions within the current block range, and try one level
* deeper if one is found.
*/
ret = kvm_pgtable_walk(pgt, start, granule, &check_perm_walker);
if (ret != -EEXIST)
break;
}
if (!ret) {
range->start = start;
range->end = end;
}
return ret;
}

113
arch/arm64/kvm/hyp/reserved_mem.c Normal file
View File

@ -0,0 +1,113 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2020 - Google LLC
* Author: Quentin Perret <qperret@google.com>
*/
#include <linux/kvm_host.h>
#include <linux/memblock.h>
#include <linux/sort.h>
#include <asm/kvm_host.h>
#include <nvhe/memory.h>
#include <nvhe/mm.h>
static struct memblock_region *hyp_memory = kvm_nvhe_sym(hyp_memory);
static unsigned int *hyp_memblock_nr_ptr = &kvm_nvhe_sym(hyp_memblock_nr);
phys_addr_t hyp_mem_base;
phys_addr_t hyp_mem_size;
static int cmp_hyp_memblock(const void *p1, const void *p2)
{
const struct memblock_region *r1 = p1;
const struct memblock_region *r2 = p2;
return r1->base < r2->base ? -1 : (r1->base > r2->base);
}
static void __init sort_memblock_regions(void)
{
sort(hyp_memory,
*hyp_memblock_nr_ptr,
sizeof(struct memblock_region),
cmp_hyp_memblock,
NULL);
}
static int __init register_memblock_regions(void)
{
struct memblock_region *reg;
for_each_mem_region(reg) {
if (*hyp_memblock_nr_ptr >= HYP_MEMBLOCK_REGIONS)
return -ENOMEM;
hyp_memory[*hyp_memblock_nr_ptr] = *reg;
(*hyp_memblock_nr_ptr)++;
}
sort_memblock_regions();
return 0;
}
void __init kvm_hyp_reserve(void)
{
u64 nr_pages, prev, hyp_mem_pages = 0;
int ret;
if (!is_hyp_mode_available() || is_kernel_in_hyp_mode())
return;
if (kvm_get_mode() != KVM_MODE_PROTECTED)
return;
ret = register_memblock_regions();
if (ret) {
*hyp_memblock_nr_ptr = 0;
kvm_err("Failed to register hyp memblocks: %d\n", ret);
return;
}
hyp_mem_pages += hyp_s1_pgtable_pages();
hyp_mem_pages += host_s2_mem_pgtable_pages();
hyp_mem_pages += host_s2_dev_pgtable_pages();
/*
* The hyp_vmemmap needs to be backed by pages, but these pages
* themselves need to be present in the vmemmap, so compute the number
* of pages needed by looking for a fixed point.
*/
nr_pages = 0;
do {
prev = nr_pages;
nr_pages = hyp_mem_pages + prev;
nr_pages = DIV_ROUND_UP(nr_pages * sizeof(struct hyp_page), PAGE_SIZE);
nr_pages += __hyp_pgtable_max_pages(nr_pages);
} while (nr_pages != prev);
hyp_mem_pages += nr_pages;
/*
* Try to allocate a PMD-aligned region to reduce TLB pressure once
* this is unmapped from the host stage-2, and fallback to PAGE_SIZE.
*/
hyp_mem_size = hyp_mem_pages << PAGE_SHIFT;
hyp_mem_base = memblock_find_in_range(0, memblock_end_of_DRAM(),
ALIGN(hyp_mem_size, PMD_SIZE),
PMD_SIZE);
if (!hyp_mem_base)
hyp_mem_base = memblock_find_in_range(0, memblock_end_of_DRAM(),
hyp_mem_size, PAGE_SIZE);
else
hyp_mem_size = ALIGN(hyp_mem_size, PMD_SIZE);
if (!hyp_mem_base) {
kvm_err("Failed to reserve hyp memory\n");
return;
}
memblock_reserve(hyp_mem_base, hyp_mem_size);
kvm_info("Reserved %lld MiB at 0x%llx\n", hyp_mem_size >> 20,
hyp_mem_base);
}

115
arch/arm64/kvm/mmu.c
View File

@ -88,6 +88,44 @@ static bool kvm_is_device_pfn(unsigned long pfn)
return !pfn_valid(pfn);
}
static void *stage2_memcache_zalloc_page(void *arg)
{
struct kvm_mmu_memory_cache *mc = arg;
/* Allocated with __GFP_ZERO, so no need to zero */
return kvm_mmu_memory_cache_alloc(mc);
}
static void *kvm_host_zalloc_pages_exact(size_t size)
{
return alloc_pages_exact(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO);
}
static void kvm_host_get_page(void *addr)
{
get_page(virt_to_page(addr));
}
static void kvm_host_put_page(void *addr)
{
put_page(virt_to_page(addr));
}
static int kvm_host_page_count(void *addr)
{
return page_count(virt_to_page(addr));
}
static phys_addr_t kvm_host_pa(void *addr)
{
return __pa(addr);
}
static void *kvm_host_va(phys_addr_t phys)
{
return __va(phys);
}
/*
* Unmapping vs dcache management:
*
@ -127,7 +165,7 @@ static bool kvm_is_device_pfn(unsigned long pfn)
static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 size,
bool may_block)
{
struct kvm *kvm = mmu->kvm;
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
phys_addr_t end = start + size;
assert_spin_locked(&kvm->mmu_lock);
@ -183,15 +221,39 @@ void free_hyp_pgds(void)
if (hyp_pgtable) {
kvm_pgtable_hyp_destroy(hyp_pgtable);
kfree(hyp_pgtable);
hyp_pgtable = NULL;
}
mutex_unlock(&kvm_hyp_pgd_mutex);
}
static bool kvm_host_owns_hyp_mappings(void)
{
if (static_branch_likely(&kvm_protected_mode_initialized))
return false;
/*
* This can happen at boot time when __create_hyp_mappings() is called
* after the hyp protection has been enabled, but the static key has
* not been flipped yet.
*/
if (!hyp_pgtable && is_protected_kvm_enabled())
return false;
WARN_ON(!hyp_pgtable);
return true;
}
static int __create_hyp_mappings(unsigned long start, unsigned long size,
unsigned long phys, enum kvm_pgtable_prot prot)
{
int err;
if (!kvm_host_owns_hyp_mappings()) {
return kvm_call_hyp_nvhe(__pkvm_create_mappings,
start, size, phys, prot);
}
mutex_lock(&kvm_hyp_pgd_mutex);
err = kvm_pgtable_hyp_map(hyp_pgtable, start, size, phys, prot);
mutex_unlock(&kvm_hyp_pgd_mutex);
@ -253,6 +315,16 @@ static int __create_hyp_private_mapping(phys_addr_t phys_addr, size_t size,
unsigned long base;
int ret = 0;
if (!kvm_host_owns_hyp_mappings()) {
base = kvm_call_hyp_nvhe(__pkvm_create_private_mapping,
phys_addr, size, prot);
if (IS_ERR_OR_NULL((void *)base))
return PTR_ERR((void *)base);
*haddr = base;
return 0;
}
mutex_lock(&kvm_hyp_pgd_mutex);
/*
@ -351,6 +423,17 @@ int create_hyp_exec_mappings(phys_addr_t phys_addr, size_t size,
return 0;
}
static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = {
.zalloc_page = stage2_memcache_zalloc_page,
.zalloc_pages_exact = kvm_host_zalloc_pages_exact,
.free_pages_exact = free_pages_exact,
.get_page = kvm_host_get_page,
.put_page = kvm_host_put_page,
.page_count = kvm_host_page_count,
.phys_to_virt = kvm_host_va,
.virt_to_phys = kvm_host_pa,
};
/**
* kvm_init_stage2_mmu - Initialise a S2 MMU strucrure
* @kvm: The pointer to the KVM structure
@ -374,7 +457,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
if (!pgt)
return -ENOMEM;
err = kvm_pgtable_stage2_init(pgt, kvm);
err = kvm_pgtable_stage2_init(pgt, &kvm->arch, &kvm_s2_mm_ops);
if (err)
goto out_free_pgtable;
@ -387,7 +470,7 @@ int kvm_init_stage2_mmu(struct kvm *kvm, struct kvm_s2_mmu *mmu)
for_each_possible_cpu(cpu)
*per_cpu_ptr(mmu->last_vcpu_ran, cpu) = -1;
mmu->kvm = kvm;
mmu->arch = &kvm->arch;
mmu->pgt = pgt;
mmu->pgd_phys = __pa(pgt->pgd);
mmu->vmid.vmid_gen = 0;
@ -469,7 +552,7 @@ void stage2_unmap_vm(struct kvm *kvm)
void kvm_free_stage2_pgd(struct kvm_s2_mmu *mmu)
{
struct kvm *kvm = mmu->kvm;
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
struct kvm_pgtable *pgt = NULL;
spin_lock(&kvm->mmu_lock);
@ -538,7 +621,7 @@ int kvm_phys_addr_ioremap(struct kvm *kvm, phys_addr_t guest_ipa,
*/
static void stage2_wp_range(struct kvm_s2_mmu *mmu, phys_addr_t addr, phys_addr_t end)
{
struct kvm *kvm = mmu->kvm;
struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
stage2_apply_range_resched(kvm, addr, end, kvm_pgtable_stage2_wrprotect);
}
@ -1208,10 +1291,22 @@ static int kvm_map_idmap_text(void)
return err;
}
int kvm_mmu_init(void)
static void *kvm_hyp_zalloc_page(void *arg)
{
return (void *)get_zeroed_page(GFP_KERNEL);
}
static struct kvm_pgtable_mm_ops kvm_hyp_mm_ops = {
.zalloc_page = kvm_hyp_zalloc_page,
.get_page = kvm_host_get_page,
.put_page = kvm_host_put_page,
.phys_to_virt = kvm_host_va,
.virt_to_phys = kvm_host_pa,
};
int kvm_mmu_init(u32 *hyp_va_bits)
{
int err;
u32 hyp_va_bits;
hyp_idmap_start = __pa_symbol(__hyp_idmap_text_start);
hyp_idmap_start = ALIGN_DOWN(hyp_idmap_start, PAGE_SIZE);
@ -1225,8 +1320,8 @@ int kvm_mmu_init(void)
*/
BUG_ON((hyp_idmap_start ^ (hyp_idmap_end - 1)) & PAGE_MASK);
hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
kvm_debug("Using %u-bit virtual addresses at EL2\n", hyp_va_bits);
*hyp_va_bits = 64 - ((idmap_t0sz & TCR_T0SZ_MASK) >> TCR_T0SZ_OFFSET);
kvm_debug("Using %u-bit virtual addresses at EL2\n", *hyp_va_bits);
kvm_debug("IDMAP page: %lx\n", hyp_idmap_start);
kvm_debug("HYP VA range: %lx:%lx\n",
kern_hyp_va(PAGE_OFFSET),
@ -1251,7 +1346,7 @@ int kvm_mmu_init(void)
goto out;
}
err = kvm_pgtable_hyp_init(hyp_pgtable, hyp_va_bits);
err = kvm_pgtable_hyp_init(hyp_pgtable, *hyp_va_bits, &kvm_hyp_mm_ops);
if (err)
goto out_free_pgtable;

3
arch/arm64/kvm/perf.c
View File

@ -55,7 +55,8 @@ int kvm_perf_init(void)
* hardware performance counters. This could ensure the presence of
* a physical PMU and CONFIG_PERF_EVENT is selected.
*/
if (IS_ENABLED(CONFIG_ARM_PMU) && perf_num_counters() > 0)
if (IS_ENABLED(CONFIG_ARM_PMU) && perf_num_counters() > 0
&& !is_protected_kvm_enabled())
static_branch_enable(&kvm_arm_pmu_available);
return perf_register_guest_info_callbacks(&kvm_guest_cbs);

8
arch/arm64/kvm/pmu.c
View File

@ -33,7 +33,7 @@ void kvm_set_pmu_events(u32 set, struct perf_event_attr *attr)
{
struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data);
if (!ctx || !kvm_pmu_switch_needed(attr))
if (!kvm_arm_support_pmu_v3() || !ctx || !kvm_pmu_switch_needed(attr))
return;
if (!attr->exclude_host)
@ -49,7 +49,7 @@ void kvm_clr_pmu_events(u32 clr)
{
struct kvm_host_data *ctx = this_cpu_ptr_hyp_sym(kvm_host_data);
if (!ctx)
if (!kvm_arm_support_pmu_v3() || !ctx)
return;
ctx->pmu_events.events_host &= ~clr;
@ -172,7 +172,7 @@ void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu)
struct kvm_host_data *host;
u32 events_guest, events_host;
if (!has_vhe())
if (!kvm_arm_support_pmu_v3() || !has_vhe())
return;
preempt_disable();
@ -193,7 +193,7 @@ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu)
struct kvm_host_data *host;
u32 events_guest, events_host;
if (!has_vhe())
if (!kvm_arm_support_pmu_v3() || !has_vhe())
return;
host = this_cpu_ptr_hyp_sym(kvm_host_data);

46
arch/arm64/kvm/reset.c
View File

@ -74,10 +74,6 @@ static int kvm_vcpu_enable_sve(struct kvm_vcpu *vcpu)
if (!system_supports_sve())
return -EINVAL;
/* Verify that KVM startup enforced this when SVE was detected: */
if (WARN_ON(!has_vhe()))
return -EINVAL;
vcpu->arch.sve_max_vl = kvm_sve_max_vl;
/*
@ -331,19 +327,10 @@ int kvm_set_ipa_limit(void)
return 0;
}
/*
* Configure the VTCR_EL2 for this VM. The VTCR value is common
* across all the physical CPUs on the system. We use system wide
* sanitised values to fill in different fields, except for Hardware
* Management of Access Flags. HA Flag is set unconditionally on
* all CPUs, as it is safe to run with or without the feature and
* the bit is RES0 on CPUs that don't support it.
*/
int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type)
{
u64 vtcr = VTCR_EL2_FLAGS, mmfr0;
u32 parange, phys_shift;
u8 lvls;
u64 mmfr0, mmfr1;
u32 phys_shift;
if (type & ~KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
return -EINVAL;
@ -363,33 +350,8 @@ int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long type)
}
mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
parange = cpuid_feature_extract_unsigned_field(mmfr0,
ID_AA64MMFR0_PARANGE_SHIFT);
if (parange > ID_AA64MMFR0_PARANGE_MAX)
parange = ID_AA64MMFR0_PARANGE_MAX;
vtcr |= parange << VTCR_EL2_PS_SHIFT;
mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
kvm->arch.vtcr = kvm_get_vtcr(mmfr0, mmfr1, phys_shift);
vtcr |= VTCR_EL2_T0SZ(phys_shift);
/*
* Use a minimum 2 level page table to prevent splitting
* host PMD huge pages at stage2.
*/
lvls = stage2_pgtable_levels(phys_shift);
if (lvls < 2)
lvls = 2;
vtcr |= VTCR_EL2_LVLS_TO_SL0(lvls);
/*
* Enable the Hardware Access Flag management, unconditionally
* on all CPUs. The features is RES0 on CPUs without the support
* and must be ignored by the CPUs.
*/
vtcr |= VTCR_EL2_HA;
/* Set the vmid bits */
vtcr |= (kvm_get_vmid_bits() == 16) ?
VTCR_EL2_VS_16BIT :
VTCR_EL2_VS_8BIT;
kvm->arch.vtcr = vtcr;
return 0;
}

22
arch/arm64/kvm/sys_regs.c
View File

@ -21,6 +21,7 @@
#include <asm/debug-monitors.h>
#include <asm/esr.h>
#include <asm/kvm_arm.h>
#include <asm/kvm_cpufeature.h>
#include <asm/kvm_emulate.h>
#include <asm/kvm_hyp.h>
#include <asm/kvm_mmu.h>
@ -2775,3 +2776,24 @@ void kvm_sys_reg_table_init(void)
/* Clear all higher bits. */
cache_levels &= (1 << (i*3))-1;
}
#define CPU_FTR_REG_HYP_COPY(id, name) \
{ .sys_id = id, .dst = (struct arm64_ftr_reg *)&kvm_nvhe_sym(name) }
struct __ftr_reg_copy_entry {
u32 sys_id;
struct arm64_ftr_reg *dst;
} hyp_ftr_regs[] __initdata = {
CPU_FTR_REG_HYP_COPY(SYS_CTR_EL0, arm64_ftr_reg_ctrel0),
CPU_FTR_REG_HYP_COPY(SYS_ID_AA64MMFR0_EL1, arm64_ftr_reg_id_aa64mmfr0_el1),
CPU_FTR_REG_HYP_COPY(SYS_ID_AA64MMFR1_EL1, arm64_ftr_reg_id_aa64mmfr1_el1),
};
void __init setup_kvm_el2_caps(void)
{
int i;
for (i = 0; i < ARRAY_SIZE(hyp_ftr_regs); i++) {
WARN(copy_ftr_reg(hyp_ftr_regs[i].sys_id, hyp_ftr_regs[i].dst),
"%u feature register not found\n", hyp_ftr_regs[i].sys_id);
}
}

4
arch/arm64/lib/clear_page.S
View File

@ -14,7 +14,7 @@
* Parameters:
* x0 - dest
*/
SYM_FUNC_START(clear_page)
SYM_FUNC_START_PI(clear_page)
mrs x1, dczid_el0
and w1, w1, #0xf
mov x2, #4
@ -25,5 +25,5 @@ SYM_FUNC_START(clear_page)
tst x0, #(PAGE_SIZE - 1)
b.ne 1b
ret
SYM_FUNC_END(clear_page)
SYM_FUNC_END_PI(clear_page)
EXPORT_SYMBOL(clear_page)

4
arch/arm64/lib/copy_page.S
View File

@ -17,7 +17,7 @@
* x0 - dest
* x1 - src
*/
SYM_FUNC_START(copy_page)
SYM_FUNC_START_PI(copy_page)
alternative_if ARM64_HAS_NO_HW_PREFETCH
// Prefetch three cache lines ahead.
prfm pldl1strm, [x1, #128]
@ -75,5 +75,5 @@ alternative_else_nop_endif
stnp x16, x17, [x0, #112 - 256]
ret
SYM_FUNC_END(copy_page)
SYM_FUNC_END_PI(copy_page)
EXPORT_SYMBOL(copy_page)

3
arch/arm64/mm/init.c
View File

@ -35,6 +35,7 @@
#include <asm/fixmap.h>
#include <asm/kasan.h>
#include <asm/kernel-pgtable.h>
#include <asm/kvm_host.h>
#include <asm/memory.h>
#include <asm/numa.h>
#include <asm/sections.h>
@ -433,6 +434,8 @@ void __init bootmem_init(void)
dma_pernuma_cma_reserve();
kvm_hyp_reserve();
/*
* sparse_init() tries to allocate memory from memblock, so must be
* done after the fixed reservations

3
arch/arm64/net/bpf_jit_comp.c
View File

@ -17,6 +17,7 @@
#include <asm/cacheflush.h>
#include <asm/debug-monitors.h>
#include <asm/set_memory.h>
#include <trace/hooks/memory.h>
#include "bpf_jit.h"
@ -1098,6 +1099,8 @@ skip_init_ctx:
goto out_off;
}
bpf_jit_binary_lock_ro(header);
trace_android_vh_set_memory_ro((unsigned long)header, header->pages);
trace_android_vh_set_memory_x((unsigned long)header, header->pages);
} else {
jit_data->ctx = ctx;
jit_data->image = image_ptr;

10
arch/x86/configs/gki_defconfig
View File

@ -56,6 +56,7 @@ CONFIG_CMDLINE="stack_depot_disable=on"
CONFIG_PM_WAKELOCKS=y
CONFIG_PM_WAKELOCKS_LIMIT=0
# CONFIG_PM_WAKELOCKS_GC is not set
CONFIG_ENERGY_MODEL=y
CONFIG_CPU_FREQ_STAT=y
CONFIG_CPU_FREQ_TIMES=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=y
@ -443,6 +444,9 @@ CONFIG_PM_DEVFREQ_EVENT=y
CONFIG_IIO=y
CONFIG_IIO_BUFFER=y
CONFIG_IIO_TRIGGER=y
CONFIG_POWERCAP=y
CONFIG_DTPM=y
CONFIG_DTPM_CPU=y
CONFIG_RAS=y
CONFIG_ANDROID=y
CONFIG_ANDROID_BINDER_IPC=y
@ -561,15 +565,11 @@ CONFIG_DEBUG_INFO_DWARF4=y
CONFIG_HEADERS_INSTALL=y
# CONFIG_SECTION_MISMATCH_WARN_ONLY is not set
CONFIG_MAGIC_SYSRQ=y
CONFIG_UBSAN=y
CONFIG_UBSAN_TRAP=y
CONFIG_UBSAN_LOCAL_BOUNDS=y
# CONFIG_UBSAN_MISC is not set
CONFIG_PAGE_OWNER=y
CONFIG_DEBUG_STACK_USAGE=y
CONFIG_DEBUG_MEMORY_INIT=y
CONFIG_KFENCE=y
CONFIG_KFENCE_SAMPLE_INTERVAL=0
CONFIG_KFENCE_SAMPLE_INTERVAL=500
CONFIG_PANIC_ON_OOPS=y
CONFIG_PANIC_TIMEOUT=-1
CONFIG_WQ_WATCHDOG=y

42
drivers/android/vendor_hooks.c
View File

@ -40,6 +40,12 @@
#include <trace/hooks/ufshcd.h>
#include <trace/hooks/cgroup.h>
#include <trace/hooks/sys.h>
#include <trace/hooks/traps.h>
#include <trace/hooks/avc.h>
#include <trace/hooks/creds.h>
#include <trace/hooks/module.h>
#include <trace/hooks/memory.h>
#include <trace/hooks/typec.h>
/*
* Export tracepoints that act as a bare tracehook (ie: have no trace event
@ -100,9 +106,7 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_busiest_queue);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_migrate_queued_task);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_resume_cpus);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_find_energy_efficient_cpu);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_sugov_sched_attr);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_iowait);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_set_sugov_update);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpu_overutilized);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_sched_setaffinity);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_update_cpus_allowed);
@ -170,12 +174,44 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_compl_command);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_uic_command);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_send_tm_command);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_check_int_errors);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_ufs_update_sdev);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_do_wake_up_sync);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_wake_flags);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_uclamp_eff_value);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_uclamp_eff_get);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_cpufreq_transition);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cgroup_set_task);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_syscall_prctl_finished);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_tick);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_check_preempt_wakeup_ignore);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_replace_next_task_fair);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_build_sched_domains);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alter_mutex_list_add);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_mutex_unlock_slowpath);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_rwsem_wake_finish);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_do_undefinstr);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_bad_mode);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_arm64_serror_panic);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_insert);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_node_delete);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_node_replace);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_selinux_avc_lookup);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_commit_creds);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_creds);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_override_creds);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_revert_creds);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_x);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_nx);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_ro);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_memory_rw);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_before_init);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_set_module_permit_after_init);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_rvh_util_est_update);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_meminfo_proc_show);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_exit_mm);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_alloc_pages_slowpath);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_show_mem);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_print_slabinfo_header);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_cache_show);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_override_toggling);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_check_contaminant);
EXPORT_TRACEPOINT_SYMBOL_GPL(android_vh_typec_tcpci_get_vbus);

6
drivers/block/zram/zram_drv.c
View File

@ -648,8 +648,8 @@ static ssize_t writeback_store(struct device *dev,
if (strncmp(buf, PAGE_WB_SIG, sizeof(PAGE_WB_SIG) - 1))
return -EINVAL;
ret = kstrtol(buf + sizeof(PAGE_WB_SIG) - 1, 10, &index);
if (ret || index >= nr_pages)
if (kstrtol(buf + sizeof(PAGE_WB_SIG) - 1, 10, &index) ||
index >= nr_pages)
return -EINVAL;
nr_pages = 1;
@ -673,7 +673,7 @@ static ssize_t writeback_store(struct device *dev,
goto release_init_lock;
}
while (nr_pages--) {
for (; nr_pages != 0; index++, nr_pages--) {
struct bio_vec bvec;
bvec.bv_page = page;

4
drivers/dma-buf/heaps/system_heap.c
View File

@ -534,13 +534,13 @@ static int system_heap_create(void)
for (i = 0; i < NUM_ORDERS; i++) {
pools[i] = dmabuf_page_pool_create(order_flags[i], orders[i]);
if (IS_ERR(pools[i])) {
if (!pools[i]) {
int j;
pr_err("%s: page pool creation failed!\n", __func__);
for (j = 0; j < i; j++)
dmabuf_page_pool_destroy(pools[j]);
return PTR_ERR(pools[i]);
return -ENOMEM;
}
}

15
drivers/mmc/core/crypto.c
View File

@ -31,18 +31,11 @@ void mmc_crypto_prepare_req(struct mmc_queue_req *mqrq)
struct request *req = mmc_queue_req_to_req(mqrq);
struct mmc_request *mrq = &mqrq->brq.mrq;
if (!req->crypt_keyslot)
if (!req->crypt_ctx)
return;
mrq->crypto_enabled = true;
mrq->crypto_key_slot = blk_ksm_get_slot_idx(req->crypt_keyslot);
/*
* For now we assume that all MMC drivers set max_dun_bytes_supported=4,
* which is the limit for CQHCI crypto. So all DUNs should be 32-bit.
*/
WARN_ON_ONCE(req->crypt_ctx->bc_dun[0] > U32_MAX);
mrq->data_unit_num = req->crypt_ctx->bc_dun[0];
mrq->crypto_ctx = req->crypt_ctx;
if (req->crypt_keyslot)
mrq->crypto_key_slot = blk_ksm_get_slot_idx(req->crypt_keyslot);
}
EXPORT_SYMBOL_GPL(mmc_crypto_prepare_req);

7
drivers/mmc/host/cqhci-crypto.h
View File

@ -22,12 +22,15 @@ int cqhci_crypto_init(struct cqhci_host *host);
*/
static inline u64 cqhci_crypto_prep_task_desc(struct mmc_request *mrq)
{
if (!mrq->crypto_enabled)
if (!mrq->crypto_ctx)
return 0;
/* We set max_dun_bytes_supported=4, so all DUNs should be 32-bit. */
WARN_ON_ONCE(mrq->crypto_ctx->bc_dun[0] > U32_MAX);
return CQHCI_CRYPTO_ENABLE_BIT |
CQHCI_CRYPTO_KEYSLOT(mrq->crypto_key_slot) |
mrq->data_unit_num;
mrq->crypto_ctx->bc_dun[0];
}
#else /* CONFIG_MMC_CRYPTO */

6
drivers/pci/pci.h
View File

@ -3,6 +3,7 @@
#define DRIVERS_PCI_H
#include <linux/pci.h>
#include <linux/android_kabi.h>
/* Number of possible devfns: 0.0 to 1f.7 inclusive */
#define MAX_NR_DEVFNS 256
@ -343,6 +344,11 @@ struct pci_sriov {
u16 subsystem_device; /* VF subsystem device */
resource_size_t barsz[PCI_SRIOV_NUM_BARS]; /* VF BAR size */
bool drivers_autoprobe; /* Auto probing of VFs by driver */
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
ANDROID_KABI_RESERVE(3);
ANDROID_KABI_RESERVE(4);
};
/**

13
drivers/powercap/Kconfig
View File

@ -43,4 +43,17 @@ config IDLE_INJECT
CPUs for power capping. Idle period can be injected
synchronously on a set of specified CPUs or alternatively
on a per CPU basis.
config DTPM
bool "Power capping for Dynamic Thermal Power Management"
help
This enables support for the power capping for the dynamic
thermal power management userspace engine.
config DTPM_CPU
bool "Add CPU power capping based on the energy model"
depends on DTPM && ENERGY_MODEL
help
This enables support for CPU power limitation based on
energy model.
endif

2
drivers/powercap/Makefile
View File

@ -1,4 +1,6 @@
# SPDX-License-Identifier: GPL-2.0-only
obj-$(CONFIG_DTPM) += dtpm.o
obj-$(CONFIG_DTPM_CPU) += dtpm_cpu.o
obj-$(CONFIG_POWERCAP) += powercap_sys.o
obj-$(CONFIG_INTEL_RAPL_CORE) += intel_rapl_common.o
obj-$(CONFIG_INTEL_RAPL) += intel_rapl_msr.o

480
drivers/powercap/dtpm.c Normal file
View File

@ -0,0 +1,480 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2020 Linaro Limited
*
* Author: Daniel Lezcano <daniel.lezcano@linaro.org>
*
* The powercap based Dynamic Thermal Power Management framework
* provides to the userspace a consistent API to set the power limit
* on some devices.
*
* DTPM defines the functions to create a tree of constraints. Each
* parent node is a virtual description of the aggregation of the
* children. It propagates the constraints set at its level to its
* children and collect the children power information. The leaves of
* the tree are the real devices which have the ability to get their
* current power consumption and set their power limit.
*/
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/dtpm.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/powercap.h>
#include <linux/slab.h>
#include <linux/mutex.h>
#define DTPM_POWER_LIMIT_FLAG 0
static const char *constraint_name[] = {
"Instantaneous",
};
static DEFINE_MUTEX(dtpm_lock);
static struct powercap_control_type *pct;
static struct dtpm *root;
static int get_time_window_us(struct powercap_zone *pcz, int cid, u64 *window)
{
return -ENOSYS;
}
static int set_time_window_us(struct powercap_zone *pcz, int cid, u64 window)
{
return -ENOSYS;
}
static int get_max_power_range_uw(struct powercap_zone *pcz, u64 *max_power_uw)
{
struct dtpm *dtpm = to_dtpm(pcz);
mutex_lock(&dtpm_lock);
*max_power_uw = dtpm->power_max - dtpm->power_min;
mutex_unlock(&dtpm_lock);
return 0;
}
static int __get_power_uw(struct dtpm *dtpm, u64 *power_uw)
{
struct dtpm *child;
u64 power;
int ret = 0;
if (dtpm->ops) {
*power_uw = dtpm->ops->get_power_uw(dtpm);
return 0;
}
*power_uw = 0;
list_for_each_entry(child, &dtpm->children, sibling) {
ret = __get_power_uw(child, &power);
if (ret)
break;
*power_uw += power;
}
return ret;
}
static int get_power_uw(struct powercap_zone *pcz, u64 *power_uw)
{
struct dtpm *dtpm = to_dtpm(pcz);
int ret;
mutex_lock(&dtpm_lock);
ret = __get_power_uw(dtpm, power_uw);
mutex_unlock(&dtpm_lock);
return ret;
}
static void __dtpm_rebalance_weight(struct dtpm *dtpm)
{
struct dtpm *child;
list_for_each_entry(child, &dtpm->children, sibling) {
pr_debug("Setting weight '%d' for '%s'\n",
child->weight, child->zone.name);
child->weight = DIV64_U64_ROUND_CLOSEST(
child->power_max * 1024, dtpm->power_max);
__dtpm_rebalance_weight(child);
}
}
static void __dtpm_sub_power(struct dtpm *dtpm)
{
struct dtpm *parent = dtpm->parent;
while (parent) {
parent->power_min -= dtpm->power_min;
parent->power_max -= dtpm->power_max;
parent->power_limit -= dtpm->power_limit;
parent = parent->parent;
}
__dtpm_rebalance_weight(root);
}
static void __dtpm_add_power(struct dtpm *dtpm)
{
struct dtpm *parent = dtpm->parent;
while (parent) {
parent->power_min += dtpm->power_min;
parent->power_max += dtpm->power_max;
parent->power_limit += dtpm->power_limit;
parent = parent->parent;
}
__dtpm_rebalance_weight(root);
}
/**
* dtpm_update_power - Update the power on the dtpm
* @dtpm: a pointer to a dtpm structure to update
* @power_min: a u64 representing the new power_min value
* @power_max: a u64 representing the new power_max value
*
* Function to update the power values of the dtpm node specified in
* parameter. These new values will be propagated to the tree.
*
* Return: zero on success, -EINVAL if the values are inconsistent
*/
int dtpm_update_power(struct dtpm *dtpm, u64 power_min, u64 power_max)
{
int ret = 0;
mutex_lock(&dtpm_lock);
if (power_min == dtpm->power_min && power_max == dtpm->power_max)
goto unlock;
if (power_max < power_min) {
ret = -EINVAL;
goto unlock;
}
__dtpm_sub_power(dtpm);
dtpm->power_min = power_min;
dtpm->power_max = power_max;
if (!test_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags))
dtpm->power_limit = power_max;
__dtpm_add_power(dtpm);
unlock:
mutex_unlock(&dtpm_lock);
return ret;
}
/**
* dtpm_release_zone - Cleanup when the node is released
* @pcz: a pointer to a powercap_zone structure
*
* Do some housecleaning and update the weight on the tree. The
* release will be denied if the node has children. This function must
* be called by the specific release callback of the different
* backends.
*
* Return: 0 on success, -EBUSY if there are children
*/
int dtpm_release_zone(struct powercap_zone *pcz)
{
struct dtpm *dtpm = to_dtpm(pcz);
struct dtpm *parent = dtpm->parent;
mutex_lock(&dtpm_lock);
if (!list_empty(&dtpm->children)) {
mutex_unlock(&dtpm_lock);
return -EBUSY;
}
if (parent)
list_del(&dtpm->sibling);
__dtpm_sub_power(dtpm);
mutex_unlock(&dtpm_lock);
if (dtpm->ops)
dtpm->ops->release(dtpm);
kfree(dtpm);
return 0;
}
static int __get_power_limit_uw(struct dtpm *dtpm, int cid, u64 *power_limit)
{
*power_limit = dtpm->power_limit;
return 0;
}
static int get_power_limit_uw(struct powercap_zone *pcz,
int cid, u64 *power_limit)
{
struct dtpm *dtpm = to_dtpm(pcz);
int ret;
mutex_lock(&dtpm_lock);
ret = __get_power_limit_uw(dtpm, cid, power_limit);
mutex_unlock(&dtpm_lock);
return ret;
}
/*
* Set the power limit on the nodes, the power limit is distributed
* given the weight of the children.
*
* The dtpm node lock must be held when calling this function.
*/
static int __set_power_limit_uw(struct dtpm *dtpm, int cid, u64 power_limit)
{
struct dtpm *child;
int ret = 0;
u64 power;
/*
* A max power limitation means we remove the power limit,
* otherwise we set a constraint and flag the dtpm node.
*/
if (power_limit == dtpm->power_max) {
clear_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags);
} else {
set_bit(DTPM_POWER_LIMIT_FLAG, &dtpm->flags);
}
pr_debug("Setting power limit for '%s': %llu uW\n",
dtpm->zone.name, power_limit);
/*
* Only leaves of the dtpm tree has ops to get/set the power
*/
if (dtpm->ops) {
dtpm->power_limit = dtpm->ops->set_power_uw(dtpm, power_limit);
} else {
dtpm->power_limit = 0;
list_for_each_entry(child, &dtpm->children, sibling) {
/*
* Integer division rounding will inevitably
* lead to a different min or max value when
* set several times. In order to restore the
* initial value, we force the child's min or
* max power every time if the constraint is
* at the boundaries.
*/
if (power_limit == dtpm->power_max) {
power = child->power_max;
} else if (power_limit == dtpm->power_min) {
power = child->power_min;
} else {
power = DIV_ROUND_CLOSEST_ULL(
power_limit * child->weight, 1024);
}
pr_debug("Setting power limit for '%s': %llu uW\n",
child->zone.name, power);
ret = __set_power_limit_uw(child, cid, power);
if (!ret)
ret = __get_power_limit_uw(child, cid, &power);
if (ret)
break;
dtpm->power_limit += power;
}
}
return ret;
}
static int set_power_limit_uw(struct powercap_zone *pcz,
int cid, u64 power_limit)
{
struct dtpm *dtpm = to_dtpm(pcz);
int ret;
mutex_lock(&dtpm_lock);
/*
* Don't allow values outside of the power range previously
* set when initializing the power numbers.
*/
power_limit = clamp_val(power_limit, dtpm->power_min, dtpm->power_max);
ret = __set_power_limit_uw(dtpm, cid, power_limit);
pr_debug("%s: power limit: %llu uW, power max: %llu uW\n",
dtpm->zone.name, dtpm->power_limit, dtpm->power_max);
mutex_unlock(&dtpm_lock);
return ret;
}
static const char *get_constraint_name(struct powercap_zone *pcz, int cid)
{
return constraint_name[cid];
}
static int get_max_power_uw(struct powercap_zone *pcz, int id, u64 *max_power)
{
struct dtpm *dtpm = to_dtpm(pcz);
mutex_lock(&dtpm_lock);
*max_power = dtpm->power_max;
mutex_unlock(&dtpm_lock);
return 0;
}
static struct powercap_zone_constraint_ops constraint_ops = {
.set_power_limit_uw = set_power_limit_uw,
.get_power_limit_uw = get_power_limit_uw,
.set_time_window_us = set_time_window_us,
.get_time_window_us = get_time_window_us,
.get_max_power_uw = get_max_power_uw,
.get_name = get_constraint_name,
};
static struct powercap_zone_ops zone_ops = {
.get_max_power_range_uw = get_max_power_range_uw,
.get_power_uw = get_power_uw,
.release = dtpm_release_zone,
};
/**
* dtpm_alloc - Allocate and initialize a dtpm struct
* @name: a string specifying the name of the node
*
* Return: a struct dtpm pointer, NULL in case of error
*/
struct dtpm *dtpm_alloc(struct dtpm_ops *ops)
{
struct dtpm *dtpm;
dtpm = kzalloc(sizeof(*dtpm), GFP_KERNEL);
if (dtpm) {
INIT_LIST_HEAD(&dtpm->children);
INIT_LIST_HEAD(&dtpm->sibling);
dtpm->weight = 1024;
dtpm->ops = ops;
}
return dtpm;
}
/**
* dtpm_unregister - Unregister a dtpm node from the hierarchy tree
* @dtpm: a pointer to a dtpm structure corresponding to the node to be removed
*
* Call the underlying powercap unregister function. That will call
* the release callback of the powercap zone.
*/
void dtpm_unregister(struct dtpm *dtpm)
{
powercap_unregister_zone(pct, &dtpm->zone);
pr_info("Unregistered dtpm node '%s'\n", dtpm->zone.name);
}
/**
* dtpm_register - Register a dtpm node in the hierarchy tree
* @name: a string specifying the name of the node
* @dtpm: a pointer to a dtpm structure corresponding to the new node
* @parent: a pointer to a dtpm structure corresponding to the parent node
*
* Create a dtpm node in the tree. If no parent is specified, the node
* is the root node of the hierarchy. If the root node already exists,
* then the registration will fail. The powercap controller must be
* initialized before calling this function.
*
* The dtpm structure must be initialized with the power numbers
* before calling this function.
*
* Return: zero on success, a negative value in case of error:
* -EAGAIN: the function is called before the framework is initialized.
* -EBUSY: the root node is already inserted
* -EINVAL: * there is no root node yet and @parent is specified
* * no all ops are defined
* * parent have ops which are reserved for leaves
* Other negative values are reported back from the powercap framework
*/
int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent)
{
struct powercap_zone *pcz;
if (!pct)
return -EAGAIN;
if (root && !parent)
return -EBUSY;
if (!root && parent)
return -EINVAL;
if (parent && parent->ops)
return -EINVAL;
if (!dtpm)
return -EINVAL;
if (dtpm->ops && !(dtpm->ops->set_power_uw &&
dtpm->ops->get_power_uw &&
dtpm->ops->release))
return -EINVAL;
pcz = powercap_register_zone(&dtpm->zone, pct, name,
parent ? &parent->zone : NULL,
&zone_ops, MAX_DTPM_CONSTRAINTS,
&constraint_ops);
if (IS_ERR(pcz))
return PTR_ERR(pcz);
mutex_lock(&dtpm_lock);
if (parent) {
list_add_tail(&dtpm->sibling, &parent->children);
dtpm->parent = parent;
} else {
root = dtpm;
}
__dtpm_add_power(dtpm);
pr_info("Registered dtpm node '%s' / %llu-%llu uW, \n",
dtpm->zone.name, dtpm->power_min, dtpm->power_max);
mutex_unlock(&dtpm_lock);
return 0;
}
static int __init dtpm_init(void)
{
struct dtpm_descr **dtpm_descr;
pct = powercap_register_control_type(NULL, "dtpm", NULL);
if (IS_ERR(pct)) {
pr_err("Failed to register control type\n");
return PTR_ERR(pct);
}
for_each_dtpm_table(dtpm_descr)
(*dtpm_descr)->init(*dtpm_descr);
return 0;
}
late_initcall(dtpm_init);

257
drivers/powercap/dtpm_cpu.c Normal file
View File

@ -0,0 +1,257 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright 2020 Linaro Limited
*
* Author: Daniel Lezcano <daniel.lezcano@linaro.org>
*
* The DTPM CPU is based on the energy model. It hooks the CPU in the
* DTPM tree which in turns update the power number by propagating the
* power number from the CPU energy model information to the parents.
*
* The association between the power and the performance state, allows
* to set the power of the CPU at the OPP granularity.
*
* The CPU hotplug is supported and the power numbers will be updated
* if a CPU is hot plugged / unplugged.
*/
#include <linux/cpumask.h>
#include <linux/cpufreq.h>
#include <linux/cpuhotplug.h>
#include <linux/dtpm.h>
#include <linux/energy_model.h>
#include <linux/pm_qos.h>
#include <linux/slab.h>
#include <linux/units.h>
static struct dtpm *__parent;
static DEFINE_PER_CPU(struct dtpm *, dtpm_per_cpu);
struct dtpm_cpu {
struct freq_qos_request qos_req;
int cpu;
};
/*
* When a new CPU is inserted at hotplug or boot time, add the power
* contribution and update the dtpm tree.
*/
static int power_add(struct dtpm *dtpm, struct em_perf_domain *em)
{
u64 power_min, power_max;
power_min = em->table[0].power;
power_min *= MICROWATT_PER_MILLIWATT;
power_min += dtpm->power_min;
power_max = em->table[em->nr_perf_states - 1].power;
power_max *= MICROWATT_PER_MILLIWATT;
power_max += dtpm->power_max;
return dtpm_update_power(dtpm, power_min, power_max);
}
/*
* When a CPU is unplugged, remove its power contribution from the
* dtpm tree.
*/
static int power_sub(struct dtpm *dtpm, struct em_perf_domain *em)
{
u64 power_min, power_max;
power_min = em->table[0].power;
power_min *= MICROWATT_PER_MILLIWATT;
power_min = dtpm->power_min - power_min;
power_max = em->table[em->nr_perf_states - 1].power;
power_max *= MICROWATT_PER_MILLIWATT;
power_max = dtpm->power_max - power_max;
return dtpm_update_power(dtpm, power_min, power_max);
}
static u64 set_pd_power_limit(struct dtpm *dtpm, u64 power_limit)
{
struct dtpm_cpu *dtpm_cpu = dtpm->private;
struct em_perf_domain *pd;
struct cpumask cpus;
unsigned long freq;
u64 power;
int i, nr_cpus;
pd = em_cpu_get(dtpm_cpu->cpu);
cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus));
nr_cpus = cpumask_weight(&cpus);
for (i = 0; i < pd->nr_perf_states; i++) {
power = pd->table[i].power * MICROWATT_PER_MILLIWATT * nr_cpus;
if (power > power_limit)
break;
}
freq = pd->table[i - 1].frequency;
freq_qos_update_request(&dtpm_cpu->qos_req, freq);
power_limit = pd->table[i - 1].power *
MICROWATT_PER_MILLIWATT * nr_cpus;
return power_limit;
}
static u64 get_pd_power_uw(struct dtpm *dtpm)
{
struct dtpm_cpu *dtpm_cpu = dtpm->private;
struct em_perf_domain *pd;
struct cpumask cpus;
unsigned long freq;
int i, nr_cpus;
pd = em_cpu_get(dtpm_cpu->cpu);
freq = cpufreq_quick_get(dtpm_cpu->cpu);
cpumask_and(&cpus, cpu_online_mask, to_cpumask(pd->cpus));
nr_cpus = cpumask_weight(&cpus);
for (i = 0; i < pd->nr_perf_states; i++) {
if (pd->table[i].frequency < freq)
continue;
return pd->table[i].power *
MICROWATT_PER_MILLIWATT * nr_cpus;
}
return 0;
}
static void pd_release(struct dtpm *dtpm)
{
struct dtpm_cpu *dtpm_cpu = dtpm->private;
if (freq_qos_request_active(&dtpm_cpu->qos_req))
freq_qos_remove_request(&dtpm_cpu->qos_req);
kfree(dtpm_cpu);
}
static struct dtpm_ops dtpm_ops = {
.set_power_uw = set_pd_power_limit,
.get_power_uw = get_pd_power_uw,
.release = pd_release,
};
static int cpuhp_dtpm_cpu_offline(unsigned int cpu)
{
struct cpufreq_policy *policy;
struct em_perf_domain *pd;
struct dtpm *dtpm;
policy = cpufreq_cpu_get(cpu);
if (!policy)
return 0;
pd = em_cpu_get(cpu);
if (!pd)
return -EINVAL;
dtpm = per_cpu(dtpm_per_cpu, cpu);
power_sub(dtpm, pd);
if (cpumask_weight(policy->cpus) != 1)
return 0;
for_each_cpu(cpu, policy->related_cpus)
per_cpu(dtpm_per_cpu, cpu) = NULL;
dtpm_unregister(dtpm);
return 0;
}
static int cpuhp_dtpm_cpu_online(unsigned int cpu)
{
struct dtpm *dtpm;
struct dtpm_cpu *dtpm_cpu;
struct cpufreq_policy *policy;
struct em_perf_domain *pd;
char name[CPUFREQ_NAME_LEN];
int ret = -ENOMEM;
policy = cpufreq_cpu_get(cpu);
if (!policy)
return 0;
pd = em_cpu_get(cpu);
if (!pd)
return -EINVAL;
dtpm = per_cpu(dtpm_per_cpu, cpu);
if (dtpm)
return power_add(dtpm, pd);
dtpm = dtpm_alloc(&dtpm_ops);
if (!dtpm)
return -EINVAL;
dtpm_cpu = kzalloc(sizeof(*dtpm_cpu), GFP_KERNEL);
if (!dtpm_cpu)
goto out_kfree_dtpm;
dtpm->private = dtpm_cpu;
dtpm_cpu->cpu = cpu;
for_each_cpu(cpu, policy->related_cpus)
per_cpu(dtpm_per_cpu, cpu) = dtpm;
sprintf(name, "cpu%d", dtpm_cpu->cpu);
ret = dtpm_register(name, dtpm, __parent);
if (ret)
goto out_kfree_dtpm_cpu;
ret = power_add(dtpm, pd);
if (ret)
goto out_dtpm_unregister;
ret = freq_qos_add_request(&policy->constraints,
&dtpm_cpu->qos_req, FREQ_QOS_MAX,
pd->table[pd->nr_perf_states - 1].frequency);
if (ret)
goto out_power_sub;
return 0;
out_power_sub:
power_sub(dtpm, pd);
out_dtpm_unregister:
dtpm_unregister(dtpm);
dtpm_cpu = NULL;
dtpm = NULL;
out_kfree_dtpm_cpu:
for_each_cpu(cpu, policy->related_cpus)
per_cpu(dtpm_per_cpu, cpu) = NULL;
kfree(dtpm_cpu);
out_kfree_dtpm:
kfree(dtpm);
return ret;
}
int dtpm_register_cpu(struct dtpm *parent)
{
__parent = parent;
return cpuhp_setup_state(CPUHP_AP_DTPM_CPU_ONLINE,
"dtpm_cpu:online",
cpuhp_dtpm_cpu_online,
cpuhp_dtpm_cpu_offline);
}

3
drivers/scsi/sd.h
View File

@ -125,6 +125,9 @@ struct scsi_disk {
unsigned urswrz : 1;
unsigned security : 1;
unsigned ignore_medium_access_errors : 1;
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
};
#define to_scsi_disk(obj) container_of(obj,struct scsi_disk,dev)

2
drivers/scsi/ufs/ufshcd.c
View File

@ -4809,6 +4809,8 @@ static int ufshcd_slave_configure(struct scsi_device *sdev)
ufshcd_crypto_setup_rq_keyslot_manager(hba, q);
trace_android_vh_ufs_update_sdev(sdev);
return 0;
}

2
drivers/thermal/thermal_netlink.c
View File

@ -232,7 +232,7 @@ static int thermal_genl_send_event(enum thermal_genl_event event,
void *hdr;
int enable_thermal_genl = 1;
trace_android_vh_enable_thermal_genl_check(event, &enable_thermal_genl);
trace_android_vh_enable_thermal_genl_check(event, p->tz_id, &enable_thermal_genl);
if (!enable_thermal_genl)
return 0;

11
drivers/usb/dwc3/gadget.c
View File

@ -783,8 +783,6 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
trace_dwc3_gadget_ep_disable(dep);
dwc3_remove_requests(dwc, dep);
/* make sure HW endpoint isn't stalled */
if (dep->flags & DWC3_EP_STALL)
__dwc3_gadget_ep_set_halt(dep, 0, false);
@ -803,6 +801,8 @@ static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
dep->endpoint.desc = NULL;
}
dwc3_remove_requests(dwc, dep);
return 0;
}
@ -1617,7 +1617,7 @@ static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
{
struct dwc3 *dwc = dep->dwc;
if (!dep->endpoint.desc || !dwc->pullups_connected) {
if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
dep->name);
return -ESHUTDOWN;
@ -2247,6 +2247,7 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
if (!is_on) {
u32 count;
dwc->connected = false;
/*
* In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
* Section 4.1.8 Table 4-7, it states that for a device-initiated
@ -2271,7 +2272,6 @@ static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
dwc->ev_buf->lpos = (dwc->ev_buf->lpos + count) %
dwc->ev_buf->length;
}
dwc->connected = false;
} else {
__dwc3_gadget_start(dwc);
}
@ -3344,8 +3344,6 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
{
u32 reg;
dwc->connected = true;
/*
* WORKAROUND: DWC3 revisions <1.88a have an issue which
* would cause a missing Disconnect Event if there's a
@ -3385,6 +3383,7 @@ static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
* transfers."
*/
dwc3_stop_active_transfers(dwc);
dwc->connected = true;
reg = dwc3_readl(dwc->regs, DWC3_DCTL);
reg &= ~DWC3_DCTL_TSTCTRL_MASK;

14
drivers/usb/gadget/configfs.c
View File

@ -129,6 +129,8 @@ struct gadget_config_name {
struct list_head list;
};
#define USB_MAX_STRING_WITH_NULL_LEN (USB_MAX_STRING_LEN+1)
static int usb_string_copy(const char *s, char **s_copy)
{
int ret;
@ -138,12 +140,16 @@ static int usb_string_copy(const char *s, char **s_copy)
if (ret > USB_MAX_STRING_LEN)
return -EOVERFLOW;
str = kstrdup(s, GFP_KERNEL);
if (!str)
return -ENOMEM;
if (copy) {
str = copy;
} else {
str = kmalloc(USB_MAX_STRING_WITH_NULL_LEN, GFP_KERNEL);
if (!str)
return -ENOMEM;
}
strcpy(str, s);
if (str[ret - 1] == '\n')
str[ret - 1] = '\0';
kfree(copy);
*s_copy = str;
return 0;
}

17
drivers/usb/host/xhci.h
View File

@ -17,6 +17,7 @@
#include <linux/kernel.h>
#include <linux/usb/hcd.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/android_kabi.h>
/* Code sharing between pci-quirks and xhci hcd */
#include "xhci-ext-caps.h"
@ -809,6 +810,9 @@ struct xhci_command {
struct completion *completion;
union xhci_trb *command_trb;
struct list_head cmd_list;
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
};
/* drop context bitmasks */
@ -1533,6 +1537,8 @@ struct xhci_segment {
void *bounce_buf;
unsigned int bounce_offs;
unsigned int bounce_len;
ANDROID_KABI_RESERVE(1);
};
enum xhci_cancelled_td_status {
@ -1622,6 +1628,9 @@ struct xhci_ring {
enum xhci_ring_type type;
bool last_td_was_short;
struct radix_tree_root *trb_address_map;
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
};
struct xhci_erst_entry {
@ -1639,6 +1648,8 @@ struct xhci_erst {
dma_addr_t erst_dma_addr;
/* Num entries the ERST can contain */
unsigned int erst_size;
ANDROID_KABI_RESERVE(1);
};
struct xhci_scratchpad {
@ -1916,6 +1927,12 @@ struct xhci_hcd {
struct list_head regset_list;
void *dbc;
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
ANDROID_KABI_RESERVE(3);
ANDROID_KABI_RESERVE(4);
/* platform-specific data -- must come last */
unsigned long priv[] __aligned(sizeof(s64));
};

36
drivers/usb/typec/tcpm/tcpci.c
View File

@ -15,6 +15,7 @@
#include <linux/usb/pd.h>
#include <linux/usb/tcpm.h>
#include <linux/usb/typec.h>
#include <trace/hooks/typec.h>
#include "tcpci.h"
@ -24,14 +25,10 @@
#define AUTO_DISCHARGE_PD_HEADROOM_MV 850
#define AUTO_DISCHARGE_PPS_HEADROOM_MV 1250
#define tcpc_presenting_cc1_rd(reg) \
#define tcpc_presenting_rd(reg, cc) \
(!(TCPC_ROLE_CTRL_DRP & (reg)) && \
(((reg) & (TCPC_ROLE_CTRL_CC1_MASK << TCPC_ROLE_CTRL_CC1_SHIFT)) == \
(TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_CC1_SHIFT)))
#define tcpc_presenting_cc2_rd(reg) \
(!(TCPC_ROLE_CTRL_DRP & (reg)) && \
(((reg) & (TCPC_ROLE_CTRL_CC2_MASK << TCPC_ROLE_CTRL_CC2_SHIFT)) == \
(TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_CC2_SHIFT)))
(((reg) & (TCPC_ROLE_CTRL_## cc ##_MASK << TCPC_ROLE_CTRL_## cc ##_SHIFT)) == \
(TCPC_ROLE_CTRL_CC_RD << TCPC_ROLE_CTRL_## cc ##_SHIFT)))
struct tcpci {
struct device *dev;
@ -126,14 +123,17 @@ static int tcpci_start_toggling(struct tcpc_dev *tcpc,
int ret;
struct tcpci *tcpci = tcpc_to_tcpci(tcpc);
unsigned int reg = TCPC_ROLE_CTRL_DRP;
int override_toggling = 0;
if (port_type != TYPEC_PORT_DRP)
return -EOPNOTSUPP;
/* Handle vendor drp toggling */
if (tcpci->data->start_drp_toggling) {
trace_android_vh_typec_tcpci_override_toggling(tcpci, tcpci->data,
&override_toggling);
ret = tcpci->data->start_drp_toggling(tcpci, tcpci->data, cc);
if (ret < 0)
if (ret < 0 || override_toggling)
return ret;
}
@ -201,11 +201,11 @@ static int tcpci_get_cc(struct tcpc_dev *tcpc,
*cc1 = tcpci_to_typec_cc((reg >> TCPC_CC_STATUS_CC1_SHIFT) &
TCPC_CC_STATUS_CC1_MASK,
reg & TCPC_CC_STATUS_TERM ||
tcpc_presenting_cc1_rd(role_control));
tcpc_presenting_rd(role_control, CC1));
*cc2 = tcpci_to_typec_cc((reg >> TCPC_CC_STATUS_CC2_SHIFT) &
TCPC_CC_STATUS_CC2_MASK,
reg & TCPC_CC_STATUS_TERM ||
tcpc_presenting_cc2_rd(role_control));
tcpc_presenting_rd(role_control, CC2));
return 0;
}
@ -417,7 +417,11 @@ static int tcpci_get_vbus(struct tcpc_dev *tcpc)
{
struct tcpci *tcpci = tcpc_to_tcpci(tcpc);
unsigned int reg;
int ret;
int ret, vbus, bypass = 0;
trace_android_vh_typec_tcpci_get_vbus(tcpci, tcpci->data, &vbus, &bypass);
if (bypass)
return vbus;
ret = regmap_read(tcpci->regmap, TCPC_POWER_STATUS, &reg);
if (ret < 0)
@ -426,6 +430,15 @@ static int tcpci_get_vbus(struct tcpc_dev *tcpc)
return !!(reg & TCPC_POWER_STATUS_VBUS_PRES);
}
static int tcpci_check_contaminant(struct tcpc_dev *tcpc)
{
struct tcpci *tcpci = tcpc_to_tcpci(tcpc);
int ret = 0;
trace_android_vh_typec_tcpci_check_contaminant(tcpci, tcpci->data, &ret);
return ret;
}
static bool tcpci_is_vbus_vsafe0v(struct tcpc_dev *tcpc)
{
struct tcpci *tcpci = tcpc_to_tcpci(tcpc);
@ -744,6 +757,7 @@ struct tcpci *tcpci_register_port(struct device *dev, struct tcpci_data *data)
tcpci->tcpc.enable_frs = tcpci_enable_frs;
tcpci->tcpc.frs_sourcing_vbus = tcpci_frs_sourcing_vbus;
tcpci->tcpc.set_partner_usb_comm_capable = tcpci_set_partner_usb_comm_capable;
tcpci->tcpc.check_contaminant = tcpci_check_contaminant;
if (tcpci->data->auto_discharge_disconnect) {
tcpci->tcpc.enable_auto_vbus_discharge = tcpci_enable_auto_vbus_discharge;

16
drivers/usb/typec/tcpm/tcpci.h
View File

@ -47,7 +47,10 @@
#define TCPC_TCPC_CTRL 0x19
#define TCPC_TCPC_CTRL_ORIENTATION BIT(0)
#define PLUG_ORNT_CC1 0
#define PLUG_ORNT_CC2 1
#define TCPC_TCPC_CTRL_BIST_TM BIT(1)
#define TCPC_TCPC_CTRL_EN_LK4CONN_ALRT BIT(6)
#define TCPC_EXTENDED_STATUS 0x20
#define TCPC_EXTENDED_STATUS_VSAFE0V BIT(0)
@ -74,21 +77,28 @@
#define TCPC_POWER_CTRL_VCONN_ENABLE BIT(0)
#define TCPC_POWER_CTRL_BLEED_DISCHARGE BIT(3)
#define TCPC_POWER_CTRL_AUTO_DISCHARGE BIT(4)
#define TCPC_DIS_VOLT_ALRM BIT(5)
#define TCPC_POWER_CTRL_VBUS_VOLT_MON BIT(6)
#define TCPC_FAST_ROLE_SWAP_EN BIT(7)
#define TCPC_CC_STATUS 0x1d
#define TCPC_CC_STATUS_TOGGLING BIT(5)
#define TCPC_CC_STATUS_TERM BIT(4)
#define TCPC_CC_STATUS_TERM_RP 0
#define TCPC_CC_STATUS_TERM_RD 1
#define TCPC_CC_STATE_SRC_OPEN 0
#define TCPC_CC_STATUS_CC2_SHIFT 2
#define TCPC_CC_STATUS_CC2_MASK 0x3
#define TCPC_CC_STATUS_CC1_SHIFT 0
#define TCPC_CC_STATUS_CC1_MASK 0x3
#define TCPC_POWER_STATUS 0x1e
#define TCPC_POWER_STATUS_DBG_ACC_CON BIT(7)
#define TCPC_POWER_STATUS_UNINIT BIT(6)
#define TCPC_POWER_STATUS_SOURCING_VBUS BIT(4)
#define TCPC_POWER_STATUS_VBUS_DET BIT(3)
#define TCPC_POWER_STATUS_VBUS_PRES BIT(2)
#define TCPC_POWER_STATUS_SINKING_VBUS BIT(0)
#define TCPC_FAULT_STATUS 0x1f
@ -121,6 +131,10 @@
#define TCPC_RX_DETECT 0x2f
#define TCPC_RX_DETECT_HARD_RESET BIT(5)
#define TCPC_RX_DETECT_SOP BIT(0)
#define TCPC_RX_DETECT_SOP1 BIT(1)
#define TCPC_RX_DETECT_SOP2 BIT(2)
#define TCPC_RX_DETECT_DBG1 BIT(3)
#define TCPC_RX_DETECT_DBG2 BIT(4)
#define TCPC_RX_BYTE_CNT 0x30
#define TCPC_RX_BUF_FRAME_TYPE 0x31
@ -139,6 +153,8 @@
#define TCPC_TX_DATA 0x54 /* through 0x6f */
#define TCPC_VBUS_VOLTAGE 0x70
#define TCPC_VBUS_VOLTAGE_MASK 0x3ff
#define TCPC_VBUS_VOLTAGE_LSB_MV 25
#define TCPC_VBUS_SINK_DISCONNECT_THRESH 0x72
#define TCPC_VBUS_SINK_DISCONNECT_THRESH_LSB_MV 25
#define TCPC_VBUS_SINK_DISCONNECT_THRESH_MAX 0x3ff

15
drivers/usb/typec/tcpm/tcpm.c
View File

@ -962,6 +962,7 @@ static int tcpm_set_current_limit(struct tcpm_port *port, u32 max_ma, u32 mv)
port->supply_voltage = mv;
port->current_limit = max_ma;
power_supply_changed(port->psy);
if (port->tcpc->set_current_limit)
ret = port->tcpc->set_current_limit(port->tcpc, max_ma, mv);
@ -2948,6 +2949,7 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo,
port->pps_data.supported = false;
port->usb_type = POWER_SUPPLY_USB_TYPE_PD;
power_supply_changed(port->psy);
/*
* Select the source PDO providing the most power which has a
@ -2972,6 +2974,7 @@ static int tcpm_pd_select_pdo(struct tcpm_port *port, int *sink_pdo,
port->pps_data.supported = true;
port->usb_type =
POWER_SUPPLY_USB_TYPE_PD_PPS;
power_supply_changed(port->psy);
}
continue;
default:
@ -3129,6 +3132,7 @@ static unsigned int tcpm_pd_select_pps_apdo(struct tcpm_port *port)
port->pps_data.out_volt));
port->pps_data.op_curr = min(port->pps_data.max_curr,
port->pps_data.op_curr);
power_supply_changed(port->psy);
}
return src_pdo;
@ -3364,6 +3368,7 @@ static int tcpm_set_charge(struct tcpm_port *port, bool charge)
return ret;
}
port->vbus_charge = charge;
power_supply_changed(port->psy);
return 0;
}
@ -3548,6 +3553,7 @@ static void tcpm_reset_port(struct tcpm_port *port)
port->try_src_count = 0;
port->try_snk_count = 0;
port->usb_type = POWER_SUPPLY_USB_TYPE_C;
power_supply_changed(port->psy);
port->nr_sink_caps = 0;
port->sink_cap_done = false;
if (port->tcpc->enable_frs)
@ -3683,8 +3689,8 @@ EXPORT_SYMBOL_GPL(tcpm_is_toggling);
static void tcpm_check_send_discover(struct tcpm_port *port)
{
if (port->data_role == TYPEC_HOST && port->send_discover &&
port->pd_capable)
if ((port->data_role == TYPEC_HOST || port->negotiated_rev > PD_REV20) &&
port->send_discover && port->pd_capable)
tcpm_send_vdm(port, USB_SID_PD, CMD_DISCOVER_IDENT, NULL, 0);
port->send_discover = false;
}
@ -5274,7 +5280,7 @@ static void tcpm_enable_frs_work(struct kthread_work *work)
goto unlock;
/* Send when the state machine is idle */
if (port->state != SNK_READY || port->vdm_state != VDM_STATE_DONE || port->send_discover)
if (port->state != SNK_READY || port->vdm_sm_running || port->send_discover)
goto resched;
port->upcoming_state = GET_SINK_CAP;
@ -6003,7 +6009,6 @@ static int tcpm_psy_get_prop(struct power_supply *psy,
ret = -EINVAL;
break;
}
return ret;
}
@ -6055,6 +6060,7 @@ static int tcpm_psy_set_prop(struct power_supply *psy,
ret = -EINVAL;
break;
}
power_supply_changed(port->psy);
return ret;
}
@ -6208,6 +6214,7 @@ struct tcpm_port *tcpm_register_port(struct device *dev, struct tcpc_dev *tcpc)
err = devm_tcpm_psy_register(port);
if (err)
goto out_role_sw_put;
power_supply_changed(port->psy);
port->typec_port = typec_register_port(port->dev, &port->typec_caps);
if (IS_ERR(port->typec_port)) {

1
drivers/usb/typec/tps6598x.c
View File

@ -62,7 +62,6 @@ enum {
struct tps6598x_rx_identity_reg {
u8 status;
struct usb_pd_identity identity;
u32 vdo[3];
} __packed;
/* Standard Task return codes */

12
fs/buffer.c
View File

@ -1265,6 +1265,14 @@ static void bh_lru_install(struct buffer_head *bh)
int i;
check_irqs_on();
/*
* buffer_head in bh_lru could increase refcount of the page
* until it will be invalidated. It causes page migraion failure.
* Skip putting upcoming bh into bh_lru until migration is done.
*/
if (lru_cache_disabled())
return;
bh_lru_lock();
b = this_cpu_ptr(&bh_lrus);
@ -1410,7 +1418,7 @@ EXPORT_SYMBOL(__bread_gfp);
* This doesn't race because it runs in each cpu either in irq
* or with preempt disabled.
*/
static void invalidate_bh_lru(void *arg)
void invalidate_bh_lru(void *arg)
{
struct bh_lru *b = &get_cpu_var(bh_lrus);
int i;
@ -1422,7 +1430,7 @@ static void invalidate_bh_lru(void *arg)
put_cpu_var(bh_lrus);
}
static bool has_bh_in_lru(int cpu, void *dummy)
bool has_bh_in_lru(int cpu, void *dummy)
{
struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
int i;

1
fs/f2fs/compress.c
View File

@ -1353,6 +1353,7 @@ unlock_continue:
if (fio.compr_blocks)
f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false);
f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true);
add_compr_block_stat(inode, cc->nr_cpages);
set_inode_flag(cc->inode, FI_APPEND_WRITE);
if (cc->cluster_idx == 0)

1
fs/f2fs/data.c
View File

@ -1909,7 +1909,6 @@ next:
}
if (size) {
flags |= FIEMAP_EXTENT_MERGED;
if (IS_ENCRYPTED(inode))
flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;

19
fs/f2fs/f2fs.h
View File

@ -1623,6 +1623,11 @@ struct f2fs_sb_info {
#ifdef CONFIG_F2FS_FS_COMPRESSION
struct kmem_cache *page_array_slab; /* page array entry */
unsigned int page_array_slab_size; /* default page array slab size */
/* For runtime compression statistics */
u64 compr_written_block;
u64 compr_saved_block;
u32 compr_new_inode;
#endif
};
@ -3954,6 +3959,18 @@ int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi);
void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi);
int __init f2fs_init_compress_cache(void);
void f2fs_destroy_compress_cache(void);
#define inc_compr_inode_stat(inode) \
do { \
struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
sbi->compr_new_inode++; \
} while (0)
#define add_compr_block_stat(inode, blocks) \
do { \
struct f2fs_sb_info *sbi = F2FS_I_SB(inode); \
int diff = F2FS_I(inode)->i_cluster_size - blocks; \
sbi->compr_written_block += blocks; \
sbi->compr_saved_block += diff; \
} while (0)
#else
static inline bool f2fs_is_compressed_page(struct page *page) { return false; }
static inline bool f2fs_is_compress_backend_ready(struct inode *inode)
@ -3982,6 +3999,7 @@ static inline int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) { return
static inline void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) { }
static inline int __init f2fs_init_compress_cache(void) { return 0; }
static inline void f2fs_destroy_compress_cache(void) { }
#define inc_compr_inode_stat(inode) do { } while (0)
#endif
static inline void set_compress_context(struct inode *inode)
@ -4005,6 +4023,7 @@ static inline void set_compress_context(struct inode *inode)
F2FS_I(inode)->i_flags |= F2FS_COMPR_FL;
set_inode_flag(inode, FI_COMPRESSED_FILE);
stat_inc_compr_inode(inode);
inc_compr_inode_stat(inode);
f2fs_mark_inode_dirty_sync(inode, true);
}

38
fs/f2fs/sysfs.c
View File

@ -12,6 +12,7 @@
#include <linux/seq_file.h>
#include <linux/unicode.h>
#include <linux/ioprio.h>
#include <linux/sysfs.h>
#include "f2fs.h"
#include "segment.h"
@ -282,6 +283,17 @@ static ssize_t f2fs_sbi_show(struct f2fs_attr *a,
return len;
}
#ifdef CONFIG_F2FS_FS_COMPRESSION
if (!strcmp(a->attr.name, "compr_written_block"))
return sysfs_emit(buf, "%llu\n", sbi->compr_written_block);
if (!strcmp(a->attr.name, "compr_saved_block"))
return sysfs_emit(buf, "%llu\n", sbi->compr_saved_block);
if (!strcmp(a->attr.name, "compr_new_inode"))
return sysfs_emit(buf, "%u\n", sbi->compr_new_inode);
#endif
ui = (unsigned int *)(ptr + a->offset);
return sprintf(buf, "%u\n", *ui);
@ -458,6 +470,24 @@ out:
return count;
}
#ifdef CONFIG_F2FS_FS_COMPRESSION
if (!strcmp(a->attr.name, "compr_written_block") ||
!strcmp(a->attr.name, "compr_saved_block")) {
if (t != 0)
return -EINVAL;
sbi->compr_written_block = 0;
sbi->compr_saved_block = 0;
return count;
}
if (!strcmp(a->attr.name, "compr_new_inode")) {
if (t != 0)
return -EINVAL;
sbi->compr_new_inode = 0;
return count;
}
#endif
*ui = (unsigned int)t;
return count;
@ -668,6 +698,9 @@ F2FS_FEATURE_RO_ATTR(sb_checksum, FEAT_SB_CHECKSUM);
F2FS_FEATURE_RO_ATTR(casefold, FEAT_CASEFOLD);
#ifdef CONFIG_F2FS_FS_COMPRESSION
F2FS_FEATURE_RO_ATTR(compression, FEAT_COMPRESSION);
F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, compr_written_block, compr_written_block);
F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, compr_saved_block, compr_saved_block);
F2FS_RW_ATTR(F2FS_SBI, f2fs_sb_info, compr_new_inode, compr_new_inode);
#endif
#define ATTR_LIST(name) (&f2fs_attr_##name.attr)
@ -730,6 +763,11 @@ static struct attribute *f2fs_attrs[] = {
ATTR_LIST(moved_blocks_foreground),
ATTR_LIST(moved_blocks_background),
ATTR_LIST(avg_vblocks),
#endif
#ifdef CONFIG_F2FS_FS_COMPRESSION
ATTR_LIST(compr_written_block),
ATTR_LIST(compr_saved_block),
ATTR_LIST(compr_new_inode),
#endif
NULL,
};

3
fs/proc/meminfo.c
View File

@ -18,7 +18,7 @@
#endif
#include <asm/page.h>
#include "internal.h"
#include <trace/hooks/mm.h>
void __attribute__((weak)) arch_report_meminfo(struct seq_file *m)
{
}
@ -145,6 +145,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
show_val_kb(m, "CmaFree: ",
global_zone_page_state(NR_FREE_CMA_PAGES));
#endif
trace_android_vh_meminfo_proc_show(m);
hugetlb_report_meminfo(m);

14
include/asm-generic/vmlinux.lds.h
View File

@ -319,6 +319,16 @@
#define THERMAL_TABLE(name)
#endif
#ifdef CONFIG_DTPM
#define DTPM_TABLE() \
. = ALIGN(8); \
__dtpm_table = .; \
KEEP(*(__dtpm_table)) \
__dtpm_table_end = .;
#else
#define DTPM_TABLE()
#endif
#define KERNEL_DTB() \
STRUCT_ALIGN(); \
__dtb_start = .; \
@ -753,6 +763,7 @@
ACPI_PROBE_TABLE(irqchip) \
ACPI_PROBE_TABLE(timer) \
THERMAL_TABLE(governor) \
DTPM_TABLE() \
EARLYCON_TABLE() \
LSM_TABLE() \
EARLY_LSM_TABLE() \
@ -1019,7 +1030,8 @@
* keep any .init_array.* sections.
* https://bugs.llvm.org/show_bug.cgi?id=46478
*/
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN)
#if defined(CONFIG_GCOV_KERNEL) || defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KCSAN) || \
defined(CONFIG_CFI_CLANG)
# ifdef CONFIG_CONSTRUCTORS
# define SANITIZER_DISCARDS \
*(.eh_frame)

113
include/linux/android_kabi.h Normal file
View File

@ -0,0 +1,113 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* android_kabi.h - Android kernel abi abstraction header
*
* Copyright (C) 2020 Google, Inc.
*
* Heavily influenced by rh_kabi.h which came from the RHEL/CENTOS kernel and
* was:
* Copyright (c) 2014 Don Zickus
* Copyright (c) 2015-2018 Jiri Benc
* Copyright (c) 2015 Sabrina Dubroca, Hannes Frederic Sowa
* Copyright (c) 2016-2018 Prarit Bhargava
* Copyright (c) 2017 Paolo Abeni, Larry Woodman
*
* These macros are to be used to try to help alleviate future kernel abi
* changes that will occur as LTS and other kernel patches are merged into the
* tree during a period in which the kernel abi is wishing to not be disturbed.
*
* There are two times these macros should be used:
* - Before the kernel abi is "frozen"
* Padding can be added to various kernel structures that have in the past
* been known to change over time. That will give "room" in the structure
* that can then be used when fields are added so that the structure size
* will not change.
*
* - After the kernel abi is "frozen"
* If a structure's field is changed to a type that is identical in size to
* the previous type, it can be changed with a union macro
* If a field is added to a structure, the padding fields can be used to add
* the new field in a "safe" way.
*/
#ifndef _ANDROID_KABI_H
#define _ANDROID_KABI_H
#include <linux/compiler.h>
/*
* Worker macros, don't use these, use the ones without a leading '_'
*/
#define __ANDROID_KABI_CHECK_SIZE_ALIGN(_orig, _new) \
union { \
_Static_assert(sizeof(struct{_new;}) <= sizeof(struct{_orig;}), \
__FILE__ ":" __stringify(__LINE__) ": " \
__stringify(_new) \
" is larger than " \
__stringify(_orig) ); \
_Static_assert(__alignof__(struct{_new;}) <= __alignof__(struct{_orig;}), \
__FILE__ ":" __stringify(__LINE__) ": " \
__stringify(_orig) \
" is not aligned the same as " \
__stringify(_new) ); \
}
#ifdef __GENKSYMS__
#define _ANDROID_KABI_REPLACE(_orig, _new) _orig
#else
#define _ANDROID_KABI_REPLACE(_orig, _new) \
union { \
_new; \
struct { \
_orig; \
} __UNIQUE_ID(android_kabi_hide); \
__ANDROID_KABI_CHECK_SIZE_ALIGN(_orig, _new); \
}
#endif /* __GENKSYMS__ */
#define _ANDROID_KABI_RESERVE(n) u64 android_kabi_reserved##n
/*
* Macros to use _before_ the ABI is frozen
*/
/*
* ANDROID_KABI_RESERVE
* Reserve some "padding" in a structure for potential future use.
* This normally placed at the end of a structure.
* number: the "number" of the padding variable in the structure. Start with
* 1 and go up.
*/
#define ANDROID_KABI_RESERVE(number) _ANDROID_KABI_RESERVE(number)
/*
* Macros to use _after_ the ABI is frozen
*/
/*
* ANDROID_KABI_USE(number, _new)
* Use a previous padding entry that was defined with ANDROID_KABI_RESERVE
* number: the previous "number" of the padding variable
* _new: the variable to use now instead of the padding variable
*/
#define ANDROID_KABI_USE(number, _new) \
_ANDROID_KABI_REPLACE(_ANDROID_KABI_RESERVE(number), _new)
/*
* ANDROID_KABI_USE2(number, _new1, _new2)
* Use a previous padding entry that was defined with ANDROID_KABI_RESERVE for
* two new variables that fit into 64 bits. This is good for when you do not
* want to "burn" a 64bit padding variable for a smaller variable size if not
* needed.
*/
#define ANDROID_KABI_USE2(number, _new1, _new2) \
_ANDROID_KABI_REPLACE(_ANDROID_KABI_RESERVE(number), struct{ _new1; _new2; })
#endif /* _ANDROID_KABI_H */

4
include/linux/blk_types.h
View File

@ -9,6 +9,7 @@
#include <linux/types.h>
#include <linux/bvec.h>
#include <linux/ktime.h>
#include <linux/android_kabi.h>
struct bio_set;
struct bio;
@ -261,6 +262,9 @@ struct bio {
struct bio_set *bi_pool;
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
/*
* We can inline a number of vecs at the end of the bio, to avoid
* double allocations for a small number of bio_vecs. This member

4
include/linux/buffer_head.h
View File

@ -194,6 +194,8 @@ void __breadahead_gfp(struct block_device *, sector_t block, unsigned int size,
struct buffer_head *__bread_gfp(struct block_device *,
sector_t block, unsigned size, gfp_t gfp);
void invalidate_bh_lrus(void);
void invalidate_bh_lru(void *arg);
bool has_bh_in_lru(int cpu, void *dummy);
struct buffer_head *alloc_buffer_head(gfp_t gfp_flags);
void free_buffer_head(struct buffer_head * bh);
void unlock_buffer(struct buffer_head *bh);
@ -406,6 +408,8 @@ static inline int inode_has_buffers(struct inode *inode) { return 0; }
static inline void invalidate_inode_buffers(struct inode *inode) {}
static inline int remove_inode_buffers(struct inode *inode) { return 1; }
static inline int sync_mapping_buffers(struct address_space *mapping) { return 0; }
static inline void invalidate_bh_lru(void *arg) {}
static inline bool has_bh_in_lru(int cpu, void *dummy) { return 0; }
#define buffer_heads_over_limit 0
#endif /* CONFIG_BLOCK */

1
include/linux/cpuhotplug.h
View File

@ -192,6 +192,7 @@ enum cpuhp_state {
CPUHP_AP_ONLINE_DYN_END = CPUHP_AP_ONLINE_DYN + 30,
CPUHP_AP_X86_HPET_ONLINE,
CPUHP_AP_X86_KVM_CLK_ONLINE,
CPUHP_AP_DTPM_CPU_ONLINE,
CPUHP_AP_ACTIVE,
CPUHP_ONLINE,
};

6
include/linux/dma-map-ops.h
View File

@ -8,6 +8,7 @@
#include <linux/dma-mapping.h>
#include <linux/pgtable.h>
#include <linux/android_kabi.h>
struct cma;
@ -69,6 +70,11 @@ struct dma_map_ops {
u64 (*get_required_mask)(struct device *dev);
size_t (*max_mapping_size)(struct device *dev);
unsigned long (*get_merge_boundary)(struct device *dev);
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
ANDROID_KABI_RESERVE(3);
ANDROID_KABI_RESERVE(4);
};
#ifdef CONFIG_DMA_OPS

77
include/linux/dtpm.h Normal file
View File

@ -0,0 +1,77 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020 Linaro Ltd
*
* Author: Daniel Lezcano <daniel.lezcano@linaro.org>
*/
#ifndef ___DTPM_H__
#define ___DTPM_H__
#include <linux/powercap.h>
#define MAX_DTPM_DESCR 8
#define MAX_DTPM_CONSTRAINTS 1
struct dtpm {
struct powercap_zone zone;
struct dtpm *parent;
struct list_head sibling;
struct list_head children;
struct dtpm_ops *ops;
unsigned long flags;
u64 power_limit;
u64 power_max;
u64 power_min;
int weight;
void *private;
};
struct dtpm_ops {
u64 (*set_power_uw)(struct dtpm *, u64);
u64 (*get_power_uw)(struct dtpm *);
void (*release)(struct dtpm *);
};
struct dtpm_descr;
typedef int (*dtpm_init_t)(struct dtpm_descr *);
struct dtpm_descr {
struct dtpm *parent;
const char *name;
dtpm_init_t init;
};
/* Init section thermal table */
extern struct dtpm_descr *__dtpm_table[];
extern struct dtpm_descr *__dtpm_table_end[];
#define DTPM_TABLE_ENTRY(name) \
static typeof(name) *__dtpm_table_entry_##name \
__used __section("__dtpm_table") = &name
#define DTPM_DECLARE(name) DTPM_TABLE_ENTRY(name)
#define for_each_dtpm_table(__dtpm) \
for (__dtpm = __dtpm_table; \
__dtpm < __dtpm_table_end; \
__dtpm++)
static inline struct dtpm *to_dtpm(struct powercap_zone *zone)
{
return container_of(zone, struct dtpm, zone);
}
int dtpm_update_power(struct dtpm *dtpm, u64 power_min, u64 power_max);
int dtpm_release_zone(struct powercap_zone *pcz);
struct dtpm *dtpm_alloc(struct dtpm_ops *ops);
void dtpm_unregister(struct dtpm *dtpm);
int dtpm_register(const char *name, struct dtpm *dtpm, struct dtpm *parent);
int dtpm_register_cpu(struct dtpm *parent);
#endif

6
include/linux/ethtool.h
View File

@ -15,6 +15,7 @@
#include <linux/bitmap.h>
#include <linux/compat.h>
#include <linux/android_kabi.h>
#include <uapi/linux/ethtool.h>
#ifdef CONFIG_COMPAT
@ -509,6 +510,11 @@ struct ethtool_ops {
const struct ethtool_tunable *, void *);
int (*set_phy_tunable)(struct net_device *,
const struct ethtool_tunable *, const void *);
ANDROID_KABI_RESERVE(1);
ANDROID_KABI_RESERVE(2);
ANDROID_KABI_RESERVE(3);
ANDROID_KABI_RESERVE(4);
};
int ethtool_check_ops(const struct ethtool_ops *ops);

Some files were not shown because too many files have changed in this diff Show More