arm64 updates for 3.20:
- reimplementation of the virtual remapping of UEFI Runtime Services in a way that is stable across kexec - emulation of the "setend" instruction for 32-bit tasks (user endianness switching trapped in the kernel, SCTLR_EL1.E0E bit set accordingly) - compat_sys_call_table implemented in C (from asm) and made it a constant array together with sys_call_table - export CPU cache information via /sys (like other architectures) - DMA API implementation clean-up in preparation for IOMMU support - macros clean-up for KVM - dropped some unnecessary cache+tlb maintenance - CONFIG_ARM64_CPU_SUSPEND clean-up - defconfig update (CPU_IDLE) -----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQIcBAABAgAGBQJU25v3AAoJEGvWsS0AyF7xYjcP/j8ESvs+z0BPgeJ6XREfOnCh cp+w/1rJ5BafJ5RRkibrciwTNOIJS4FGMivWyURtoh430lS0Rh7fxZ3Ouna3xjrT Nf7AxenWoA8Lo6wHh+FlNUeGk3iWfX6WwA2tYrbKudK+LBJ1wHjwpE7cWQO0FgwJ aFDahu+QD5/u45p/VcVctMtiEDvOxBdO8gfat6r+YkLm7pbRxQkZnpA/JE4Gps1p Td5jvMNH9pXI5pffSbeR9Q+vs/r0yqKLXQg01Eb2bZgGDgwf9yzADrHuaKamZt35 X5flmLiTGC6swJCJvUkZC1Nuue33bXcvW5+vgvar+MNGyXsxv+B/wARLqGhiWhQZ nLGwFpuNu6wdY9tGHb/XR8khcewkw1/lRH1hHKhchrmRyUqHvXcPgC5tamjLrY8C BV3BAeQvRho8OKwWUmbXIlyON1vPux6CJdj4D/A5NL+qph2WHeVWJCXg6nVFx0Wc Eb3bXbI4QRwTFL7pGRF8RyZJBAQtgYhQMKWMW2GHgUgn+r1EixG73BZoSwvpHrrw FOR9AVNfVBqmNON8xiIb3DN4EViq76EF0jrsZh5I9EoWS2w5qtk60kJQgXE+M4EE vOlmh3dhEVfCN2SxOn0bgoQmTulyjqGauTSSJKQbIBuinPFveukrJfGNFIWt0SZs f38FBMo6sgU4VG85B+Fr =X5x/ -----END PGP SIGNATURE----- Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux Pull arm64 updates from Catalin Marinas: "arm64 updates for 3.20: - reimplementation of the virtual remapping of UEFI Runtime Services in a way that is stable across kexec - emulation of the "setend" instruction for 32-bit tasks (user endianness switching trapped in the kernel, SCTLR_EL1.E0E bit set accordingly) - compat_sys_call_table implemented in C (from asm) and made it a constant array together with sys_call_table - export CPU cache information via /sys (like other architectures) - DMA API implementation clean-up in preparation for IOMMU support - macros clean-up for KVM - dropped some unnecessary cache+tlb maintenance - CONFIG_ARM64_CPU_SUSPEND clean-up - defconfig update (CPU_IDLE) The EFI changes going via the arm64 tree have been acked by Matt Fleming. There is also a patch adding sys_*stat64 prototypes to include/linux/syscalls.h, acked by Andrew Morton" * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (47 commits) arm64: compat: Remove incorrect comment in compat_siginfo arm64: Fix section mismatch on alloc_init_p[mu]d() arm64: Avoid breakage caused by .altmacro in fpsimd save/restore macros arm64: mm: use *_sect to check for section maps arm64: drop unnecessary cache+tlb maintenance arm64:mm: free the useless initial page table arm64: Enable CPU_IDLE in defconfig arm64: kernel: remove ARM64_CPU_SUSPEND config option arm64: make sys_call_table const arm64: Remove asm/syscalls.h arm64: Implement the compat_sys_call_table in C syscalls: Declare sys_*stat64 prototypes if __ARCH_WANT_(COMPAT_)STAT64 compat: Declare compat_sys_sigpending and compat_sys_sigprocmask prototypes arm64: uapi: expose our struct ucontext to the uapi headers smp, ARM64: Kill SMP single function call interrupt arm64: Emulate SETEND for AArch32 tasks arm64: Consolidate hotplug notifier for instruction emulation arm64: Track system support for mixed endian EL0 arm64: implement generic IOMMU configuration arm64: Combine coherent and non-coherent swiotlb dma_ops ...
This commit is contained in:
Коммит
6b00f7efb5
|
@ -32,6 +32,9 @@ The default mode depends on the status of the instruction in the
|
|||
architecture. Deprecated instructions should default to emulation
|
||||
while obsolete instructions must be undefined by default.
|
||||
|
||||
Note: Instruction emulation may not be possible in all cases. See
|
||||
individual instruction notes for further information.
|
||||
|
||||
Supported legacy instructions
|
||||
-----------------------------
|
||||
* SWP{B}
|
||||
|
@ -43,3 +46,12 @@ Default: Undef (0)
|
|||
Node: /proc/sys/abi/cp15_barrier
|
||||
Status: Deprecated
|
||||
Default: Emulate (1)
|
||||
|
||||
* SETEND
|
||||
Node: /proc/sys/abi/setend
|
||||
Status: Deprecated
|
||||
Default: Emulate (1)*
|
||||
Note: All the cpus on the system must have mixed endian support at EL0
|
||||
for this feature to be enabled. If a new CPU - which doesn't support mixed
|
||||
endian - is hotplugged in after this feature has been enabled, there could
|
||||
be unexpected results in the application.
|
||||
|
|
|
@ -540,6 +540,21 @@ config CP15_BARRIER_EMULATION
|
|||
|
||||
If unsure, say Y
|
||||
|
||||
config SETEND_EMULATION
|
||||
bool "Emulate SETEND instruction"
|
||||
help
|
||||
The SETEND instruction alters the data-endianness of the
|
||||
AArch32 EL0, and is deprecated in ARMv8.
|
||||
|
||||
Say Y here to enable software emulation of the instruction
|
||||
for AArch32 userspace code.
|
||||
|
||||
Note: All the cpus on the system must have mixed endian support at EL0
|
||||
for this feature to be enabled. If a new CPU - which doesn't support mixed
|
||||
endian - is hotplugged in after this feature has been enabled, there could
|
||||
be unexpected results in the applications.
|
||||
|
||||
If unsure, say Y
|
||||
endif
|
||||
|
||||
endmenu
|
||||
|
@ -627,9 +642,6 @@ source "kernel/power/Kconfig"
|
|||
config ARCH_SUSPEND_POSSIBLE
|
||||
def_bool y
|
||||
|
||||
config ARM64_CPU_SUSPEND
|
||||
def_bool PM_SLEEP
|
||||
|
||||
endmenu
|
||||
|
||||
menu "CPU Power Management"
|
||||
|
|
|
@ -66,4 +66,27 @@ config DEBUG_SET_MODULE_RONX
|
|||
against certain classes of kernel exploits.
|
||||
If in doubt, say "N".
|
||||
|
||||
config DEBUG_RODATA
|
||||
bool "Make kernel text and rodata read-only"
|
||||
help
|
||||
If this is set, kernel text and rodata will be made read-only. This
|
||||
is to help catch accidental or malicious attempts to change the
|
||||
kernel's executable code. Additionally splits rodata from kernel
|
||||
text so it can be made explicitly non-executable.
|
||||
|
||||
If in doubt, say Y
|
||||
|
||||
config DEBUG_ALIGN_RODATA
|
||||
depends on DEBUG_RODATA && !ARM64_64K_PAGES
|
||||
bool "Align linker sections up to SECTION_SIZE"
|
||||
help
|
||||
If this option is enabled, sections that may potentially be marked as
|
||||
read only or non-executable will be aligned up to the section size of
|
||||
the kernel. This prevents sections from being split into pages and
|
||||
avoids a potential TLB penalty. The downside is an increase in
|
||||
alignment and potentially wasted space. Turn on this option if
|
||||
performance is more important than memory pressure.
|
||||
|
||||
If in doubt, say N
|
||||
|
||||
endmenu
|
||||
|
|
|
@ -15,8 +15,6 @@ CPPFLAGS_vmlinux.lds = -DTEXT_OFFSET=$(TEXT_OFFSET)
|
|||
OBJCOPYFLAGS :=-O binary -R .note -R .note.gnu.build-id -R .comment -S
|
||||
GZFLAGS :=-9
|
||||
|
||||
LIBGCC := $(shell $(CC) $(KBUILD_CFLAGS) -print-libgcc-file-name)
|
||||
|
||||
KBUILD_DEFCONFIG := defconfig
|
||||
|
||||
KBUILD_CFLAGS += -mgeneral-regs-only
|
||||
|
@ -50,7 +48,6 @@ core-$(CONFIG_KVM) += arch/arm64/kvm/
|
|||
core-$(CONFIG_XEN) += arch/arm64/xen/
|
||||
core-$(CONFIG_CRYPTO) += arch/arm64/crypto/
|
||||
libs-y := arch/arm64/lib/ $(libs-y)
|
||||
libs-y += $(LIBGCC)
|
||||
libs-$(CONFIG_EFI_STUB) += drivers/firmware/efi/libstub/
|
||||
|
||||
# Default target when executing plain make
|
||||
|
|
|
@ -45,6 +45,8 @@ CONFIG_CMA=y
|
|||
CONFIG_CMDLINE="console=ttyAMA0"
|
||||
# CONFIG_CORE_DUMP_DEFAULT_ELF_HEADERS is not set
|
||||
CONFIG_COMPAT=y
|
||||
CONFIG_CPU_IDLE=y
|
||||
CONFIG_ARM64_CPUIDLE=y
|
||||
CONFIG_NET=y
|
||||
CONFIG_PACKET=y
|
||||
CONFIG_UNIX=y
|
||||
|
|
|
@ -152,4 +152,9 @@ int set_memory_ro(unsigned long addr, int numpages);
|
|||
int set_memory_rw(unsigned long addr, int numpages);
|
||||
int set_memory_x(unsigned long addr, int numpages);
|
||||
int set_memory_nx(unsigned long addr, int numpages);
|
||||
|
||||
#ifdef CONFIG_DEBUG_RODATA
|
||||
void mark_rodata_ro(void);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
|
@ -39,24 +39,41 @@
|
|||
|
||||
extern unsigned long __icache_flags;
|
||||
|
||||
/*
|
||||
* NumSets, bits[27:13] - (Number of sets in cache) - 1
|
||||
* Associativity, bits[12:3] - (Associativity of cache) - 1
|
||||
* LineSize, bits[2:0] - (Log2(Number of words in cache line)) - 2
|
||||
*/
|
||||
#define CCSIDR_EL1_WRITE_THROUGH BIT(31)
|
||||
#define CCSIDR_EL1_WRITE_BACK BIT(30)
|
||||
#define CCSIDR_EL1_READ_ALLOCATE BIT(29)
|
||||
#define CCSIDR_EL1_WRITE_ALLOCATE BIT(28)
|
||||
#define CCSIDR_EL1_LINESIZE_MASK 0x7
|
||||
#define CCSIDR_EL1_LINESIZE(x) ((x) & CCSIDR_EL1_LINESIZE_MASK)
|
||||
|
||||
#define CCSIDR_EL1_ASSOCIATIVITY_SHIFT 3
|
||||
#define CCSIDR_EL1_ASSOCIATIVITY_MASK 0x3ff
|
||||
#define CCSIDR_EL1_ASSOCIATIVITY(x) \
|
||||
(((x) >> CCSIDR_EL1_ASSOCIATIVITY_SHIFT) & CCSIDR_EL1_ASSOCIATIVITY_MASK)
|
||||
#define CCSIDR_EL1_NUMSETS_SHIFT 13
|
||||
#define CCSIDR_EL1_NUMSETS_MASK (0x7fff << CCSIDR_EL1_NUMSETS_SHIFT)
|
||||
#define CCSIDR_EL1_NUMSETS_MASK 0x7fff
|
||||
#define CCSIDR_EL1_NUMSETS(x) \
|
||||
(((x) & CCSIDR_EL1_NUMSETS_MASK) >> CCSIDR_EL1_NUMSETS_SHIFT)
|
||||
(((x) >> CCSIDR_EL1_NUMSETS_SHIFT) & CCSIDR_EL1_NUMSETS_MASK)
|
||||
|
||||
extern u64 __attribute_const__ icache_get_ccsidr(void);
|
||||
#define CACHE_LINESIZE(x) (16 << CCSIDR_EL1_LINESIZE(x))
|
||||
#define CACHE_NUMSETS(x) (CCSIDR_EL1_NUMSETS(x) + 1)
|
||||
#define CACHE_ASSOCIATIVITY(x) (CCSIDR_EL1_ASSOCIATIVITY(x) + 1)
|
||||
|
||||
extern u64 __attribute_const__ cache_get_ccsidr(u64 csselr);
|
||||
|
||||
/* Helpers for Level 1 Instruction cache csselr = 1L */
|
||||
static inline int icache_get_linesize(void)
|
||||
{
|
||||
return 16 << CCSIDR_EL1_LINESIZE(icache_get_ccsidr());
|
||||
return CACHE_LINESIZE(cache_get_ccsidr(1L));
|
||||
}
|
||||
|
||||
static inline int icache_get_numsets(void)
|
||||
{
|
||||
return 1 + CCSIDR_EL1_NUMSETS(icache_get_ccsidr());
|
||||
return CACHE_NUMSETS(cache_get_ccsidr(1L));
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
@ -161,7 +161,6 @@ typedef struct compat_siginfo {
|
|||
int si_code;
|
||||
|
||||
union {
|
||||
/* The padding is the same size as AArch64. */
|
||||
int _pad[128/sizeof(int) - 3];
|
||||
|
||||
/* kill() */
|
||||
|
|
|
@ -28,8 +28,6 @@ struct device_node;
|
|||
* enable-method property.
|
||||
* @cpu_init: Reads any data necessary for a specific enable-method from the
|
||||
* devicetree, for a given cpu node and proposed logical id.
|
||||
* @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
|
||||
* devicetree, for a given cpu node and proposed logical id.
|
||||
* @cpu_prepare: Early one-time preparation step for a cpu. If there is a
|
||||
* mechanism for doing so, tests whether it is possible to boot
|
||||
* the given CPU.
|
||||
|
@ -42,6 +40,8 @@ struct device_node;
|
|||
* @cpu_die: Makes a cpu leave the kernel. Must not fail. Called from the
|
||||
* cpu being killed.
|
||||
* @cpu_kill: Ensures a cpu has left the kernel. Called from another cpu.
|
||||
* @cpu_init_idle: Reads any data necessary to initialize CPU idle states from
|
||||
* devicetree, for a given cpu node and proposed logical id.
|
||||
* @cpu_suspend: Suspends a cpu and saves the required context. May fail owing
|
||||
* to wrong parameters or error conditions. Called from the
|
||||
* CPU being suspended. Must be called with IRQs disabled.
|
||||
|
@ -49,7 +49,6 @@ struct device_node;
|
|||
struct cpu_operations {
|
||||
const char *name;
|
||||
int (*cpu_init)(struct device_node *, unsigned int);
|
||||
int (*cpu_init_idle)(struct device_node *, unsigned int);
|
||||
int (*cpu_prepare)(unsigned int);
|
||||
int (*cpu_boot)(unsigned int);
|
||||
void (*cpu_postboot)(void);
|
||||
|
@ -58,7 +57,8 @@ struct cpu_operations {
|
|||
void (*cpu_die)(unsigned int cpu);
|
||||
int (*cpu_kill)(unsigned int cpu);
|
||||
#endif
|
||||
#ifdef CONFIG_ARM64_CPU_SUSPEND
|
||||
#ifdef CONFIG_CPU_IDLE
|
||||
int (*cpu_init_idle)(struct device_node *, unsigned int);
|
||||
int (*cpu_suspend)(unsigned long);
|
||||
#endif
|
||||
};
|
||||
|
|
|
@ -52,6 +52,8 @@ static inline void cpus_set_cap(unsigned int num)
|
|||
}
|
||||
|
||||
void check_local_cpu_errata(void);
|
||||
bool cpu_supports_mixed_endian_el0(void);
|
||||
bool system_supports_mixed_endian_el0(void);
|
||||
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
|
|
|
@ -3,11 +3,17 @@
|
|||
|
||||
#ifdef CONFIG_CPU_IDLE
|
||||
extern int cpu_init_idle(unsigned int cpu);
|
||||
extern int cpu_suspend(unsigned long arg);
|
||||
#else
|
||||
static inline int cpu_init_idle(unsigned int cpu)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
|
||||
static inline int cpu_suspend(unsigned long arg)
|
||||
{
|
||||
return -EOPNOTSUPP;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
|
|
@ -72,6 +72,18 @@
|
|||
|
||||
#define APM_CPU_PART_POTENZA 0x000
|
||||
|
||||
#define ID_AA64MMFR0_BIGENDEL0_SHIFT 16
|
||||
#define ID_AA64MMFR0_BIGENDEL0_MASK (0xf << ID_AA64MMFR0_BIGENDEL0_SHIFT)
|
||||
#define ID_AA64MMFR0_BIGENDEL0(mmfr0) \
|
||||
(((mmfr0) & ID_AA64MMFR0_BIGENDEL0_MASK) >> ID_AA64MMFR0_BIGENDEL0_SHIFT)
|
||||
#define ID_AA64MMFR0_BIGEND_SHIFT 8
|
||||
#define ID_AA64MMFR0_BIGEND_MASK (0xf << ID_AA64MMFR0_BIGEND_SHIFT)
|
||||
#define ID_AA64MMFR0_BIGEND(mmfr0) \
|
||||
(((mmfr0) & ID_AA64MMFR0_BIGEND_MASK) >> ID_AA64MMFR0_BIGEND_SHIFT)
|
||||
|
||||
#define SCTLR_EL1_CP15BEN (0x1 << 5)
|
||||
#define SCTLR_EL1_SED (0x1 << 8)
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
/*
|
||||
|
@ -104,6 +116,11 @@ static inline u32 __attribute_const__ read_cpuid_cachetype(void)
|
|||
return read_cpuid(CTR_EL0);
|
||||
}
|
||||
|
||||
static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
|
||||
{
|
||||
return (ID_AA64MMFR0_BIGEND(mmfr0) == 0x1) ||
|
||||
(ID_AA64MMFR0_BIGENDEL0(mmfr0) == 0x1);
|
||||
}
|
||||
#endif /* __ASSEMBLY__ */
|
||||
|
||||
#endif
|
||||
|
|
|
@ -28,8 +28,6 @@
|
|||
|
||||
#define DMA_ERROR_CODE (~(dma_addr_t)0)
|
||||
extern struct dma_map_ops *dma_ops;
|
||||
extern struct dma_map_ops coherent_swiotlb_dma_ops;
|
||||
extern struct dma_map_ops noncoherent_swiotlb_dma_ops;
|
||||
|
||||
static inline struct dma_map_ops *__generic_dma_ops(struct device *dev)
|
||||
{
|
||||
|
@ -47,23 +45,18 @@ static inline struct dma_map_ops *get_dma_ops(struct device *dev)
|
|||
return __generic_dma_ops(dev);
|
||||
}
|
||||
|
||||
static inline void set_dma_ops(struct device *dev, struct dma_map_ops *ops)
|
||||
{
|
||||
dev->archdata.dma_ops = ops;
|
||||
}
|
||||
|
||||
static inline void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
|
||||
struct iommu_ops *iommu, bool coherent)
|
||||
{
|
||||
dev->archdata.dma_coherent = coherent;
|
||||
if (coherent)
|
||||
set_dma_ops(dev, &coherent_swiotlb_dma_ops);
|
||||
}
|
||||
#define arch_setup_dma_ops arch_setup_dma_ops
|
||||
|
||||
/* do not use this function in a driver */
|
||||
static inline bool is_device_dma_coherent(struct device *dev)
|
||||
{
|
||||
if (!dev)
|
||||
return false;
|
||||
return dev->archdata.dma_coherent;
|
||||
}
|
||||
|
||||
|
|
|
@ -6,29 +6,33 @@
|
|||
|
||||
#ifdef CONFIG_EFI
|
||||
extern void efi_init(void);
|
||||
extern void efi_idmap_init(void);
|
||||
#else
|
||||
#define efi_init()
|
||||
#define efi_idmap_init()
|
||||
#endif
|
||||
|
||||
#define efi_call_virt(f, ...) \
|
||||
({ \
|
||||
efi_##f##_t *__f = efi.systab->runtime->f; \
|
||||
efi_##f##_t *__f; \
|
||||
efi_status_t __s; \
|
||||
\
|
||||
kernel_neon_begin(); \
|
||||
efi_virtmap_load(); \
|
||||
__f = efi.systab->runtime->f; \
|
||||
__s = __f(__VA_ARGS__); \
|
||||
efi_virtmap_unload(); \
|
||||
kernel_neon_end(); \
|
||||
__s; \
|
||||
})
|
||||
|
||||
#define __efi_call_virt(f, ...) \
|
||||
({ \
|
||||
efi_##f##_t *__f = efi.systab->runtime->f; \
|
||||
efi_##f##_t *__f; \
|
||||
\
|
||||
kernel_neon_begin(); \
|
||||
efi_virtmap_load(); \
|
||||
__f = efi.systab->runtime->f; \
|
||||
__f(__VA_ARGS__); \
|
||||
efi_virtmap_unload(); \
|
||||
kernel_neon_end(); \
|
||||
})
|
||||
|
||||
|
@ -44,4 +48,22 @@ extern void efi_idmap_init(void);
|
|||
|
||||
#define efi_call_early(f, ...) sys_table_arg->boottime->f(__VA_ARGS__)
|
||||
|
||||
#define EFI_ALLOC_ALIGN SZ_64K
|
||||
|
||||
/*
|
||||
* On ARM systems, virtually remapped UEFI runtime services are set up in two
|
||||
* distinct stages:
|
||||
* - The stub retrieves the final version of the memory map from UEFI, populates
|
||||
* the virt_addr fields and calls the SetVirtualAddressMap() [SVAM] runtime
|
||||
* service to communicate the new mapping to the firmware (Note that the new
|
||||
* mapping is not live at this time)
|
||||
* - During an early initcall(), the EFI system table is permanently remapped
|
||||
* and the virtual remapping of the UEFI Runtime Services regions is loaded
|
||||
* into a private set of page tables. If this all succeeds, the Runtime
|
||||
* Services are enabled and the EFI_RUNTIME_SERVICES bit set.
|
||||
*/
|
||||
|
||||
void efi_virtmap_load(void);
|
||||
void efi_virtmap_unload(void);
|
||||
|
||||
#endif /* _ASM_EFI_H */
|
||||
|
|
|
@ -18,40 +18,89 @@
|
|||
#ifndef __ASM_ESR_H
|
||||
#define __ASM_ESR_H
|
||||
|
||||
#define ESR_EL1_WRITE (1 << 6)
|
||||
#define ESR_EL1_CM (1 << 8)
|
||||
#define ESR_EL1_IL (1 << 25)
|
||||
#define ESR_ELx_EC_UNKNOWN (0x00)
|
||||
#define ESR_ELx_EC_WFx (0x01)
|
||||
/* Unallocated EC: 0x02 */
|
||||
#define ESR_ELx_EC_CP15_32 (0x03)
|
||||
#define ESR_ELx_EC_CP15_64 (0x04)
|
||||
#define ESR_ELx_EC_CP14_MR (0x05)
|
||||
#define ESR_ELx_EC_CP14_LS (0x06)
|
||||
#define ESR_ELx_EC_FP_ASIMD (0x07)
|
||||
#define ESR_ELx_EC_CP10_ID (0x08)
|
||||
/* Unallocated EC: 0x09 - 0x0B */
|
||||
#define ESR_ELx_EC_CP14_64 (0x0C)
|
||||
/* Unallocated EC: 0x0d */
|
||||
#define ESR_ELx_EC_ILL (0x0E)
|
||||
/* Unallocated EC: 0x0F - 0x10 */
|
||||
#define ESR_ELx_EC_SVC32 (0x11)
|
||||
#define ESR_ELx_EC_HVC32 (0x12)
|
||||
#define ESR_ELx_EC_SMC32 (0x13)
|
||||
/* Unallocated EC: 0x14 */
|
||||
#define ESR_ELx_EC_SVC64 (0x15)
|
||||
#define ESR_ELx_EC_HVC64 (0x16)
|
||||
#define ESR_ELx_EC_SMC64 (0x17)
|
||||
#define ESR_ELx_EC_SYS64 (0x18)
|
||||
/* Unallocated EC: 0x19 - 0x1E */
|
||||
#define ESR_ELx_EC_IMP_DEF (0x1f)
|
||||
#define ESR_ELx_EC_IABT_LOW (0x20)
|
||||
#define ESR_ELx_EC_IABT_CUR (0x21)
|
||||
#define ESR_ELx_EC_PC_ALIGN (0x22)
|
||||
/* Unallocated EC: 0x23 */
|
||||
#define ESR_ELx_EC_DABT_LOW (0x24)
|
||||
#define ESR_ELx_EC_DABT_CUR (0x25)
|
||||
#define ESR_ELx_EC_SP_ALIGN (0x26)
|
||||
/* Unallocated EC: 0x27 */
|
||||
#define ESR_ELx_EC_FP_EXC32 (0x28)
|
||||
/* Unallocated EC: 0x29 - 0x2B */
|
||||
#define ESR_ELx_EC_FP_EXC64 (0x2C)
|
||||
/* Unallocated EC: 0x2D - 0x2E */
|
||||
#define ESR_ELx_EC_SERROR (0x2F)
|
||||
#define ESR_ELx_EC_BREAKPT_LOW (0x30)
|
||||
#define ESR_ELx_EC_BREAKPT_CUR (0x31)
|
||||
#define ESR_ELx_EC_SOFTSTP_LOW (0x32)
|
||||
#define ESR_ELx_EC_SOFTSTP_CUR (0x33)
|
||||
#define ESR_ELx_EC_WATCHPT_LOW (0x34)
|
||||
#define ESR_ELx_EC_WATCHPT_CUR (0x35)
|
||||
/* Unallocated EC: 0x36 - 0x37 */
|
||||
#define ESR_ELx_EC_BKPT32 (0x38)
|
||||
/* Unallocated EC: 0x39 */
|
||||
#define ESR_ELx_EC_VECTOR32 (0x3A)
|
||||
/* Unallocted EC: 0x3B */
|
||||
#define ESR_ELx_EC_BRK64 (0x3C)
|
||||
/* Unallocated EC: 0x3D - 0x3F */
|
||||
#define ESR_ELx_EC_MAX (0x3F)
|
||||
|
||||
#define ESR_EL1_EC_SHIFT (26)
|
||||
#define ESR_EL1_EC_UNKNOWN (0x00)
|
||||
#define ESR_EL1_EC_WFI (0x01)
|
||||
#define ESR_EL1_EC_CP15_32 (0x03)
|
||||
#define ESR_EL1_EC_CP15_64 (0x04)
|
||||
#define ESR_EL1_EC_CP14_MR (0x05)
|
||||
#define ESR_EL1_EC_CP14_LS (0x06)
|
||||
#define ESR_EL1_EC_FP_ASIMD (0x07)
|
||||
#define ESR_EL1_EC_CP10_ID (0x08)
|
||||
#define ESR_EL1_EC_CP14_64 (0x0C)
|
||||
#define ESR_EL1_EC_ILL_ISS (0x0E)
|
||||
#define ESR_EL1_EC_SVC32 (0x11)
|
||||
#define ESR_EL1_EC_SVC64 (0x15)
|
||||
#define ESR_EL1_EC_SYS64 (0x18)
|
||||
#define ESR_EL1_EC_IABT_EL0 (0x20)
|
||||
#define ESR_EL1_EC_IABT_EL1 (0x21)
|
||||
#define ESR_EL1_EC_PC_ALIGN (0x22)
|
||||
#define ESR_EL1_EC_DABT_EL0 (0x24)
|
||||
#define ESR_EL1_EC_DABT_EL1 (0x25)
|
||||
#define ESR_EL1_EC_SP_ALIGN (0x26)
|
||||
#define ESR_EL1_EC_FP_EXC32 (0x28)
|
||||
#define ESR_EL1_EC_FP_EXC64 (0x2C)
|
||||
#define ESR_EL1_EC_SERROR (0x2F)
|
||||
#define ESR_EL1_EC_BREAKPT_EL0 (0x30)
|
||||
#define ESR_EL1_EC_BREAKPT_EL1 (0x31)
|
||||
#define ESR_EL1_EC_SOFTSTP_EL0 (0x32)
|
||||
#define ESR_EL1_EC_SOFTSTP_EL1 (0x33)
|
||||
#define ESR_EL1_EC_WATCHPT_EL0 (0x34)
|
||||
#define ESR_EL1_EC_WATCHPT_EL1 (0x35)
|
||||
#define ESR_EL1_EC_BKPT32 (0x38)
|
||||
#define ESR_EL1_EC_BRK64 (0x3C)
|
||||
#define ESR_ELx_EC_SHIFT (26)
|
||||
#define ESR_ELx_EC_MASK (UL(0x3F) << ESR_ELx_EC_SHIFT)
|
||||
|
||||
#define ESR_ELx_IL (UL(1) << 25)
|
||||
#define ESR_ELx_ISS_MASK (ESR_ELx_IL - 1)
|
||||
#define ESR_ELx_ISV (UL(1) << 24)
|
||||
#define ESR_ELx_SAS_SHIFT (22)
|
||||
#define ESR_ELx_SAS (UL(3) << ESR_ELx_SAS_SHIFT)
|
||||
#define ESR_ELx_SSE (UL(1) << 21)
|
||||
#define ESR_ELx_SRT_SHIFT (16)
|
||||
#define ESR_ELx_SRT_MASK (UL(0x1F) << ESR_ELx_SRT_SHIFT)
|
||||
#define ESR_ELx_SF (UL(1) << 15)
|
||||
#define ESR_ELx_AR (UL(1) << 14)
|
||||
#define ESR_ELx_EA (UL(1) << 9)
|
||||
#define ESR_ELx_CM (UL(1) << 8)
|
||||
#define ESR_ELx_S1PTW (UL(1) << 7)
|
||||
#define ESR_ELx_WNR (UL(1) << 6)
|
||||
#define ESR_ELx_FSC (0x3F)
|
||||
#define ESR_ELx_FSC_TYPE (0x3C)
|
||||
#define ESR_ELx_FSC_EXTABT (0x10)
|
||||
#define ESR_ELx_FSC_FAULT (0x04)
|
||||
#define ESR_ELx_FSC_PERM (0x0C)
|
||||
#define ESR_ELx_CV (UL(1) << 24)
|
||||
#define ESR_ELx_COND_SHIFT (20)
|
||||
#define ESR_ELx_COND_MASK (UL(0xF) << ESR_ELx_COND_SHIFT)
|
||||
#define ESR_ELx_WFx_ISS_WFE (UL(1) << 0)
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
#include <asm/types.h>
|
||||
|
||||
const char *esr_get_class_string(u32 esr);
|
||||
#endif /* __ASSEMBLY */
|
||||
|
||||
#endif /* __ASM_ESR_H */
|
||||
|
|
|
@ -49,6 +49,7 @@ enum fixed_addresses {
|
|||
|
||||
FIX_BTMAP_END = __end_of_permanent_fixed_addresses,
|
||||
FIX_BTMAP_BEGIN = FIX_BTMAP_END + TOTAL_FIX_BTMAPS - 1,
|
||||
FIX_TEXT_POKE0,
|
||||
__end_of_fixed_addresses
|
||||
};
|
||||
|
||||
|
|
|
@ -76,7 +76,6 @@
|
|||
fpsimd_restore_fpcr x\tmpnr, \state
|
||||
.endm
|
||||
|
||||
.altmacro
|
||||
.macro fpsimd_save_partial state, numnr, tmpnr1, tmpnr2
|
||||
mrs x\tmpnr1, fpsr
|
||||
str w\numnr, [\state, #8]
|
||||
|
@ -86,11 +85,22 @@
|
|||
add \state, \state, x\numnr, lsl #4
|
||||
sub x\tmpnr1, x\tmpnr1, x\numnr, lsl #1
|
||||
br x\tmpnr1
|
||||
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0
|
||||
.irp qb, %(qa + 1)
|
||||
stp q\qa, q\qb, [\state, # -16 * \qa - 16]
|
||||
.endr
|
||||
.endr
|
||||
stp q30, q31, [\state, #-16 * 30 - 16]
|
||||
stp q28, q29, [\state, #-16 * 28 - 16]
|
||||
stp q26, q27, [\state, #-16 * 26 - 16]
|
||||
stp q24, q25, [\state, #-16 * 24 - 16]
|
||||
stp q22, q23, [\state, #-16 * 22 - 16]
|
||||
stp q20, q21, [\state, #-16 * 20 - 16]
|
||||
stp q18, q19, [\state, #-16 * 18 - 16]
|
||||
stp q16, q17, [\state, #-16 * 16 - 16]
|
||||
stp q14, q15, [\state, #-16 * 14 - 16]
|
||||
stp q12, q13, [\state, #-16 * 12 - 16]
|
||||
stp q10, q11, [\state, #-16 * 10 - 16]
|
||||
stp q8, q9, [\state, #-16 * 8 - 16]
|
||||
stp q6, q7, [\state, #-16 * 6 - 16]
|
||||
stp q4, q5, [\state, #-16 * 4 - 16]
|
||||
stp q2, q3, [\state, #-16 * 2 - 16]
|
||||
stp q0, q1, [\state, #-16 * 0 - 16]
|
||||
0:
|
||||
.endm
|
||||
|
||||
|
@ -103,10 +113,21 @@
|
|||
add \state, \state, x\tmpnr2, lsl #4
|
||||
sub x\tmpnr1, x\tmpnr1, x\tmpnr2, lsl #1
|
||||
br x\tmpnr1
|
||||
.irp qa, 30, 28, 26, 24, 22, 20, 18, 16, 14, 12, 10, 8, 6, 4, 2, 0
|
||||
.irp qb, %(qa + 1)
|
||||
ldp q\qa, q\qb, [\state, # -16 * \qa - 16]
|
||||
.endr
|
||||
.endr
|
||||
ldp q30, q31, [\state, #-16 * 30 - 16]
|
||||
ldp q28, q29, [\state, #-16 * 28 - 16]
|
||||
ldp q26, q27, [\state, #-16 * 26 - 16]
|
||||
ldp q24, q25, [\state, #-16 * 24 - 16]
|
||||
ldp q22, q23, [\state, #-16 * 22 - 16]
|
||||
ldp q20, q21, [\state, #-16 * 20 - 16]
|
||||
ldp q18, q19, [\state, #-16 * 18 - 16]
|
||||
ldp q16, q17, [\state, #-16 * 16 - 16]
|
||||
ldp q14, q15, [\state, #-16 * 14 - 16]
|
||||
ldp q12, q13, [\state, #-16 * 12 - 16]
|
||||
ldp q10, q11, [\state, #-16 * 10 - 16]
|
||||
ldp q8, q9, [\state, #-16 * 8 - 16]
|
||||
ldp q6, q7, [\state, #-16 * 6 - 16]
|
||||
ldp q4, q5, [\state, #-16 * 4 - 16]
|
||||
ldp q2, q3, [\state, #-16 * 2 - 16]
|
||||
ldp q0, q1, [\state, #-16 * 0 - 16]
|
||||
0:
|
||||
.endm
|
||||
|
|
|
@ -20,7 +20,7 @@
|
|||
#include <linux/threads.h>
|
||||
#include <asm/irq.h>
|
||||
|
||||
#define NR_IPI 6
|
||||
#define NR_IPI 5
|
||||
|
||||
typedef struct {
|
||||
unsigned int __softirq_pending;
|
||||
|
|
|
@ -26,6 +26,7 @@
|
|||
|
||||
#include <asm/byteorder.h>
|
||||
#include <asm/barrier.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/early_ioremap.h>
|
||||
#include <asm/alternative.h>
|
||||
|
@ -145,8 +146,8 @@ static inline u64 __raw_readq(const volatile void __iomem *addr)
|
|||
* I/O port access primitives.
|
||||
*/
|
||||
#define arch_has_dev_port() (1)
|
||||
#define IO_SPACE_LIMIT (SZ_32M - 1)
|
||||
#define PCI_IOBASE ((void __iomem *)(MODULES_VADDR - SZ_32M))
|
||||
#define IO_SPACE_LIMIT (PCI_IO_SIZE - 1)
|
||||
#define PCI_IOBASE ((void __iomem *)PCI_IO_START)
|
||||
|
||||
/*
|
||||
* String version of I/O memory access operations.
|
||||
|
|
|
@ -18,6 +18,7 @@
|
|||
#ifndef __ARM64_KVM_ARM_H__
|
||||
#define __ARM64_KVM_ARM_H__
|
||||
|
||||
#include <asm/esr.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/types.h>
|
||||
|
||||
|
@ -184,77 +185,11 @@
|
|||
#define MDCR_EL2_TPMCR (1 << 5)
|
||||
#define MDCR_EL2_HPMN_MASK (0x1F)
|
||||
|
||||
/* Exception Syndrome Register (ESR) bits */
|
||||
#define ESR_EL2_EC_SHIFT (26)
|
||||
#define ESR_EL2_EC (UL(0x3f) << ESR_EL2_EC_SHIFT)
|
||||
#define ESR_EL2_IL (UL(1) << 25)
|
||||
#define ESR_EL2_ISS (ESR_EL2_IL - 1)
|
||||
#define ESR_EL2_ISV_SHIFT (24)
|
||||
#define ESR_EL2_ISV (UL(1) << ESR_EL2_ISV_SHIFT)
|
||||
#define ESR_EL2_SAS_SHIFT (22)
|
||||
#define ESR_EL2_SAS (UL(3) << ESR_EL2_SAS_SHIFT)
|
||||
#define ESR_EL2_SSE (1 << 21)
|
||||
#define ESR_EL2_SRT_SHIFT (16)
|
||||
#define ESR_EL2_SRT_MASK (0x1f << ESR_EL2_SRT_SHIFT)
|
||||
#define ESR_EL2_SF (1 << 15)
|
||||
#define ESR_EL2_AR (1 << 14)
|
||||
#define ESR_EL2_EA (1 << 9)
|
||||
#define ESR_EL2_CM (1 << 8)
|
||||
#define ESR_EL2_S1PTW (1 << 7)
|
||||
#define ESR_EL2_WNR (1 << 6)
|
||||
#define ESR_EL2_FSC (0x3f)
|
||||
#define ESR_EL2_FSC_TYPE (0x3c)
|
||||
|
||||
#define ESR_EL2_CV_SHIFT (24)
|
||||
#define ESR_EL2_CV (UL(1) << ESR_EL2_CV_SHIFT)
|
||||
#define ESR_EL2_COND_SHIFT (20)
|
||||
#define ESR_EL2_COND (UL(0xf) << ESR_EL2_COND_SHIFT)
|
||||
|
||||
|
||||
#define FSC_FAULT (0x04)
|
||||
#define FSC_PERM (0x0c)
|
||||
/* For compatibility with fault code shared with 32-bit */
|
||||
#define FSC_FAULT ESR_ELx_FSC_FAULT
|
||||
#define FSC_PERM ESR_ELx_FSC_PERM
|
||||
|
||||
/* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */
|
||||
#define HPFAR_MASK (~UL(0xf))
|
||||
|
||||
#define ESR_EL2_EC_UNKNOWN (0x00)
|
||||
#define ESR_EL2_EC_WFI (0x01)
|
||||
#define ESR_EL2_EC_CP15_32 (0x03)
|
||||
#define ESR_EL2_EC_CP15_64 (0x04)
|
||||
#define ESR_EL2_EC_CP14_MR (0x05)
|
||||
#define ESR_EL2_EC_CP14_LS (0x06)
|
||||
#define ESR_EL2_EC_FP_ASIMD (0x07)
|
||||
#define ESR_EL2_EC_CP10_ID (0x08)
|
||||
#define ESR_EL2_EC_CP14_64 (0x0C)
|
||||
#define ESR_EL2_EC_ILL_ISS (0x0E)
|
||||
#define ESR_EL2_EC_SVC32 (0x11)
|
||||
#define ESR_EL2_EC_HVC32 (0x12)
|
||||
#define ESR_EL2_EC_SMC32 (0x13)
|
||||
#define ESR_EL2_EC_SVC64 (0x15)
|
||||
#define ESR_EL2_EC_HVC64 (0x16)
|
||||
#define ESR_EL2_EC_SMC64 (0x17)
|
||||
#define ESR_EL2_EC_SYS64 (0x18)
|
||||
#define ESR_EL2_EC_IABT (0x20)
|
||||
#define ESR_EL2_EC_IABT_HYP (0x21)
|
||||
#define ESR_EL2_EC_PC_ALIGN (0x22)
|
||||
#define ESR_EL2_EC_DABT (0x24)
|
||||
#define ESR_EL2_EC_DABT_HYP (0x25)
|
||||
#define ESR_EL2_EC_SP_ALIGN (0x26)
|
||||
#define ESR_EL2_EC_FP_EXC32 (0x28)
|
||||
#define ESR_EL2_EC_FP_EXC64 (0x2C)
|
||||
#define ESR_EL2_EC_SERROR (0x2F)
|
||||
#define ESR_EL2_EC_BREAKPT (0x30)
|
||||
#define ESR_EL2_EC_BREAKPT_HYP (0x31)
|
||||
#define ESR_EL2_EC_SOFTSTP (0x32)
|
||||
#define ESR_EL2_EC_SOFTSTP_HYP (0x33)
|
||||
#define ESR_EL2_EC_WATCHPT (0x34)
|
||||
#define ESR_EL2_EC_WATCHPT_HYP (0x35)
|
||||
#define ESR_EL2_EC_BKPT32 (0x38)
|
||||
#define ESR_EL2_EC_VECTOR32 (0x3A)
|
||||
#define ESR_EL2_EC_BRK64 (0x3C)
|
||||
|
||||
#define ESR_EL2_EC_xABT_xFSR_EXTABT 0x10
|
||||
|
||||
#define ESR_EL2_EC_WFI_ISS_WFE (1 << 0)
|
||||
|
||||
#endif /* __ARM64_KVM_ARM_H__ */
|
||||
|
|
|
@ -23,8 +23,10 @@
|
|||
#define __ARM64_KVM_EMULATE_H__
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_mmio.h>
|
||||
#include <asm/ptrace.h>
|
||||
|
||||
|
@ -140,63 +142,63 @@ static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
|
|||
|
||||
static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_ISV);
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_WNR);
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR);
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SSE);
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE);
|
||||
}
|
||||
|
||||
static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SRT_MASK) >> ESR_EL2_SRT_SHIFT;
|
||||
return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EA);
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_EA);
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_dabt_iss1tw(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_S1PTW);
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW);
|
||||
}
|
||||
|
||||
static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_EL2_SAS) >> ESR_EL2_SAS_SHIFT);
|
||||
return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
|
||||
}
|
||||
|
||||
/* This one is not specific to Data Abort */
|
||||
static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_EL2_IL);
|
||||
return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL);
|
||||
}
|
||||
|
||||
static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_get_hsr(vcpu) >> ESR_EL2_EC_SHIFT;
|
||||
return kvm_vcpu_get_hsr(vcpu) >> ESR_ELx_EC_SHIFT;
|
||||
}
|
||||
|
||||
static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_trap_get_class(vcpu) == ESR_EL2_EC_IABT;
|
||||
return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
|
||||
}
|
||||
|
||||
static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC;
|
||||
return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC;
|
||||
}
|
||||
|
||||
static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return kvm_vcpu_get_hsr(vcpu) & ESR_EL2_FSC_TYPE;
|
||||
return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
|
||||
}
|
||||
|
||||
static inline unsigned long kvm_vcpu_get_mpidr(struct kvm_vcpu *vcpu)
|
||||
|
|
|
@ -32,6 +32,12 @@
|
|||
*/
|
||||
#define UL(x) _AC(x, UL)
|
||||
|
||||
/*
|
||||
* Size of the PCI I/O space. This must remain a power of two so that
|
||||
* IO_SPACE_LIMIT acts as a mask for the low bits of I/O addresses.
|
||||
*/
|
||||
#define PCI_IO_SIZE SZ_16M
|
||||
|
||||
/*
|
||||
* PAGE_OFFSET - the virtual address of the start of the kernel image (top
|
||||
* (VA_BITS - 1))
|
||||
|
@ -45,7 +51,9 @@
|
|||
#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
|
||||
#define MODULES_END (PAGE_OFFSET)
|
||||
#define MODULES_VADDR (MODULES_END - SZ_64M)
|
||||
#define FIXADDR_TOP (MODULES_VADDR - SZ_2M - PAGE_SIZE)
|
||||
#define PCI_IO_END (MODULES_VADDR - SZ_2M)
|
||||
#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
|
||||
#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
|
||||
#define TASK_SIZE_64 (UL(1) << VA_BITS)
|
||||
|
||||
#ifdef CONFIG_COMPAT
|
||||
|
|
|
@ -31,7 +31,8 @@ extern void paging_init(void);
|
|||
extern void setup_mm_for_reboot(void);
|
||||
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
|
||||
extern void init_mem_pgprot(void);
|
||||
/* create an identity mapping for memory (or io if map_io is true) */
|
||||
extern void create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io);
|
||||
extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
|
||||
unsigned long virt, phys_addr_t size,
|
||||
pgprot_t prot);
|
||||
|
||||
#endif
|
||||
|
|
|
@ -263,6 +263,11 @@ static inline pmd_t pte_pmd(pte_t pte)
|
|||
return __pmd(pte_val(pte));
|
||||
}
|
||||
|
||||
static inline pgprot_t mk_sect_prot(pgprot_t prot)
|
||||
{
|
||||
return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
|
||||
}
|
||||
|
||||
/*
|
||||
* THP definitions.
|
||||
*/
|
||||
|
@ -336,9 +341,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
|
|||
|
||||
#ifdef CONFIG_ARM64_64K_PAGES
|
||||
#define pud_sect(pud) (0)
|
||||
#define pud_table(pud) (1)
|
||||
#else
|
||||
#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
||||
PUD_TYPE_SECT)
|
||||
#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
||||
PUD_TYPE_TABLE)
|
||||
#endif
|
||||
|
||||
static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
|
||||
|
|
|
@ -58,6 +58,13 @@
|
|||
#define COMPAT_PSR_Z_BIT 0x40000000
|
||||
#define COMPAT_PSR_N_BIT 0x80000000
|
||||
#define COMPAT_PSR_IT_MASK 0x0600fc00 /* If-Then execution state mask */
|
||||
|
||||
#ifdef CONFIG_CPU_BIG_ENDIAN
|
||||
#define COMPAT_PSR_ENDSTATE COMPAT_PSR_E_BIT
|
||||
#else
|
||||
#define COMPAT_PSR_ENDSTATE 0
|
||||
#endif
|
||||
|
||||
/*
|
||||
* These are 'magic' values for PTRACE_PEEKUSR that return info about where a
|
||||
* process is located in memory.
|
||||
|
|
|
@ -23,6 +23,4 @@ struct sleep_save_sp {
|
|||
|
||||
extern int __cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
|
||||
extern void cpu_resume(void);
|
||||
extern int cpu_suspend(unsigned long);
|
||||
|
||||
#endif
|
||||
|
|
|
@ -1,30 +0,0 @@
|
|||
/*
|
||||
* Copyright (C) 2012 ARM Ltd.
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#ifndef __ASM_SYSCALLS_H
|
||||
#define __ASM_SYSCALLS_H
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/signal.h>
|
||||
|
||||
/*
|
||||
* System call wrappers implemented in kernel/entry.S.
|
||||
*/
|
||||
asmlinkage long sys_rt_sigreturn_wrapper(void);
|
||||
|
||||
#include <asm-generic/syscalls.h>
|
||||
|
||||
#endif /* __ASM_SYSCALLS_H */
|
|
@ -48,6 +48,9 @@
|
|||
#endif
|
||||
|
||||
#define __ARCH_WANT_SYS_CLONE
|
||||
|
||||
#ifndef __COMPAT_SYSCALL_NR
|
||||
#include <uapi/asm/unistd.h>
|
||||
#endif
|
||||
|
||||
#define NR_syscalls (__NR_syscalls)
|
||||
|
|
|
@ -18,4 +18,5 @@ header-y += siginfo.h
|
|||
header-y += signal.h
|
||||
header-y += stat.h
|
||||
header-y += statfs.h
|
||||
header-y += ucontext.h
|
||||
header-y += unistd.h
|
||||
|
|
|
@ -13,8 +13,10 @@
|
|||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#ifndef __ASM_UCONTEXT_H
|
||||
#define __ASM_UCONTEXT_H
|
||||
#ifndef _UAPI__ASM_UCONTEXT_H
|
||||
#define _UAPI__ASM_UCONTEXT_H
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
struct ucontext {
|
||||
unsigned long uc_flags;
|
||||
|
@ -27,4 +29,4 @@ struct ucontext {
|
|||
struct sigcontext uc_mcontext;
|
||||
};
|
||||
|
||||
#endif /* __ASM_UCONTEXT_H */
|
||||
#endif /* _UAPI__ASM_UCONTEXT_H */
|
|
@ -16,10 +16,10 @@ arm64-obj-y := cputable.o debug-monitors.o entry.o irq.o fpsimd.o \
|
|||
entry-fpsimd.o process.o ptrace.o setup.o signal.o \
|
||||
sys.o stacktrace.o time.o traps.o io.o vdso.o \
|
||||
hyp-stub.o psci.o cpu_ops.o insn.o return_address.o \
|
||||
cpuinfo.o cpu_errata.o alternative.o
|
||||
cpuinfo.o cpu_errata.o alternative.o cacheinfo.o
|
||||
|
||||
arm64-obj-$(CONFIG_COMPAT) += sys32.o kuser32.o signal32.o \
|
||||
sys_compat.o \
|
||||
sys_compat.o entry32.o \
|
||||
../../arm/kernel/opcodes.o
|
||||
arm64-obj-$(CONFIG_FUNCTION_TRACER) += ftrace.o entry-ftrace.o
|
||||
arm64-obj-$(CONFIG_MODULES) += arm64ksyms.o module.o
|
||||
|
@ -27,7 +27,7 @@ arm64-obj-$(CONFIG_SMP) += smp.o smp_spin_table.o topology.o
|
|||
arm64-obj-$(CONFIG_PERF_EVENTS) += perf_regs.o
|
||||
arm64-obj-$(CONFIG_HW_PERF_EVENTS) += perf_event.o
|
||||
arm64-obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
|
||||
arm64-obj-$(CONFIG_ARM64_CPU_SUSPEND) += sleep.o suspend.o
|
||||
arm64-obj-$(CONFIG_CPU_PM) += sleep.o suspend.o
|
||||
arm64-obj-$(CONFIG_CPU_IDLE) += cpuidle.o
|
||||
arm64-obj-$(CONFIG_JUMP_LABEL) += jump_label.o
|
||||
arm64-obj-$(CONFIG_KGDB) += kgdb.o
|
||||
|
|
|
@ -19,6 +19,7 @@
|
|||
#include <asm/system_misc.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/uaccess.h>
|
||||
#include <asm/cpufeature.h>
|
||||
|
||||
#define CREATE_TRACE_POINTS
|
||||
#include "trace-events-emulation.h"
|
||||
|
@ -85,6 +86,57 @@ static void remove_emulation_hooks(struct insn_emulation_ops *ops)
|
|||
pr_notice("Removed %s emulation handler\n", ops->name);
|
||||
}
|
||||
|
||||
static void enable_insn_hw_mode(void *data)
|
||||
{
|
||||
struct insn_emulation *insn = (struct insn_emulation *)data;
|
||||
if (insn->ops->set_hw_mode)
|
||||
insn->ops->set_hw_mode(true);
|
||||
}
|
||||
|
||||
static void disable_insn_hw_mode(void *data)
|
||||
{
|
||||
struct insn_emulation *insn = (struct insn_emulation *)data;
|
||||
if (insn->ops->set_hw_mode)
|
||||
insn->ops->set_hw_mode(false);
|
||||
}
|
||||
|
||||
/* Run set_hw_mode(mode) on all active CPUs */
|
||||
static int run_all_cpu_set_hw_mode(struct insn_emulation *insn, bool enable)
|
||||
{
|
||||
if (!insn->ops->set_hw_mode)
|
||||
return -EINVAL;
|
||||
if (enable)
|
||||
on_each_cpu(enable_insn_hw_mode, (void *)insn, true);
|
||||
else
|
||||
on_each_cpu(disable_insn_hw_mode, (void *)insn, true);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Run set_hw_mode for all insns on a starting CPU.
|
||||
* Returns:
|
||||
* 0 - If all the hooks ran successfully.
|
||||
* -EINVAL - At least one hook is not supported by the CPU.
|
||||
*/
|
||||
static int run_all_insn_set_hw_mode(unsigned long cpu)
|
||||
{
|
||||
int rc = 0;
|
||||
unsigned long flags;
|
||||
struct insn_emulation *insn;
|
||||
|
||||
raw_spin_lock_irqsave(&insn_emulation_lock, flags);
|
||||
list_for_each_entry(insn, &insn_emulation, node) {
|
||||
bool enable = (insn->current_mode == INSN_HW);
|
||||
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(enable)) {
|
||||
pr_warn("CPU[%ld] cannot support the emulation of %s",
|
||||
cpu, insn->ops->name);
|
||||
rc = -EINVAL;
|
||||
}
|
||||
}
|
||||
raw_spin_unlock_irqrestore(&insn_emulation_lock, flags);
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int update_insn_emulation_mode(struct insn_emulation *insn,
|
||||
enum insn_emulation_mode prev)
|
||||
{
|
||||
|
@ -97,10 +149,8 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
|
|||
remove_emulation_hooks(insn->ops);
|
||||
break;
|
||||
case INSN_HW:
|
||||
if (insn->ops->set_hw_mode) {
|
||||
insn->ops->set_hw_mode(false);
|
||||
if (!run_all_cpu_set_hw_mode(insn, false))
|
||||
pr_notice("Disabled %s support\n", insn->ops->name);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -111,10 +161,9 @@ static int update_insn_emulation_mode(struct insn_emulation *insn,
|
|||
register_emulation_hooks(insn->ops);
|
||||
break;
|
||||
case INSN_HW:
|
||||
if (insn->ops->set_hw_mode && insn->ops->set_hw_mode(true))
|
||||
ret = run_all_cpu_set_hw_mode(insn, true);
|
||||
if (!ret)
|
||||
pr_notice("Enabled %s support\n", insn->ops->name);
|
||||
else
|
||||
ret = -EINVAL;
|
||||
break;
|
||||
}
|
||||
|
||||
|
@ -133,6 +182,8 @@ static void register_insn_emulation(struct insn_emulation_ops *ops)
|
|||
switch (ops->status) {
|
||||
case INSN_DEPRECATED:
|
||||
insn->current_mode = INSN_EMULATE;
|
||||
/* Disable the HW mode if it was turned on at early boot time */
|
||||
run_all_cpu_set_hw_mode(insn, false);
|
||||
insn->max = INSN_HW;
|
||||
break;
|
||||
case INSN_OBSOLETE:
|
||||
|
@ -453,8 +504,6 @@ ret:
|
|||
return 0;
|
||||
}
|
||||
|
||||
#define SCTLR_EL1_CP15BEN (1 << 5)
|
||||
|
||||
static inline void config_sctlr_el1(u32 clear, u32 set)
|
||||
{
|
||||
u32 val;
|
||||
|
@ -465,48 +514,13 @@ static inline void config_sctlr_el1(u32 clear, u32 set)
|
|||
asm volatile("msr sctlr_el1, %0" : : "r" (val));
|
||||
}
|
||||
|
||||
static void enable_cp15_ben(void *info)
|
||||
{
|
||||
config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
|
||||
}
|
||||
|
||||
static void disable_cp15_ben(void *info)
|
||||
{
|
||||
config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
|
||||
}
|
||||
|
||||
static int cpu_hotplug_notify(struct notifier_block *b,
|
||||
unsigned long action, void *hcpu)
|
||||
{
|
||||
switch (action) {
|
||||
case CPU_STARTING:
|
||||
case CPU_STARTING_FROZEN:
|
||||
enable_cp15_ben(NULL);
|
||||
return NOTIFY_DONE;
|
||||
case CPU_DYING:
|
||||
case CPU_DYING_FROZEN:
|
||||
disable_cp15_ben(NULL);
|
||||
return NOTIFY_DONE;
|
||||
}
|
||||
|
||||
return NOTIFY_OK;
|
||||
}
|
||||
|
||||
static struct notifier_block cpu_hotplug_notifier = {
|
||||
.notifier_call = cpu_hotplug_notify,
|
||||
};
|
||||
|
||||
static int cp15_barrier_set_hw_mode(bool enable)
|
||||
{
|
||||
if (enable) {
|
||||
register_cpu_notifier(&cpu_hotplug_notifier);
|
||||
on_each_cpu(enable_cp15_ben, NULL, true);
|
||||
} else {
|
||||
unregister_cpu_notifier(&cpu_hotplug_notifier);
|
||||
on_each_cpu(disable_cp15_ben, NULL, true);
|
||||
}
|
||||
|
||||
return true;
|
||||
if (enable)
|
||||
config_sctlr_el1(0, SCTLR_EL1_CP15BEN);
|
||||
else
|
||||
config_sctlr_el1(SCTLR_EL1_CP15BEN, 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct undef_hook cp15_barrier_hooks[] = {
|
||||
|
@ -534,6 +548,93 @@ static struct insn_emulation_ops cp15_barrier_ops = {
|
|||
.set_hw_mode = cp15_barrier_set_hw_mode,
|
||||
};
|
||||
|
||||
static int setend_set_hw_mode(bool enable)
|
||||
{
|
||||
if (!cpu_supports_mixed_endian_el0())
|
||||
return -EINVAL;
|
||||
|
||||
if (enable)
|
||||
config_sctlr_el1(SCTLR_EL1_SED, 0);
|
||||
else
|
||||
config_sctlr_el1(0, SCTLR_EL1_SED);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int compat_setend_handler(struct pt_regs *regs, u32 big_endian)
|
||||
{
|
||||
char *insn;
|
||||
|
||||
perf_sw_event(PERF_COUNT_SW_EMULATION_FAULTS, 1, regs, regs->pc);
|
||||
|
||||
if (big_endian) {
|
||||
insn = "setend be";
|
||||
regs->pstate |= COMPAT_PSR_E_BIT;
|
||||
} else {
|
||||
insn = "setend le";
|
||||
regs->pstate &= ~COMPAT_PSR_E_BIT;
|
||||
}
|
||||
|
||||
trace_instruction_emulation(insn, regs->pc);
|
||||
pr_warn_ratelimited("\"%s\" (%ld) uses deprecated setend instruction at 0x%llx\n",
|
||||
current->comm, (unsigned long)current->pid, regs->pc);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int a32_setend_handler(struct pt_regs *regs, u32 instr)
|
||||
{
|
||||
int rc = compat_setend_handler(regs, (instr >> 9) & 1);
|
||||
regs->pc += 4;
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int t16_setend_handler(struct pt_regs *regs, u32 instr)
|
||||
{
|
||||
int rc = compat_setend_handler(regs, (instr >> 3) & 1);
|
||||
regs->pc += 2;
|
||||
return rc;
|
||||
}
|
||||
|
||||
static struct undef_hook setend_hooks[] = {
|
||||
{
|
||||
.instr_mask = 0xfffffdff,
|
||||
.instr_val = 0xf1010000,
|
||||
.pstate_mask = COMPAT_PSR_MODE_MASK,
|
||||
.pstate_val = COMPAT_PSR_MODE_USR,
|
||||
.fn = a32_setend_handler,
|
||||
},
|
||||
{
|
||||
/* Thumb mode */
|
||||
.instr_mask = 0x0000fff7,
|
||||
.instr_val = 0x0000b650,
|
||||
.pstate_mask = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_MASK),
|
||||
.pstate_val = (COMPAT_PSR_T_BIT | COMPAT_PSR_MODE_USR),
|
||||
.fn = t16_setend_handler,
|
||||
},
|
||||
{}
|
||||
};
|
||||
|
||||
static struct insn_emulation_ops setend_ops = {
|
||||
.name = "setend",
|
||||
.status = INSN_DEPRECATED,
|
||||
.hooks = setend_hooks,
|
||||
.set_hw_mode = setend_set_hw_mode,
|
||||
};
|
||||
|
||||
static int insn_cpu_hotplug_notify(struct notifier_block *b,
|
||||
unsigned long action, void *hcpu)
|
||||
{
|
||||
int rc = 0;
|
||||
if ((action & ~CPU_TASKS_FROZEN) == CPU_STARTING)
|
||||
rc = run_all_insn_set_hw_mode((unsigned long)hcpu);
|
||||
|
||||
return notifier_from_errno(rc);
|
||||
}
|
||||
|
||||
static struct notifier_block insn_cpu_hotplug_notifier = {
|
||||
.notifier_call = insn_cpu_hotplug_notify,
|
||||
};
|
||||
|
||||
/*
|
||||
* Invoked as late_initcall, since not needed before init spawned.
|
||||
*/
|
||||
|
@ -545,6 +646,14 @@ static int __init armv8_deprecated_init(void)
|
|||
if (IS_ENABLED(CONFIG_CP15_BARRIER_EMULATION))
|
||||
register_insn_emulation(&cp15_barrier_ops);
|
||||
|
||||
if (IS_ENABLED(CONFIG_SETEND_EMULATION)) {
|
||||
if(system_supports_mixed_endian_el0())
|
||||
register_insn_emulation(&setend_ops);
|
||||
else
|
||||
pr_info("setend instruction emulation is not supported on the system");
|
||||
}
|
||||
|
||||
register_cpu_notifier(&insn_cpu_hotplug_notifier);
|
||||
register_insn_emulation_sysctl(ctl_abi);
|
||||
|
||||
return 0;
|
||||
|
|
|
@ -152,7 +152,7 @@ int main(void)
|
|||
DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr));
|
||||
DEFINE(KVM_VGIC_VCTRL, offsetof(struct kvm, arch.vgic.vctrl_base));
|
||||
#endif
|
||||
#ifdef CONFIG_ARM64_CPU_SUSPEND
|
||||
#ifdef CONFIG_CPU_PM
|
||||
DEFINE(CPU_SUSPEND_SZ, sizeof(struct cpu_suspend_ctx));
|
||||
DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
|
||||
DEFINE(MPIDR_HASH_MASK, offsetof(struct mpidr_hash, mask));
|
||||
|
|
|
@ -0,0 +1,128 @@
|
|||
/*
|
||||
* ARM64 cacheinfo support
|
||||
*
|
||||
* Copyright (C) 2015 ARM Ltd.
|
||||
* All Rights Reserved
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed "as is" WITHOUT ANY WARRANTY of any
|
||||
* kind, whether express or implied; without even the implied warranty
|
||||
* of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/cacheinfo.h>
|
||||
#include <linux/cpu.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/of.h>
|
||||
|
||||
#include <asm/cachetype.h>
|
||||
#include <asm/processor.h>
|
||||
|
||||
#define MAX_CACHE_LEVEL 7 /* Max 7 level supported */
|
||||
/* Ctypen, bits[3(n - 1) + 2 : 3(n - 1)], for n = 1 to 7 */
|
||||
#define CLIDR_CTYPE_SHIFT(level) (3 * (level - 1))
|
||||
#define CLIDR_CTYPE_MASK(level) (7 << CLIDR_CTYPE_SHIFT(level))
|
||||
#define CLIDR_CTYPE(clidr, level) \
|
||||
(((clidr) & CLIDR_CTYPE_MASK(level)) >> CLIDR_CTYPE_SHIFT(level))
|
||||
|
||||
static inline enum cache_type get_cache_type(int level)
|
||||
{
|
||||
u64 clidr;
|
||||
|
||||
if (level > MAX_CACHE_LEVEL)
|
||||
return CACHE_TYPE_NOCACHE;
|
||||
asm volatile ("mrs %x0, clidr_el1" : "=r" (clidr));
|
||||
return CLIDR_CTYPE(clidr, level);
|
||||
}
|
||||
|
||||
/*
|
||||
* Cache Size Selection Register(CSSELR) selects which Cache Size ID
|
||||
* Register(CCSIDR) is accessible by specifying the required cache
|
||||
* level and the cache type. We need to ensure that no one else changes
|
||||
* CSSELR by calling this in non-preemtible context
|
||||
*/
|
||||
u64 __attribute_const__ cache_get_ccsidr(u64 csselr)
|
||||
{
|
||||
u64 ccsidr;
|
||||
|
||||
WARN_ON(preemptible());
|
||||
|
||||
/* Put value into CSSELR */
|
||||
asm volatile("msr csselr_el1, %x0" : : "r" (csselr));
|
||||
isb();
|
||||
/* Read result out of CCSIDR */
|
||||
asm volatile("mrs %x0, ccsidr_el1" : "=r" (ccsidr));
|
||||
|
||||
return ccsidr;
|
||||
}
|
||||
|
||||
static void ci_leaf_init(struct cacheinfo *this_leaf,
|
||||
enum cache_type type, unsigned int level)
|
||||
{
|
||||
bool is_icache = type & CACHE_TYPE_INST;
|
||||
u64 tmp = cache_get_ccsidr((level - 1) << 1 | is_icache);
|
||||
|
||||
this_leaf->level = level;
|
||||
this_leaf->type = type;
|
||||
this_leaf->coherency_line_size = CACHE_LINESIZE(tmp);
|
||||
this_leaf->number_of_sets = CACHE_NUMSETS(tmp);
|
||||
this_leaf->ways_of_associativity = CACHE_ASSOCIATIVITY(tmp);
|
||||
this_leaf->size = this_leaf->number_of_sets *
|
||||
this_leaf->coherency_line_size * this_leaf->ways_of_associativity;
|
||||
this_leaf->attributes =
|
||||
((tmp & CCSIDR_EL1_WRITE_THROUGH) ? CACHE_WRITE_THROUGH : 0) |
|
||||
((tmp & CCSIDR_EL1_WRITE_BACK) ? CACHE_WRITE_BACK : 0) |
|
||||
((tmp & CCSIDR_EL1_READ_ALLOCATE) ? CACHE_READ_ALLOCATE : 0) |
|
||||
((tmp & CCSIDR_EL1_WRITE_ALLOCATE) ? CACHE_WRITE_ALLOCATE : 0);
|
||||
}
|
||||
|
||||
static int __init_cache_level(unsigned int cpu)
|
||||
{
|
||||
unsigned int ctype, level, leaves;
|
||||
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
|
||||
|
||||
for (level = 1, leaves = 0; level <= MAX_CACHE_LEVEL; level++) {
|
||||
ctype = get_cache_type(level);
|
||||
if (ctype == CACHE_TYPE_NOCACHE) {
|
||||
level--;
|
||||
break;
|
||||
}
|
||||
/* Separate instruction and data caches */
|
||||
leaves += (ctype == CACHE_TYPE_SEPARATE) ? 2 : 1;
|
||||
}
|
||||
|
||||
this_cpu_ci->num_levels = level;
|
||||
this_cpu_ci->num_leaves = leaves;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int __populate_cache_leaves(unsigned int cpu)
|
||||
{
|
||||
unsigned int level, idx;
|
||||
enum cache_type type;
|
||||
struct cpu_cacheinfo *this_cpu_ci = get_cpu_cacheinfo(cpu);
|
||||
struct cacheinfo *this_leaf = this_cpu_ci->info_list;
|
||||
|
||||
for (idx = 0, level = 1; level <= this_cpu_ci->num_levels &&
|
||||
idx < this_cpu_ci->num_leaves; idx++, level++) {
|
||||
type = get_cache_type(level);
|
||||
if (type == CACHE_TYPE_SEPARATE) {
|
||||
ci_leaf_init(this_leaf++, CACHE_TYPE_DATA, level);
|
||||
ci_leaf_init(this_leaf++, CACHE_TYPE_INST, level);
|
||||
} else {
|
||||
ci_leaf_init(this_leaf++, type, level);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
DEFINE_SMP_CALL_CACHE_FUNCTION(init_cache_level)
|
||||
DEFINE_SMP_CALL_CACHE_FUNCTION(populate_cache_leaves)
|
|
@ -29,3 +29,23 @@ int cpu_init_idle(unsigned int cpu)
|
|||
of_node_put(cpu_node);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* cpu_suspend() - function to enter a low-power idle state
|
||||
* @arg: argument to pass to CPU suspend operations
|
||||
*
|
||||
* Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
|
||||
* operations back-end error code otherwise.
|
||||
*/
|
||||
int cpu_suspend(unsigned long arg)
|
||||
{
|
||||
int cpu = smp_processor_id();
|
||||
|
||||
/*
|
||||
* If cpu_ops have not been registered or suspend
|
||||
* has not been initialized, cpu_suspend call fails early.
|
||||
*/
|
||||
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
|
||||
return -EOPNOTSUPP;
|
||||
return cpu_ops[cpu]->cpu_suspend(arg);
|
||||
}
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
*/
|
||||
DEFINE_PER_CPU(struct cpuinfo_arm64, cpu_data);
|
||||
static struct cpuinfo_arm64 boot_cpu_data;
|
||||
static bool mixed_endian_el0 = true;
|
||||
|
||||
static char *icache_policy_str[] = {
|
||||
[ICACHE_POLICY_RESERVED] = "RESERVED/UNKNOWN",
|
||||
|
@ -68,6 +69,26 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info)
|
|||
pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu);
|
||||
}
|
||||
|
||||
bool cpu_supports_mixed_endian_el0(void)
|
||||
{
|
||||
return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
|
||||
}
|
||||
|
||||
bool system_supports_mixed_endian_el0(void)
|
||||
{
|
||||
return mixed_endian_el0;
|
||||
}
|
||||
|
||||
static void update_mixed_endian_el0_support(struct cpuinfo_arm64 *info)
|
||||
{
|
||||
mixed_endian_el0 &= id_aa64mmfr0_mixed_endian_el0(info->reg_id_aa64mmfr0);
|
||||
}
|
||||
|
||||
static void update_cpu_features(struct cpuinfo_arm64 *info)
|
||||
{
|
||||
update_mixed_endian_el0_support(info);
|
||||
}
|
||||
|
||||
static int check_reg_mask(char *name, u64 mask, u64 boot, u64 cur, int cpu)
|
||||
{
|
||||
if ((boot & mask) == (cur & mask))
|
||||
|
@ -215,6 +236,7 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info)
|
|||
cpuinfo_detect_icache_policy(info);
|
||||
|
||||
check_local_cpu_errata();
|
||||
update_cpu_features(info);
|
||||
}
|
||||
|
||||
void cpuinfo_store_cpu(void)
|
||||
|
@ -231,15 +253,3 @@ void __init cpuinfo_store_boot_cpu(void)
|
|||
|
||||
boot_cpu_data = *info;
|
||||
}
|
||||
|
||||
u64 __attribute_const__ icache_get_ccsidr(void)
|
||||
{
|
||||
u64 ccsidr;
|
||||
|
||||
WARN_ON(preemptible());
|
||||
|
||||
/* Select L1 I-cache and read its size ID register */
|
||||
asm("msr csselr_el1, %1; isb; mrs %0, ccsidr_el1"
|
||||
: "=r"(ccsidr) : "r"(1L));
|
||||
return ccsidr;
|
||||
}
|
||||
|
|
|
@ -11,27 +11,46 @@
|
|||
*
|
||||
*/
|
||||
|
||||
#include <linux/atomic.h>
|
||||
#include <linux/dmi.h>
|
||||
#include <linux/efi.h>
|
||||
#include <linux/export.h>
|
||||
#include <linux/memblock.h>
|
||||
#include <linux/mm_types.h>
|
||||
#include <linux/bootmem.h>
|
||||
#include <linux/of.h>
|
||||
#include <linux/of_fdt.h>
|
||||
#include <linux/preempt.h>
|
||||
#include <linux/rbtree.h>
|
||||
#include <linux/rwsem.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/spinlock.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/efi.h>
|
||||
#include <asm/tlbflush.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/mmu.h>
|
||||
#include <asm/pgtable.h>
|
||||
|
||||
struct efi_memory_map memmap;
|
||||
|
||||
static efi_runtime_services_t *runtime;
|
||||
|
||||
static u64 efi_system_table;
|
||||
|
||||
static pgd_t efi_pgd[PTRS_PER_PGD] __page_aligned_bss;
|
||||
|
||||
static struct mm_struct efi_mm = {
|
||||
.mm_rb = RB_ROOT,
|
||||
.pgd = efi_pgd,
|
||||
.mm_users = ATOMIC_INIT(2),
|
||||
.mm_count = ATOMIC_INIT(1),
|
||||
.mmap_sem = __RWSEM_INITIALIZER(efi_mm.mmap_sem),
|
||||
.page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock),
|
||||
.mmlist = LIST_HEAD_INIT(efi_mm.mmlist),
|
||||
INIT_MM_CONTEXT(efi_mm)
|
||||
};
|
||||
|
||||
static int uefi_debug __initdata;
|
||||
static int __init uefi_debug_setup(char *str)
|
||||
{
|
||||
|
@ -48,30 +67,33 @@ static int __init is_normal_ram(efi_memory_desc_t *md)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void __init efi_setup_idmap(void)
|
||||
/*
|
||||
* Translate a EFI virtual address into a physical address: this is necessary,
|
||||
* as some data members of the EFI system table are virtually remapped after
|
||||
* SetVirtualAddressMap() has been called.
|
||||
*/
|
||||
static phys_addr_t efi_to_phys(unsigned long addr)
|
||||
{
|
||||
struct memblock_region *r;
|
||||
efi_memory_desc_t *md;
|
||||
u64 paddr, npages, size;
|
||||
|
||||
for_each_memblock(memory, r)
|
||||
create_id_mapping(r->base, r->size, 0);
|
||||
|
||||
/* map runtime io spaces */
|
||||
for_each_efi_memory_desc(&memmap, md) {
|
||||
if (!(md->attribute & EFI_MEMORY_RUNTIME) || is_normal_ram(md))
|
||||
if (!(md->attribute & EFI_MEMORY_RUNTIME))
|
||||
continue;
|
||||
paddr = md->phys_addr;
|
||||
npages = md->num_pages;
|
||||
memrange_efi_to_native(&paddr, &npages);
|
||||
size = npages << PAGE_SHIFT;
|
||||
create_id_mapping(paddr, size, 1);
|
||||
if (md->virt_addr == 0)
|
||||
/* no virtual mapping has been installed by the stub */
|
||||
break;
|
||||
if (md->virt_addr <= addr &&
|
||||
(addr - md->virt_addr) < (md->num_pages << EFI_PAGE_SHIFT))
|
||||
return md->phys_addr + addr - md->virt_addr;
|
||||
}
|
||||
return addr;
|
||||
}
|
||||
|
||||
static int __init uefi_init(void)
|
||||
{
|
||||
efi_char16_t *c16;
|
||||
void *config_tables;
|
||||
u64 table_size;
|
||||
char vendor[100] = "unknown";
|
||||
int i, retval;
|
||||
|
||||
|
@ -99,7 +121,7 @@ static int __init uefi_init(void)
|
|||
efi.systab->hdr.revision & 0xffff);
|
||||
|
||||
/* Show what we know for posterity */
|
||||
c16 = early_memremap(efi.systab->fw_vendor,
|
||||
c16 = early_memremap(efi_to_phys(efi.systab->fw_vendor),
|
||||
sizeof(vendor));
|
||||
if (c16) {
|
||||
for (i = 0; i < (int) sizeof(vendor) - 1 && *c16; ++i)
|
||||
|
@ -112,8 +134,14 @@ static int __init uefi_init(void)
|
|||
efi.systab->hdr.revision >> 16,
|
||||
efi.systab->hdr.revision & 0xffff, vendor);
|
||||
|
||||
retval = efi_config_init(NULL);
|
||||
table_size = sizeof(efi_config_table_64_t) * efi.systab->nr_tables;
|
||||
config_tables = early_memremap(efi_to_phys(efi.systab->tables),
|
||||
table_size);
|
||||
|
||||
retval = efi_config_parse_tables(config_tables, efi.systab->nr_tables,
|
||||
sizeof(efi_config_table_64_t), NULL);
|
||||
|
||||
early_memunmap(config_tables, table_size);
|
||||
out:
|
||||
early_memunmap(efi.systab, sizeof(efi_system_table_t));
|
||||
return retval;
|
||||
|
@ -163,9 +191,7 @@ static __init void reserve_regions(void)
|
|||
if (is_normal_ram(md))
|
||||
early_init_dt_add_memory_arch(paddr, size);
|
||||
|
||||
if (is_reserve_region(md) ||
|
||||
md->type == EFI_BOOT_SERVICES_CODE ||
|
||||
md->type == EFI_BOOT_SERVICES_DATA) {
|
||||
if (is_reserve_region(md)) {
|
||||
memblock_reserve(paddr, size);
|
||||
if (uefi_debug)
|
||||
pr_cont("*");
|
||||
|
@ -178,123 +204,6 @@ static __init void reserve_regions(void)
|
|||
set_bit(EFI_MEMMAP, &efi.flags);
|
||||
}
|
||||
|
||||
|
||||
static u64 __init free_one_region(u64 start, u64 end)
|
||||
{
|
||||
u64 size = end - start;
|
||||
|
||||
if (uefi_debug)
|
||||
pr_info(" EFI freeing: 0x%012llx-0x%012llx\n", start, end - 1);
|
||||
|
||||
free_bootmem_late(start, size);
|
||||
return size;
|
||||
}
|
||||
|
||||
static u64 __init free_region(u64 start, u64 end)
|
||||
{
|
||||
u64 map_start, map_end, total = 0;
|
||||
|
||||
if (end <= start)
|
||||
return total;
|
||||
|
||||
map_start = (u64)memmap.phys_map;
|
||||
map_end = PAGE_ALIGN(map_start + (memmap.map_end - memmap.map));
|
||||
map_start &= PAGE_MASK;
|
||||
|
||||
if (start < map_end && end > map_start) {
|
||||
/* region overlaps UEFI memmap */
|
||||
if (start < map_start)
|
||||
total += free_one_region(start, map_start);
|
||||
|
||||
if (map_end < end)
|
||||
total += free_one_region(map_end, end);
|
||||
} else
|
||||
total += free_one_region(start, end);
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
static void __init free_boot_services(void)
|
||||
{
|
||||
u64 total_freed = 0;
|
||||
u64 keep_end, free_start, free_end;
|
||||
efi_memory_desc_t *md;
|
||||
|
||||
/*
|
||||
* If kernel uses larger pages than UEFI, we have to be careful
|
||||
* not to inadvertantly free memory we want to keep if there is
|
||||
* overlap at the kernel page size alignment. We do not want to
|
||||
* free is_reserve_region() memory nor the UEFI memmap itself.
|
||||
*
|
||||
* The memory map is sorted, so we keep track of the end of
|
||||
* any previous region we want to keep, remember any region
|
||||
* we want to free and defer freeing it until we encounter
|
||||
* the next region we want to keep. This way, before freeing
|
||||
* it, we can clip it as needed to avoid freeing memory we
|
||||
* want to keep for UEFI.
|
||||
*/
|
||||
|
||||
keep_end = 0;
|
||||
free_start = 0;
|
||||
|
||||
for_each_efi_memory_desc(&memmap, md) {
|
||||
u64 paddr, npages, size;
|
||||
|
||||
if (is_reserve_region(md)) {
|
||||
/*
|
||||
* We don't want to free any memory from this region.
|
||||
*/
|
||||
if (free_start) {
|
||||
/* adjust free_end then free region */
|
||||
if (free_end > md->phys_addr)
|
||||
free_end -= PAGE_SIZE;
|
||||
total_freed += free_region(free_start, free_end);
|
||||
free_start = 0;
|
||||
}
|
||||
keep_end = md->phys_addr + (md->num_pages << EFI_PAGE_SHIFT);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (md->type != EFI_BOOT_SERVICES_CODE &&
|
||||
md->type != EFI_BOOT_SERVICES_DATA) {
|
||||
/* no need to free this region */
|
||||
continue;
|
||||
}
|
||||
|
||||
/*
|
||||
* We want to free memory from this region.
|
||||
*/
|
||||
paddr = md->phys_addr;
|
||||
npages = md->num_pages;
|
||||
memrange_efi_to_native(&paddr, &npages);
|
||||
size = npages << PAGE_SHIFT;
|
||||
|
||||
if (free_start) {
|
||||
if (paddr <= free_end)
|
||||
free_end = paddr + size;
|
||||
else {
|
||||
total_freed += free_region(free_start, free_end);
|
||||
free_start = paddr;
|
||||
free_end = paddr + size;
|
||||
}
|
||||
} else {
|
||||
free_start = paddr;
|
||||
free_end = paddr + size;
|
||||
}
|
||||
if (free_start < keep_end) {
|
||||
free_start += PAGE_SIZE;
|
||||
if (free_start >= free_end)
|
||||
free_start = 0;
|
||||
}
|
||||
}
|
||||
if (free_start)
|
||||
total_freed += free_region(free_start, free_end);
|
||||
|
||||
if (total_freed)
|
||||
pr_info("Freed 0x%llx bytes of EFI boot services memory",
|
||||
total_freed);
|
||||
}
|
||||
|
||||
void __init efi_init(void)
|
||||
{
|
||||
struct efi_fdt_params params;
|
||||
|
@ -317,159 +226,100 @@ void __init efi_init(void)
|
|||
return;
|
||||
|
||||
reserve_regions();
|
||||
early_memunmap(memmap.map, params.mmap_size);
|
||||
}
|
||||
|
||||
void __init efi_idmap_init(void)
|
||||
static bool __init efi_virtmap_init(void)
|
||||
{
|
||||
if (!efi_enabled(EFI_BOOT))
|
||||
return;
|
||||
efi_memory_desc_t *md;
|
||||
|
||||
/* boot time idmap_pg_dir is incomplete, so fill in missing parts */
|
||||
efi_setup_idmap();
|
||||
early_memunmap(memmap.map, memmap.map_end - memmap.map);
|
||||
}
|
||||
for_each_efi_memory_desc(&memmap, md) {
|
||||
u64 paddr, npages, size;
|
||||
pgprot_t prot;
|
||||
|
||||
static int __init remap_region(efi_memory_desc_t *md, void **new)
|
||||
{
|
||||
u64 paddr, vaddr, npages, size;
|
||||
if (!(md->attribute & EFI_MEMORY_RUNTIME))
|
||||
continue;
|
||||
if (md->virt_addr == 0)
|
||||
return false;
|
||||
|
||||
paddr = md->phys_addr;
|
||||
npages = md->num_pages;
|
||||
memrange_efi_to_native(&paddr, &npages);
|
||||
size = npages << PAGE_SHIFT;
|
||||
paddr = md->phys_addr;
|
||||
npages = md->num_pages;
|
||||
memrange_efi_to_native(&paddr, &npages);
|
||||
size = npages << PAGE_SHIFT;
|
||||
|
||||
if (is_normal_ram(md))
|
||||
vaddr = (__force u64)ioremap_cache(paddr, size);
|
||||
else
|
||||
vaddr = (__force u64)ioremap(paddr, size);
|
||||
|
||||
if (!vaddr) {
|
||||
pr_err("Unable to remap 0x%llx pages @ %p\n",
|
||||
npages, (void *)paddr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* adjust for any rounding when EFI and system pagesize differs */
|
||||
md->virt_addr = vaddr + (md->phys_addr - paddr);
|
||||
|
||||
if (uefi_debug)
|
||||
pr_info(" EFI remap 0x%012llx => %p\n",
|
||||
pr_info(" EFI remap 0x%016llx => %p\n",
|
||||
md->phys_addr, (void *)md->virt_addr);
|
||||
|
||||
memcpy(*new, md, memmap.desc_size);
|
||||
*new += memmap.desc_size;
|
||||
/*
|
||||
* Only regions of type EFI_RUNTIME_SERVICES_CODE need to be
|
||||
* executable, everything else can be mapped with the XN bits
|
||||
* set.
|
||||
*/
|
||||
if (!is_normal_ram(md))
|
||||
prot = __pgprot(PROT_DEVICE_nGnRE);
|
||||
else if (md->type == EFI_RUNTIME_SERVICES_CODE)
|
||||
prot = PAGE_KERNEL_EXEC;
|
||||
else
|
||||
prot = PAGE_KERNEL;
|
||||
|
||||
return 1;
|
||||
create_pgd_mapping(&efi_mm, paddr, md->virt_addr, size, prot);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Switch UEFI from an identity map to a kernel virtual map
|
||||
* Enable the UEFI Runtime Services if all prerequisites are in place, i.e.,
|
||||
* non-early mapping of the UEFI system table and virtual mappings for all
|
||||
* EFI_MEMORY_RUNTIME regions.
|
||||
*/
|
||||
static int __init arm64_enter_virtual_mode(void)
|
||||
static int __init arm64_enable_runtime_services(void)
|
||||
{
|
||||
efi_memory_desc_t *md;
|
||||
phys_addr_t virtmap_phys;
|
||||
void *virtmap, *virt_md;
|
||||
efi_status_t status;
|
||||
u64 mapsize;
|
||||
int count = 0;
|
||||
unsigned long flags;
|
||||
|
||||
if (!efi_enabled(EFI_BOOT)) {
|
||||
pr_info("EFI services will not be available.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
mapsize = memmap.map_end - memmap.map;
|
||||
|
||||
if (efi_runtime_disabled()) {
|
||||
pr_info("EFI runtime services will be disabled.\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
pr_info("Remapping and enabling EFI services.\n");
|
||||
/* replace early memmap mapping with permanent mapping */
|
||||
|
||||
mapsize = memmap.map_end - memmap.map;
|
||||
memmap.map = (__force void *)ioremap_cache((phys_addr_t)memmap.phys_map,
|
||||
mapsize);
|
||||
memmap.map_end = memmap.map + mapsize;
|
||||
|
||||
efi.memmap = &memmap;
|
||||
|
||||
/* Map the runtime regions */
|
||||
virtmap = kmalloc(mapsize, GFP_KERNEL);
|
||||
if (!virtmap) {
|
||||
pr_err("Failed to allocate EFI virtual memmap\n");
|
||||
if (!memmap.map) {
|
||||
pr_err("Failed to remap EFI memory map\n");
|
||||
return -1;
|
||||
}
|
||||
virtmap_phys = virt_to_phys(virtmap);
|
||||
virt_md = virtmap;
|
||||
memmap.map_end = memmap.map + mapsize;
|
||||
efi.memmap = &memmap;
|
||||
|
||||
for_each_efi_memory_desc(&memmap, md) {
|
||||
if (!(md->attribute & EFI_MEMORY_RUNTIME))
|
||||
continue;
|
||||
if (!remap_region(md, &virt_md))
|
||||
goto err_unmap;
|
||||
++count;
|
||||
}
|
||||
|
||||
efi.systab = (__force void *)efi_lookup_mapped_addr(efi_system_table);
|
||||
efi.systab = (__force void *)ioremap_cache(efi_system_table,
|
||||
sizeof(efi_system_table_t));
|
||||
if (!efi.systab) {
|
||||
/*
|
||||
* If we have no virtual mapping for the System Table at this
|
||||
* point, the memory map doesn't cover the physical offset where
|
||||
* it resides. This means the System Table will be inaccessible
|
||||
* to Runtime Services themselves once the virtual mapping is
|
||||
* installed.
|
||||
*/
|
||||
pr_err("Failed to remap EFI System Table -- buggy firmware?\n");
|
||||
goto err_unmap;
|
||||
pr_err("Failed to remap EFI System Table\n");
|
||||
return -1;
|
||||
}
|
||||
set_bit(EFI_SYSTEM_TABLES, &efi.flags);
|
||||
|
||||
local_irq_save(flags);
|
||||
cpu_switch_mm(idmap_pg_dir, &init_mm);
|
||||
|
||||
/* Call SetVirtualAddressMap with the physical address of the map */
|
||||
runtime = efi.systab->runtime;
|
||||
efi.set_virtual_address_map = runtime->set_virtual_address_map;
|
||||
|
||||
status = efi.set_virtual_address_map(count * memmap.desc_size,
|
||||
memmap.desc_size,
|
||||
memmap.desc_version,
|
||||
(efi_memory_desc_t *)virtmap_phys);
|
||||
cpu_set_reserved_ttbr0();
|
||||
flush_tlb_all();
|
||||
local_irq_restore(flags);
|
||||
|
||||
kfree(virtmap);
|
||||
|
||||
free_boot_services();
|
||||
|
||||
if (status != EFI_SUCCESS) {
|
||||
pr_err("Failed to set EFI virtual address map! [%lx]\n",
|
||||
status);
|
||||
if (!efi_virtmap_init()) {
|
||||
pr_err("No UEFI virtual mapping was installed -- runtime services will not be available\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
/* Set up runtime services function pointers */
|
||||
runtime = efi.systab->runtime;
|
||||
efi_native_runtime_setup();
|
||||
set_bit(EFI_RUNTIME_SERVICES, &efi.flags);
|
||||
|
||||
efi.runtime_version = efi.systab->hdr.revision;
|
||||
|
||||
return 0;
|
||||
|
||||
err_unmap:
|
||||
/* unmap all mappings that succeeded: there are 'count' of those */
|
||||
for (virt_md = virtmap; count--; virt_md += memmap.desc_size) {
|
||||
md = virt_md;
|
||||
iounmap((__force void __iomem *)md->virt_addr);
|
||||
}
|
||||
kfree(virtmap);
|
||||
return -1;
|
||||
}
|
||||
early_initcall(arm64_enter_virtual_mode);
|
||||
early_initcall(arm64_enable_runtime_services);
|
||||
|
||||
static int __init arm64_dmi_init(void)
|
||||
{
|
||||
|
@ -484,3 +334,23 @@ static int __init arm64_dmi_init(void)
|
|||
return 0;
|
||||
}
|
||||
core_initcall(arm64_dmi_init);
|
||||
|
||||
static void efi_set_pgd(struct mm_struct *mm)
|
||||
{
|
||||
cpu_switch_mm(mm->pgd, mm);
|
||||
flush_tlb_all();
|
||||
if (icache_is_aivivt())
|
||||
__flush_icache_all();
|
||||
}
|
||||
|
||||
void efi_virtmap_load(void)
|
||||
{
|
||||
preempt_disable();
|
||||
efi_set_pgd(&efi_mm);
|
||||
}
|
||||
|
||||
void efi_virtmap_unload(void)
|
||||
{
|
||||
efi_set_pgd(current->active_mm);
|
||||
preempt_enable();
|
||||
}
|
||||
|
|
|
@ -269,18 +269,18 @@ ENDPROC(el1_error_invalid)
|
|||
el1_sync:
|
||||
kernel_entry 1
|
||||
mrs x1, esr_el1 // read the syndrome register
|
||||
lsr x24, x1, #ESR_EL1_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_EL1_EC_DABT_EL1 // data abort in EL1
|
||||
lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
|
||||
b.eq el1_da
|
||||
cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
|
||||
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
|
||||
b.eq el1_undef
|
||||
cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
|
||||
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
|
||||
b.eq el1_sp_pc
|
||||
cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
|
||||
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
||||
b.eq el1_sp_pc
|
||||
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL1
|
||||
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
|
||||
b.eq el1_undef
|
||||
cmp x24, #ESR_EL1_EC_BREAKPT_EL1 // debug exception in EL1
|
||||
cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
|
||||
b.ge el1_dbg
|
||||
b el1_inv
|
||||
el1_da:
|
||||
|
@ -318,7 +318,7 @@ el1_dbg:
|
|||
/*
|
||||
* Debug exception handling
|
||||
*/
|
||||
cmp x24, #ESR_EL1_EC_BRK64 // if BRK64
|
||||
cmp x24, #ESR_ELx_EC_BRK64 // if BRK64
|
||||
cinc x24, x24, eq // set bit '0'
|
||||
tbz x24, #0, el1_inv // EL1 only
|
||||
mrs x0, far_el1
|
||||
|
@ -375,26 +375,26 @@ el1_preempt:
|
|||
el0_sync:
|
||||
kernel_entry 0
|
||||
mrs x25, esr_el1 // read the syndrome register
|
||||
lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_EL1_EC_SVC64 // SVC in 64-bit state
|
||||
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_ELx_EC_SVC64 // SVC in 64-bit state
|
||||
b.eq el0_svc
|
||||
cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
|
||||
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
|
||||
b.eq el0_da
|
||||
cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
|
||||
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
|
||||
b.eq el0_ia
|
||||
cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
|
||||
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
|
||||
b.eq el0_fpsimd_acc
|
||||
cmp x24, #ESR_EL1_EC_FP_EXC64 // FP/ASIMD exception
|
||||
cmp x24, #ESR_ELx_EC_FP_EXC64 // FP/ASIMD exception
|
||||
b.eq el0_fpsimd_exc
|
||||
cmp x24, #ESR_EL1_EC_SYS64 // configurable trap
|
||||
cmp x24, #ESR_ELx_EC_SYS64 // configurable trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_SP_ALIGN // stack alignment exception
|
||||
cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
|
||||
b.eq el0_sp_pc
|
||||
cmp x24, #ESR_EL1_EC_PC_ALIGN // pc alignment exception
|
||||
cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
|
||||
b.eq el0_sp_pc
|
||||
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
|
||||
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
|
||||
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
|
||||
b.ge el0_dbg
|
||||
b el0_inv
|
||||
|
||||
|
@ -403,37 +403,37 @@ el0_sync:
|
|||
el0_sync_compat:
|
||||
kernel_entry 0, 32
|
||||
mrs x25, esr_el1 // read the syndrome register
|
||||
lsr x24, x25, #ESR_EL1_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_EL1_EC_SVC32 // SVC in 32-bit state
|
||||
lsr x24, x25, #ESR_ELx_EC_SHIFT // exception class
|
||||
cmp x24, #ESR_ELx_EC_SVC32 // SVC in 32-bit state
|
||||
b.eq el0_svc_compat
|
||||
cmp x24, #ESR_EL1_EC_DABT_EL0 // data abort in EL0
|
||||
cmp x24, #ESR_ELx_EC_DABT_LOW // data abort in EL0
|
||||
b.eq el0_da
|
||||
cmp x24, #ESR_EL1_EC_IABT_EL0 // instruction abort in EL0
|
||||
cmp x24, #ESR_ELx_EC_IABT_LOW // instruction abort in EL0
|
||||
b.eq el0_ia
|
||||
cmp x24, #ESR_EL1_EC_FP_ASIMD // FP/ASIMD access
|
||||
cmp x24, #ESR_ELx_EC_FP_ASIMD // FP/ASIMD access
|
||||
b.eq el0_fpsimd_acc
|
||||
cmp x24, #ESR_EL1_EC_FP_EXC32 // FP/ASIMD exception
|
||||
cmp x24, #ESR_ELx_EC_FP_EXC32 // FP/ASIMD exception
|
||||
b.eq el0_fpsimd_exc
|
||||
cmp x24, #ESR_EL1_EC_UNKNOWN // unknown exception in EL0
|
||||
cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL0
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_CP15_32 // CP15 MRC/MCR trap
|
||||
cmp x24, #ESR_ELx_EC_CP15_32 // CP15 MRC/MCR trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_CP15_64 // CP15 MRRC/MCRR trap
|
||||
cmp x24, #ESR_ELx_EC_CP15_64 // CP15 MRRC/MCRR trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_CP14_MR // CP14 MRC/MCR trap
|
||||
cmp x24, #ESR_ELx_EC_CP14_MR // CP14 MRC/MCR trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_CP14_LS // CP14 LDC/STC trap
|
||||
cmp x24, #ESR_ELx_EC_CP14_LS // CP14 LDC/STC trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_CP14_64 // CP14 MRRC/MCRR trap
|
||||
cmp x24, #ESR_ELx_EC_CP14_64 // CP14 MRRC/MCRR trap
|
||||
b.eq el0_undef
|
||||
cmp x24, #ESR_EL1_EC_BREAKPT_EL0 // debug exception in EL0
|
||||
cmp x24, #ESR_ELx_EC_BREAKPT_LOW // debug exception in EL0
|
||||
b.ge el0_dbg
|
||||
b el0_inv
|
||||
el0_svc_compat:
|
||||
/*
|
||||
* AArch32 syscall handling
|
||||
*/
|
||||
adr stbl, compat_sys_call_table // load compat syscall table pointer
|
||||
adrp stbl, compat_sys_call_table // load compat syscall table pointer
|
||||
uxtw scno, w7 // syscall number in w7 (r7)
|
||||
mov sc_nr, #__NR_compat_syscalls
|
||||
b el0_svc_naked
|
||||
|
|
|
@ -27,26 +27,26 @@
|
|||
* System call wrappers for the AArch32 compatibility layer.
|
||||
*/
|
||||
|
||||
compat_sys_sigreturn_wrapper:
|
||||
ENTRY(compat_sys_sigreturn_wrapper)
|
||||
mov x0, sp
|
||||
mov x27, #0 // prevent syscall restart handling (why)
|
||||
b compat_sys_sigreturn
|
||||
ENDPROC(compat_sys_sigreturn_wrapper)
|
||||
|
||||
compat_sys_rt_sigreturn_wrapper:
|
||||
ENTRY(compat_sys_rt_sigreturn_wrapper)
|
||||
mov x0, sp
|
||||
mov x27, #0 // prevent syscall restart handling (why)
|
||||
b compat_sys_rt_sigreturn
|
||||
ENDPROC(compat_sys_rt_sigreturn_wrapper)
|
||||
|
||||
compat_sys_statfs64_wrapper:
|
||||
ENTRY(compat_sys_statfs64_wrapper)
|
||||
mov w3, #84
|
||||
cmp w1, #88
|
||||
csel w1, w3, w1, eq
|
||||
b compat_sys_statfs64
|
||||
ENDPROC(compat_sys_statfs64_wrapper)
|
||||
|
||||
compat_sys_fstatfs64_wrapper:
|
||||
ENTRY(compat_sys_fstatfs64_wrapper)
|
||||
mov w3, #84
|
||||
cmp w1, #88
|
||||
csel w1, w3, w1, eq
|
||||
|
@ -58,33 +58,33 @@ ENDPROC(compat_sys_fstatfs64_wrapper)
|
|||
* in registers or that take 32-bit parameters which require sign
|
||||
* extension.
|
||||
*/
|
||||
compat_sys_pread64_wrapper:
|
||||
ENTRY(compat_sys_pread64_wrapper)
|
||||
regs_to_64 x3, x4, x5
|
||||
b sys_pread64
|
||||
ENDPROC(compat_sys_pread64_wrapper)
|
||||
|
||||
compat_sys_pwrite64_wrapper:
|
||||
ENTRY(compat_sys_pwrite64_wrapper)
|
||||
regs_to_64 x3, x4, x5
|
||||
b sys_pwrite64
|
||||
ENDPROC(compat_sys_pwrite64_wrapper)
|
||||
|
||||
compat_sys_truncate64_wrapper:
|
||||
ENTRY(compat_sys_truncate64_wrapper)
|
||||
regs_to_64 x1, x2, x3
|
||||
b sys_truncate
|
||||
ENDPROC(compat_sys_truncate64_wrapper)
|
||||
|
||||
compat_sys_ftruncate64_wrapper:
|
||||
ENTRY(compat_sys_ftruncate64_wrapper)
|
||||
regs_to_64 x1, x2, x3
|
||||
b sys_ftruncate
|
||||
ENDPROC(compat_sys_ftruncate64_wrapper)
|
||||
|
||||
compat_sys_readahead_wrapper:
|
||||
ENTRY(compat_sys_readahead_wrapper)
|
||||
regs_to_64 x1, x2, x3
|
||||
mov w2, w4
|
||||
b sys_readahead
|
||||
ENDPROC(compat_sys_readahead_wrapper)
|
||||
|
||||
compat_sys_fadvise64_64_wrapper:
|
||||
ENTRY(compat_sys_fadvise64_64_wrapper)
|
||||
mov w6, w1
|
||||
regs_to_64 x1, x2, x3
|
||||
regs_to_64 x2, x4, x5
|
||||
|
@ -92,24 +92,14 @@ compat_sys_fadvise64_64_wrapper:
|
|||
b sys_fadvise64_64
|
||||
ENDPROC(compat_sys_fadvise64_64_wrapper)
|
||||
|
||||
compat_sys_sync_file_range2_wrapper:
|
||||
ENTRY(compat_sys_sync_file_range2_wrapper)
|
||||
regs_to_64 x2, x2, x3
|
||||
regs_to_64 x3, x4, x5
|
||||
b sys_sync_file_range2
|
||||
ENDPROC(compat_sys_sync_file_range2_wrapper)
|
||||
|
||||
compat_sys_fallocate_wrapper:
|
||||
ENTRY(compat_sys_fallocate_wrapper)
|
||||
regs_to_64 x2, x2, x3
|
||||
regs_to_64 x3, x4, x5
|
||||
b sys_fallocate
|
||||
ENDPROC(compat_sys_fallocate_wrapper)
|
||||
|
||||
#undef __SYSCALL
|
||||
#define __SYSCALL(x, y) .quad y // x
|
||||
|
||||
/*
|
||||
* The system calls table must be 4KB aligned.
|
||||
*/
|
||||
.align 12
|
||||
ENTRY(compat_sys_call_table)
|
||||
#include <asm/unistd32.h>
|
|
@ -894,7 +894,7 @@ static struct notifier_block hw_breakpoint_reset_nb = {
|
|||
.notifier_call = hw_breakpoint_reset_notify,
|
||||
};
|
||||
|
||||
#ifdef CONFIG_ARM64_CPU_SUSPEND
|
||||
#ifdef CONFIG_CPU_PM
|
||||
extern void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *));
|
||||
#else
|
||||
static inline void cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
|
||||
|
|
|
@ -17,14 +17,19 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
#include <linux/bitops.h>
|
||||
#include <linux/bug.h>
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/smp.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/stop_machine.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/fixmap.h>
|
||||
#include <asm/insn.h>
|
||||
|
||||
#define AARCH64_INSN_SF_BIT BIT(31)
|
||||
|
@ -72,6 +77,29 @@ bool __kprobes aarch64_insn_is_nop(u32 insn)
|
|||
}
|
||||
}
|
||||
|
||||
static DEFINE_SPINLOCK(patch_lock);
|
||||
|
||||
static void __kprobes *patch_map(void *addr, int fixmap)
|
||||
{
|
||||
unsigned long uintaddr = (uintptr_t) addr;
|
||||
bool module = !core_kernel_text(uintaddr);
|
||||
struct page *page;
|
||||
|
||||
if (module && IS_ENABLED(CONFIG_DEBUG_SET_MODULE_RONX))
|
||||
page = vmalloc_to_page(addr);
|
||||
else
|
||||
page = virt_to_page(addr);
|
||||
|
||||
BUG_ON(!page);
|
||||
set_fixmap(fixmap, page_to_phys(page));
|
||||
|
||||
return (void *) (__fix_to_virt(fixmap) + (uintaddr & ~PAGE_MASK));
|
||||
}
|
||||
|
||||
static void __kprobes patch_unmap(int fixmap)
|
||||
{
|
||||
clear_fixmap(fixmap);
|
||||
}
|
||||
/*
|
||||
* In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always
|
||||
* little-endian.
|
||||
|
@ -88,10 +116,27 @@ int __kprobes aarch64_insn_read(void *addr, u32 *insnp)
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int __kprobes __aarch64_insn_write(void *addr, u32 insn)
|
||||
{
|
||||
void *waddr = addr;
|
||||
unsigned long flags = 0;
|
||||
int ret;
|
||||
|
||||
spin_lock_irqsave(&patch_lock, flags);
|
||||
waddr = patch_map(addr, FIX_TEXT_POKE0);
|
||||
|
||||
ret = probe_kernel_write(waddr, &insn, AARCH64_INSN_SIZE);
|
||||
|
||||
patch_unmap(FIX_TEXT_POKE0);
|
||||
spin_unlock_irqrestore(&patch_lock, flags);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int __kprobes aarch64_insn_write(void *addr, u32 insn)
|
||||
{
|
||||
insn = cpu_to_le32(insn);
|
||||
return probe_kernel_write(addr, &insn, AARCH64_INSN_SIZE);
|
||||
return __aarch64_insn_write(addr, insn);
|
||||
}
|
||||
|
||||
static bool __kprobes __aarch64_insn_hotpatch_safe(u32 insn)
|
||||
|
|
|
@ -540,8 +540,6 @@ const struct cpu_operations cpu_psci_ops = {
|
|||
.name = "psci",
|
||||
#ifdef CONFIG_CPU_IDLE
|
||||
.cpu_init_idle = cpu_psci_cpu_init_idle,
|
||||
#endif
|
||||
#ifdef CONFIG_ARM64_CPU_SUSPEND
|
||||
.cpu_suspend = cpu_psci_cpu_suspend,
|
||||
#endif
|
||||
#ifdef CONFIG_SMP
|
||||
|
|
|
@ -40,6 +40,7 @@
|
|||
#include <linux/fs.h>
|
||||
#include <linux/proc_fs.h>
|
||||
#include <linux/memblock.h>
|
||||
#include <linux/of_iommu.h>
|
||||
#include <linux/of_fdt.h>
|
||||
#include <linux/of_platform.h>
|
||||
#include <linux/efi.h>
|
||||
|
@ -322,25 +323,6 @@ static void __init setup_machine_fdt(phys_addr_t dt_phys)
|
|||
dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name());
|
||||
}
|
||||
|
||||
/*
|
||||
* Limit the memory size that was specified via FDT.
|
||||
*/
|
||||
static int __init early_mem(char *p)
|
||||
{
|
||||
phys_addr_t limit;
|
||||
|
||||
if (!p)
|
||||
return 1;
|
||||
|
||||
limit = memparse(p, &p) & PAGE_MASK;
|
||||
pr_notice("Memory limited to %lldMB\n", limit >> 20);
|
||||
|
||||
memblock_enforce_memory_limit(limit);
|
||||
|
||||
return 0;
|
||||
}
|
||||
early_param("mem", early_mem);
|
||||
|
||||
static void __init request_standard_resources(void)
|
||||
{
|
||||
struct memblock_region *region;
|
||||
|
@ -401,7 +383,6 @@ void __init setup_arch(char **cmdline_p)
|
|||
paging_init();
|
||||
request_standard_resources();
|
||||
|
||||
efi_idmap_init();
|
||||
early_ioremap_reset();
|
||||
|
||||
unflatten_device_tree();
|
||||
|
@ -425,6 +406,7 @@ void __init setup_arch(char **cmdline_p)
|
|||
|
||||
static int __init arm64_device_init(void)
|
||||
{
|
||||
of_iommu_init();
|
||||
of_platform_populate(NULL, of_default_bus_match_table, NULL, NULL);
|
||||
return 0;
|
||||
}
|
||||
|
|
|
@ -440,7 +440,7 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
|
|||
{
|
||||
compat_ulong_t handler = ptr_to_compat(ka->sa.sa_handler);
|
||||
compat_ulong_t retcode;
|
||||
compat_ulong_t spsr = regs->pstate & ~PSR_f;
|
||||
compat_ulong_t spsr = regs->pstate & ~(PSR_f | COMPAT_PSR_E_BIT);
|
||||
int thumb;
|
||||
|
||||
/* Check if the handler is written for ARM or Thumb */
|
||||
|
@ -454,6 +454,9 @@ static void compat_setup_return(struct pt_regs *regs, struct k_sigaction *ka,
|
|||
/* The IT state must be cleared for both ARM and Thumb-2 */
|
||||
spsr &= ~COMPAT_PSR_IT_MASK;
|
||||
|
||||
/* Restore the original endianness */
|
||||
spsr |= COMPAT_PSR_ENDSTATE;
|
||||
|
||||
if (ka->sa.sa_flags & SA_RESTORER) {
|
||||
retcode = ptr_to_compat(ka->sa.sa_restorer);
|
||||
} else {
|
||||
|
@ -501,7 +504,7 @@ static int compat_setup_sigframe(struct compat_sigframe __user *sf,
|
|||
|
||||
__put_user_error((compat_ulong_t)0, &sf->uc.uc_mcontext.trap_no, err);
|
||||
/* set the compat FSR WnR */
|
||||
__put_user_error(!!(current->thread.fault_code & ESR_EL1_WRITE) <<
|
||||
__put_user_error(!!(current->thread.fault_code & ESR_ELx_WNR) <<
|
||||
FSR_WRITE_SHIFT, &sf->uc.uc_mcontext.error_code, err);
|
||||
__put_user_error(current->thread.fault_address, &sf->uc.uc_mcontext.fault_address, err);
|
||||
__put_user_error(set->sig[0], &sf->uc.uc_mcontext.oldmask, err);
|
||||
|
|
|
@ -65,7 +65,6 @@ struct secondary_data secondary_data;
|
|||
enum ipi_msg_type {
|
||||
IPI_RESCHEDULE,
|
||||
IPI_CALL_FUNC,
|
||||
IPI_CALL_FUNC_SINGLE,
|
||||
IPI_CPU_STOP,
|
||||
IPI_TIMER,
|
||||
IPI_IRQ_WORK,
|
||||
|
@ -483,7 +482,6 @@ static const char *ipi_types[NR_IPI] __tracepoint_string = {
|
|||
#define S(x,s) [x] = s
|
||||
S(IPI_RESCHEDULE, "Rescheduling interrupts"),
|
||||
S(IPI_CALL_FUNC, "Function call interrupts"),
|
||||
S(IPI_CALL_FUNC_SINGLE, "Single function call interrupts"),
|
||||
S(IPI_CPU_STOP, "CPU stop interrupts"),
|
||||
S(IPI_TIMER, "Timer broadcast interrupts"),
|
||||
S(IPI_IRQ_WORK, "IRQ work interrupts"),
|
||||
|
@ -527,7 +525,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask)
|
|||
|
||||
void arch_send_call_function_single_ipi(int cpu)
|
||||
{
|
||||
smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC_SINGLE);
|
||||
smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_IRQ_WORK
|
||||
|
@ -585,12 +583,6 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
|
|||
irq_exit();
|
||||
break;
|
||||
|
||||
case IPI_CALL_FUNC_SINGLE:
|
||||
irq_enter();
|
||||
generic_smp_call_function_single_interrupt();
|
||||
irq_exit();
|
||||
break;
|
||||
|
||||
case IPI_CPU_STOP:
|
||||
irq_enter();
|
||||
ipi_cpu_stop(cpu);
|
||||
|
|
|
@ -1,7 +1,6 @@
|
|||
#include <linux/percpu.h>
|
||||
#include <linux/slab.h>
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/cpu_ops.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/memory.h>
|
||||
|
@ -51,26 +50,6 @@ void __init cpu_suspend_set_dbg_restorer(void (*hw_bp_restore)(void *))
|
|||
hw_breakpoint_restore = hw_bp_restore;
|
||||
}
|
||||
|
||||
/**
|
||||
* cpu_suspend() - function to enter a low-power state
|
||||
* @arg: argument to pass to CPU suspend operations
|
||||
*
|
||||
* Return: 0 on success, -EOPNOTSUPP if CPU suspend hook not initialized, CPU
|
||||
* operations back-end error code otherwise.
|
||||
*/
|
||||
int cpu_suspend(unsigned long arg)
|
||||
{
|
||||
int cpu = smp_processor_id();
|
||||
|
||||
/*
|
||||
* If cpu_ops have not been registered or suspend
|
||||
* has not been initialized, cpu_suspend call fails early.
|
||||
*/
|
||||
if (!cpu_ops[cpu] || !cpu_ops[cpu]->cpu_suspend)
|
||||
return -EOPNOTSUPP;
|
||||
return cpu_ops[cpu]->cpu_suspend(arg);
|
||||
}
|
||||
|
||||
/*
|
||||
* __cpu_suspend
|
||||
*
|
||||
|
|
|
@ -39,10 +39,9 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
|
|||
/*
|
||||
* Wrappers to pass the pt_regs argument.
|
||||
*/
|
||||
asmlinkage long sys_rt_sigreturn_wrapper(void);
|
||||
#define sys_rt_sigreturn sys_rt_sigreturn_wrapper
|
||||
|
||||
#include <asm/syscalls.h>
|
||||
|
||||
#undef __SYSCALL
|
||||
#define __SYSCALL(nr, sym) [nr] = sym,
|
||||
|
||||
|
@ -50,7 +49,7 @@ asmlinkage long sys_mmap(unsigned long addr, unsigned long len,
|
|||
* The sys_call_table array must be 4K aligned to be accessible from
|
||||
* kernel/entry.S.
|
||||
*/
|
||||
void *sys_call_table[__NR_syscalls] __aligned(4096) = {
|
||||
void * const sys_call_table[__NR_syscalls] __aligned(4096) = {
|
||||
[0 ... __NR_syscalls - 1] = sys_ni_syscall,
|
||||
#include <asm/unistd.h>
|
||||
};
|
||||
|
|
|
@ -0,0 +1,51 @@
|
|||
/*
|
||||
* arch/arm64/kernel/sys32.c
|
||||
*
|
||||
* Copyright (C) 2015 ARM Ltd.
|
||||
*
|
||||
* This program is free software(void); you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http(void);//www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Needed to avoid conflicting __NR_* macros between uapi/asm/unistd.h and
|
||||
* asm/unistd32.h.
|
||||
*/
|
||||
#define __COMPAT_SYSCALL_NR
|
||||
|
||||
#include <linux/compiler.h>
|
||||
#include <linux/syscalls.h>
|
||||
|
||||
asmlinkage long compat_sys_sigreturn_wrapper(void);
|
||||
asmlinkage long compat_sys_rt_sigreturn_wrapper(void);
|
||||
asmlinkage long compat_sys_statfs64_wrapper(void);
|
||||
asmlinkage long compat_sys_fstatfs64_wrapper(void);
|
||||
asmlinkage long compat_sys_pread64_wrapper(void);
|
||||
asmlinkage long compat_sys_pwrite64_wrapper(void);
|
||||
asmlinkage long compat_sys_truncate64_wrapper(void);
|
||||
asmlinkage long compat_sys_ftruncate64_wrapper(void);
|
||||
asmlinkage long compat_sys_readahead_wrapper(void);
|
||||
asmlinkage long compat_sys_fadvise64_64_wrapper(void);
|
||||
asmlinkage long compat_sys_sync_file_range2_wrapper(void);
|
||||
asmlinkage long compat_sys_fallocate_wrapper(void);
|
||||
|
||||
#undef __SYSCALL
|
||||
#define __SYSCALL(nr, sym) [nr] = sym,
|
||||
|
||||
/*
|
||||
* The sys_call_table array must be 4K aligned to be accessible from
|
||||
* kernel/entry.S.
|
||||
*/
|
||||
void * const compat_sys_call_table[__NR_compat_syscalls] __aligned(4096) = {
|
||||
[0 ... __NR_compat_syscalls - 1] = sys_ni_syscall,
|
||||
#include <asm/unistd32.h>
|
||||
};
|
|
@ -33,6 +33,7 @@
|
|||
|
||||
#include <asm/atomic.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/traps.h>
|
||||
#include <asm/stacktrace.h>
|
||||
#include <asm/exception.h>
|
||||
|
@ -373,6 +374,51 @@ asmlinkage long do_ni_syscall(struct pt_regs *regs)
|
|||
return sys_ni_syscall();
|
||||
}
|
||||
|
||||
static const char *esr_class_str[] = {
|
||||
[0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
|
||||
[ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
|
||||
[ESR_ELx_EC_WFx] = "WFI/WFE",
|
||||
[ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
|
||||
[ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
|
||||
[ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
|
||||
[ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
|
||||
[ESR_ELx_EC_FP_ASIMD] = "ASIMD",
|
||||
[ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
|
||||
[ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
|
||||
[ESR_ELx_EC_ILL] = "PSTATE.IL",
|
||||
[ESR_ELx_EC_SVC32] = "SVC (AArch32)",
|
||||
[ESR_ELx_EC_HVC32] = "HVC (AArch32)",
|
||||
[ESR_ELx_EC_SMC32] = "SMC (AArch32)",
|
||||
[ESR_ELx_EC_SVC64] = "SVC (AArch64)",
|
||||
[ESR_ELx_EC_HVC64] = "HVC (AArch64)",
|
||||
[ESR_ELx_EC_SMC64] = "SMC (AArch64)",
|
||||
[ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
|
||||
[ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
|
||||
[ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
|
||||
[ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
|
||||
[ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
|
||||
[ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
|
||||
[ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
|
||||
[ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
|
||||
[ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
|
||||
[ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
|
||||
[ESR_ELx_EC_SERROR] = "SError",
|
||||
[ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
|
||||
[ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
|
||||
[ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
|
||||
[ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
|
||||
[ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
|
||||
[ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
|
||||
[ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
|
||||
[ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
|
||||
[ESR_ELx_EC_BRK64] = "BRK (AArch64)",
|
||||
};
|
||||
|
||||
const char *esr_get_class_string(u32 esr)
|
||||
{
|
||||
return esr_class_str[esr >> ESR_ELx_EC_SHIFT];
|
||||
}
|
||||
|
||||
/*
|
||||
* bad_mode handles the impossible case in the exception vector.
|
||||
*/
|
||||
|
@ -382,8 +428,8 @@ asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
|
|||
void __user *pc = (void __user *)instruction_pointer(regs);
|
||||
console_verbose();
|
||||
|
||||
pr_crit("Bad mode in %s handler detected, code 0x%08x\n",
|
||||
handler[reason], esr);
|
||||
pr_crit("Bad mode in %s handler detected, code 0x%08x -- %s\n",
|
||||
handler[reason], esr, esr_get_class_string(esr));
|
||||
__show_regs(regs);
|
||||
|
||||
info.si_signo = SIGILL;
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
#include <asm/thread_info.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/pgtable.h>
|
||||
|
||||
#include "image.h"
|
||||
|
||||
|
@ -49,6 +50,14 @@ PECOFF_FILE_ALIGNMENT = 0x200;
|
|||
#define PECOFF_EDATA_PADDING
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_DEBUG_ALIGN_RODATA
|
||||
#define ALIGN_DEBUG_RO . = ALIGN(1<<SECTION_SHIFT);
|
||||
#define ALIGN_DEBUG_RO_MIN(min) ALIGN_DEBUG_RO
|
||||
#else
|
||||
#define ALIGN_DEBUG_RO
|
||||
#define ALIGN_DEBUG_RO_MIN(min) . = ALIGN(min);
|
||||
#endif
|
||||
|
||||
SECTIONS
|
||||
{
|
||||
/*
|
||||
|
@ -71,6 +80,7 @@ SECTIONS
|
|||
_text = .;
|
||||
HEAD_TEXT
|
||||
}
|
||||
ALIGN_DEBUG_RO
|
||||
.text : { /* Real text segment */
|
||||
_stext = .; /* Text and read-only data */
|
||||
__exception_text_start = .;
|
||||
|
@ -87,19 +97,22 @@ SECTIONS
|
|||
*(.got) /* Global offset table */
|
||||
}
|
||||
|
||||
ALIGN_DEBUG_RO
|
||||
RO_DATA(PAGE_SIZE)
|
||||
EXCEPTION_TABLE(8)
|
||||
NOTES
|
||||
ALIGN_DEBUG_RO
|
||||
_etext = .; /* End of text and rodata section */
|
||||
|
||||
. = ALIGN(PAGE_SIZE);
|
||||
ALIGN_DEBUG_RO_MIN(PAGE_SIZE)
|
||||
__init_begin = .;
|
||||
|
||||
INIT_TEXT_SECTION(8)
|
||||
.exit.text : {
|
||||
ARM_EXIT_KEEP(EXIT_TEXT)
|
||||
}
|
||||
. = ALIGN(16);
|
||||
|
||||
ALIGN_DEBUG_RO_MIN(16)
|
||||
.init.data : {
|
||||
INIT_DATA
|
||||
INIT_SETUP(16)
|
||||
|
|
|
@ -22,6 +22,7 @@
|
|||
*/
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
|
||||
/*
|
||||
|
@ -55,8 +56,8 @@ static int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
|
|||
{
|
||||
u32 esr = kvm_vcpu_get_hsr(vcpu);
|
||||
|
||||
if (esr & ESR_EL2_CV)
|
||||
return (esr & ESR_EL2_COND) >> ESR_EL2_COND_SHIFT;
|
||||
if (esr & ESR_ELx_CV)
|
||||
return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
|
|
@ -21,8 +21,10 @@
|
|||
|
||||
#include <linux/kvm.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kvm_coproc.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
#include <asm/kvm_psci.h>
|
||||
|
||||
|
@ -61,7 +63,7 @@ static int handle_smc(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|||
*/
|
||||
static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
||||
{
|
||||
if (kvm_vcpu_get_hsr(vcpu) & ESR_EL2_EC_WFI_ISS_WFE)
|
||||
if (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WFx_ISS_WFE)
|
||||
kvm_vcpu_on_spin(vcpu);
|
||||
else
|
||||
kvm_vcpu_block(vcpu);
|
||||
|
@ -72,29 +74,30 @@ static int kvm_handle_wfx(struct kvm_vcpu *vcpu, struct kvm_run *run)
|
|||
}
|
||||
|
||||
static exit_handle_fn arm_exit_handlers[] = {
|
||||
[ESR_EL2_EC_WFI] = kvm_handle_wfx,
|
||||
[ESR_EL2_EC_CP15_32] = kvm_handle_cp15_32,
|
||||
[ESR_EL2_EC_CP15_64] = kvm_handle_cp15_64,
|
||||
[ESR_EL2_EC_CP14_MR] = kvm_handle_cp14_32,
|
||||
[ESR_EL2_EC_CP14_LS] = kvm_handle_cp14_load_store,
|
||||
[ESR_EL2_EC_CP14_64] = kvm_handle_cp14_64,
|
||||
[ESR_EL2_EC_HVC32] = handle_hvc,
|
||||
[ESR_EL2_EC_SMC32] = handle_smc,
|
||||
[ESR_EL2_EC_HVC64] = handle_hvc,
|
||||
[ESR_EL2_EC_SMC64] = handle_smc,
|
||||
[ESR_EL2_EC_SYS64] = kvm_handle_sys_reg,
|
||||
[ESR_EL2_EC_IABT] = kvm_handle_guest_abort,
|
||||
[ESR_EL2_EC_DABT] = kvm_handle_guest_abort,
|
||||
[ESR_ELx_EC_WFx] = kvm_handle_wfx,
|
||||
[ESR_ELx_EC_CP15_32] = kvm_handle_cp15_32,
|
||||
[ESR_ELx_EC_CP15_64] = kvm_handle_cp15_64,
|
||||
[ESR_ELx_EC_CP14_MR] = kvm_handle_cp14_32,
|
||||
[ESR_ELx_EC_CP14_LS] = kvm_handle_cp14_load_store,
|
||||
[ESR_ELx_EC_CP14_64] = kvm_handle_cp14_64,
|
||||
[ESR_ELx_EC_HVC32] = handle_hvc,
|
||||
[ESR_ELx_EC_SMC32] = handle_smc,
|
||||
[ESR_ELx_EC_HVC64] = handle_hvc,
|
||||
[ESR_ELx_EC_SMC64] = handle_smc,
|
||||
[ESR_ELx_EC_SYS64] = kvm_handle_sys_reg,
|
||||
[ESR_ELx_EC_IABT_LOW] = kvm_handle_guest_abort,
|
||||
[ESR_ELx_EC_DABT_LOW] = kvm_handle_guest_abort,
|
||||
};
|
||||
|
||||
static exit_handle_fn kvm_get_exit_handler(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
u8 hsr_ec = kvm_vcpu_trap_get_class(vcpu);
|
||||
u32 hsr = kvm_vcpu_get_hsr(vcpu);
|
||||
u8 hsr_ec = hsr >> ESR_ELx_EC_SHIFT;
|
||||
|
||||
if (hsr_ec >= ARRAY_SIZE(arm_exit_handlers) ||
|
||||
!arm_exit_handlers[hsr_ec]) {
|
||||
kvm_err("Unknown exception class: hsr: %#08x\n",
|
||||
(unsigned int)kvm_vcpu_get_hsr(vcpu));
|
||||
kvm_err("Unknown exception class: hsr: %#08x -- %s\n",
|
||||
hsr, esr_get_class_string(hsr));
|
||||
BUG();
|
||||
}
|
||||
|
||||
|
|
|
@ -17,15 +17,16 @@
|
|||
|
||||
#include <linux/linkage.h>
|
||||
|
||||
#include <asm/assembler.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/assembler.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/fpsimdmacros.h>
|
||||
#include <asm/kvm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_asm.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
#include <asm/memory.h>
|
||||
|
||||
#define CPU_GP_REG_OFFSET(x) (CPU_GP_REGS + x)
|
||||
#define CPU_XREG_OFFSET(x) CPU_GP_REG_OFFSET(CPU_USER_PT_REGS + 8*x)
|
||||
|
@ -1141,9 +1142,9 @@ el1_sync: // Guest trapped into EL2
|
|||
push x2, x3
|
||||
|
||||
mrs x1, esr_el2
|
||||
lsr x2, x1, #ESR_EL2_EC_SHIFT
|
||||
lsr x2, x1, #ESR_ELx_EC_SHIFT
|
||||
|
||||
cmp x2, #ESR_EL2_EC_HVC64
|
||||
cmp x2, #ESR_ELx_EC_HVC64
|
||||
b.ne el1_trap
|
||||
|
||||
mrs x3, vttbr_el2 // If vttbr is valid, the 64bit guest
|
||||
|
@ -1178,13 +1179,13 @@ el1_trap:
|
|||
* x1: ESR
|
||||
* x2: ESR_EC
|
||||
*/
|
||||
cmp x2, #ESR_EL2_EC_DABT
|
||||
mov x0, #ESR_EL2_EC_IABT
|
||||
cmp x2, #ESR_ELx_EC_DABT_LOW
|
||||
mov x0, #ESR_ELx_EC_IABT_LOW
|
||||
ccmp x2, x0, #4, ne
|
||||
b.ne 1f // Not an abort we care about
|
||||
|
||||
/* This is an abort. Check for permission fault */
|
||||
and x2, x1, #ESR_EL2_FSC_TYPE
|
||||
and x2, x1, #ESR_ELx_FSC_TYPE
|
||||
cmp x2, #FSC_PERM
|
||||
b.ne 1f // Not a permission fault
|
||||
|
||||
|
|
|
@ -118,27 +118,27 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
|
|||
* instruction set. Report an external synchronous abort.
|
||||
*/
|
||||
if (kvm_vcpu_trap_il_is32bit(vcpu))
|
||||
esr |= ESR_EL1_IL;
|
||||
esr |= ESR_ELx_IL;
|
||||
|
||||
/*
|
||||
* Here, the guest runs in AArch64 mode when in EL1. If we get
|
||||
* an AArch32 fault, it means we managed to trap an EL0 fault.
|
||||
*/
|
||||
if (is_aarch32 || (cpsr & PSR_MODE_MASK) == PSR_MODE_EL0t)
|
||||
esr |= (ESR_EL1_EC_IABT_EL0 << ESR_EL1_EC_SHIFT);
|
||||
esr |= (ESR_ELx_EC_IABT_LOW << ESR_ELx_EC_SHIFT);
|
||||
else
|
||||
esr |= (ESR_EL1_EC_IABT_EL1 << ESR_EL1_EC_SHIFT);
|
||||
esr |= (ESR_ELx_EC_IABT_CUR << ESR_ELx_EC_SHIFT);
|
||||
|
||||
if (!is_iabt)
|
||||
esr |= ESR_EL1_EC_DABT_EL0;
|
||||
esr |= ESR_ELx_EC_DABT_LOW;
|
||||
|
||||
vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_EL2_EC_xABT_xFSR_EXTABT;
|
||||
vcpu_sys_reg(vcpu, ESR_EL1) = esr | ESR_ELx_FSC_EXTABT;
|
||||
}
|
||||
|
||||
static void inject_undef64(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
unsigned long cpsr = *vcpu_cpsr(vcpu);
|
||||
u32 esr = (ESR_EL1_EC_UNKNOWN << ESR_EL1_EC_SHIFT);
|
||||
u32 esr = (ESR_ELx_EC_UNKNOWN << ESR_ELx_EC_SHIFT);
|
||||
|
||||
*vcpu_spsr(vcpu) = cpsr;
|
||||
*vcpu_elr_el1(vcpu) = *vcpu_pc(vcpu);
|
||||
|
@ -151,7 +151,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
|
|||
* set.
|
||||
*/
|
||||
if (kvm_vcpu_trap_il_is32bit(vcpu))
|
||||
esr |= ESR_EL1_IL;
|
||||
esr |= ESR_ELx_IL;
|
||||
|
||||
vcpu_sys_reg(vcpu, ESR_EL1) = esr;
|
||||
}
|
||||
|
|
|
@ -20,17 +20,20 @@
|
|||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include <linux/mm.h>
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/uaccess.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_host.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
#include <asm/kvm_coproc.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
|
||||
#include <asm/cacheflush.h>
|
||||
#include <asm/cputype.h>
|
||||
#include <asm/debug-monitors.h>
|
||||
#include <asm/esr.h>
|
||||
#include <asm/kvm_arm.h>
|
||||
#include <asm/kvm_coproc.h>
|
||||
#include <asm/kvm_emulate.h>
|
||||
#include <asm/kvm_host.h>
|
||||
#include <asm/kvm_mmu.h>
|
||||
|
||||
#include <trace/events/kvm.h>
|
||||
|
||||
#include "sys_regs.h"
|
||||
|
@ -760,12 +763,12 @@ static void unhandled_cp_access(struct kvm_vcpu *vcpu,
|
|||
int cp;
|
||||
|
||||
switch(hsr_ec) {
|
||||
case ESR_EL2_EC_CP15_32:
|
||||
case ESR_EL2_EC_CP15_64:
|
||||
case ESR_ELx_EC_CP15_32:
|
||||
case ESR_ELx_EC_CP15_64:
|
||||
cp = 15;
|
||||
break;
|
||||
case ESR_EL2_EC_CP14_MR:
|
||||
case ESR_EL2_EC_CP14_64:
|
||||
case ESR_ELx_EC_CP14_MR:
|
||||
case ESR_ELx_EC_CP14_64:
|
||||
cp = 14;
|
||||
break;
|
||||
default:
|
||||
|
|
|
@ -134,16 +134,17 @@ static void __dma_free_coherent(struct device *dev, size_t size,
|
|||
swiotlb_free_coherent(dev, size, vaddr, dma_handle);
|
||||
}
|
||||
|
||||
static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
|
||||
dma_addr_t *dma_handle, gfp_t flags,
|
||||
struct dma_attrs *attrs)
|
||||
static void *__dma_alloc(struct device *dev, size_t size,
|
||||
dma_addr_t *dma_handle, gfp_t flags,
|
||||
struct dma_attrs *attrs)
|
||||
{
|
||||
struct page *page;
|
||||
void *ptr, *coherent_ptr;
|
||||
bool coherent = is_device_dma_coherent(dev);
|
||||
|
||||
size = PAGE_ALIGN(size);
|
||||
|
||||
if (!(flags & __GFP_WAIT)) {
|
||||
if (!coherent && !(flags & __GFP_WAIT)) {
|
||||
struct page *page = NULL;
|
||||
void *addr = __alloc_from_pool(size, &page);
|
||||
|
||||
|
@ -151,13 +152,16 @@ static void *__dma_alloc_noncoherent(struct device *dev, size_t size,
|
|||
*dma_handle = phys_to_dma(dev, page_to_phys(page));
|
||||
|
||||
return addr;
|
||||
|
||||
}
|
||||
|
||||
ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
|
||||
if (!ptr)
|
||||
goto no_mem;
|
||||
|
||||
/* no need for non-cacheable mapping if coherent */
|
||||
if (coherent)
|
||||
return ptr;
|
||||
|
||||
/* remove any dirty cache lines on the kernel alias */
|
||||
__dma_flush_range(ptr, ptr + size);
|
||||
|
||||
|
@ -179,15 +183,17 @@ no_mem:
|
|||
return NULL;
|
||||
}
|
||||
|
||||
static void __dma_free_noncoherent(struct device *dev, size_t size,
|
||||
void *vaddr, dma_addr_t dma_handle,
|
||||
struct dma_attrs *attrs)
|
||||
static void __dma_free(struct device *dev, size_t size,
|
||||
void *vaddr, dma_addr_t dma_handle,
|
||||
struct dma_attrs *attrs)
|
||||
{
|
||||
void *swiotlb_addr = phys_to_virt(dma_to_phys(dev, dma_handle));
|
||||
|
||||
if (__free_from_pool(vaddr, size))
|
||||
return;
|
||||
vunmap(vaddr);
|
||||
if (!is_device_dma_coherent(dev)) {
|
||||
if (__free_from_pool(vaddr, size))
|
||||
return;
|
||||
vunmap(vaddr);
|
||||
}
|
||||
__dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
|
||||
}
|
||||
|
||||
|
@ -199,7 +205,8 @@ static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
|
|||
dma_addr_t dev_addr;
|
||||
|
||||
dev_addr = swiotlb_map_page(dev, page, offset, size, dir, attrs);
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
|
||||
return dev_addr;
|
||||
}
|
||||
|
@ -209,7 +216,8 @@ static void __swiotlb_unmap_page(struct device *dev, dma_addr_t dev_addr,
|
|||
size_t size, enum dma_data_direction dir,
|
||||
struct dma_attrs *attrs)
|
||||
{
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
swiotlb_unmap_page(dev, dev_addr, size, dir, attrs);
|
||||
}
|
||||
|
||||
|
@ -221,9 +229,10 @@ static int __swiotlb_map_sg_attrs(struct device *dev, struct scatterlist *sgl,
|
|||
int i, ret;
|
||||
|
||||
ret = swiotlb_map_sg_attrs(dev, sgl, nelems, dir, attrs);
|
||||
for_each_sg(sgl, sg, ret, i)
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
for_each_sg(sgl, sg, ret, i)
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
@ -236,9 +245,10 @@ static void __swiotlb_unmap_sg_attrs(struct device *dev,
|
|||
struct scatterlist *sg;
|
||||
int i;
|
||||
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
swiotlb_unmap_sg_attrs(dev, sgl, nelems, dir, attrs);
|
||||
}
|
||||
|
||||
|
@ -246,7 +256,8 @@ static void __swiotlb_sync_single_for_cpu(struct device *dev,
|
|||
dma_addr_t dev_addr, size_t size,
|
||||
enum dma_data_direction dir)
|
||||
{
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
swiotlb_sync_single_for_cpu(dev, dev_addr, size, dir);
|
||||
}
|
||||
|
||||
|
@ -255,7 +266,8 @@ static void __swiotlb_sync_single_for_device(struct device *dev,
|
|||
enum dma_data_direction dir)
|
||||
{
|
||||
swiotlb_sync_single_for_device(dev, dev_addr, size, dir);
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, dev_addr)), size, dir);
|
||||
}
|
||||
|
||||
static void __swiotlb_sync_sg_for_cpu(struct device *dev,
|
||||
|
@ -265,9 +277,10 @@ static void __swiotlb_sync_sg_for_cpu(struct device *dev,
|
|||
struct scatterlist *sg;
|
||||
int i;
|
||||
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_unmap_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
swiotlb_sync_sg_for_cpu(dev, sgl, nelems, dir);
|
||||
}
|
||||
|
||||
|
@ -279,9 +292,10 @@ static void __swiotlb_sync_sg_for_device(struct device *dev,
|
|||
int i;
|
||||
|
||||
swiotlb_sync_sg_for_device(dev, sgl, nelems, dir);
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
if (!is_device_dma_coherent(dev))
|
||||
for_each_sg(sgl, sg, nelems, i)
|
||||
__dma_map_area(phys_to_virt(dma_to_phys(dev, sg->dma_address)),
|
||||
sg->length, dir);
|
||||
}
|
||||
|
||||
/* vma->vm_page_prot must be set appropriately before calling this function */
|
||||
|
@ -308,28 +322,20 @@ static int __dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int __swiotlb_mmap_noncoherent(struct device *dev,
|
||||
struct vm_area_struct *vma,
|
||||
void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
||||
struct dma_attrs *attrs)
|
||||
static int __swiotlb_mmap(struct device *dev,
|
||||
struct vm_area_struct *vma,
|
||||
void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
||||
struct dma_attrs *attrs)
|
||||
{
|
||||
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot, false);
|
||||
vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot,
|
||||
is_device_dma_coherent(dev));
|
||||
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
|
||||
}
|
||||
|
||||
static int __swiotlb_mmap_coherent(struct device *dev,
|
||||
struct vm_area_struct *vma,
|
||||
void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
||||
struct dma_attrs *attrs)
|
||||
{
|
||||
/* Just use whatever page_prot attributes were specified */
|
||||
return __dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
|
||||
}
|
||||
|
||||
struct dma_map_ops noncoherent_swiotlb_dma_ops = {
|
||||
.alloc = __dma_alloc_noncoherent,
|
||||
.free = __dma_free_noncoherent,
|
||||
.mmap = __swiotlb_mmap_noncoherent,
|
||||
static struct dma_map_ops swiotlb_dma_ops = {
|
||||
.alloc = __dma_alloc,
|
||||
.free = __dma_free,
|
||||
.mmap = __swiotlb_mmap,
|
||||
.map_page = __swiotlb_map_page,
|
||||
.unmap_page = __swiotlb_unmap_page,
|
||||
.map_sg = __swiotlb_map_sg_attrs,
|
||||
|
@ -341,24 +347,6 @@ struct dma_map_ops noncoherent_swiotlb_dma_ops = {
|
|||
.dma_supported = swiotlb_dma_supported,
|
||||
.mapping_error = swiotlb_dma_mapping_error,
|
||||
};
|
||||
EXPORT_SYMBOL(noncoherent_swiotlb_dma_ops);
|
||||
|
||||
struct dma_map_ops coherent_swiotlb_dma_ops = {
|
||||
.alloc = __dma_alloc_coherent,
|
||||
.free = __dma_free_coherent,
|
||||
.mmap = __swiotlb_mmap_coherent,
|
||||
.map_page = swiotlb_map_page,
|
||||
.unmap_page = swiotlb_unmap_page,
|
||||
.map_sg = swiotlb_map_sg_attrs,
|
||||
.unmap_sg = swiotlb_unmap_sg_attrs,
|
||||
.sync_single_for_cpu = swiotlb_sync_single_for_cpu,
|
||||
.sync_single_for_device = swiotlb_sync_single_for_device,
|
||||
.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
|
||||
.sync_sg_for_device = swiotlb_sync_sg_for_device,
|
||||
.dma_supported = swiotlb_dma_supported,
|
||||
.mapping_error = swiotlb_dma_mapping_error,
|
||||
};
|
||||
EXPORT_SYMBOL(coherent_swiotlb_dma_ops);
|
||||
|
||||
extern int swiotlb_late_init_with_default_size(size_t default_size);
|
||||
|
||||
|
@ -427,7 +415,7 @@ static int __init swiotlb_late_init(void)
|
|||
{
|
||||
size_t swiotlb_size = min(SZ_64M, MAX_ORDER_NR_PAGES << PAGE_SHIFT);
|
||||
|
||||
dma_ops = &noncoherent_swiotlb_dma_ops;
|
||||
dma_ops = &swiotlb_dma_ops;
|
||||
|
||||
return swiotlb_late_init_with_default_size(swiotlb_size);
|
||||
}
|
||||
|
|
|
@ -14,14 +14,18 @@
|
|||
* of the License.
|
||||
*/
|
||||
#include <linux/debugfs.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/io.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/seq_file.h>
|
||||
|
||||
#include <asm/fixmap.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/pgtable-hwdef.h>
|
||||
|
||||
#define LOWEST_ADDR (UL(0xffffffffffffffff) << VA_BITS)
|
||||
|
||||
|
@ -37,10 +41,10 @@ enum address_markers_idx {
|
|||
VMEMMAP_START_NR,
|
||||
VMEMMAP_END_NR,
|
||||
#endif
|
||||
PCI_START_NR,
|
||||
PCI_END_NR,
|
||||
FIXADDR_START_NR,
|
||||
FIXADDR_END_NR,
|
||||
PCI_START_NR,
|
||||
PCI_END_NR,
|
||||
MODULES_START_NR,
|
||||
MODUELS_END_NR,
|
||||
KERNEL_SPACE_NR,
|
||||
|
@ -53,10 +57,10 @@ static struct addr_marker address_markers[] = {
|
|||
{ 0, "vmemmap start" },
|
||||
{ 0, "vmemmap end" },
|
||||
#endif
|
||||
{ (unsigned long) PCI_IOBASE, "PCI I/O start" },
|
||||
{ (unsigned long) PCI_IOBASE + SZ_16M, "PCI I/O end" },
|
||||
{ FIXADDR_START, "Fixmap start" },
|
||||
{ FIXADDR_TOP, "Fixmap end" },
|
||||
{ PCI_IO_START, "PCI I/O start" },
|
||||
{ PCI_IO_END, "PCI I/O end" },
|
||||
{ MODULES_VADDR, "Modules start" },
|
||||
{ MODULES_END, "Modules end" },
|
||||
{ PAGE_OFFSET, "Kernel Mapping" },
|
||||
|
@ -246,10 +250,12 @@ static void walk_pmd(struct pg_state *st, pud_t *pud, unsigned long start)
|
|||
|
||||
for (i = 0; i < PTRS_PER_PMD; i++, pmd++) {
|
||||
addr = start + i * PMD_SIZE;
|
||||
if (pmd_none(*pmd) || pmd_sect(*pmd) || pmd_bad(*pmd))
|
||||
if (pmd_none(*pmd) || pmd_sect(*pmd)) {
|
||||
note_page(st, addr, 3, pmd_val(*pmd));
|
||||
else
|
||||
} else {
|
||||
BUG_ON(pmd_bad(*pmd));
|
||||
walk_pte(st, pmd, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -261,10 +267,12 @@ static void walk_pud(struct pg_state *st, pgd_t *pgd, unsigned long start)
|
|||
|
||||
for (i = 0; i < PTRS_PER_PUD; i++, pud++) {
|
||||
addr = start + i * PUD_SIZE;
|
||||
if (pud_none(*pud) || pud_sect(*pud) || pud_bad(*pud))
|
||||
if (pud_none(*pud) || pud_sect(*pud)) {
|
||||
note_page(st, addr, 2, pud_val(*pud));
|
||||
else
|
||||
} else {
|
||||
BUG_ON(pud_bad(*pud));
|
||||
walk_pmd(st, pud, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -276,10 +284,12 @@ static void walk_pgd(struct pg_state *st, struct mm_struct *mm, unsigned long st
|
|||
|
||||
for (i = 0; i < PTRS_PER_PGD; i++, pgd++) {
|
||||
addr = start + i * PGDIR_SIZE;
|
||||
if (pgd_none(*pgd) || pgd_bad(*pgd))
|
||||
if (pgd_none(*pgd)) {
|
||||
note_page(st, addr, 1, pgd_val(*pgd));
|
||||
else
|
||||
} else {
|
||||
BUG_ON(pgd_bad(*pgd));
|
||||
walk_pud(st, pgd, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -219,7 +219,7 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr,
|
|||
|
||||
if (esr & ESR_LNX_EXEC) {
|
||||
vm_flags = VM_EXEC;
|
||||
} else if ((esr & ESR_EL1_WRITE) && !(esr & ESR_EL1_CM)) {
|
||||
} else if ((esr & ESR_ELx_WNR) && !(esr & ESR_ELx_CM)) {
|
||||
vm_flags = VM_WRITE;
|
||||
mm_flags |= FAULT_FLAG_WRITE;
|
||||
}
|
||||
|
|
|
@ -35,6 +35,7 @@
|
|||
#include <linux/efi.h>
|
||||
|
||||
#include <asm/fixmap.h>
|
||||
#include <asm/memory.h>
|
||||
#include <asm/sections.h>
|
||||
#include <asm/setup.h>
|
||||
#include <asm/sizes.h>
|
||||
|
@ -136,10 +137,29 @@ static void arm64_memory_present(void)
|
|||
}
|
||||
#endif
|
||||
|
||||
static phys_addr_t memory_limit = (phys_addr_t)ULLONG_MAX;
|
||||
|
||||
/*
|
||||
* Limit the memory size that was specified via FDT.
|
||||
*/
|
||||
static int __init early_mem(char *p)
|
||||
{
|
||||
if (!p)
|
||||
return 1;
|
||||
|
||||
memory_limit = memparse(p, &p) & PAGE_MASK;
|
||||
pr_notice("Memory limited to %lldMB\n", memory_limit >> 20);
|
||||
|
||||
return 0;
|
||||
}
|
||||
early_param("mem", early_mem);
|
||||
|
||||
void __init arm64_memblock_init(void)
|
||||
{
|
||||
phys_addr_t dma_phys_limit = 0;
|
||||
|
||||
memblock_enforce_memory_limit(memory_limit);
|
||||
|
||||
/*
|
||||
* Register the kernel text, kernel data, initrd, and initial
|
||||
* pagetables with memblock.
|
||||
|
@ -277,8 +297,8 @@ void __init mem_init(void)
|
|||
" vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
|
||||
" 0x%16lx - 0x%16lx (%6ld MB actual)\n"
|
||||
#endif
|
||||
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
|
||||
" fixed : 0x%16lx - 0x%16lx (%6ld KB)\n"
|
||||
" PCI I/O : 0x%16lx - 0x%16lx (%6ld MB)\n"
|
||||
" modules : 0x%16lx - 0x%16lx (%6ld MB)\n"
|
||||
" memory : 0x%16lx - 0x%16lx (%6ld MB)\n"
|
||||
" .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
|
||||
|
@ -291,8 +311,8 @@ void __init mem_init(void)
|
|||
MLM((unsigned long)virt_to_page(PAGE_OFFSET),
|
||||
(unsigned long)virt_to_page(high_memory)),
|
||||
#endif
|
||||
MLM((unsigned long)PCI_IOBASE, (unsigned long)PCI_IOBASE + SZ_16M),
|
||||
MLK(FIXADDR_START, FIXADDR_TOP),
|
||||
MLM(PCI_IO_START, PCI_IO_END),
|
||||
MLM(MODULES_VADDR, MODULES_END),
|
||||
MLM(PAGE_OFFSET, (unsigned long)high_memory),
|
||||
MLK_ROUNDUP(__init_begin, __init_end),
|
||||
|
@ -325,6 +345,7 @@ void __init mem_init(void)
|
|||
|
||||
void free_initmem(void)
|
||||
{
|
||||
fixup_init();
|
||||
free_initmem_default(0);
|
||||
free_alternatives_memory();
|
||||
}
|
||||
|
|
|
@ -62,6 +62,7 @@ static void __iomem *__ioremap_caller(phys_addr_t phys_addr, size_t size,
|
|||
if (!area)
|
||||
return NULL;
|
||||
addr = (unsigned long)area->addr;
|
||||
area->phys_addr = phys_addr;
|
||||
|
||||
err = ioremap_page_range(addr, addr + size, phys_addr, prot);
|
||||
if (err) {
|
||||
|
|
|
@ -1 +1,3 @@
|
|||
extern void __init bootmem_init(void);
|
||||
|
||||
void fixup_init(void);
|
||||
|
|
|
@ -26,6 +26,8 @@
|
|||
#include <linux/memblock.h>
|
||||
#include <linux/fs.h>
|
||||
#include <linux/io.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/stop_machine.h>
|
||||
|
||||
#include <asm/cputype.h>
|
||||
#include <asm/fixmap.h>
|
||||
|
@ -45,80 +47,6 @@
|
|||
struct page *empty_zero_page;
|
||||
EXPORT_SYMBOL(empty_zero_page);
|
||||
|
||||
struct cachepolicy {
|
||||
const char policy[16];
|
||||
u64 mair;
|
||||
u64 tcr;
|
||||
};
|
||||
|
||||
static struct cachepolicy cache_policies[] __initdata = {
|
||||
{
|
||||
.policy = "uncached",
|
||||
.mair = 0x44, /* inner, outer non-cacheable */
|
||||
.tcr = TCR_IRGN_NC | TCR_ORGN_NC,
|
||||
}, {
|
||||
.policy = "writethrough",
|
||||
.mair = 0xaa, /* inner, outer write-through, read-allocate */
|
||||
.tcr = TCR_IRGN_WT | TCR_ORGN_WT,
|
||||
}, {
|
||||
.policy = "writeback",
|
||||
.mair = 0xee, /* inner, outer write-back, read-allocate */
|
||||
.tcr = TCR_IRGN_WBnWA | TCR_ORGN_WBnWA,
|
||||
}
|
||||
};
|
||||
|
||||
/*
|
||||
* These are useful for identifying cache coherency problems by allowing the
|
||||
* cache or the cache and writebuffer to be turned off. It changes the Normal
|
||||
* memory caching attributes in the MAIR_EL1 register.
|
||||
*/
|
||||
static int __init early_cachepolicy(char *p)
|
||||
{
|
||||
int i;
|
||||
u64 tmp;
|
||||
|
||||
for (i = 0; i < ARRAY_SIZE(cache_policies); i++) {
|
||||
int len = strlen(cache_policies[i].policy);
|
||||
|
||||
if (memcmp(p, cache_policies[i].policy, len) == 0)
|
||||
break;
|
||||
}
|
||||
if (i == ARRAY_SIZE(cache_policies)) {
|
||||
pr_err("ERROR: unknown or unsupported cache policy: %s\n", p);
|
||||
return 0;
|
||||
}
|
||||
|
||||
flush_cache_all();
|
||||
|
||||
/*
|
||||
* Modify MT_NORMAL attributes in MAIR_EL1.
|
||||
*/
|
||||
asm volatile(
|
||||
" mrs %0, mair_el1\n"
|
||||
" bfi %0, %1, %2, #8\n"
|
||||
" msr mair_el1, %0\n"
|
||||
" isb\n"
|
||||
: "=&r" (tmp)
|
||||
: "r" (cache_policies[i].mair), "i" (MT_NORMAL * 8));
|
||||
|
||||
/*
|
||||
* Modify TCR PTW cacheability attributes.
|
||||
*/
|
||||
asm volatile(
|
||||
" mrs %0, tcr_el1\n"
|
||||
" bic %0, %0, %2\n"
|
||||
" orr %0, %0, %1\n"
|
||||
" msr tcr_el1, %0\n"
|
||||
" isb\n"
|
||||
: "=&r" (tmp)
|
||||
: "r" (cache_policies[i].tcr), "r" (TCR_IRGN_MASK | TCR_ORGN_MASK));
|
||||
|
||||
flush_cache_all();
|
||||
|
||||
return 0;
|
||||
}
|
||||
early_param("cachepolicy", early_cachepolicy);
|
||||
|
||||
pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
|
||||
unsigned long size, pgprot_t vma_prot)
|
||||
{
|
||||
|
@ -133,19 +61,42 @@ EXPORT_SYMBOL(phys_mem_access_prot);
|
|||
static void __init *early_alloc(unsigned long sz)
|
||||
{
|
||||
void *ptr = __va(memblock_alloc(sz, sz));
|
||||
BUG_ON(!ptr);
|
||||
memset(ptr, 0, sz);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
|
||||
/*
|
||||
* remap a PMD into pages
|
||||
*/
|
||||
static void split_pmd(pmd_t *pmd, pte_t *pte)
|
||||
{
|
||||
unsigned long pfn = pmd_pfn(*pmd);
|
||||
int i = 0;
|
||||
|
||||
do {
|
||||
/*
|
||||
* Need to have the least restrictive permissions available
|
||||
* permissions will be fixed up later
|
||||
*/
|
||||
set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC));
|
||||
pfn++;
|
||||
} while (pte++, i++, i < PTRS_PER_PTE);
|
||||
}
|
||||
|
||||
static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
|
||||
unsigned long end, unsigned long pfn,
|
||||
pgprot_t prot)
|
||||
pgprot_t prot,
|
||||
void *(*alloc)(unsigned long size))
|
||||
{
|
||||
pte_t *pte;
|
||||
|
||||
if (pmd_none(*pmd)) {
|
||||
pte = early_alloc(PTRS_PER_PTE * sizeof(pte_t));
|
||||
if (pmd_none(*pmd) || pmd_sect(*pmd)) {
|
||||
pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
|
||||
if (pmd_sect(*pmd))
|
||||
split_pmd(pmd, pte);
|
||||
__pmd_populate(pmd, __pa(pte), PMD_TYPE_TABLE);
|
||||
flush_tlb_all();
|
||||
}
|
||||
BUG_ON(pmd_bad(*pmd));
|
||||
|
||||
|
@ -156,30 +107,42 @@ static void __init alloc_init_pte(pmd_t *pmd, unsigned long addr,
|
|||
} while (pte++, addr += PAGE_SIZE, addr != end);
|
||||
}
|
||||
|
||||
static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
|
||||
unsigned long end, phys_addr_t phys,
|
||||
int map_io)
|
||||
void split_pud(pud_t *old_pud, pmd_t *pmd)
|
||||
{
|
||||
unsigned long addr = pud_pfn(*old_pud) << PAGE_SHIFT;
|
||||
pgprot_t prot = __pgprot(pud_val(*old_pud) ^ addr);
|
||||
int i = 0;
|
||||
|
||||
do {
|
||||
set_pmd(pmd, __pmd(addr | prot));
|
||||
addr += PMD_SIZE;
|
||||
} while (pmd++, i++, i < PTRS_PER_PMD);
|
||||
}
|
||||
|
||||
static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
|
||||
unsigned long addr, unsigned long end,
|
||||
phys_addr_t phys, pgprot_t prot,
|
||||
void *(*alloc)(unsigned long size))
|
||||
{
|
||||
pmd_t *pmd;
|
||||
unsigned long next;
|
||||
pmdval_t prot_sect;
|
||||
pgprot_t prot_pte;
|
||||
|
||||
if (map_io) {
|
||||
prot_sect = PROT_SECT_DEVICE_nGnRE;
|
||||
prot_pte = __pgprot(PROT_DEVICE_nGnRE);
|
||||
} else {
|
||||
prot_sect = PROT_SECT_NORMAL_EXEC;
|
||||
prot_pte = PAGE_KERNEL_EXEC;
|
||||
}
|
||||
|
||||
/*
|
||||
* Check for initial section mappings in the pgd/pud and remove them.
|
||||
*/
|
||||
if (pud_none(*pud) || pud_bad(*pud)) {
|
||||
pmd = early_alloc(PTRS_PER_PMD * sizeof(pmd_t));
|
||||
pud_populate(&init_mm, pud, pmd);
|
||||
if (pud_none(*pud) || pud_sect(*pud)) {
|
||||
pmd = alloc(PTRS_PER_PMD * sizeof(pmd_t));
|
||||
if (pud_sect(*pud)) {
|
||||
/*
|
||||
* need to have the 1G of mappings continue to be
|
||||
* present
|
||||
*/
|
||||
split_pud(pud, pmd);
|
||||
}
|
||||
pud_populate(mm, pud, pmd);
|
||||
flush_tlb_all();
|
||||
}
|
||||
BUG_ON(pud_bad(*pud));
|
||||
|
||||
pmd = pmd_offset(pud, addr);
|
||||
do {
|
||||
|
@ -187,31 +150,51 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr,
|
|||
/* try section mapping first */
|
||||
if (((addr | next | phys) & ~SECTION_MASK) == 0) {
|
||||
pmd_t old_pmd =*pmd;
|
||||
set_pmd(pmd, __pmd(phys | prot_sect));
|
||||
set_pmd(pmd, __pmd(phys |
|
||||
pgprot_val(mk_sect_prot(prot))));
|
||||
/*
|
||||
* Check for previous table entries created during
|
||||
* boot (__create_page_tables) and flush them.
|
||||
*/
|
||||
if (!pmd_none(old_pmd))
|
||||
if (!pmd_none(old_pmd)) {
|
||||
flush_tlb_all();
|
||||
if (pmd_table(old_pmd)) {
|
||||
phys_addr_t table = __pa(pte_offset_map(&old_pmd, 0));
|
||||
if (!WARN_ON_ONCE(slab_is_available()))
|
||||
memblock_free(table, PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys),
|
||||
prot_pte);
|
||||
prot, alloc);
|
||||
}
|
||||
phys += next - addr;
|
||||
} while (pmd++, addr = next, addr != end);
|
||||
}
|
||||
|
||||
static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
|
||||
unsigned long end, phys_addr_t phys,
|
||||
int map_io)
|
||||
static inline bool use_1G_block(unsigned long addr, unsigned long next,
|
||||
unsigned long phys)
|
||||
{
|
||||
if (PAGE_SHIFT != 12)
|
||||
return false;
|
||||
|
||||
if (((addr | next | phys) & ~PUD_MASK) != 0)
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void alloc_init_pud(struct mm_struct *mm, pgd_t *pgd,
|
||||
unsigned long addr, unsigned long end,
|
||||
phys_addr_t phys, pgprot_t prot,
|
||||
void *(*alloc)(unsigned long size))
|
||||
{
|
||||
pud_t *pud;
|
||||
unsigned long next;
|
||||
|
||||
if (pgd_none(*pgd)) {
|
||||
pud = early_alloc(PTRS_PER_PUD * sizeof(pud_t));
|
||||
pgd_populate(&init_mm, pgd, pud);
|
||||
pud = alloc(PTRS_PER_PUD * sizeof(pud_t));
|
||||
pgd_populate(mm, pgd, pud);
|
||||
}
|
||||
BUG_ON(pgd_bad(*pgd));
|
||||
|
||||
|
@ -222,10 +205,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
|
|||
/*
|
||||
* For 4K granule only, attempt to put down a 1GB block
|
||||
*/
|
||||
if (!map_io && (PAGE_SHIFT == 12) &&
|
||||
((addr | next | phys) & ~PUD_MASK) == 0) {
|
||||
if (use_1G_block(addr, next, phys)) {
|
||||
pud_t old_pud = *pud;
|
||||
set_pud(pud, __pud(phys | PROT_SECT_NORMAL_EXEC));
|
||||
set_pud(pud, __pud(phys |
|
||||
pgprot_val(mk_sect_prot(prot))));
|
||||
|
||||
/*
|
||||
* If we have an old value for a pud, it will
|
||||
|
@ -235,12 +218,15 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
|
|||
* Look up the old pmd table and free it.
|
||||
*/
|
||||
if (!pud_none(old_pud)) {
|
||||
phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
|
||||
memblock_free(table, PAGE_SIZE);
|
||||
flush_tlb_all();
|
||||
if (pud_table(old_pud)) {
|
||||
phys_addr_t table = __pa(pmd_offset(&old_pud, 0));
|
||||
if (!WARN_ON_ONCE(slab_is_available()))
|
||||
memblock_free(table, PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
alloc_init_pmd(pud, addr, next, phys, map_io);
|
||||
alloc_init_pmd(mm, pud, addr, next, phys, prot, alloc);
|
||||
}
|
||||
phys += next - addr;
|
||||
} while (pud++, addr = next, addr != end);
|
||||
|
@ -250,9 +236,10 @@ static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr,
|
|||
* Create the page directory entries and any necessary page tables for the
|
||||
* mapping specified by 'md'.
|
||||
*/
|
||||
static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
|
||||
unsigned long virt, phys_addr_t size,
|
||||
int map_io)
|
||||
static void __create_mapping(struct mm_struct *mm, pgd_t *pgd,
|
||||
phys_addr_t phys, unsigned long virt,
|
||||
phys_addr_t size, pgprot_t prot,
|
||||
void *(*alloc)(unsigned long size))
|
||||
{
|
||||
unsigned long addr, length, end, next;
|
||||
|
||||
|
@ -262,32 +249,96 @@ static void __init __create_mapping(pgd_t *pgd, phys_addr_t phys,
|
|||
end = addr + length;
|
||||
do {
|
||||
next = pgd_addr_end(addr, end);
|
||||
alloc_init_pud(pgd, addr, next, phys, map_io);
|
||||
alloc_init_pud(mm, pgd, addr, next, phys, prot, alloc);
|
||||
phys += next - addr;
|
||||
} while (pgd++, addr = next, addr != end);
|
||||
}
|
||||
|
||||
static void __init create_mapping(phys_addr_t phys, unsigned long virt,
|
||||
phys_addr_t size)
|
||||
static void *late_alloc(unsigned long size)
|
||||
{
|
||||
void *ptr;
|
||||
|
||||
BUG_ON(size > PAGE_SIZE);
|
||||
ptr = (void *)__get_free_page(PGALLOC_GFP);
|
||||
BUG_ON(!ptr);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
static void __ref create_mapping(phys_addr_t phys, unsigned long virt,
|
||||
phys_addr_t size, pgprot_t prot)
|
||||
{
|
||||
if (virt < VMALLOC_START) {
|
||||
pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
|
||||
&phys, virt);
|
||||
return;
|
||||
}
|
||||
__create_mapping(pgd_offset_k(virt & PAGE_MASK), phys, virt, size, 0);
|
||||
__create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK), phys, virt,
|
||||
size, prot, early_alloc);
|
||||
}
|
||||
|
||||
void __init create_id_mapping(phys_addr_t addr, phys_addr_t size, int map_io)
|
||||
void __init create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
|
||||
unsigned long virt, phys_addr_t size,
|
||||
pgprot_t prot)
|
||||
{
|
||||
if ((addr >> PGDIR_SHIFT) >= ARRAY_SIZE(idmap_pg_dir)) {
|
||||
pr_warn("BUG: not creating id mapping for %pa\n", &addr);
|
||||
__create_mapping(mm, pgd_offset(mm, virt), phys, virt, size, prot,
|
||||
late_alloc);
|
||||
}
|
||||
|
||||
static void create_mapping_late(phys_addr_t phys, unsigned long virt,
|
||||
phys_addr_t size, pgprot_t prot)
|
||||
{
|
||||
if (virt < VMALLOC_START) {
|
||||
pr_warn("BUG: not creating mapping for %pa at 0x%016lx - outside kernel range\n",
|
||||
&phys, virt);
|
||||
return;
|
||||
}
|
||||
__create_mapping(&idmap_pg_dir[pgd_index(addr)],
|
||||
addr, addr, size, map_io);
|
||||
|
||||
return __create_mapping(&init_mm, pgd_offset_k(virt & PAGE_MASK),
|
||||
phys, virt, size, prot, late_alloc);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEBUG_RODATA
|
||||
static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
|
||||
{
|
||||
/*
|
||||
* Set up the executable regions using the existing section mappings
|
||||
* for now. This will get more fine grained later once all memory
|
||||
* is mapped
|
||||
*/
|
||||
unsigned long kernel_x_start = round_down(__pa(_stext), SECTION_SIZE);
|
||||
unsigned long kernel_x_end = round_up(__pa(__init_end), SECTION_SIZE);
|
||||
|
||||
if (end < kernel_x_start) {
|
||||
create_mapping(start, __phys_to_virt(start),
|
||||
end - start, PAGE_KERNEL);
|
||||
} else if (start >= kernel_x_end) {
|
||||
create_mapping(start, __phys_to_virt(start),
|
||||
end - start, PAGE_KERNEL);
|
||||
} else {
|
||||
if (start < kernel_x_start)
|
||||
create_mapping(start, __phys_to_virt(start),
|
||||
kernel_x_start - start,
|
||||
PAGE_KERNEL);
|
||||
create_mapping(kernel_x_start,
|
||||
__phys_to_virt(kernel_x_start),
|
||||
kernel_x_end - kernel_x_start,
|
||||
PAGE_KERNEL_EXEC);
|
||||
if (kernel_x_end < end)
|
||||
create_mapping(kernel_x_end,
|
||||
__phys_to_virt(kernel_x_end),
|
||||
end - kernel_x_end,
|
||||
PAGE_KERNEL);
|
||||
}
|
||||
|
||||
}
|
||||
#else
|
||||
static void __init __map_memblock(phys_addr_t start, phys_addr_t end)
|
||||
{
|
||||
create_mapping(start, __phys_to_virt(start), end - start,
|
||||
PAGE_KERNEL_EXEC);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void __init map_mem(void)
|
||||
{
|
||||
struct memblock_region *reg;
|
||||
|
@ -332,14 +383,53 @@ static void __init map_mem(void)
|
|||
memblock_set_current_limit(limit);
|
||||
}
|
||||
#endif
|
||||
|
||||
create_mapping(start, __phys_to_virt(start), end - start);
|
||||
__map_memblock(start, end);
|
||||
}
|
||||
|
||||
/* Limit no longer required. */
|
||||
memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
|
||||
}
|
||||
|
||||
void __init fixup_executable(void)
|
||||
{
|
||||
#ifdef CONFIG_DEBUG_RODATA
|
||||
/* now that we are actually fully mapped, make the start/end more fine grained */
|
||||
if (!IS_ALIGNED((unsigned long)_stext, SECTION_SIZE)) {
|
||||
unsigned long aligned_start = round_down(__pa(_stext),
|
||||
SECTION_SIZE);
|
||||
|
||||
create_mapping(aligned_start, __phys_to_virt(aligned_start),
|
||||
__pa(_stext) - aligned_start,
|
||||
PAGE_KERNEL);
|
||||
}
|
||||
|
||||
if (!IS_ALIGNED((unsigned long)__init_end, SECTION_SIZE)) {
|
||||
unsigned long aligned_end = round_up(__pa(__init_end),
|
||||
SECTION_SIZE);
|
||||
create_mapping(__pa(__init_end), (unsigned long)__init_end,
|
||||
aligned_end - __pa(__init_end),
|
||||
PAGE_KERNEL);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef CONFIG_DEBUG_RODATA
|
||||
void mark_rodata_ro(void)
|
||||
{
|
||||
create_mapping_late(__pa(_stext), (unsigned long)_stext,
|
||||
(unsigned long)_etext - (unsigned long)_stext,
|
||||
PAGE_KERNEL_EXEC | PTE_RDONLY);
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
void fixup_init(void)
|
||||
{
|
||||
create_mapping_late(__pa(__init_begin), (unsigned long)__init_begin,
|
||||
(unsigned long)__init_end - (unsigned long)__init_begin,
|
||||
PAGE_KERNEL);
|
||||
}
|
||||
|
||||
/*
|
||||
* paging_init() sets up the page tables, initialises the zone memory
|
||||
* maps and sets up the zero page.
|
||||
|
@ -349,13 +439,7 @@ void __init paging_init(void)
|
|||
void *zero_page;
|
||||
|
||||
map_mem();
|
||||
|
||||
/*
|
||||
* Finally flush the caches and tlb to ensure that we're in a
|
||||
* consistent state.
|
||||
*/
|
||||
flush_cache_all();
|
||||
flush_tlb_all();
|
||||
fixup_executable();
|
||||
|
||||
/* allocate the zero page. */
|
||||
zero_page = early_alloc(PAGE_SIZE);
|
||||
|
|
|
@ -102,7 +102,7 @@ ENTRY(cpu_do_idle)
|
|||
ret
|
||||
ENDPROC(cpu_do_idle)
|
||||
|
||||
#ifdef CONFIG_ARM64_CPU_SUSPEND
|
||||
#ifdef CONFIG_CPU_PM
|
||||
/**
|
||||
* cpu_do_suspend - save CPU registers context
|
||||
*
|
||||
|
@ -244,14 +244,18 @@ ENTRY(__cpu_setup)
|
|||
ENDPROC(__cpu_setup)
|
||||
|
||||
/*
|
||||
* We set the desired value explicitly, including those of the
|
||||
* reserved bits. The values of bits EE & E0E were set early in
|
||||
* el2_setup, which are left untouched below.
|
||||
*
|
||||
* n n T
|
||||
* U E WT T UD US IHBS
|
||||
* CE0 XWHW CZ ME TEEA S
|
||||
* .... .IEE .... NEAI TE.I ..AD DEN0 ACAM
|
||||
* 0011 0... 1101 ..0. ..0. 10.. .... .... < hardware reserved
|
||||
* .... .1.. .... 01.1 11.1 ..01 0001 1101 < software settings
|
||||
* 0011 0... 1101 ..0. ..0. 10.. .0.. .... < hardware reserved
|
||||
* .... .1.. .... 01.1 11.1 ..01 0.01 1101 < software settings
|
||||
*/
|
||||
.type crval, #object
|
||||
crval:
|
||||
.word 0x000802e2 // clear
|
||||
.word 0x0405d11d // set
|
||||
.word 0xfcffffff // clear
|
||||
.word 0x34d5d91d // set
|
||||
|
|
|
@ -4,7 +4,6 @@
|
|||
|
||||
config ARM64_CPUIDLE
|
||||
bool "Generic ARM64 CPU idle Driver"
|
||||
select ARM64_CPU_SUSPEND
|
||||
select DT_IDLE_STATES
|
||||
help
|
||||
Select this to enable generic cpuidle driver for ARM64.
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
#include <linux/of.h>
|
||||
|
||||
#include <asm/cpuidle.h>
|
||||
#include <asm/suspend.h>
|
||||
|
||||
#include "dt_idle_states.h"
|
||||
|
||||
|
|
|
@ -297,10 +297,49 @@ static __init int match_config_table(efi_guid_t *guid,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int __init efi_config_parse_tables(void *config_tables, int count, int sz,
|
||||
efi_config_table_type_t *arch_tables)
|
||||
{
|
||||
void *tablep;
|
||||
int i;
|
||||
|
||||
tablep = config_tables;
|
||||
pr_info("");
|
||||
for (i = 0; i < count; i++) {
|
||||
efi_guid_t guid;
|
||||
unsigned long table;
|
||||
|
||||
if (efi_enabled(EFI_64BIT)) {
|
||||
u64 table64;
|
||||
guid = ((efi_config_table_64_t *)tablep)->guid;
|
||||
table64 = ((efi_config_table_64_t *)tablep)->table;
|
||||
table = table64;
|
||||
#ifndef CONFIG_64BIT
|
||||
if (table64 >> 32) {
|
||||
pr_cont("\n");
|
||||
pr_err("Table located above 4GB, disabling EFI.\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
guid = ((efi_config_table_32_t *)tablep)->guid;
|
||||
table = ((efi_config_table_32_t *)tablep)->table;
|
||||
}
|
||||
|
||||
if (!match_config_table(&guid, table, common_tables))
|
||||
match_config_table(&guid, table, arch_tables);
|
||||
|
||||
tablep += sz;
|
||||
}
|
||||
pr_cont("\n");
|
||||
set_bit(EFI_CONFIG_TABLES, &efi.flags);
|
||||
return 0;
|
||||
}
|
||||
|
||||
int __init efi_config_init(efi_config_table_type_t *arch_tables)
|
||||
{
|
||||
void *config_tables, *tablep;
|
||||
int i, sz;
|
||||
void *config_tables;
|
||||
int sz, ret;
|
||||
|
||||
if (efi_enabled(EFI_64BIT))
|
||||
sz = sizeof(efi_config_table_64_t);
|
||||
|
@ -317,42 +356,11 @@ int __init efi_config_init(efi_config_table_type_t *arch_tables)
|
|||
return -ENOMEM;
|
||||
}
|
||||
|
||||
tablep = config_tables;
|
||||
pr_info("");
|
||||
for (i = 0; i < efi.systab->nr_tables; i++) {
|
||||
efi_guid_t guid;
|
||||
unsigned long table;
|
||||
ret = efi_config_parse_tables(config_tables, efi.systab->nr_tables, sz,
|
||||
arch_tables);
|
||||
|
||||
if (efi_enabled(EFI_64BIT)) {
|
||||
u64 table64;
|
||||
guid = ((efi_config_table_64_t *)tablep)->guid;
|
||||
table64 = ((efi_config_table_64_t *)tablep)->table;
|
||||
table = table64;
|
||||
#ifndef CONFIG_64BIT
|
||||
if (table64 >> 32) {
|
||||
pr_cont("\n");
|
||||
pr_err("Table located above 4GB, disabling EFI.\n");
|
||||
early_memunmap(config_tables,
|
||||
efi.systab->nr_tables * sz);
|
||||
return -EINVAL;
|
||||
}
|
||||
#endif
|
||||
} else {
|
||||
guid = ((efi_config_table_32_t *)tablep)->guid;
|
||||
table = ((efi_config_table_32_t *)tablep)->table;
|
||||
}
|
||||
|
||||
if (!match_config_table(&guid, table, common_tables))
|
||||
match_config_table(&guid, table, arch_tables);
|
||||
|
||||
tablep += sz;
|
||||
}
|
||||
pr_cont("\n");
|
||||
early_memunmap(config_tables, efi.systab->nr_tables * sz);
|
||||
|
||||
set_bit(EFI_CONFIG_TABLES, &efi.flags);
|
||||
|
||||
return 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_EFI_VARS_MODULE
|
||||
|
|
|
@ -295,3 +295,62 @@ fail_free_image:
|
|||
fail:
|
||||
return EFI_ERROR;
|
||||
}
|
||||
|
||||
/*
|
||||
* This is the base address at which to start allocating virtual memory ranges
|
||||
* for UEFI Runtime Services. This is in the low TTBR0 range so that we can use
|
||||
* any allocation we choose, and eliminate the risk of a conflict after kexec.
|
||||
* The value chosen is the largest non-zero power of 2 suitable for this purpose
|
||||
* both on 32-bit and 64-bit ARM CPUs, to maximize the likelihood that it can
|
||||
* be mapped efficiently.
|
||||
*/
|
||||
#define EFI_RT_VIRTUAL_BASE 0x40000000
|
||||
|
||||
/*
|
||||
* efi_get_virtmap() - create a virtual mapping for the EFI memory map
|
||||
*
|
||||
* This function populates the virt_addr fields of all memory region descriptors
|
||||
* in @memory_map whose EFI_MEMORY_RUNTIME attribute is set. Those descriptors
|
||||
* are also copied to @runtime_map, and their total count is returned in @count.
|
||||
*/
|
||||
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
|
||||
unsigned long desc_size, efi_memory_desc_t *runtime_map,
|
||||
int *count)
|
||||
{
|
||||
u64 efi_virt_base = EFI_RT_VIRTUAL_BASE;
|
||||
efi_memory_desc_t *out = runtime_map;
|
||||
int l;
|
||||
|
||||
for (l = 0; l < map_size; l += desc_size) {
|
||||
efi_memory_desc_t *in = (void *)memory_map + l;
|
||||
u64 paddr, size;
|
||||
|
||||
if (!(in->attribute & EFI_MEMORY_RUNTIME))
|
||||
continue;
|
||||
|
||||
/*
|
||||
* Make the mapping compatible with 64k pages: this allows
|
||||
* a 4k page size kernel to kexec a 64k page size kernel and
|
||||
* vice versa.
|
||||
*/
|
||||
paddr = round_down(in->phys_addr, SZ_64K);
|
||||
size = round_up(in->num_pages * EFI_PAGE_SIZE +
|
||||
in->phys_addr - paddr, SZ_64K);
|
||||
|
||||
/*
|
||||
* Avoid wasting memory on PTEs by choosing a virtual base that
|
||||
* is compatible with section mappings if this region has the
|
||||
* appropriate size and physical alignment. (Sections are 2 MB
|
||||
* on 4k granule kernels)
|
||||
*/
|
||||
if (IS_ALIGNED(in->phys_addr, SZ_2M) && size >= SZ_2M)
|
||||
efi_virt_base = round_up(efi_virt_base, SZ_2M);
|
||||
|
||||
in->virt_addr = efi_virt_base + in->phys_addr - paddr;
|
||||
efi_virt_base += size;
|
||||
|
||||
memcpy(out, in, desc_size);
|
||||
out = (void *)out + desc_size;
|
||||
++*count;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -32,6 +32,15 @@
|
|||
|
||||
static unsigned long __chunk_size = EFI_READ_CHUNK_SIZE;
|
||||
|
||||
/*
|
||||
* Allow the platform to override the allocation granularity: this allows
|
||||
* systems that have the capability to run with a larger page size to deal
|
||||
* with the allocations for initrd and fdt more efficiently.
|
||||
*/
|
||||
#ifndef EFI_ALLOC_ALIGN
|
||||
#define EFI_ALLOC_ALIGN EFI_PAGE_SIZE
|
||||
#endif
|
||||
|
||||
struct file_info {
|
||||
efi_file_handle_t *handle;
|
||||
u64 size;
|
||||
|
@ -154,10 +163,10 @@ efi_status_t efi_high_alloc(efi_system_table_t *sys_table_arg,
|
|||
* a specific address. We are doing page-based allocations,
|
||||
* so we must be aligned to a page.
|
||||
*/
|
||||
if (align < EFI_PAGE_SIZE)
|
||||
align = EFI_PAGE_SIZE;
|
||||
if (align < EFI_ALLOC_ALIGN)
|
||||
align = EFI_ALLOC_ALIGN;
|
||||
|
||||
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
|
||||
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
|
||||
again:
|
||||
for (i = 0; i < map_size / desc_size; i++) {
|
||||
efi_memory_desc_t *desc;
|
||||
|
@ -239,10 +248,10 @@ efi_status_t efi_low_alloc(efi_system_table_t *sys_table_arg,
|
|||
* a specific address. We are doing page-based allocations,
|
||||
* so we must be aligned to a page.
|
||||
*/
|
||||
if (align < EFI_PAGE_SIZE)
|
||||
align = EFI_PAGE_SIZE;
|
||||
if (align < EFI_ALLOC_ALIGN)
|
||||
align = EFI_ALLOC_ALIGN;
|
||||
|
||||
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
|
||||
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
|
||||
for (i = 0; i < map_size / desc_size; i++) {
|
||||
efi_memory_desc_t *desc;
|
||||
unsigned long m = (unsigned long)map;
|
||||
|
@ -296,7 +305,7 @@ void efi_free(efi_system_table_t *sys_table_arg, unsigned long size,
|
|||
if (!size)
|
||||
return;
|
||||
|
||||
nr_pages = round_up(size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
|
||||
nr_pages = round_up(size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
|
||||
efi_call_early(free_pages, addr, nr_pages);
|
||||
}
|
||||
|
||||
|
@ -565,7 +574,7 @@ efi_status_t efi_relocate_kernel(efi_system_table_t *sys_table_arg,
|
|||
* to the preferred address. If that fails, allocate as low
|
||||
* as possible while respecting the required alignment.
|
||||
*/
|
||||
nr_pages = round_up(alloc_size, EFI_PAGE_SIZE) / EFI_PAGE_SIZE;
|
||||
nr_pages = round_up(alloc_size, EFI_ALLOC_ALIGN) / EFI_PAGE_SIZE;
|
||||
status = efi_call_early(allocate_pages,
|
||||
EFI_ALLOCATE_ADDRESS, EFI_LOADER_DATA,
|
||||
nr_pages, &efi_addr);
|
||||
|
|
|
@ -39,4 +39,8 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
|
|||
|
||||
void *get_fdt(efi_system_table_t *sys_table);
|
||||
|
||||
void efi_get_virtmap(efi_memory_desc_t *memory_map, unsigned long map_size,
|
||||
unsigned long desc_size, efi_memory_desc_t *runtime_map,
|
||||
int *count);
|
||||
|
||||
#endif
|
||||
|
|
|
@ -14,6 +14,8 @@
|
|||
#include <linux/libfdt.h>
|
||||
#include <asm/efi.h>
|
||||
|
||||
#include "efistub.h"
|
||||
|
||||
efi_status_t update_fdt(efi_system_table_t *sys_table, void *orig_fdt,
|
||||
unsigned long orig_fdt_size,
|
||||
void *fdt, int new_fdt_size, char *cmdline_ptr,
|
||||
|
@ -193,9 +195,26 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
|
|||
unsigned long map_size, desc_size;
|
||||
u32 desc_ver;
|
||||
unsigned long mmap_key;
|
||||
efi_memory_desc_t *memory_map;
|
||||
efi_memory_desc_t *memory_map, *runtime_map;
|
||||
unsigned long new_fdt_size;
|
||||
efi_status_t status;
|
||||
int runtime_entry_count = 0;
|
||||
|
||||
/*
|
||||
* Get a copy of the current memory map that we will use to prepare
|
||||
* the input for SetVirtualAddressMap(). We don't have to worry about
|
||||
* subsequent allocations adding entries, since they could not affect
|
||||
* the number of EFI_MEMORY_RUNTIME regions.
|
||||
*/
|
||||
status = efi_get_memory_map(sys_table, &runtime_map, &map_size,
|
||||
&desc_size, &desc_ver, &mmap_key);
|
||||
if (status != EFI_SUCCESS) {
|
||||
pr_efi_err(sys_table, "Unable to retrieve UEFI memory map.\n");
|
||||
return status;
|
||||
}
|
||||
|
||||
pr_efi(sys_table,
|
||||
"Exiting boot services and installing virtual address map...\n");
|
||||
|
||||
/*
|
||||
* Estimate size of new FDT, and allocate memory for it. We
|
||||
|
@ -248,12 +267,48 @@ efi_status_t allocate_new_fdt_and_exit_boot(efi_system_table_t *sys_table,
|
|||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Update the memory map with virtual addresses. The function will also
|
||||
* populate @runtime_map with copies of just the EFI_MEMORY_RUNTIME
|
||||
* entries so that we can pass it straight into SetVirtualAddressMap()
|
||||
*/
|
||||
efi_get_virtmap(memory_map, map_size, desc_size, runtime_map,
|
||||
&runtime_entry_count);
|
||||
|
||||
/* Now we are ready to exit_boot_services.*/
|
||||
status = sys_table->boottime->exit_boot_services(handle, mmap_key);
|
||||
|
||||
if (status == EFI_SUCCESS) {
|
||||
efi_set_virtual_address_map_t *svam;
|
||||
|
||||
if (status == EFI_SUCCESS)
|
||||
return status;
|
||||
/* Install the new virtual address map */
|
||||
svam = sys_table->runtime->set_virtual_address_map;
|
||||
status = svam(runtime_entry_count * desc_size, desc_size,
|
||||
desc_ver, runtime_map);
|
||||
|
||||
/*
|
||||
* We are beyond the point of no return here, so if the call to
|
||||
* SetVirtualAddressMap() failed, we need to signal that to the
|
||||
* incoming kernel but proceed normally otherwise.
|
||||
*/
|
||||
if (status != EFI_SUCCESS) {
|
||||
int l;
|
||||
|
||||
/*
|
||||
* Set the virtual address field of all
|
||||
* EFI_MEMORY_RUNTIME entries to 0. This will signal
|
||||
* the incoming kernel that no virtual translation has
|
||||
* been installed.
|
||||
*/
|
||||
for (l = 0; l < map_size; l += desc_size) {
|
||||
efi_memory_desc_t *p = (void *)memory_map + l;
|
||||
|
||||
if (p->attribute & EFI_MEMORY_RUNTIME)
|
||||
p->virt_addr = 0;
|
||||
}
|
||||
}
|
||||
return EFI_SUCCESS;
|
||||
}
|
||||
|
||||
pr_efi_err(sys_table, "Exit boot services failed.\n");
|
||||
|
||||
|
@ -264,6 +319,7 @@ fail_free_new_fdt:
|
|||
efi_free(sys_table, new_fdt_size, *new_fdt_addr);
|
||||
|
||||
fail:
|
||||
sys_table->boottime->free_pool(runtime_map);
|
||||
return EFI_LOAD_ERROR;
|
||||
}
|
||||
|
||||
|
|
|
@ -689,6 +689,15 @@ asmlinkage long compat_sys_sendfile64(int out_fd, int in_fd,
|
|||
asmlinkage long compat_sys_sigaltstack(const compat_stack_t __user *uss_ptr,
|
||||
compat_stack_t __user *uoss_ptr);
|
||||
|
||||
#ifdef __ARCH_WANT_SYS_SIGPENDING
|
||||
asmlinkage long compat_sys_sigpending(compat_old_sigset_t __user *set);
|
||||
#endif
|
||||
|
||||
#ifdef __ARCH_WANT_SYS_SIGPROCMASK
|
||||
asmlinkage long compat_sys_sigprocmask(int how, compat_old_sigset_t __user *nset,
|
||||
compat_old_sigset_t __user *oset);
|
||||
#endif
|
||||
|
||||
int compat_restore_altstack(const compat_stack_t __user *uss);
|
||||
int __compat_save_altstack(compat_stack_t __user *, unsigned long);
|
||||
#define compat_save_altstack_ex(uss, sp) do { \
|
||||
|
|
|
@ -875,6 +875,8 @@ static inline efi_status_t efi_query_variable_store(u32 attributes, unsigned lon
|
|||
#endif
|
||||
extern void __iomem *efi_lookup_mapped_addr(u64 phys_addr);
|
||||
extern int efi_config_init(efi_config_table_type_t *arch_tables);
|
||||
extern int efi_config_parse_tables(void *config_tables, int count, int sz,
|
||||
efi_config_table_type_t *arch_tables);
|
||||
extern u64 efi_get_iobase (void);
|
||||
extern u32 efi_mem_type (unsigned long phys_addr);
|
||||
extern u64 efi_mem_attributes (unsigned long phys_addr);
|
||||
|
|
|
@ -410,12 +410,16 @@ asmlinkage long sys_newlstat(const char __user *filename,
|
|||
struct stat __user *statbuf);
|
||||
asmlinkage long sys_newfstat(unsigned int fd, struct stat __user *statbuf);
|
||||
asmlinkage long sys_ustat(unsigned dev, struct ustat __user *ubuf);
|
||||
#if BITS_PER_LONG == 32
|
||||
#if defined(__ARCH_WANT_STAT64) || defined(__ARCH_WANT_COMPAT_STAT64)
|
||||
asmlinkage long sys_stat64(const char __user *filename,
|
||||
struct stat64 __user *statbuf);
|
||||
asmlinkage long sys_fstat64(unsigned long fd, struct stat64 __user *statbuf);
|
||||
asmlinkage long sys_lstat64(const char __user *filename,
|
||||
struct stat64 __user *statbuf);
|
||||
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
|
||||
struct stat64 __user *statbuf, int flag);
|
||||
#endif
|
||||
#if BITS_PER_LONG == 32
|
||||
asmlinkage long sys_truncate64(const char __user *path, loff_t length);
|
||||
asmlinkage long sys_ftruncate64(unsigned int fd, loff_t length);
|
||||
#endif
|
||||
|
@ -771,8 +775,6 @@ asmlinkage long sys_openat(int dfd, const char __user *filename, int flags,
|
|||
umode_t mode);
|
||||
asmlinkage long sys_newfstatat(int dfd, const char __user *filename,
|
||||
struct stat __user *statbuf, int flag);
|
||||
asmlinkage long sys_fstatat64(int dfd, const char __user *filename,
|
||||
struct stat64 __user *statbuf, int flag);
|
||||
asmlinkage long sys_readlinkat(int dfd, const char __user *path, char __user *buf,
|
||||
int bufsiz);
|
||||
asmlinkage long sys_utimensat(int dfd, const char __user *filename,
|
||||
|
|
Загрузка…
Ссылка в новой задаче