Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 updates from Will Deacon: - virt_to_page/page_address optimisations - support for NUMA systems described using device-tree - support for hibernate/suspend-to-disk - proper support for maxcpus= command line parameter - detection and graceful handling of AArch64-only CPUs - miscellaneous cleanups and non-critical fixes * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (92 commits) arm64: do not enforce strict 16 byte alignment to stack pointer arm64: kernel: Fix incorrect brk randomization arm64: cpuinfo: Missing NULL terminator in compat_hwcap_str arm64: secondary_start_kernel: Remove unnecessary barrier arm64: Ensure pmd_present() returns false after pmd_mknotpresent() arm64: Replace hard-coded values in the pmd/pud_bad() macros arm64: Implement pmdp_set_access_flags() for hardware AF/DBM arm64: Fix typo in the pmdp_huge_get_and_clear() definition arm64: mm: remove unnecessary EXPORT_SYMBOL_GPL arm64: always use STRICT_MM_TYPECHECKS arm64: kvm: Fix kvm teardown for systems using the extended idmap arm64: kaslr: increase randomization granularity arm64: kconfig: drop CONFIG_RTC_LIB dependency arm64: make ARCH_SUPPORTS_DEBUG_PAGEALLOC depend on !HIBERNATION arm64: hibernate: Refuse to hibernate if the boot cpu is offline arm64: kernel: Add support for hibernate/suspend-to-disk PM / Hibernate: Call flush_icache_range() on pages restored in-place arm64: Add new asm macro copy_page arm64: Promote KERNEL_START/KERNEL_END definitions to a header file arm64: kernel: Include _AC definition in page.h ...
This commit is contained in:
Linus Torvalds
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Based on arch/arm/include/asm/assembler.h
|
||||
* Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S
|
||||
*
|
||||
* Copyright (C) 1996-2000 Russell King
|
||||
* Copyright (C) 2012 ARM Ltd.
|
||||
@@ -23,21 +23,12 @@
|
||||
#ifndef __ASM_ASSEMBLER_H
|
||||
#define __ASM_ASSEMBLER_H
|
||||
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/pgtable-hwdef.h>
|
||||
#include <asm/ptrace.h>
|
||||
#include <asm/thread_info.h>
|
||||
|
||||
/*
|
||||
* Stack pushing/popping (register pairs only). Equivalent to store decrement
|
||||
* before, load increment after.
|
||||
*/
|
||||
.macro push, xreg1, xreg2
|
||||
stp \xreg1, \xreg2, [sp, #-16]!
|
||||
.endm
|
||||
|
||||
.macro pop, xreg1, xreg2
|
||||
ldp \xreg1, \xreg2, [sp], #16
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Enable and disable interrupts.
|
||||
*/
|
||||
@@ -211,6 +202,102 @@ lr .req x30 // link register
|
||||
add \reg, \reg, \tmp
|
||||
.endm
|
||||
|
||||
/*
|
||||
* vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
|
||||
*/
|
||||
.macro vma_vm_mm, rd, rn
|
||||
ldr \rd, [\rn, #VMA_VM_MM]
|
||||
.endm
|
||||
|
||||
/*
|
||||
* mmid - get context id from mm pointer (mm->context.id)
|
||||
*/
|
||||
.macro mmid, rd, rn
|
||||
ldr \rd, [\rn, #MM_CONTEXT_ID]
|
||||
.endm
|
||||
|
||||
/*
|
||||
* dcache_line_size - get the minimum D-cache line size from the CTR register.
|
||||
*/
|
||||
.macro dcache_line_size, reg, tmp
|
||||
mrs \tmp, ctr_el0 // read CTR
|
||||
ubfm \tmp, \tmp, #16, #19 // cache line size encoding
|
||||
mov \reg, #4 // bytes per word
|
||||
lsl \reg, \reg, \tmp // actual cache line size
|
||||
.endm
|
||||
|
||||
/*
|
||||
* icache_line_size - get the minimum I-cache line size from the CTR register.
|
||||
*/
|
||||
.macro icache_line_size, reg, tmp
|
||||
mrs \tmp, ctr_el0 // read CTR
|
||||
and \tmp, \tmp, #0xf // cache line size encoding
|
||||
mov \reg, #4 // bytes per word
|
||||
lsl \reg, \reg, \tmp // actual cache line size
|
||||
.endm
|
||||
|
||||
/*
|
||||
* tcr_set_idmap_t0sz - update TCR.T0SZ so that we can load the ID map
|
||||
*/
|
||||
.macro tcr_set_idmap_t0sz, valreg, tmpreg
|
||||
#ifndef CONFIG_ARM64_VA_BITS_48
|
||||
ldr_l \tmpreg, idmap_t0sz
|
||||
bfi \valreg, \tmpreg, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
|
||||
#endif
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Macro to perform a data cache maintenance for the interval
|
||||
* [kaddr, kaddr + size)
|
||||
*
|
||||
* op: operation passed to dc instruction
|
||||
* domain: domain used in dsb instruciton
|
||||
* kaddr: starting virtual address of the region
|
||||
* size: size of the region
|
||||
* Corrupts: kaddr, size, tmp1, tmp2
|
||||
*/
|
||||
.macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
|
||||
dcache_line_size \tmp1, \tmp2
|
||||
add \size, \kaddr, \size
|
||||
sub \tmp2, \tmp1, #1
|
||||
bic \kaddr, \kaddr, \tmp2
|
||||
9998: dc \op, \kaddr
|
||||
add \kaddr, \kaddr, \tmp1
|
||||
cmp \kaddr, \size
|
||||
b.lo 9998b
|
||||
dsb \domain
|
||||
.endm
|
||||
|
||||
/*
|
||||
* reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
|
||||
*/
|
||||
.macro reset_pmuserenr_el0, tmpreg
|
||||
mrs \tmpreg, id_aa64dfr0_el1 // Check ID_AA64DFR0_EL1 PMUVer
|
||||
sbfx \tmpreg, \tmpreg, #8, #4
|
||||
cmp \tmpreg, #1 // Skip if no PMU present
|
||||
b.lt 9000f
|
||||
msr pmuserenr_el0, xzr // Disable PMU access from EL0
|
||||
9000:
|
||||
.endm
|
||||
|
||||
/*
|
||||
* copy_page - copy src to dest using temp registers t1-t8
|
||||
*/
|
||||
.macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
|
||||
9998: ldp \t1, \t2, [\src]
|
||||
ldp \t3, \t4, [\src, #16]
|
||||
ldp \t5, \t6, [\src, #32]
|
||||
ldp \t7, \t8, [\src, #48]
|
||||
add \src, \src, #64
|
||||
stnp \t1, \t2, [\dest]
|
||||
stnp \t3, \t4, [\dest, #16]
|
||||
stnp \t5, \t6, [\dest, #32]
|
||||
stnp \t7, \t8, [\dest, #48]
|
||||
add \dest, \dest, #64
|
||||
tst \src, #(PAGE_SIZE - 1)
|
||||
b.ne 9998b
|
||||
.endm
|
||||
|
||||
/*
|
||||
* Annotate a function as position independent, i.e., safe to be called before
|
||||
* the kernel virtual mapping is activated.
|
||||
@@ -233,4 +320,24 @@ lr .req x30 // link register
|
||||
.long \sym\()_hi32
|
||||
.endm
|
||||
|
||||
/*
|
||||
* mov_q - move an immediate constant into a 64-bit register using
|
||||
* between 2 and 4 movz/movk instructions (depending on the
|
||||
* magnitude and sign of the operand)
|
||||
*/
|
||||
.macro mov_q, reg, val
|
||||
.if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
|
||||
movz \reg, :abs_g1_s:\val
|
||||
.else
|
||||
.if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
|
||||
movz \reg, :abs_g2_s:\val
|
||||
.else
|
||||
movz \reg, :abs_g3:\val
|
||||
movk \reg, :abs_g2_nc:\val
|
||||
.endif
|
||||
movk \reg, :abs_g1_nc:\val
|
||||
.endif
|
||||
movk \reg, :abs_g0_nc:\val
|
||||
.endm
|
||||
|
||||
#endif /* __ASM_ASSEMBLER_H */
|
||||
|
||||
@@ -35,8 +35,9 @@
|
||||
#define ARM64_ALT_PAN_NOT_UAO 10
|
||||
#define ARM64_HAS_VIRT_HOST_EXTN 11
|
||||
#define ARM64_WORKAROUND_CAVIUM_27456 12
|
||||
#define ARM64_HAS_32BIT_EL0 13
|
||||
|
||||
#define ARM64_NCAPS 13
|
||||
#define ARM64_NCAPS 14
|
||||
|
||||
#ifndef __ASSEMBLY__
|
||||
|
||||
@@ -77,10 +78,17 @@ struct arm64_ftr_reg {
|
||||
struct arm64_ftr_bits *ftr_bits;
|
||||
};
|
||||
|
||||
/* scope of capability check */
|
||||
enum {
|
||||
SCOPE_SYSTEM,
|
||||
SCOPE_LOCAL_CPU,
|
||||
};
|
||||
|
||||
struct arm64_cpu_capabilities {
|
||||
const char *desc;
|
||||
u16 capability;
|
||||
bool (*matches)(const struct arm64_cpu_capabilities *);
|
||||
int def_scope; /* default scope */
|
||||
bool (*matches)(const struct arm64_cpu_capabilities *caps, int scope);
|
||||
void (*enable)(void *); /* Called on all active CPUs */
|
||||
union {
|
||||
struct { /* To be used for erratum handling only */
|
||||
@@ -101,6 +109,8 @@ struct arm64_cpu_capabilities {
|
||||
|
||||
extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
|
||||
|
||||
bool this_cpu_has_cap(unsigned int cap);
|
||||
|
||||
static inline bool cpu_have_feature(unsigned int num)
|
||||
{
|
||||
return elf_hwcap & (1UL << num);
|
||||
@@ -170,12 +180,20 @@ static inline bool id_aa64mmfr0_mixed_endian_el0(u64 mmfr0)
|
||||
cpuid_feature_extract_unsigned_field(mmfr0, ID_AA64MMFR0_BIGENDEL0_SHIFT) == 0x1;
|
||||
}
|
||||
|
||||
static inline bool id_aa64pfr0_32bit_el0(u64 pfr0)
|
||||
{
|
||||
u32 val = cpuid_feature_extract_unsigned_field(pfr0, ID_AA64PFR0_EL0_SHIFT);
|
||||
|
||||
return val == ID_AA64PFR0_EL0_32BIT_64BIT;
|
||||
}
|
||||
|
||||
void __init setup_cpu_features(void);
|
||||
|
||||
void update_cpu_capabilities(const struct arm64_cpu_capabilities *caps,
|
||||
const char *info);
|
||||
void check_local_cpu_errata(void);
|
||||
|
||||
void verify_local_cpu_errata(void);
|
||||
void verify_local_cpu_capabilities(void);
|
||||
|
||||
u64 read_system_reg(u32 id);
|
||||
@@ -185,6 +203,11 @@ static inline bool cpu_supports_mixed_endian_el0(void)
|
||||
return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1));
|
||||
}
|
||||
|
||||
static inline bool system_supports_32bit_el0(void)
|
||||
{
|
||||
return cpus_have_cap(ARM64_HAS_32BIT_EL0);
|
||||
}
|
||||
|
||||
static inline bool system_supports_mixed_endian_el0(void)
|
||||
{
|
||||
return id_aa64mmfr0_mixed_endian_el0(read_system_reg(SYS_ID_AA64MMFR0_EL1));
|
||||
|
||||
@@ -177,7 +177,8 @@ typedef compat_elf_greg_t compat_elf_gregset_t[COMPAT_ELF_NGREG];
|
||||
|
||||
/* AArch32 EABI. */
|
||||
#define EF_ARM_EABI_MASK 0xff000000
|
||||
#define compat_elf_check_arch(x) (((x)->e_machine == EM_ARM) && \
|
||||
#define compat_elf_check_arch(x) (system_supports_32bit_el0() && \
|
||||
((x)->e_machine == EM_ARM) && \
|
||||
((x)->e_flags & EF_ARM_EABI_MASK))
|
||||
|
||||
#define compat_start_thread compat_start_thread
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#ifndef __ASM_KERNEL_PGTABLE_H
|
||||
#define __ASM_KERNEL_PGTABLE_H
|
||||
|
||||
#include <asm/sparsemem.h>
|
||||
|
||||
/*
|
||||
* The linear mapping and the start of memory are both 2M aligned (per
|
||||
@@ -86,10 +87,24 @@
|
||||
* (64k granule), or a multiple that can be mapped using contiguous bits
|
||||
* in the page tables: 32 * PMD_SIZE (16k granule)
|
||||
*/
|
||||
#ifdef CONFIG_ARM64_64K_PAGES
|
||||
#define ARM64_MEMSTART_ALIGN SZ_512M
|
||||
#if defined(CONFIG_ARM64_4K_PAGES)
|
||||
#define ARM64_MEMSTART_SHIFT PUD_SHIFT
|
||||
#elif defined(CONFIG_ARM64_16K_PAGES)
|
||||
#define ARM64_MEMSTART_SHIFT (PMD_SHIFT + 5)
|
||||
#else
|
||||
#define ARM64_MEMSTART_ALIGN SZ_1G
|
||||
#define ARM64_MEMSTART_SHIFT PMD_SHIFT
|
||||
#endif
|
||||
|
||||
/*
|
||||
* sparsemem vmemmap imposes an additional requirement on the alignment of
|
||||
* memstart_addr, due to the fact that the base of the vmemmap region
|
||||
* has a direct correspondence, and needs to appear sufficiently aligned
|
||||
* in the virtual address space.
|
||||
*/
|
||||
#if defined(CONFIG_SPARSEMEM_VMEMMAP) && ARM64_MEMSTART_SHIFT < SECTION_SIZE_BITS
|
||||
#define ARM64_MEMSTART_ALIGN (1UL << SECTION_SIZE_BITS)
|
||||
#else
|
||||
#define ARM64_MEMSTART_ALIGN (1UL << ARM64_MEMSTART_SHIFT)
|
||||
#endif
|
||||
|
||||
#endif /* __ASM_KERNEL_PGTABLE_H */
|
||||
|
||||
@@ -84,17 +84,6 @@
|
||||
#define HCR_INT_OVERRIDE (HCR_FMO | HCR_IMO)
|
||||
#define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H)
|
||||
|
||||
/* Hyp System Control Register (SCTLR_EL2) bits */
|
||||
#define SCTLR_EL2_EE (1 << 25)
|
||||
#define SCTLR_EL2_WXN (1 << 19)
|
||||
#define SCTLR_EL2_I (1 << 12)
|
||||
#define SCTLR_EL2_SA (1 << 3)
|
||||
#define SCTLR_EL2_C (1 << 2)
|
||||
#define SCTLR_EL2_A (1 << 1)
|
||||
#define SCTLR_EL2_M 1
|
||||
#define SCTLR_EL2_FLAGS (SCTLR_EL2_M | SCTLR_EL2_A | SCTLR_EL2_C | \
|
||||
SCTLR_EL2_SA | SCTLR_EL2_I)
|
||||
|
||||
/* TCR_EL2 Registers bits */
|
||||
#define TCR_EL2_RES1 ((1 << 31) | (1 << 23))
|
||||
#define TCR_EL2_TBI (1 << 20)
|
||||
|
||||
@@ -22,6 +22,8 @@
|
||||
|
||||
#define ARM_EXCEPTION_IRQ 0
|
||||
#define ARM_EXCEPTION_TRAP 1
|
||||
/* The hyp-stub will return this for any kvm_call_hyp() call */
|
||||
#define ARM_EXCEPTION_HYP_GONE 2
|
||||
|
||||
#define KVM_ARM64_DEBUG_DIRTY_SHIFT 0
|
||||
#define KVM_ARM64_DEBUG_DIRTY (1 << KVM_ARM64_DEBUG_DIRTY_SHIFT)
|
||||
@@ -40,6 +42,7 @@ struct kvm_vcpu;
|
||||
|
||||
extern char __kvm_hyp_init[];
|
||||
extern char __kvm_hyp_init_end[];
|
||||
extern char __kvm_hyp_reset[];
|
||||
|
||||
extern char __kvm_hyp_vector[];
|
||||
|
||||
|
||||
@@ -46,6 +46,8 @@
|
||||
int __attribute_const__ kvm_target_cpu(void);
|
||||
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
|
||||
int kvm_arch_dev_ioctl_check_extension(long ext);
|
||||
unsigned long kvm_hyp_reset_entry(void);
|
||||
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
|
||||
|
||||
struct kvm_arch {
|
||||
/* The VMID generation used for the virt. memory system */
|
||||
@@ -352,7 +354,17 @@ static inline void __cpu_init_hyp_mode(phys_addr_t boot_pgd_ptr,
|
||||
hyp_stack_ptr, vector_ptr);
|
||||
}
|
||||
|
||||
static inline void kvm_arch_hardware_disable(void) {}
|
||||
static inline void __cpu_reset_hyp_mode(phys_addr_t boot_pgd_ptr,
|
||||
phys_addr_t phys_idmap_start)
|
||||
{
|
||||
/*
|
||||
* Call reset code, and switch back to stub hyp vectors.
|
||||
* Uses __kvm_call_hyp() to avoid kaslr's kvm_ksym_ref() translation.
|
||||
*/
|
||||
__kvm_call_hyp((void *)kvm_hyp_reset_entry(),
|
||||
boot_pgd_ptr, phys_idmap_start);
|
||||
}
|
||||
|
||||
static inline void kvm_arch_hardware_unsetup(void) {}
|
||||
static inline void kvm_arch_sync_events(struct kvm *kvm) {}
|
||||
static inline void kvm_arch_vcpu_uninit(struct kvm_vcpu *vcpu) {}
|
||||
|
||||
@@ -109,6 +109,7 @@ void kvm_mmu_free_memory_caches(struct kvm_vcpu *vcpu);
|
||||
phys_addr_t kvm_mmu_get_httbr(void);
|
||||
phys_addr_t kvm_mmu_get_boot_httbr(void);
|
||||
phys_addr_t kvm_get_idmap_vector(void);
|
||||
phys_addr_t kvm_get_idmap_start(void);
|
||||
int kvm_mmu_init(void);
|
||||
void kvm_clear_hyp_idmap(void);
|
||||
|
||||
|
||||
@@ -39,6 +39,21 @@
|
||||
*/
|
||||
#define PCI_IO_SIZE SZ_16M
|
||||
|
||||
/*
|
||||
* Log2 of the upper bound of the size of a struct page. Used for sizing
|
||||
* the vmemmap region only, does not affect actual memory footprint.
|
||||
* We don't use sizeof(struct page) directly since taking its size here
|
||||
* requires its definition to be available at this point in the inclusion
|
||||
* chain, and it may not be a power of 2 in the first place.
|
||||
*/
|
||||
#define STRUCT_PAGE_MAX_SHIFT 6
|
||||
|
||||
/*
|
||||
* VMEMMAP_SIZE - allows the whole linear region to be covered by
|
||||
* a struct page array
|
||||
*/
|
||||
#define VMEMMAP_SIZE (UL(1) << (VA_BITS - PAGE_SHIFT - 1 + STRUCT_PAGE_MAX_SHIFT))
|
||||
|
||||
/*
|
||||
* PAGE_OFFSET - the virtual address of the start of the kernel image (top
|
||||
* (VA_BITS - 1))
|
||||
@@ -54,7 +69,8 @@
|
||||
#define MODULES_END (MODULES_VADDR + MODULES_VSIZE)
|
||||
#define MODULES_VADDR (VA_START + KASAN_SHADOW_SIZE)
|
||||
#define MODULES_VSIZE (SZ_128M)
|
||||
#define PCI_IO_END (PAGE_OFFSET - SZ_2M)
|
||||
#define VMEMMAP_START (PAGE_OFFSET - VMEMMAP_SIZE)
|
||||
#define PCI_IO_END (VMEMMAP_START - SZ_2M)
|
||||
#define PCI_IO_START (PCI_IO_END - PCI_IO_SIZE)
|
||||
#define FIXADDR_TOP (PCI_IO_START - SZ_2M)
|
||||
#define TASK_SIZE_64 (UL(1) << VA_BITS)
|
||||
@@ -71,6 +87,9 @@
|
||||
|
||||
#define TASK_UNMAPPED_BASE (PAGE_ALIGN(TASK_SIZE / 4))
|
||||
|
||||
#define KERNEL_START _text
|
||||
#define KERNEL_END _end
|
||||
|
||||
/*
|
||||
* The size of the KASAN shadow region. This should be 1/8th of the
|
||||
* size of the entire kernel virtual address space.
|
||||
@@ -192,9 +211,19 @@ static inline void *phys_to_virt(phys_addr_t x)
|
||||
*/
|
||||
#define ARCH_PFN_OFFSET ((unsigned long)PHYS_PFN_OFFSET)
|
||||
|
||||
#ifndef CONFIG_SPARSEMEM_VMEMMAP
|
||||
#define virt_to_page(kaddr) pfn_to_page(__pa(kaddr) >> PAGE_SHIFT)
|
||||
#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
|
||||
#define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT)
|
||||
#else
|
||||
#define __virt_to_pgoff(kaddr) (((u64)(kaddr) & ~PAGE_OFFSET) / PAGE_SIZE * sizeof(struct page))
|
||||
#define __page_to_voff(kaddr) (((u64)(page) & ~VMEMMAP_START) * PAGE_SIZE / sizeof(struct page))
|
||||
|
||||
#define page_to_virt(page) ((void *)((__page_to_voff(page)) | PAGE_OFFSET))
|
||||
#define virt_to_page(vaddr) ((struct page *)((__virt_to_pgoff(vaddr)) | VMEMMAP_START))
|
||||
|
||||
#define virt_addr_valid(kaddr) pfn_valid((((u64)(kaddr) & ~PAGE_OFFSET) \
|
||||
+ PHYS_OFFSET) >> PAGE_SHIFT)
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#include <asm-generic/memory_model.h>
|
||||
|
||||
@@ -29,6 +29,7 @@ typedef struct {
|
||||
#define ASID(mm) ((mm)->context.id.counter & 0xffff)
|
||||
|
||||
extern void paging_init(void);
|
||||
extern void bootmem_init(void);
|
||||
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
|
||||
extern void init_mem_pgprot(void);
|
||||
extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
|
||||
|
||||
12
arch/arm64/include/asm/mmzone.h
Normal file
12
arch/arm64/include/asm/mmzone.h
Normal file
@@ -0,0 +1,12 @@
|
||||
#ifndef __ASM_MMZONE_H
|
||||
#define __ASM_MMZONE_H
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
|
||||
#include <asm/numa.h>
|
||||
|
||||
extern struct pglist_data *node_data[];
|
||||
#define NODE_DATA(nid) (node_data[(nid)])
|
||||
|
||||
#endif /* CONFIG_NUMA */
|
||||
#endif /* __ASM_MMZONE_H */
|
||||
45
arch/arm64/include/asm/numa.h
Normal file
45
arch/arm64/include/asm/numa.h
Normal file
@@ -0,0 +1,45 @@
|
||||
#ifndef __ASM_NUMA_H
|
||||
#define __ASM_NUMA_H
|
||||
|
||||
#include <asm/topology.h>
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
|
||||
/* currently, arm64 implements flat NUMA topology */
|
||||
#define parent_node(node) (node)
|
||||
|
||||
int __node_distance(int from, int to);
|
||||
#define node_distance(a, b) __node_distance(a, b)
|
||||
|
||||
extern nodemask_t numa_nodes_parsed __initdata;
|
||||
|
||||
/* Mappings between node number and cpus on that node. */
|
||||
extern cpumask_var_t node_to_cpumask_map[MAX_NUMNODES];
|
||||
void numa_clear_node(unsigned int cpu);
|
||||
|
||||
#ifdef CONFIG_DEBUG_PER_CPU_MAPS
|
||||
const struct cpumask *cpumask_of_node(int node);
|
||||
#else
|
||||
/* Returns a pointer to the cpumask of CPUs on Node 'node'. */
|
||||
static inline const struct cpumask *cpumask_of_node(int node)
|
||||
{
|
||||
return node_to_cpumask_map[node];
|
||||
}
|
||||
#endif
|
||||
|
||||
void __init arm64_numa_init(void);
|
||||
int __init numa_add_memblk(int nodeid, u64 start, u64 end);
|
||||
void __init numa_set_distance(int from, int to, int distance);
|
||||
void __init numa_free_distance(void);
|
||||
void __init early_map_cpu_to_node(unsigned int cpu, int nid);
|
||||
void numa_store_cpu_info(unsigned int cpu);
|
||||
|
||||
#else /* CONFIG_NUMA */
|
||||
|
||||
static inline void numa_store_cpu_info(unsigned int cpu) { }
|
||||
static inline void arm64_numa_init(void) { }
|
||||
static inline void early_map_cpu_to_node(unsigned int cpu, int nid) { }
|
||||
|
||||
#endif /* CONFIG_NUMA */
|
||||
|
||||
#endif /* __ASM_NUMA_H */
|
||||
@@ -19,6 +19,8 @@
|
||||
#ifndef __ASM_PAGE_H
|
||||
#define __ASM_PAGE_H
|
||||
|
||||
#include <linux/const.h>
|
||||
|
||||
/* PAGE_SHIFT determines the page size */
|
||||
/* CONT_SHIFT determines the number of pages which can be tracked together */
|
||||
#ifdef CONFIG_ARM64_64K_PAGES
|
||||
|
||||
@@ -133,7 +133,6 @@
|
||||
* Section
|
||||
*/
|
||||
#define PMD_SECT_VALID (_AT(pmdval_t, 1) << 0)
|
||||
#define PMD_SECT_PROT_NONE (_AT(pmdval_t, 1) << 58)
|
||||
#define PMD_SECT_USER (_AT(pmdval_t, 1) << 6) /* AP[1] */
|
||||
#define PMD_SECT_RDONLY (_AT(pmdval_t, 1) << 7) /* AP[2] */
|
||||
#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
|
||||
|
||||
@@ -27,10 +27,6 @@ typedef u64 pmdval_t;
|
||||
typedef u64 pudval_t;
|
||||
typedef u64 pgdval_t;
|
||||
|
||||
#undef STRICT_MM_TYPECHECKS
|
||||
|
||||
#ifdef STRICT_MM_TYPECHECKS
|
||||
|
||||
/*
|
||||
* These are used to make use of C type-checking..
|
||||
*/
|
||||
@@ -58,34 +54,6 @@ typedef struct { pteval_t pgprot; } pgprot_t;
|
||||
#define pgprot_val(x) ((x).pgprot)
|
||||
#define __pgprot(x) ((pgprot_t) { (x) } )
|
||||
|
||||
#else /* !STRICT_MM_TYPECHECKS */
|
||||
|
||||
typedef pteval_t pte_t;
|
||||
#define pte_val(x) (x)
|
||||
#define __pte(x) (x)
|
||||
|
||||
#if CONFIG_PGTABLE_LEVELS > 2
|
||||
typedef pmdval_t pmd_t;
|
||||
#define pmd_val(x) (x)
|
||||
#define __pmd(x) (x)
|
||||
#endif
|
||||
|
||||
#if CONFIG_PGTABLE_LEVELS > 3
|
||||
typedef pudval_t pud_t;
|
||||
#define pud_val(x) (x)
|
||||
#define __pud(x) (x)
|
||||
#endif
|
||||
|
||||
typedef pgdval_t pgd_t;
|
||||
#define pgd_val(x) (x)
|
||||
#define __pgd(x) (x)
|
||||
|
||||
typedef pteval_t pgprot_t;
|
||||
#define pgprot_val(x) (x)
|
||||
#define __pgprot(x) (x)
|
||||
|
||||
#endif /* STRICT_MM_TYPECHECKS */
|
||||
|
||||
#if CONFIG_PGTABLE_LEVELS == 2
|
||||
#include <asm-generic/pgtable-nopmd.h>
|
||||
#elif CONFIG_PGTABLE_LEVELS == 3
|
||||
|
||||
@@ -24,22 +24,16 @@
|
||||
#include <asm/pgtable-prot.h>
|
||||
|
||||
/*
|
||||
* VMALLOC and SPARSEMEM_VMEMMAP ranges.
|
||||
* VMALLOC range.
|
||||
*
|
||||
* VMEMAP_SIZE: allows the whole linear region to be covered by a struct page array
|
||||
* (rounded up to PUD_SIZE).
|
||||
* VMALLOC_START: beginning of the kernel vmalloc space
|
||||
* VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space,
|
||||
* fixed mappings and modules
|
||||
* VMALLOC_END: extends to the available space below vmmemmap, PCI I/O space
|
||||
* and fixed mappings
|
||||
*/
|
||||
#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE)
|
||||
|
||||
#define VMALLOC_START (MODULES_END)
|
||||
#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
|
||||
|
||||
#define VMEMMAP_START (VMALLOC_END + SZ_64K)
|
||||
#define vmemmap ((struct page *)VMEMMAP_START - \
|
||||
SECTION_ALIGN_DOWN(memstart_addr >> PAGE_SHIFT))
|
||||
#define vmemmap ((struct page *)VMEMMAP_START - (memstart_addr >> PAGE_SHIFT))
|
||||
|
||||
#define FIRST_USER_ADDRESS 0UL
|
||||
|
||||
@@ -58,7 +52,7 @@ extern void __pgd_error(const char *file, int line, unsigned long val);
|
||||
* for zero-mapped memory areas etc..
|
||||
*/
|
||||
extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)];
|
||||
#define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page)
|
||||
#define ZERO_PAGE(vaddr) pfn_to_page(PHYS_PFN(__pa(empty_zero_page)))
|
||||
|
||||
#define pte_ERROR(pte) __pte_error(__FILE__, __LINE__, pte_val(pte))
|
||||
|
||||
@@ -272,6 +266,21 @@ static inline pgprot_t mk_sect_prot(pgprot_t prot)
|
||||
return __pgprot(pgprot_val(prot) & ~PTE_TABLE_BIT);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
/*
|
||||
* See the comment in include/asm-generic/pgtable.h
|
||||
*/
|
||||
static inline int pte_protnone(pte_t pte)
|
||||
{
|
||||
return (pte_val(pte) & (PTE_VALID | PTE_PROT_NONE)) == PTE_PROT_NONE;
|
||||
}
|
||||
|
||||
static inline int pmd_protnone(pmd_t pmd)
|
||||
{
|
||||
return pte_protnone(pmd_pte(pmd));
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* THP definitions.
|
||||
*/
|
||||
@@ -280,15 +289,16 @@ static inline pgprot_t mk_sect_prot(pgprot_t prot)
|
||||
#define pmd_trans_huge(pmd) (pmd_val(pmd) && !(pmd_val(pmd) & PMD_TABLE_BIT))
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
||||
#define pmd_present(pmd) pte_present(pmd_pte(pmd))
|
||||
#define pmd_dirty(pmd) pte_dirty(pmd_pte(pmd))
|
||||
#define pmd_young(pmd) pte_young(pmd_pte(pmd))
|
||||
#define pmd_wrprotect(pmd) pte_pmd(pte_wrprotect(pmd_pte(pmd)))
|
||||
#define pmd_mkold(pmd) pte_pmd(pte_mkold(pmd_pte(pmd)))
|
||||
#define pmd_mkwrite(pmd) pte_pmd(pte_mkwrite(pmd_pte(pmd)))
|
||||
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
|
||||
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
|
||||
#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
|
||||
#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
|
||||
#define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_TYPE_MASK))
|
||||
#define pmd_mknotpresent(pmd) (__pmd(pmd_val(pmd) & ~PMD_SECT_VALID))
|
||||
|
||||
#define __HAVE_ARCH_PMD_WRITE
|
||||
#define pmd_write(pmd) pte_write(pmd_pte(pmd))
|
||||
@@ -327,9 +337,8 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
|
||||
unsigned long size, pgprot_t vma_prot);
|
||||
|
||||
#define pmd_none(pmd) (!pmd_val(pmd))
|
||||
#define pmd_present(pmd) (pmd_val(pmd))
|
||||
|
||||
#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
|
||||
#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT))
|
||||
|
||||
#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
||||
PMD_TYPE_TABLE)
|
||||
@@ -394,7 +403,7 @@ static inline phys_addr_t pmd_page_paddr(pmd_t pmd)
|
||||
#define pmd_ERROR(pmd) __pmd_error(__FILE__, __LINE__, pmd_val(pmd))
|
||||
|
||||
#define pud_none(pud) (!pud_val(pud))
|
||||
#define pud_bad(pud) (!(pud_val(pud) & 2))
|
||||
#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT))
|
||||
#define pud_present(pud) (pud_val(pud))
|
||||
|
||||
static inline void set_pud(pud_t *pudp, pud_t pud)
|
||||
@@ -526,6 +535,21 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_HW_AFDBM
|
||||
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
|
||||
extern int ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty);
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
|
||||
static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp,
|
||||
pmd_t entry, int dirty)
|
||||
{
|
||||
return ptep_set_access_flags(vma, address, (pte_t *)pmdp, pmd_pte(entry), dirty);
|
||||
}
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Atomic pte/pmd modifications.
|
||||
*/
|
||||
@@ -578,9 +602,9 @@ static inline pte_t ptep_get_and_clear(struct mm_struct *mm,
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
#define __HAVE_ARCH_PMDP_GET_AND_CLEAR
|
||||
static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
#define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR
|
||||
static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm,
|
||||
unsigned long address, pmd_t *pmdp)
|
||||
{
|
||||
return pte_pmd(ptep_get_and_clear(mm, address, (pte_t *)pmdp));
|
||||
}
|
||||
|
||||
@@ -113,6 +113,17 @@ static inline void update_cpu_boot_status(int val)
|
||||
dsb(ishst);
|
||||
}
|
||||
|
||||
/*
|
||||
* The calling secondary CPU has detected serious configuration mismatch,
|
||||
* which calls for a kernel panic. Update the boot status and park the calling
|
||||
* CPU.
|
||||
*/
|
||||
static inline void cpu_panic_kernel(void)
|
||||
{
|
||||
update_cpu_boot_status(CPU_PANIC_KERNEL);
|
||||
cpu_park_loop();
|
||||
}
|
||||
|
||||
#endif /* ifndef __ASSEMBLY__ */
|
||||
|
||||
#endif /* ifndef __ASM_SMP_H */
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
#ifndef __ASM_SUSPEND_H
|
||||
#define __ASM_SUSPEND_H
|
||||
|
||||
#define NR_CTX_REGS 11
|
||||
#define NR_CTX_REGS 10
|
||||
#define NR_CALLEE_SAVED_REGS 12
|
||||
|
||||
/*
|
||||
* struct cpu_suspend_ctx must be 16-byte aligned since it is allocated on
|
||||
@@ -16,11 +17,34 @@ struct cpu_suspend_ctx {
|
||||
u64 sp;
|
||||
} __aligned(16);
|
||||
|
||||
struct sleep_save_sp {
|
||||
phys_addr_t *save_ptr_stash;
|
||||
phys_addr_t save_ptr_stash_phys;
|
||||
/*
|
||||
* Memory to save the cpu state is allocated on the stack by
|
||||
* __cpu_suspend_enter()'s caller, and populated by __cpu_suspend_enter().
|
||||
* This data must survive until cpu_resume() is called.
|
||||
*
|
||||
* This struct desribes the size and the layout of the saved cpu state.
|
||||
* The layout of the callee_saved_regs is defined by the implementation
|
||||
* of __cpu_suspend_enter(), and cpu_resume(). This struct must be passed
|
||||
* in by the caller as __cpu_suspend_enter()'s stack-frame is gone once it
|
||||
* returns, and the data would be subsequently corrupted by the call to the
|
||||
* finisher.
|
||||
*/
|
||||
struct sleep_stack_data {
|
||||
struct cpu_suspend_ctx system_regs;
|
||||
unsigned long callee_saved_regs[NR_CALLEE_SAVED_REGS];
|
||||
};
|
||||
|
||||
extern unsigned long *sleep_save_stash;
|
||||
|
||||
extern int cpu_suspend(unsigned long arg, int (*fn)(unsigned long));
|
||||
extern void cpu_resume(void);
|
||||
int __cpu_suspend_enter(struct sleep_stack_data *state);
|
||||
void __cpu_suspend_exit(void);
|
||||
void _cpu_resume(void);
|
||||
|
||||
int swsusp_arch_suspend(void);
|
||||
int swsusp_arch_resume(void);
|
||||
int arch_hibernation_header_save(void *addr, unsigned int max_size);
|
||||
int arch_hibernation_header_restore(void *addr);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -86,10 +86,21 @@
|
||||
#define SET_PSTATE_UAO(x) __inst_arm(0xd5000000 | REG_PSTATE_UAO_IMM |\
|
||||
(!!x)<<8 | 0x1f)
|
||||
|
||||
/* SCTLR_EL1 */
|
||||
#define SCTLR_EL1_CP15BEN (0x1 << 5)
|
||||
#define SCTLR_EL1_SED (0x1 << 8)
|
||||
#define SCTLR_EL1_SPAN (0x1 << 23)
|
||||
/* Common SCTLR_ELx flags. */
|
||||
#define SCTLR_ELx_EE (1 << 25)
|
||||
#define SCTLR_ELx_I (1 << 12)
|
||||
#define SCTLR_ELx_SA (1 << 3)
|
||||
#define SCTLR_ELx_C (1 << 2)
|
||||
#define SCTLR_ELx_A (1 << 1)
|
||||
#define SCTLR_ELx_M 1
|
||||
|
||||
#define SCTLR_ELx_FLAGS (SCTLR_ELx_M | SCTLR_ELx_A | SCTLR_ELx_C | \
|
||||
SCTLR_ELx_SA | SCTLR_ELx_I)
|
||||
|
||||
/* SCTLR_EL1 specific flags. */
|
||||
#define SCTLR_EL1_SPAN (1 << 23)
|
||||
#define SCTLR_EL1_SED (1 << 8)
|
||||
#define SCTLR_EL1_CP15BEN (1 << 5)
|
||||
|
||||
|
||||
/* id_aa64isar0 */
|
||||
@@ -115,6 +126,7 @@
|
||||
#define ID_AA64PFR0_ASIMD_SUPPORTED 0x0
|
||||
#define ID_AA64PFR0_EL1_64BIT_ONLY 0x1
|
||||
#define ID_AA64PFR0_EL0_64BIT_ONLY 0x1
|
||||
#define ID_AA64PFR0_EL0_32BIT_64BIT 0x2
|
||||
|
||||
/* id_aa64mmfr0 */
|
||||
#define ID_AA64MMFR0_TGRAN4_SHIFT 28
|
||||
@@ -145,7 +157,11 @@
|
||||
#define ID_AA64MMFR1_VMIDBITS_16 2
|
||||
|
||||
/* id_aa64mmfr2 */
|
||||
#define ID_AA64MMFR2_LVA_SHIFT 16
|
||||
#define ID_AA64MMFR2_IESB_SHIFT 12
|
||||
#define ID_AA64MMFR2_LSM_SHIFT 8
|
||||
#define ID_AA64MMFR2_UAO_SHIFT 4
|
||||
#define ID_AA64MMFR2_CNP_SHIFT 0
|
||||
|
||||
/* id_aa64dfr0 */
|
||||
#define ID_AA64DFR0_CTX_CMPS_SHIFT 28
|
||||
|
||||
@@ -22,6 +22,16 @@ void init_cpu_topology(void);
|
||||
void store_cpu_topology(unsigned int cpuid);
|
||||
const struct cpumask *cpu_coregroup_mask(int cpu);
|
||||
|
||||
#ifdef CONFIG_NUMA
|
||||
|
||||
struct pci_bus;
|
||||
int pcibus_to_node(struct pci_bus *bus);
|
||||
#define cpumask_of_pcibus(bus) (pcibus_to_node(bus) == -1 ? \
|
||||
cpu_all_mask : \
|
||||
cpumask_of_node(pcibus_to_node(bus)))
|
||||
|
||||
#endif /* CONFIG_NUMA */
|
||||
|
||||
#include <asm-generic/topology.h>
|
||||
|
||||
#endif /* _ASM_ARM_TOPOLOGY_H */
|
||||
|
||||
@@ -18,6 +18,22 @@
|
||||
#ifndef __ASM__VIRT_H
|
||||
#define __ASM__VIRT_H
|
||||
|
||||
/*
|
||||
* The arm64 hcall implementation uses x0 to specify the hcall type. A value
|
||||
* less than 0xfff indicates a special hcall, such as get/set vector.
|
||||
* Any other value is used as a pointer to the function to call.
|
||||
*/
|
||||
|
||||
/* HVC_GET_VECTORS - Return the value of the vbar_el2 register. */
|
||||
#define HVC_GET_VECTORS 0
|
||||
|
||||
/*
|
||||
* HVC_SET_VECTORS - Set the value of the vbar_el2 register.
|
||||
*
|
||||
* @x1: Physical address of the new vector table.
|
||||
*/
|
||||
#define HVC_SET_VECTORS 1
|
||||
|
||||
#define BOOT_CPU_MODE_EL1 (0xe11)
|
||||
#define BOOT_CPU_MODE_EL2 (0xe12)
|
||||
|
||||
@@ -60,6 +76,12 @@ static inline bool is_kernel_in_hyp_mode(void)
|
||||
return el == CurrentEL_EL2;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_VHE
|
||||
extern void verify_cpu_run_el(void);
|
||||
#else
|
||||
static inline void verify_cpu_run_el(void) {}
|
||||
#endif
|
||||
|
||||
/* The section containing the hypervisor text */
|
||||
extern char __hyp_text_start[];
|
||||
extern char __hyp_text_end[];
|
||||
|
||||
Reference in New Issue
Block a user