Merge tag 'x86-mm-2020-06-05' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 mm updates from Ingo Molnar: "Misc changes: - Unexport various PAT primitives - Unexport per-CPU tlbstate and uninline TLB helpers" * tag 'x86-mm-2020-06-05' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (23 commits) x86/tlb/uv: Add a forward declaration for struct flush_tlb_info x86/cpu: Export native_write_cr4() only when CONFIG_LKTDM=m x86/tlb: Restrict access to tlbstate xen/privcmd: Remove unneeded asm/tlb.h include x86/tlb: Move PCID helpers where they are used x86/tlb: Uninline nmi_uaccess_okay() x86/tlb: Move cr4_set_bits_and_update_boot() to the usage site x86/tlb: Move paravirt_tlb_remove_table() to the usage site x86/tlb: Move __flush_tlb_all() out of line x86/tlb: Move flush_tlb_others() out of line x86/tlb: Move __flush_tlb_one_kernel() out of line x86/tlb: Move __flush_tlb_one_user() out of line x86/tlb: Move __flush_tlb_global() out of line x86/tlb: Move __flush_tlb() out of line x86/alternatives: Move temporary_mm helpers into C x86/cr4: Sanitize CR4.PCE update x86/cpu: Uninline CR4 accessors x86/tlb: Uninline __get_current_cr3_fast() x86/mm: Use pgprotval_t in protval_4k_2_large() and protval_large_2_4k() x86/mm: Unexport __cachemode2pte_tbl ...
This commit is contained in:
Linus Torvalds
@@ -18,6 +18,16 @@
|
||||
|
||||
#include "mm_internal.h"
|
||||
|
||||
#ifdef CONFIG_PARAVIRT
|
||||
# define STATIC_NOPV
|
||||
#else
|
||||
# define STATIC_NOPV static
|
||||
# define __flush_tlb_local native_flush_tlb_local
|
||||
# define __flush_tlb_global native_flush_tlb_global
|
||||
# define __flush_tlb_one_user(addr) native_flush_tlb_one_user(addr)
|
||||
# define __flush_tlb_others(msk, info) native_flush_tlb_others(msk, info)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* TLB flushing, formerly SMP-only
|
||||
* c/o Linus Torvalds.
|
||||
@@ -38,6 +48,126 @@
|
||||
*/
|
||||
#define LAST_USER_MM_IBPB 0x1UL
|
||||
|
||||
/*
|
||||
* The x86 feature is called PCID (Process Context IDentifier). It is similar
|
||||
* to what is traditionally called ASID on the RISC processors.
|
||||
*
|
||||
* We don't use the traditional ASID implementation, where each process/mm gets
|
||||
* its own ASID and flush/restart when we run out of ASID space.
|
||||
*
|
||||
* Instead we have a small per-cpu array of ASIDs and cache the last few mm's
|
||||
* that came by on this CPU, allowing cheaper switch_mm between processes on
|
||||
* this CPU.
|
||||
*
|
||||
* We end up with different spaces for different things. To avoid confusion we
|
||||
* use different names for each of them:
|
||||
*
|
||||
* ASID - [0, TLB_NR_DYN_ASIDS-1]
|
||||
* the canonical identifier for an mm
|
||||
*
|
||||
* kPCID - [1, TLB_NR_DYN_ASIDS]
|
||||
* the value we write into the PCID part of CR3; corresponds to the
|
||||
* ASID+1, because PCID 0 is special.
|
||||
*
|
||||
* uPCID - [2048 + 1, 2048 + TLB_NR_DYN_ASIDS]
|
||||
* for KPTI each mm has two address spaces and thus needs two
|
||||
* PCID values, but we can still do with a single ASID denomination
|
||||
* for each mm. Corresponds to kPCID + 2048.
|
||||
*
|
||||
*/
|
||||
|
||||
/* There are 12 bits of space for ASIDS in CR3 */
|
||||
#define CR3_HW_ASID_BITS 12
|
||||
|
||||
/*
|
||||
* When enabled, PAGE_TABLE_ISOLATION consumes a single bit for
|
||||
* user/kernel switches
|
||||
*/
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
# define PTI_CONSUMED_PCID_BITS 1
|
||||
#else
|
||||
# define PTI_CONSUMED_PCID_BITS 0
|
||||
#endif
|
||||
|
||||
#define CR3_AVAIL_PCID_BITS (X86_CR3_PCID_BITS - PTI_CONSUMED_PCID_BITS)
|
||||
|
||||
/*
|
||||
* ASIDs are zero-based: 0->MAX_AVAIL_ASID are valid. -1 below to account
|
||||
* for them being zero-based. Another -1 is because PCID 0 is reserved for
|
||||
* use by non-PCID-aware users.
|
||||
*/
|
||||
#define MAX_ASID_AVAILABLE ((1 << CR3_AVAIL_PCID_BITS) - 2)
|
||||
|
||||
/*
|
||||
* Given @asid, compute kPCID
|
||||
*/
|
||||
static inline u16 kern_pcid(u16 asid)
|
||||
{
|
||||
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/*
|
||||
* Make sure that the dynamic ASID space does not confict with the
|
||||
* bit we are using to switch between user and kernel ASIDs.
|
||||
*/
|
||||
BUILD_BUG_ON(TLB_NR_DYN_ASIDS >= (1 << X86_CR3_PTI_PCID_USER_BIT));
|
||||
|
||||
/*
|
||||
* The ASID being passed in here should have respected the
|
||||
* MAX_ASID_AVAILABLE and thus never have the switch bit set.
|
||||
*/
|
||||
VM_WARN_ON_ONCE(asid & (1 << X86_CR3_PTI_PCID_USER_BIT));
|
||||
#endif
|
||||
/*
|
||||
* The dynamically-assigned ASIDs that get passed in are small
|
||||
* (<TLB_NR_DYN_ASIDS). They never have the high switch bit set,
|
||||
* so do not bother to clear it.
|
||||
*
|
||||
* If PCID is on, ASID-aware code paths put the ASID+1 into the
|
||||
* PCID bits. This serves two purposes. It prevents a nasty
|
||||
* situation in which PCID-unaware code saves CR3, loads some other
|
||||
* value (with PCID == 0), and then restores CR3, thus corrupting
|
||||
* the TLB for ASID 0 if the saved ASID was nonzero. It also means
|
||||
* that any bugs involving loading a PCID-enabled CR3 with
|
||||
* CR4.PCIDE off will trigger deterministically.
|
||||
*/
|
||||
return asid + 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* Given @asid, compute uPCID
|
||||
*/
|
||||
static inline u16 user_pcid(u16 asid)
|
||||
{
|
||||
u16 ret = kern_pcid(asid);
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
ret |= 1 << X86_CR3_PTI_PCID_USER_BIT;
|
||||
#endif
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline unsigned long build_cr3(pgd_t *pgd, u16 asid)
|
||||
{
|
||||
if (static_cpu_has(X86_FEATURE_PCID)) {
|
||||
return __sme_pa(pgd) | kern_pcid(asid);
|
||||
} else {
|
||||
VM_WARN_ON_ONCE(asid != 0);
|
||||
return __sme_pa(pgd);
|
||||
}
|
||||
}
|
||||
|
||||
static inline unsigned long build_cr3_noflush(pgd_t *pgd, u16 asid)
|
||||
{
|
||||
VM_WARN_ON_ONCE(asid > MAX_ASID_AVAILABLE);
|
||||
/*
|
||||
* Use boot_cpu_has() instead of this_cpu_has() as this function
|
||||
* might be called during early boot. This should work even after
|
||||
* boot because all CPU's the have same capabilities:
|
||||
*/
|
||||
VM_WARN_ON_ONCE(!boot_cpu_has(X86_FEATURE_PCID));
|
||||
return __sme_pa(pgd) | kern_pcid(asid) | CR3_NOFLUSH;
|
||||
}
|
||||
|
||||
/*
|
||||
* We get here when we do something requiring a TLB invalidation
|
||||
* but could not go invalidate all of the contexts. We do the
|
||||
@@ -110,6 +240,32 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen,
|
||||
*need_flush = true;
|
||||
}
|
||||
|
||||
/*
|
||||
* Given an ASID, flush the corresponding user ASID. We can delay this
|
||||
* until the next time we switch to it.
|
||||
*
|
||||
* See SWITCH_TO_USER_CR3.
|
||||
*/
|
||||
static inline void invalidate_user_asid(u16 asid)
|
||||
{
|
||||
/* There is no user ASID if address space separation is off */
|
||||
if (!IS_ENABLED(CONFIG_PAGE_TABLE_ISOLATION))
|
||||
return;
|
||||
|
||||
/*
|
||||
* We only have a single ASID if PCID is off and the CR3
|
||||
* write will have flushed it.
|
||||
*/
|
||||
if (!cpu_feature_enabled(X86_FEATURE_PCID))
|
||||
return;
|
||||
|
||||
if (!static_cpu_has(X86_FEATURE_PTI))
|
||||
return;
|
||||
|
||||
__set_bit(kern_pcid(asid),
|
||||
(unsigned long *)this_cpu_ptr(&cpu_tlbstate.user_pcid_flush_mask));
|
||||
}
|
||||
|
||||
static void load_new_mm_cr3(pgd_t *pgdir, u16 new_asid, bool need_flush)
|
||||
{
|
||||
unsigned long new_mm_cr3;
|
||||
@@ -244,6 +400,26 @@ static void cond_ibpb(struct task_struct *next)
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PERF_EVENTS
|
||||
static inline void cr4_update_pce_mm(struct mm_struct *mm)
|
||||
{
|
||||
if (static_branch_unlikely(&rdpmc_always_available_key) ||
|
||||
(!static_branch_unlikely(&rdpmc_never_available_key) &&
|
||||
atomic_read(&mm->context.perf_rdpmc_allowed)))
|
||||
cr4_set_bits_irqsoff(X86_CR4_PCE);
|
||||
else
|
||||
cr4_clear_bits_irqsoff(X86_CR4_PCE);
|
||||
}
|
||||
|
||||
void cr4_update_pce(void *ignored)
|
||||
{
|
||||
cr4_update_pce_mm(this_cpu_read(cpu_tlbstate.loaded_mm));
|
||||
}
|
||||
|
||||
#else
|
||||
static inline void cr4_update_pce_mm(struct mm_struct *mm) { }
|
||||
#endif
|
||||
|
||||
void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
|
||||
struct task_struct *tsk)
|
||||
{
|
||||
@@ -403,7 +579,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next,
|
||||
this_cpu_write(cpu_tlbstate.loaded_mm_asid, new_asid);
|
||||
|
||||
if (next != real_prev) {
|
||||
load_mm_cr4_irqsoff(next);
|
||||
cr4_update_pce_mm(next);
|
||||
switch_ldt(real_prev, next);
|
||||
}
|
||||
}
|
||||
@@ -580,7 +756,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
|
||||
unsigned long addr = f->start;
|
||||
|
||||
while (addr < f->end) {
|
||||
__flush_tlb_one_user(addr);
|
||||
flush_tlb_one_user(addr);
|
||||
addr += 1UL << f->stride_shift;
|
||||
}
|
||||
if (local)
|
||||
@@ -588,7 +764,7 @@ static void flush_tlb_func_common(const struct flush_tlb_info *f,
|
||||
trace_tlb_flush(reason, nr_invalidate);
|
||||
} else {
|
||||
/* Full flush. */
|
||||
local_flush_tlb();
|
||||
flush_tlb_local();
|
||||
if (local)
|
||||
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ALL);
|
||||
trace_tlb_flush(reason, TLB_FLUSH_ALL);
|
||||
@@ -623,8 +799,8 @@ static bool tlb_is_not_lazy(int cpu, void *data)
|
||||
return !per_cpu(cpu_tlbstate.is_lazy, cpu);
|
||||
}
|
||||
|
||||
void native_flush_tlb_others(const struct cpumask *cpumask,
|
||||
const struct flush_tlb_info *info)
|
||||
STATIC_NOPV void native_flush_tlb_others(const struct cpumask *cpumask,
|
||||
const struct flush_tlb_info *info)
|
||||
{
|
||||
count_vm_tlb_event(NR_TLB_REMOTE_FLUSH);
|
||||
if (info->end == TLB_FLUSH_ALL)
|
||||
@@ -674,6 +850,12 @@ void native_flush_tlb_others(const struct cpumask *cpumask,
|
||||
(void *)info, 1, cpumask);
|
||||
}
|
||||
|
||||
void flush_tlb_others(const struct cpumask *cpumask,
|
||||
const struct flush_tlb_info *info)
|
||||
{
|
||||
__flush_tlb_others(cpumask, info);
|
||||
}
|
||||
|
||||
/*
|
||||
* See Documentation/x86/tlb.rst for details. We choose 33
|
||||
* because it is large enough to cover the vast majority (at
|
||||
@@ -784,7 +966,7 @@ static void do_kernel_range_flush(void *info)
|
||||
|
||||
/* flush range by one by one 'invlpg' */
|
||||
for (addr = f->start; addr < f->end; addr += PAGE_SIZE)
|
||||
__flush_tlb_one_kernel(addr);
|
||||
flush_tlb_one_kernel(addr);
|
||||
}
|
||||
|
||||
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
|
||||
@@ -806,6 +988,164 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* This can be used from process context to figure out what the value of
|
||||
* CR3 is without needing to do a (slow) __read_cr3().
|
||||
*
|
||||
* It's intended to be used for code like KVM that sneakily changes CR3
|
||||
* and needs to restore it. It needs to be used very carefully.
|
||||
*/
|
||||
unsigned long __get_current_cr3_fast(void)
|
||||
{
|
||||
unsigned long cr3 = build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
|
||||
this_cpu_read(cpu_tlbstate.loaded_mm_asid));
|
||||
|
||||
/* For now, be very restrictive about when this can be called. */
|
||||
VM_WARN_ON(in_nmi() || preemptible());
|
||||
|
||||
VM_BUG_ON(cr3 != __read_cr3());
|
||||
return cr3;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__get_current_cr3_fast);
|
||||
|
||||
/*
|
||||
* Flush one page in the kernel mapping
|
||||
*/
|
||||
void flush_tlb_one_kernel(unsigned long addr)
|
||||
{
|
||||
count_vm_tlb_event(NR_TLB_LOCAL_FLUSH_ONE);
|
||||
|
||||
/*
|
||||
* If PTI is off, then __flush_tlb_one_user() is just INVLPG or its
|
||||
* paravirt equivalent. Even with PCID, this is sufficient: we only
|
||||
* use PCID if we also use global PTEs for the kernel mapping, and
|
||||
* INVLPG flushes global translations across all address spaces.
|
||||
*
|
||||
* If PTI is on, then the kernel is mapped with non-global PTEs, and
|
||||
* __flush_tlb_one_user() will flush the given address for the current
|
||||
* kernel address space and for its usermode counterpart, but it does
|
||||
* not flush it for other address spaces.
|
||||
*/
|
||||
flush_tlb_one_user(addr);
|
||||
|
||||
if (!static_cpu_has(X86_FEATURE_PTI))
|
||||
return;
|
||||
|
||||
/*
|
||||
* See above. We need to propagate the flush to all other address
|
||||
* spaces. In principle, we only need to propagate it to kernelmode
|
||||
* address spaces, but the extra bookkeeping we would need is not
|
||||
* worth it.
|
||||
*/
|
||||
this_cpu_write(cpu_tlbstate.invalidate_other, true);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush one page in the user mapping
|
||||
*/
|
||||
STATIC_NOPV void native_flush_tlb_one_user(unsigned long addr)
|
||||
{
|
||||
u32 loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
|
||||
|
||||
asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
|
||||
|
||||
if (!static_cpu_has(X86_FEATURE_PTI))
|
||||
return;
|
||||
|
||||
/*
|
||||
* Some platforms #GP if we call invpcid(type=1/2) before CR4.PCIDE=1.
|
||||
* Just use invalidate_user_asid() in case we are called early.
|
||||
*/
|
||||
if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE))
|
||||
invalidate_user_asid(loaded_mm_asid);
|
||||
else
|
||||
invpcid_flush_one(user_pcid(loaded_mm_asid), addr);
|
||||
}
|
||||
|
||||
void flush_tlb_one_user(unsigned long addr)
|
||||
{
|
||||
__flush_tlb_one_user(addr);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush everything
|
||||
*/
|
||||
STATIC_NOPV void native_flush_tlb_global(void)
|
||||
{
|
||||
unsigned long cr4, flags;
|
||||
|
||||
if (static_cpu_has(X86_FEATURE_INVPCID)) {
|
||||
/*
|
||||
* Using INVPCID is considerably faster than a pair of writes
|
||||
* to CR4 sandwiched inside an IRQ flag save/restore.
|
||||
*
|
||||
* Note, this works with CR4.PCIDE=0 or 1.
|
||||
*/
|
||||
invpcid_flush_all();
|
||||
return;
|
||||
}
|
||||
|
||||
/*
|
||||
* Read-modify-write to CR4 - protect it from preemption and
|
||||
* from interrupts. (Use the raw variant because this code can
|
||||
* be called from deep inside debugging code.)
|
||||
*/
|
||||
raw_local_irq_save(flags);
|
||||
|
||||
cr4 = this_cpu_read(cpu_tlbstate.cr4);
|
||||
/* toggle PGE */
|
||||
native_write_cr4(cr4 ^ X86_CR4_PGE);
|
||||
/* write old PGE again and flush TLBs */
|
||||
native_write_cr4(cr4);
|
||||
|
||||
raw_local_irq_restore(flags);
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush the entire current user mapping
|
||||
*/
|
||||
STATIC_NOPV void native_flush_tlb_local(void)
|
||||
{
|
||||
/*
|
||||
* Preemption or interrupts must be disabled to protect the access
|
||||
* to the per CPU variable and to prevent being preempted between
|
||||
* read_cr3() and write_cr3().
|
||||
*/
|
||||
WARN_ON_ONCE(preemptible());
|
||||
|
||||
invalidate_user_asid(this_cpu_read(cpu_tlbstate.loaded_mm_asid));
|
||||
|
||||
/* If current->mm == NULL then the read_cr3() "borrows" an mm */
|
||||
native_write_cr3(__native_read_cr3());
|
||||
}
|
||||
|
||||
void flush_tlb_local(void)
|
||||
{
|
||||
__flush_tlb_local();
|
||||
}
|
||||
|
||||
/*
|
||||
* Flush everything
|
||||
*/
|
||||
void __flush_tlb_all(void)
|
||||
{
|
||||
/*
|
||||
* This is to catch users with enabled preemption and the PGE feature
|
||||
* and don't trigger the warning in __native_flush_tlb().
|
||||
*/
|
||||
VM_WARN_ON_ONCE(preemptible());
|
||||
|
||||
if (boot_cpu_has(X86_FEATURE_PGE)) {
|
||||
__flush_tlb_global();
|
||||
} else {
|
||||
/*
|
||||
* !PGE -> !PCID (setup_pcid()), thus every flush is total.
|
||||
*/
|
||||
flush_tlb_local();
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(__flush_tlb_all);
|
||||
|
||||
/*
|
||||
* arch_tlbbatch_flush() performs a full TLB flush regardless of the active mm.
|
||||
* This means that the 'struct flush_tlb_info' that describes which mappings to
|
||||
@@ -837,6 +1177,38 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch)
|
||||
put_cpu();
|
||||
}
|
||||
|
||||
/*
|
||||
* Blindly accessing user memory from NMI context can be dangerous
|
||||
* if we're in the middle of switching the current user task or
|
||||
* switching the loaded mm. It can also be dangerous if we
|
||||
* interrupted some kernel code that was temporarily using a
|
||||
* different mm.
|
||||
*/
|
||||
bool nmi_uaccess_okay(void)
|
||||
{
|
||||
struct mm_struct *loaded_mm = this_cpu_read(cpu_tlbstate.loaded_mm);
|
||||
struct mm_struct *current_mm = current->mm;
|
||||
|
||||
VM_WARN_ON_ONCE(!loaded_mm);
|
||||
|
||||
/*
|
||||
* The condition we want to check is
|
||||
* current_mm->pgd == __va(read_cr3_pa()). This may be slow, though,
|
||||
* if we're running in a VM with shadow paging, and nmi_uaccess_okay()
|
||||
* is supposed to be reasonably fast.
|
||||
*
|
||||
* Instead, we check the almost equivalent but somewhat conservative
|
||||
* condition below, and we rely on the fact that switch_mm_irqs_off()
|
||||
* sets loaded_mm to LOADED_MM_SWITCHING before writing to CR3.
|
||||
*/
|
||||
if (loaded_mm != current_mm)
|
||||
return false;
|
||||
|
||||
VM_WARN_ON_ONCE(current_mm->pgd != __va(read_cr3_pa()));
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
|
||||
size_t count, loff_t *ppos)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user