diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index c853f612a815..e7bab1c7a452 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -404,6 +404,7 @@ struct core_state { }; struct kioctx_table; +struct percpu_rw_semaphore; struct mm_struct { struct { struct vm_area_struct *mmap; /* list of VMAs */ @@ -561,6 +562,9 @@ struct mm_struct { struct file __rcu *exe_file; #ifdef CONFIG_MMU_NOTIFIER struct mmu_notifier_subscriptions *notifier_subscriptions; +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + struct percpu_rw_semaphore *mmu_notifier_lock; +#endif #endif #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS pgtable_t pmd_huge_pte; /* protected by page_table_lock */ diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index a7255f582d7c..de5e37f95d99 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -502,9 +504,50 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, __mmu_notifier_invalidate_range(mm, start, end); } -static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm) +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + +static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm) +{ + mm->mmu_notifier_lock = kzalloc(sizeof(struct percpu_rw_semaphore), GFP_KERNEL); + if (!mm->mmu_notifier_lock) + return false; + + percpu_init_rwsem(mm->mmu_notifier_lock); + mm->notifier_subscriptions = NULL; + + return true; +} + +static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) +{ + if (mm_has_notifiers(mm)) + __mmu_notifier_subscriptions_destroy(mm); + + if (in_atomic()) { + percpu_rwsem_async_destroy(mm->mmu_notifier_lock); + } else { + percpu_free_rwsem(mm->mmu_notifier_lock); + kfree(mm->mmu_notifier_lock); + } + mm->mmu_notifier_lock = NULL; +} + +static inline bool mmu_notifier_trylock(struct mm_struct *mm) +{ + return percpu_down_read_trylock(mm->mmu_notifier_lock); +} + +static inline void mmu_notifier_unlock(struct mm_struct *mm) +{ + percpu_up_read(mm->mmu_notifier_lock); +} + +#else /* CONFIG_SPECULATIVE_PAGE_FAULT */ + +static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm) { mm->notifier_subscriptions = NULL; + return true; } static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) @@ -513,6 +556,16 @@ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) __mmu_notifier_subscriptions_destroy(mm); } +static inline bool mmu_notifier_trylock(struct mm_struct *mm) +{ + return true; +} + +static inline void mmu_notifier_unlock(struct mm_struct *mm) +{ +} + +#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */ static inline void mmu_notifier_range_init(struct mmu_notifier_range *range, enum mmu_notifier_event event, @@ -727,14 +780,23 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm, { } -static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm) +static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm) { + return true; } static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm) { } +static inline void mmu_notifier_lock(struct mm_struct *mm) +{ +} + +static inline void mmu_notifier_unlock(struct mm_struct *mm) +{ +} + #define mmu_notifier_range_update_to_read_only(r) false #define ptep_clear_flush_young_notify ptep_clear_flush_young diff --git a/kernel/fork.c b/kernel/fork.c index 45f79721ebd5..c4252e61e313 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1072,7 +1072,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, mm_init_owner(mm, p); mm_init_pasid(mm); RCU_INIT_POINTER(mm->exe_file, NULL); - mmu_notifier_subscriptions_init(mm); + if (!mmu_notifier_subscriptions_init(mm)) + goto fail_nopgd; init_tlb_flush_pending(mm); #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS mm->pmd_huge_pte = NULL; diff --git a/mm/memory.c b/mm/memory.c index 5db2a3ed395b..dfe88223c483 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -4717,8 +4717,19 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) goto unlock; } if (vmf->flags & FAULT_FLAG_WRITE) { - if (!pte_write(entry)) - return do_wp_page(vmf); + if (!pte_write(entry)) { + if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) + return do_wp_page(vmf); + + if (!mmu_notifier_trylock(vmf->vma->vm_mm)) { + ret = VM_FAULT_RETRY; + goto unlock; + } + + ret = do_wp_page(vmf); + mmu_notifier_unlock(vmf->vma->vm_mm); + return ret; + } entry = pte_mkdirty(entry); } entry = pte_mkyoung(entry); diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 07f42a7a6065..ce161cd0c8a1 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -621,6 +621,25 @@ void __mmu_notifier_invalidate_range(struct mm_struct *mm, srcu_read_unlock(&srcu, id); } +#ifdef CONFIG_SPECULATIVE_PAGE_FAULT + +static inline void mmu_notifier_write_lock(struct mm_struct *mm) +{ + percpu_down_write(mm->mmu_notifier_lock); +} + +static inline void mmu_notifier_write_unlock(struct mm_struct *mm) +{ + percpu_up_write(mm->mmu_notifier_lock); +} + +#else /* CONFIG_SPECULATIVE_PAGE_FAULT */ + +static inline void mmu_notifier_write_lock(struct mm_struct *mm) {} +static inline void mmu_notifier_write_unlock(struct mm_struct *mm) {} + +#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */ + /* * Same as mmu_notifier_register but here the caller must hold the mmap_lock in * write mode. A NULL mn signals the notifier is being registered for itree @@ -661,9 +680,13 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, INIT_HLIST_HEAD(&subscriptions->deferred_list); } + mmu_notifier_write_lock(mm); + ret = mm_take_all_locks(mm); - if (unlikely(ret)) + if (unlikely(ret)) { + mmu_notifier_write_unlock(mm); goto out_clean; + } /* * Serialize the update against mmu_notifier_unregister. A @@ -698,6 +721,7 @@ int __mmu_notifier_register(struct mmu_notifier *subscription, mm->notifier_subscriptions->has_itree = true; mm_drop_all_locks(mm); + mmu_notifier_write_unlock(mm); BUG_ON(atomic_read(&mm->mm_users) <= 0); return 0;