xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ANDROID: userfaultfd: add MMAP_TRYLOCK mode for COPY/ZEROPAGE

Browse Source 
In case mmap_lock is contended, it is possible that userspace can spend
time performing other tasks rather than waiting in uninterruptible-sleep
state for the lock to become available. Even if no other task is
available, it is better to yield or sleep rather than adding contention
to already contended lock.

We introduce MMAP_TRYLOCK mode so that when possible, userspace can
request to use mmap_read_trylock(), returning -EAGAIN if and when it
fails.

Bug: 320478828
Change-Id: I2d196fd317e054af03dbd35ac1b0c7634cb370dc
Signed-off-by: Lokesh Gidra <lokeshgidra@google.com>
This commit is contained in:
Lokesh Gidra
2024-03-08 10:04:39 -08:00
parent 0ebc4699bd
commit e3aabbf867
4 changed files with 22 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
fs/userfaultfd.c
View File

@@ -1769,7 +1769,9 @@ static int userfaultfd_copy(struct userfaultfd_ctx *ctx,
ret = -EINVAL; ret = -EINVAL;
if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src) if (uffdio_copy.src + uffdio_copy.len <= uffdio_copy.src)
goto out; goto out;
if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|UFFDIO_COPY_MODE_WP)) if (uffdio_copy.mode & ~(UFFDIO_COPY_MODE_DONTWAKE|
UFFDIO_COPY_MODE_WP|
UFFDIO_COPY_MODE_MMAP_TRYLOCK))
goto out; goto out;
if (mmget_not_zero(ctx->mm)) { if (mmget_not_zero(ctx->mm)) {
ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src, ret = mcopy_atomic(ctx->mm, uffdio_copy.dst, uffdio_copy.src,
@@ -1820,13 +1822,14 @@ static int userfaultfd_zeropage(struct userfaultfd_ctx *ctx,
if (ret) if (ret)
goto out; goto out;
ret = -EINVAL; ret = -EINVAL;
if (uffdio_zeropage.mode & ~UFFDIO_ZEROPAGE_MODE_DONTWAKE) if (uffdio_zeropage.mode & ~(UFFDIO_ZEROPAGE_MODE_DONTWAKE|
UFFDIO_ZEROPAGE_MODE_MMAP_TRYLOCK))
goto out; goto out;
if (mmget_not_zero(ctx->mm)) { if (mmget_not_zero(ctx->mm)) {
ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start, ret = mfill_zeropage(ctx->mm, uffdio_zeropage.range.start,
uffdio_zeropage.range.len, uffdio_zeropage.range.len,
&ctx->mmap_changing); &ctx->mmap_changing, uffdio_zeropage.mode);
mmput(ctx->mm); mmput(ctx->mm);
} else { } else {
return -ESRCH; return -ESRCH;

8
include/linux/userfaultfd_k.h
View File

@@ -33,6 +33,9 @@
#define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK) #define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
#define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS) #define UFFD_FLAGS_SET (EFD_SHARED_FCNTL_FLAGS)
static_assert(UFFDIO_ZEROPAGE_MODE_MMAP_TRYLOCK == UFFDIO_COPY_MODE_MMAP_TRYLOCK);
#define UFFDIO_MODE_MMAP_TRYLOCK UFFDIO_COPY_MODE_MMAP_TRYLOCK
extern int sysctl_unprivileged_userfaultfd; extern int sysctl_unprivileged_userfaultfd;
extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason); extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
@@ -65,9 +68,8 @@ extern ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
unsigned long src_start, unsigned long len, unsigned long src_start, unsigned long len,
bool *mmap_changing, __u64 mode); bool *mmap_changing, __u64 mode);
extern ssize_t mfill_zeropage(struct mm_struct *dst_mm, extern ssize_t mfill_zeropage(struct mm_struct *dst_mm,
unsigned long dst_start, unsigned long dst_start, unsigned long len,
unsigned long len, bool *mmap_changing, __u64 mode);
bool *mmap_changing);
extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start, extern ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long dst_start,
unsigned long len, bool *mmap_changing); unsigned long len, bool *mmap_changing);
extern int mwriteprotect_range(struct mm_struct *dst_mm, extern int mwriteprotect_range(struct mm_struct *dst_mm,

2
include/uapi/linux/userfaultfd.h
View File

@@ -237,6 +237,7 @@ struct uffdio_copy {
* according to the uffdio_register.ioctls. * according to the uffdio_register.ioctls.
*/ */
#define UFFDIO_COPY_MODE_WP ((__u64)1<<1) #define UFFDIO_COPY_MODE_WP ((__u64)1<<1)
#define UFFDIO_COPY_MODE_MMAP_TRYLOCK ((__u64)1<<63)
__u64 mode; __u64 mode;
/* /*
@@ -249,6 +250,7 @@ struct uffdio_copy {
struct uffdio_zeropage { struct uffdio_zeropage {
struct uffdio_range range; struct uffdio_range range;
#define UFFDIO_ZEROPAGE_MODE_DONTWAKE ((__u64)1<<0) #define UFFDIO_ZEROPAGE_MODE_DONTWAKE ((__u64)1<<0)
#define UFFDIO_ZEROPAGE_MODE_MMAP_TRYLOCK ((__u64)1<<63)
__u64 mode; __u64 mode;
/* /*

13
mm/userfaultfd.c
View File

@@ -559,14 +559,19 @@ static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
copied = 0; copied = 0;
page = NULL; page = NULL;
retry: retry:
mmap_read_lock(dst_mm); err = -EAGAIN;
if (mode & UFFDIO_MODE_MMAP_TRYLOCK) {
if (!mmap_read_trylock(dst_mm))
goto out;
} else {
mmap_read_lock(dst_mm);
}
/* /*
* If memory mappings are changing because of non-cooperative * If memory mappings are changing because of non-cooperative
* operation (e.g. mremap) running in parallel, bail out and * operation (e.g. mremap) running in parallel, bail out and
* request the user to retry later * request the user to retry later
*/ */
err = -EAGAIN;
if (mmap_changing && READ_ONCE(*mmap_changing)) if (mmap_changing && READ_ONCE(*mmap_changing))
goto out_unlock; goto out_unlock;
@@ -708,10 +713,10 @@ ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
} }
ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start, ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,
unsigned long len, bool *mmap_changing) unsigned long len, bool *mmap_changing, __u64 mode)
{ {
return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE, return __mcopy_atomic(dst_mm, start, 0, len, MCOPY_ATOMIC_ZEROPAGE,
mmap_changing, 0); mmap_changing, mode);
} }
ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start, ssize_t mcopy_continue(struct mm_struct *dst_mm, unsigned long start,