xiaomi-sm8450/android_kernel_xiaomi_sm8450
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

random: remove batched entropy locking

Browse Source 
commit 77760fd7f7ae3dfd03668204e708d1568d75447d upstream.

Rather than use spinlocks to protect batched entropy, we can instead
disable interrupts locally, since we're dealing with per-cpu data, and
manage resets with a basic generation counter. At the same time, we
can't quite do this on PREEMPT_RT, where we still want spinlocks-as-
mutexes semantics. So we use a local_lock_t, which provides the right
behavior for each. Because this is a per-cpu lock, that generation
counter is still doing the necessary CPU-to-CPU communication.

This should improve performance a bit. It will also fix the linked splat
that Jonathan received with a PROVE_RAW_LOCK_NESTING=y.

Reviewed-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Reviewed-by: Dominik Brodowski <linux@dominikbrodowski.net>
Reviewed-by: Eric Biggers <ebiggers@google.com>
Suggested-by: Andy Lutomirski <luto@kernel.org>
Reported-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net>
Tested-by: Jonathan Neuschäfer <j.neuschaefer@gmx.net>
Link: https://lore.kernel.org/lkml/YfMa0QgsjCVdRAvJ@latitude/
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
This commit is contained in:
Jason A. Donenfeld
2022-01-28 23:29:45 +01:00
committed by Greg Kroah-Hartman
parent 8d07e2a226
commit 0762b7d1f1
1 changed files with 28 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
drivers/char/random.c
View File

@@ -1721,13 +1721,16 @@ struct ctl_table random_table[] = {
}; };
#endif /* CONFIG_SYSCTL */ #endif /* CONFIG_SYSCTL */
static atomic_t batch_generation = ATOMIC_INIT(0);
struct batched_entropy { struct batched_entropy {
union { union {
u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)]; u64 entropy_u64[CHACHA_BLOCK_SIZE / sizeof(u64)];
u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)]; u32 entropy_u32[CHACHA_BLOCK_SIZE / sizeof(u32)];
}; };
local_lock_t lock;
unsigned int position; unsigned int position;
spinlock_t batch_lock; int generation;
}; };
/* /*
@@ -1739,7 +1742,7 @@ struct batched_entropy {
* point prior. * point prior.
*/ */
static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = { static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u64) = {
.batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u64.lock), .lock = INIT_LOCAL_LOCK(batched_entropy_u64.lock)
}; };
u64 get_random_u64(void) u64 get_random_u64(void)
@@ -1748,67 +1751,65 @@ u64 get_random_u64(void)
unsigned long flags; unsigned long flags;
struct batched_entropy *batch; struct batched_entropy *batch;
static void *previous; static void *previous;
int next_gen;
warn_unseeded_randomness(&previous); warn_unseeded_randomness(&previous);
local_lock_irqsave(&batched_entropy_u64.lock, flags);
batch = raw_cpu_ptr(&batched_entropy_u64); batch = raw_cpu_ptr(&batched_entropy_u64);
spin_lock_irqsave(&batch->batch_lock, flags);
if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0) { next_gen = atomic_read(&batch_generation);
if (batch->position % ARRAY_SIZE(batch->entropy_u64) == 0 ||
next_gen != batch->generation) {
extract_crng((u8 *)batch->entropy_u64); extract_crng((u8 *)batch->entropy_u64);
batch->position = 0; batch->position = 0;
batch->generation = next_gen;
} }
ret = batch->entropy_u64[batch->position++]; ret = batch->entropy_u64[batch->position++];
spin_unlock_irqrestore(&batch->batch_lock, flags); local_unlock_irqrestore(&batched_entropy_u64.lock, flags);
return ret; return ret;
} }
EXPORT_SYMBOL(get_random_u64); EXPORT_SYMBOL(get_random_u64);
static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = { static DEFINE_PER_CPU(struct batched_entropy, batched_entropy_u32) = {
.batch_lock = __SPIN_LOCK_UNLOCKED(batched_entropy_u32.lock), .lock = INIT_LOCAL_LOCK(batched_entropy_u32.lock)
}; };
u32 get_random_u32(void) u32 get_random_u32(void)
{ {
u32 ret; u32 ret;
unsigned long flags; unsigned long flags;
struct batched_entropy *batch; struct batched_entropy *batch;
static void *previous; static void *previous;
int next_gen;
warn_unseeded_randomness(&previous); warn_unseeded_randomness(&previous);
local_lock_irqsave(&batched_entropy_u32.lock, flags);
batch = raw_cpu_ptr(&batched_entropy_u32); batch = raw_cpu_ptr(&batched_entropy_u32);
spin_lock_irqsave(&batch->batch_lock, flags);
if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0) { next_gen = atomic_read(&batch_generation);
if (batch->position % ARRAY_SIZE(batch->entropy_u32) == 0 ||
next_gen != batch->generation) {
extract_crng((u8 *)batch->entropy_u32); extract_crng((u8 *)batch->entropy_u32);
batch->position = 0; batch->position = 0;
batch->generation = next_gen;
} }
ret = batch->entropy_u32[batch->position++]; ret = batch->entropy_u32[batch->position++];
spin_unlock_irqrestore(&batch->batch_lock, flags); local_unlock_irqrestore(&batched_entropy_u32.lock, flags);
return ret; return ret;
} }
EXPORT_SYMBOL(get_random_u32); EXPORT_SYMBOL(get_random_u32);
/* It's important to invalidate all potential batched entropy that might /* It's important to invalidate all potential batched entropy that might
* be stored before the crng is initialized, which we can do lazily by * be stored before the crng is initialized, which we can do lazily by
* simply resetting the counter to zero so that it's re-extracted on the * bumping the generation counter.
* next usage. */ */
static void invalidate_batched_entropy(void) static void invalidate_batched_entropy(void)
{ {
int cpu; atomic_inc(&batch_generation);
unsigned long flags;
for_each_possible_cpu(cpu) {
struct batched_entropy *batched_entropy;
batched_entropy = per_cpu_ptr(&batched_entropy_u32, cpu);
spin_lock_irqsave(&batched_entropy->batch_lock, flags);
batched_entropy->position = 0;
spin_unlock(&batched_entropy->batch_lock);
batched_entropy = per_cpu_ptr(&batched_entropy_u64, cpu);
spin_lock(&batched_entropy->batch_lock);
batched_entropy->position = 0;
spin_unlock_irqrestore(&batched_entropy->batch_lock, flags);
}
} }
/** /**