|
@@ -5967,25 +5967,27 @@ static struct pending_free *get_pending_free(void)
|
|
|
static void free_zapped_rcu(struct rcu_head *cb);
|
|
|
|
|
|
/*
|
|
|
- * Schedule an RCU callback if no RCU callback is pending. Must be called with
|
|
|
- * the graph lock held.
|
|
|
- */
|
|
|
-static void call_rcu_zapped(struct pending_free *pf)
|
|
|
+* See if we need to queue an RCU callback, must called with
|
|
|
+* the lockdep lock held, returns false if either we don't have
|
|
|
+* any pending free or the callback is already scheduled.
|
|
|
+* Otherwise, a call_rcu() must follow this function call.
|
|
|
+*/
|
|
|
+static bool prepare_call_rcu_zapped(struct pending_free *pf)
|
|
|
{
|
|
|
WARN_ON_ONCE(inside_selftest());
|
|
|
|
|
|
if (list_empty(&pf->zapped))
|
|
|
- return;
|
|
|
+ return false;
|
|
|
|
|
|
if (delayed_free.scheduled)
|
|
|
- return;
|
|
|
+ return false;
|
|
|
|
|
|
delayed_free.scheduled = true;
|
|
|
|
|
|
WARN_ON_ONCE(delayed_free.pf + delayed_free.index != pf);
|
|
|
delayed_free.index ^= 1;
|
|
|
|
|
|
- call_rcu(&delayed_free.rcu_head, free_zapped_rcu);
|
|
|
+ return true;
|
|
|
}
|
|
|
|
|
|
/* The caller must hold the graph lock. May be called from RCU context. */
|
|
@@ -6011,6 +6013,7 @@ static void free_zapped_rcu(struct rcu_head *ch)
|
|
|
{
|
|
|
struct pending_free *pf;
|
|
|
unsigned long flags;
|
|
|
+ bool need_callback;
|
|
|
|
|
|
if (WARN_ON_ONCE(ch != &delayed_free.rcu_head))
|
|
|
return;
|
|
@@ -6022,14 +6025,18 @@ static void free_zapped_rcu(struct rcu_head *ch)
|
|
|
pf = delayed_free.pf + (delayed_free.index ^ 1);
|
|
|
__free_zapped_classes(pf);
|
|
|
delayed_free.scheduled = false;
|
|
|
+ need_callback =
|
|
|
+ prepare_call_rcu_zapped(delayed_free.pf + delayed_free.index);
|
|
|
+ lockdep_unlock();
|
|
|
+ raw_local_irq_restore(flags);
|
|
|
|
|
|
/*
|
|
|
- * If there's anything on the open list, close and start a new callback.
|
|
|
- */
|
|
|
- call_rcu_zapped(delayed_free.pf + delayed_free.index);
|
|
|
+ * If there's pending free and its callback has not been scheduled,
|
|
|
+ * queue an RCU callback.
|
|
|
+ */
|
|
|
+ if (need_callback)
|
|
|
+ call_rcu(&delayed_free.rcu_head, free_zapped_rcu);
|
|
|
|
|
|
- lockdep_unlock();
|
|
|
- raw_local_irq_restore(flags);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -6069,6 +6076,7 @@ static void lockdep_free_key_range_reg(void *start, unsigned long size)
|
|
|
{
|
|
|
struct pending_free *pf;
|
|
|
unsigned long flags;
|
|
|
+ bool need_callback;
|
|
|
|
|
|
init_data_structures_once();
|
|
|
|
|
@@ -6076,10 +6084,11 @@ static void lockdep_free_key_range_reg(void *start, unsigned long size)
|
|
|
lockdep_lock();
|
|
|
pf = get_pending_free();
|
|
|
__lockdep_free_key_range(pf, start, size);
|
|
|
- call_rcu_zapped(pf);
|
|
|
+ need_callback = prepare_call_rcu_zapped(pf);
|
|
|
lockdep_unlock();
|
|
|
raw_local_irq_restore(flags);
|
|
|
-
|
|
|
+ if (need_callback)
|
|
|
+ call_rcu(&delayed_free.rcu_head, free_zapped_rcu);
|
|
|
/*
|
|
|
* Wait for any possible iterators from look_up_lock_class() to pass
|
|
|
* before continuing to free the memory they refer to.
|
|
@@ -6173,6 +6182,7 @@ static void lockdep_reset_lock_reg(struct lockdep_map *lock)
|
|
|
struct pending_free *pf;
|
|
|
unsigned long flags;
|
|
|
int locked;
|
|
|
+ bool need_callback = false;
|
|
|
|
|
|
raw_local_irq_save(flags);
|
|
|
locked = graph_lock();
|
|
@@ -6181,11 +6191,13 @@ static void lockdep_reset_lock_reg(struct lockdep_map *lock)
|
|
|
|
|
|
pf = get_pending_free();
|
|
|
__lockdep_reset_lock(pf, lock);
|
|
|
- call_rcu_zapped(pf);
|
|
|
+ need_callback = prepare_call_rcu_zapped(pf);
|
|
|
|
|
|
graph_unlock();
|
|
|
out_irq:
|
|
|
raw_local_irq_restore(flags);
|
|
|
+ if (need_callback)
|
|
|
+ call_rcu(&delayed_free.rcu_head, free_zapped_rcu);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -6229,6 +6241,7 @@ void lockdep_unregister_key(struct lock_class_key *key)
|
|
|
struct pending_free *pf;
|
|
|
unsigned long flags;
|
|
|
bool found = false;
|
|
|
+ bool need_callback = false;
|
|
|
|
|
|
might_sleep();
|
|
|
|
|
@@ -6249,11 +6262,14 @@ void lockdep_unregister_key(struct lock_class_key *key)
|
|
|
if (found) {
|
|
|
pf = get_pending_free();
|
|
|
__lockdep_free_key_range(pf, key, 1);
|
|
|
- call_rcu_zapped(pf);
|
|
|
+ need_callback = prepare_call_rcu_zapped(pf);
|
|
|
}
|
|
|
lockdep_unlock();
|
|
|
raw_local_irq_restore(flags);
|
|
|
|
|
|
+ if (need_callback)
|
|
|
+ call_rcu(&delayed_free.rcu_head, free_zapped_rcu);
|
|
|
+
|
|
|
/* Wait until is_dynamic_key() has finished accessing k->hash_entry. */
|
|
|
synchronize_rcu();
|
|
|
}
|