|
@@ -34,6 +34,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp)
|
|
|
ssp->srcu_gp_running = false;
|
|
|
ssp->srcu_gp_waiting = false;
|
|
|
ssp->srcu_idx = 0;
|
|
|
+ ssp->srcu_idx_max = 0;
|
|
|
INIT_WORK(&ssp->srcu_work, srcu_drive_gp);
|
|
|
INIT_LIST_HEAD(&ssp->srcu_work.entry);
|
|
|
return 0;
|
|
@@ -84,6 +85,8 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
|
|
|
WARN_ON(ssp->srcu_gp_waiting);
|
|
|
WARN_ON(ssp->srcu_cb_head);
|
|
|
WARN_ON(&ssp->srcu_cb_head != ssp->srcu_cb_tail);
|
|
|
+ WARN_ON(ssp->srcu_idx != ssp->srcu_idx_max);
|
|
|
+ WARN_ON(ssp->srcu_idx & 0x1);
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(cleanup_srcu_struct);
|
|
|
|
|
@@ -114,7 +117,7 @@ void srcu_drive_gp(struct work_struct *wp)
|
|
|
struct srcu_struct *ssp;
|
|
|
|
|
|
ssp = container_of(wp, struct srcu_struct, srcu_work);
|
|
|
- if (ssp->srcu_gp_running || !READ_ONCE(ssp->srcu_cb_head))
|
|
|
+ if (ssp->srcu_gp_running || USHORT_CMP_GE(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
|
|
|
return; /* Already running or nothing to do. */
|
|
|
|
|
|
/* Remove recently arrived callbacks and wait for readers. */
|
|
@@ -124,11 +127,12 @@ void srcu_drive_gp(struct work_struct *wp)
|
|
|
ssp->srcu_cb_head = NULL;
|
|
|
ssp->srcu_cb_tail = &ssp->srcu_cb_head;
|
|
|
local_irq_enable();
|
|
|
- idx = ssp->srcu_idx;
|
|
|
- WRITE_ONCE(ssp->srcu_idx, !ssp->srcu_idx);
|
|
|
+ idx = (ssp->srcu_idx & 0x2) / 2;
|
|
|
+ WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
|
|
|
WRITE_ONCE(ssp->srcu_gp_waiting, true); /* srcu_read_unlock() wakes! */
|
|
|
swait_event_exclusive(ssp->srcu_wq, !READ_ONCE(ssp->srcu_lock_nesting[idx]));
|
|
|
WRITE_ONCE(ssp->srcu_gp_waiting, false); /* srcu_read_unlock() cheap. */
|
|
|
+ WRITE_ONCE(ssp->srcu_idx, ssp->srcu_idx + 1);
|
|
|
|
|
|
/* Invoke the callbacks we removed above. */
|
|
|
while (lh) {
|
|
@@ -146,11 +150,27 @@ void srcu_drive_gp(struct work_struct *wp)
|
|
|
* straighten that out.
|
|
|
*/
|
|
|
WRITE_ONCE(ssp->srcu_gp_running, false);
|
|
|
- if (READ_ONCE(ssp->srcu_cb_head))
|
|
|
+ if (USHORT_CMP_LT(ssp->srcu_idx, READ_ONCE(ssp->srcu_idx_max)))
|
|
|
schedule_work(&ssp->srcu_work);
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(srcu_drive_gp);
|
|
|
|
|
|
+static void srcu_gp_start_if_needed(struct srcu_struct *ssp)
|
|
|
+{
|
|
|
+ unsigned short cookie;
|
|
|
+
|
|
|
+ cookie = get_state_synchronize_srcu(ssp);
|
|
|
+ if (USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx_max), cookie))
|
|
|
+ return;
|
|
|
+ WRITE_ONCE(ssp->srcu_idx_max, cookie);
|
|
|
+ if (!READ_ONCE(ssp->srcu_gp_running)) {
|
|
|
+ if (likely(srcu_init_done))
|
|
|
+ schedule_work(&ssp->srcu_work);
|
|
|
+ else if (list_empty(&ssp->srcu_work.entry))
|
|
|
+ list_add(&ssp->srcu_work.entry, &srcu_boot_list);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* Enqueue an SRCU callback on the specified srcu_struct structure,
|
|
|
* initiating grace-period processing if it is not already running.
|
|
@@ -166,12 +186,7 @@ void call_srcu(struct srcu_struct *ssp, struct rcu_head *rhp,
|
|
|
*ssp->srcu_cb_tail = rhp;
|
|
|
ssp->srcu_cb_tail = &rhp->next;
|
|
|
local_irq_restore(flags);
|
|
|
- if (!READ_ONCE(ssp->srcu_gp_running)) {
|
|
|
- if (likely(srcu_init_done))
|
|
|
- schedule_work(&ssp->srcu_work);
|
|
|
- else if (list_empty(&ssp->srcu_work.entry))
|
|
|
- list_add(&ssp->srcu_work.entry, &srcu_boot_list);
|
|
|
- }
|
|
|
+ srcu_gp_start_if_needed(ssp);
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(call_srcu);
|
|
|
|
|
@@ -190,6 +205,48 @@ void synchronize_srcu(struct srcu_struct *ssp)
|
|
|
}
|
|
|
EXPORT_SYMBOL_GPL(synchronize_srcu);
|
|
|
|
|
|
+/*
|
|
|
+ * get_state_synchronize_srcu - Provide an end-of-grace-period cookie
|
|
|
+ */
|
|
|
+unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp)
|
|
|
+{
|
|
|
+ unsigned long ret;
|
|
|
+
|
|
|
+ barrier();
|
|
|
+ ret = (READ_ONCE(ssp->srcu_idx) + 3) & ~0x1;
|
|
|
+ barrier();
|
|
|
+ return ret & USHRT_MAX;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(get_state_synchronize_srcu);
|
|
|
+
|
|
|
+/*
|
|
|
+ * start_poll_synchronize_srcu - Provide cookie and start grace period
|
|
|
+ *
|
|
|
+ * The difference between this and get_state_synchronize_srcu() is that
|
|
|
+ * this function ensures that the poll_state_synchronize_srcu() will
|
|
|
+ * eventually return the value true.
|
|
|
+ */
|
|
|
+unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp)
|
|
|
+{
|
|
|
+ unsigned long ret = get_state_synchronize_srcu(ssp);
|
|
|
+
|
|
|
+ srcu_gp_start_if_needed(ssp);
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(start_poll_synchronize_srcu);
|
|
|
+
|
|
|
+/*
|
|
|
+ * poll_state_synchronize_srcu - Has cookie's grace period ended?
|
|
|
+ */
|
|
|
+bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie)
|
|
|
+{
|
|
|
+ bool ret = USHORT_CMP_GE(READ_ONCE(ssp->srcu_idx), cookie);
|
|
|
+
|
|
|
+ barrier();
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+EXPORT_SYMBOL_GPL(poll_state_synchronize_srcu);
|
|
|
+
|
|
|
/* Lockdep diagnostics. */
|
|
|
void __init rcu_scheduler_starting(void)
|
|
|
{
|