3e31009898
This commit defers reporting of RCU-preempt quiescent states at rcu_read_unlock_special() time when any of interrupts, softirq, or preemption are disabled. These deferred quiescent states are reported at a later RCU_SOFTIRQ, context switch, idle entry, or CPU-hotplug offline operation. Of course, if another RCU read-side critical section has started in the meantime, the reporting of the quiescent state will be further deferred. This also means that disabling preemption, interrupts, and/or softirqs will act as an RCU-preempt read-side critical section. This is enforced by checking preempt_count() as needed. Some special cases must be handled on an ad-hoc basis, for example, context switch is a quiescent state even though both the scheduler and do_exit() disable preemption. In these cases, additional calls to rcu_preempt_deferred_qs() override the preemption disabling. Similar logic overrides disabled interrupts in rcu_preempt_check_callbacks() because in this case the quiescent state happened just before the corresponding scheduling-clock interrupt. In theory, this change lifts a long-standing restriction that required that if interrupts were disabled across a call to rcu_read_unlock() that the matching rcu_read_lock() also be contained within that interrupts-disabled region of code. Because the reporting of the corresponding RCU-preempt quiescent state is now deferred until after interrupts have been enabled, it is no longer possible for this situation to result in deadlocks involving the scheduler's runqueue and priority-inheritance locks. This may allow some code simplification that might reduce interrupt latency a bit. Unfortunately, in practice this would also defer deboosting a low-priority task that had been subjected to RCU priority boosting, so real-time-response considerations might well force this restriction to remain in place. Because RCU-preempt grace periods are now blocked not only by RCU read-side critical sections, but also by disabling of interrupts, preemption, and softirqs, it will be possible to eliminate RCU-bh and RCU-sched in favor of RCU-preempt in CONFIG_PREEMPT=y kernels. This may require some additional plumbing to provide the network denial-of-service guarantees that have been traditionally provided by RCU-bh. Once these are in place, CONFIG_PREEMPT=n kernels will be able to fold RCU-bh into RCU-sched. This would mean that all kernels would have but one flavor of RCU, which would open the door to significant code cleanup. Moving to a single flavor of RCU would also have the beneficial effect of reducing the NOCB kthreads by at least a factor of two. Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com> [ paulmck: Apply rcu_read_unlock_special() preempt_count() feedback from Joel Fernandes. ] [ paulmck: Adjust rcu_eqs_enter() call to rcu_preempt_deferred_qs() in response to bug reports from kbuild test robot. ] [ paulmck: Fix bug located by kbuild test robot involving recursion via rcu_preempt_deferred_qs(). ]
143 lines
3.8 KiB
C
143 lines
3.8 KiB
C
/*
|
|
* Read-Copy Update mechanism for mutual exclusion, the Bloatwatch edition.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation; either version 2 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program; if not, you can access it online at
|
|
* http://www.gnu.org/licenses/gpl-2.0.html.
|
|
*
|
|
* Copyright IBM Corporation, 2008
|
|
*
|
|
* Author: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
|
|
*
|
|
* For detailed explanation of Read-Copy Update mechanism see -
|
|
* Documentation/RCU
|
|
*/
|
|
#ifndef __LINUX_TINY_H
|
|
#define __LINUX_TINY_H
|
|
|
|
#include <linux/ktime.h>
|
|
|
|
struct rcu_dynticks;
|
|
static inline int rcu_dynticks_snap(struct rcu_dynticks *rdtp)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
/* Never flag non-existent other CPUs! */
|
|
static inline bool rcu_eqs_special_set(int cpu) { return false; }
|
|
|
|
static inline unsigned long get_state_synchronize_rcu(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void cond_synchronize_rcu(unsigned long oldstate)
|
|
{
|
|
might_sleep();
|
|
}
|
|
|
|
static inline unsigned long get_state_synchronize_sched(void)
|
|
{
|
|
return 0;
|
|
}
|
|
|
|
static inline void cond_synchronize_sched(unsigned long oldstate)
|
|
{
|
|
might_sleep();
|
|
}
|
|
|
|
extern void rcu_barrier_bh(void);
|
|
extern void rcu_barrier_sched(void);
|
|
|
|
static inline void synchronize_rcu_expedited(void)
|
|
{
|
|
synchronize_sched(); /* Only one CPU, so pretty fast anyway!!! */
|
|
}
|
|
|
|
static inline void rcu_barrier(void)
|
|
{
|
|
rcu_barrier_sched(); /* Only one CPU, so only one list of callbacks! */
|
|
}
|
|
|
|
static inline void synchronize_rcu_bh(void)
|
|
{
|
|
synchronize_sched();
|
|
}
|
|
|
|
static inline void synchronize_rcu_bh_expedited(void)
|
|
{
|
|
synchronize_sched();
|
|
}
|
|
|
|
static inline void synchronize_sched_expedited(void)
|
|
{
|
|
synchronize_sched();
|
|
}
|
|
|
|
static inline void kfree_call_rcu(struct rcu_head *head,
|
|
rcu_callback_t func)
|
|
{
|
|
call_rcu(head, func);
|
|
}
|
|
|
|
#define rcu_note_context_switch(preempt) \
|
|
do { \
|
|
rcu_sched_qs(); \
|
|
rcu_tasks_qs(current); \
|
|
} while (0)
|
|
|
|
static inline int rcu_needs_cpu(u64 basemono, u64 *nextevt)
|
|
{
|
|
*nextevt = KTIME_MAX;
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* Take advantage of the fact that there is only one CPU, which
|
|
* allows us to ignore virtualization-based context switches.
|
|
*/
|
|
static inline void rcu_virt_note_context_switch(int cpu) { }
|
|
static inline void rcu_cpu_stall_reset(void) { }
|
|
static inline void rcu_idle_enter(void) { }
|
|
static inline void rcu_idle_exit(void) { }
|
|
static inline void rcu_irq_enter(void) { }
|
|
static inline void rcu_irq_exit_irqson(void) { }
|
|
static inline void rcu_irq_enter_irqson(void) { }
|
|
static inline void rcu_irq_exit(void) { }
|
|
static inline void exit_rcu(void) { }
|
|
static inline bool rcu_preempt_need_deferred_qs(struct task_struct *t)
|
|
{
|
|
return false;
|
|
}
|
|
static inline void rcu_preempt_deferred_qs(struct task_struct *t) { }
|
|
#ifdef CONFIG_SRCU
|
|
void rcu_scheduler_starting(void);
|
|
#else /* #ifndef CONFIG_SRCU */
|
|
static inline void rcu_scheduler_starting(void) { }
|
|
#endif /* #else #ifndef CONFIG_SRCU */
|
|
static inline void rcu_end_inkernel_boot(void) { }
|
|
static inline bool rcu_is_watching(void) { return true; }
|
|
|
|
/* Avoid RCU read-side critical sections leaking across. */
|
|
static inline void rcu_all_qs(void) { barrier(); }
|
|
|
|
/* RCUtree hotplug events */
|
|
#define rcutree_prepare_cpu NULL
|
|
#define rcutree_online_cpu NULL
|
|
#define rcutree_offline_cpu NULL
|
|
#define rcutree_dead_cpu NULL
|
|
#define rcutree_dying_cpu NULL
|
|
static inline void rcu_cpu_starting(unsigned int cpu) { }
|
|
|
|
#endif /* __LINUX_RCUTINY_H */
|