rcu_segcblist.h 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153
  1. /* SPDX-License-Identifier: GPL-2.0+ */
  2. /*
  3. * RCU segmented callback lists, internal-to-rcu header file
  4. *
  5. * Copyright IBM Corporation, 2017
  6. *
  7. * Authors: Paul E. McKenney <[email protected]>
  8. */
  9. #include <linux/rcu_segcblist.h>
  10. /* Return number of callbacks in the specified callback list. */
  11. static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
  12. {
  13. return READ_ONCE(rclp->len);
  14. }
  15. /* Return number of callbacks in segmented callback list by summing seglen. */
  16. long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp);
  17. void rcu_cblist_init(struct rcu_cblist *rclp);
  18. void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
  19. void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
  20. struct rcu_cblist *srclp,
  21. struct rcu_head *rhp);
  22. struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
  23. /*
  24. * Is the specified rcu_segcblist structure empty?
  25. *
  26. * But careful! The fact that the ->head field is NULL does not
  27. * necessarily imply that there are no callbacks associated with
  28. * this structure. When callbacks are being invoked, they are
  29. * removed as a group. If callback invocation must be preempted,
  30. * the remaining callbacks will be added back to the list. Either
  31. * way, the counts are updated later.
  32. *
  33. * So it is often the case that rcu_segcblist_n_cbs() should be used
  34. * instead.
  35. */
  36. static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
  37. {
  38. return !READ_ONCE(rsclp->head);
  39. }
  40. /* Return number of callbacks in segmented callback list. */
  41. static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
  42. {
  43. #ifdef CONFIG_RCU_NOCB_CPU
  44. return atomic_long_read(&rsclp->len);
  45. #else
  46. return READ_ONCE(rsclp->len);
  47. #endif
  48. }
  49. static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp,
  50. int flags)
  51. {
  52. WRITE_ONCE(rsclp->flags, rsclp->flags | flags);
  53. }
  54. static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp,
  55. int flags)
  56. {
  57. WRITE_ONCE(rsclp->flags, rsclp->flags & ~flags);
  58. }
  59. static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp,
  60. int flags)
  61. {
  62. return READ_ONCE(rsclp->flags) & flags;
  63. }
  64. /*
  65. * Is the specified rcu_segcblist enabled, for example, not corresponding
  66. * to an offline CPU?
  67. */
  68. static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
  69. {
  70. return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED);
  71. }
  72. /*
  73. * Is the specified rcu_segcblist NOCB offloaded (or in the middle of the
  74. * [de]offloading process)?
  75. */
  76. static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
  77. {
  78. if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  79. rcu_segcblist_test_flags(rsclp, SEGCBLIST_LOCKING))
  80. return true;
  81. return false;
  82. }
  83. static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp)
  84. {
  85. if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
  86. !rcu_segcblist_test_flags(rsclp, SEGCBLIST_RCU_CORE))
  87. return true;
  88. return false;
  89. }
  90. /*
  91. * Are all segments following the specified segment of the specified
  92. * rcu_segcblist structure empty of callbacks? (The specified
  93. * segment might well contain callbacks.)
  94. */
  95. static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
  96. {
  97. return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
  98. }
  99. /*
  100. * Is the specified segment of the specified rcu_segcblist structure
  101. * empty of callbacks?
  102. */
  103. static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg)
  104. {
  105. if (seg == RCU_DONE_TAIL)
  106. return &rsclp->head == rsclp->tails[RCU_DONE_TAIL];
  107. return rsclp->tails[seg - 1] == rsclp->tails[seg];
  108. }
  109. void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
  110. void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v);
  111. void rcu_segcblist_init(struct rcu_segcblist *rsclp);
  112. void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
  113. void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload);
  114. bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
  115. bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
  116. struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
  117. struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
  118. bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
  119. void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
  120. struct rcu_head *rhp);
  121. bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
  122. struct rcu_head *rhp);
  123. void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
  124. struct rcu_cblist *rclp);
  125. void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
  126. struct rcu_cblist *rclp);
  127. void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
  128. struct rcu_cblist *rclp);
  129. void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
  130. struct rcu_cblist *rclp);
  131. void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
  132. struct rcu_cblist *rclp);
  133. void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
  134. bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
  135. void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
  136. struct rcu_segcblist *src_rsclp);