completion.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Generic wait-for-completion handler;
  4. *
  5. * It differs from semaphores in that their default case is the opposite,
  6. * wait_for_completion default blocks whereas semaphore default non-block. The
  7. * interface also makes it easy to 'complete' multiple waiting threads,
  8. * something which isn't entirely natural for semaphores.
  9. *
  10. * But more importantly, the primitive documents the usage. Semaphores would
  11. * typically be used for exclusion which gives rise to priority inversion.
  12. * Waiting for completion is a typically sync point, but not an exclusion point.
  13. */
  14. /**
  15. * complete: - signals a single thread waiting on this completion
  16. * @x: holds the state of this particular completion
  17. *
  18. * This will wake up a single thread waiting on this completion. Threads will be
  19. * awakened in the same order in which they were queued.
  20. *
  21. * See also complete_all(), wait_for_completion() and related routines.
  22. *
  23. * If this function wakes up a task, it executes a full memory barrier before
  24. * accessing the task state.
  25. */
  26. void complete(struct completion *x)
  27. {
  28. unsigned long flags;
  29. raw_spin_lock_irqsave(&x->wait.lock, flags);
  30. if (x->done != UINT_MAX)
  31. x->done++;
  32. swake_up_locked(&x->wait);
  33. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  34. }
  35. EXPORT_SYMBOL(complete);
  36. /**
  37. * complete_all: - signals all threads waiting on this completion
  38. * @x: holds the state of this particular completion
  39. *
  40. * This will wake up all threads waiting on this particular completion event.
  41. *
  42. * If this function wakes up a task, it executes a full memory barrier before
  43. * accessing the task state.
  44. *
  45. * Since complete_all() sets the completion of @x permanently to done
  46. * to allow multiple waiters to finish, a call to reinit_completion()
  47. * must be used on @x if @x is to be used again. The code must make
  48. * sure that all waiters have woken and finished before reinitializing
  49. * @x. Also note that the function completion_done() can not be used
  50. * to know if there are still waiters after complete_all() has been called.
  51. */
  52. void complete_all(struct completion *x)
  53. {
  54. unsigned long flags;
  55. lockdep_assert_RT_in_threaded_ctx();
  56. raw_spin_lock_irqsave(&x->wait.lock, flags);
  57. x->done = UINT_MAX;
  58. swake_up_all_locked(&x->wait);
  59. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  60. }
  61. EXPORT_SYMBOL(complete_all);
  62. static inline long __sched
  63. do_wait_for_common(struct completion *x,
  64. long (*action)(long), long timeout, int state)
  65. {
  66. if (!x->done) {
  67. DECLARE_SWAITQUEUE(wait);
  68. do {
  69. if (signal_pending_state(state, current)) {
  70. timeout = -ERESTARTSYS;
  71. break;
  72. }
  73. __prepare_to_swait(&x->wait, &wait);
  74. __set_current_state(state);
  75. raw_spin_unlock_irq(&x->wait.lock);
  76. timeout = action(timeout);
  77. raw_spin_lock_irq(&x->wait.lock);
  78. } while (!x->done && timeout);
  79. __finish_swait(&x->wait, &wait);
  80. if (!x->done)
  81. return timeout;
  82. }
  83. if (x->done != UINT_MAX)
  84. x->done--;
  85. return timeout ?: 1;
  86. }
  87. static inline long __sched
  88. __wait_for_common(struct completion *x,
  89. long (*action)(long), long timeout, int state)
  90. {
  91. might_sleep();
  92. complete_acquire(x);
  93. raw_spin_lock_irq(&x->wait.lock);
  94. timeout = do_wait_for_common(x, action, timeout, state);
  95. raw_spin_unlock_irq(&x->wait.lock);
  96. complete_release(x);
  97. return timeout;
  98. }
  99. static long __sched
  100. wait_for_common(struct completion *x, long timeout, int state)
  101. {
  102. return __wait_for_common(x, schedule_timeout, timeout, state);
  103. }
  104. static long __sched
  105. wait_for_common_io(struct completion *x, long timeout, int state)
  106. {
  107. return __wait_for_common(x, io_schedule_timeout, timeout, state);
  108. }
  109. /**
  110. * wait_for_completion: - waits for completion of a task
  111. * @x: holds the state of this particular completion
  112. *
  113. * This waits to be signaled for completion of a specific task. It is NOT
  114. * interruptible and there is no timeout.
  115. *
  116. * See also similar routines (i.e. wait_for_completion_timeout()) with timeout
  117. * and interrupt capability. Also see complete().
  118. */
  119. void __sched wait_for_completion(struct completion *x)
  120. {
  121. wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  122. }
  123. EXPORT_SYMBOL(wait_for_completion);
  124. /**
  125. * wait_for_completion_timeout: - waits for completion of a task (w/timeout)
  126. * @x: holds the state of this particular completion
  127. * @timeout: timeout value in jiffies
  128. *
  129. * This waits for either a completion of a specific task to be signaled or for a
  130. * specified timeout to expire. The timeout is in jiffies. It is not
  131. * interruptible.
  132. *
  133. * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
  134. * till timeout) if completed.
  135. */
  136. unsigned long __sched
  137. wait_for_completion_timeout(struct completion *x, unsigned long timeout)
  138. {
  139. return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE);
  140. }
  141. EXPORT_SYMBOL(wait_for_completion_timeout);
  142. /**
  143. * wait_for_completion_io: - waits for completion of a task
  144. * @x: holds the state of this particular completion
  145. *
  146. * This waits to be signaled for completion of a specific task. It is NOT
  147. * interruptible and there is no timeout. The caller is accounted as waiting
  148. * for IO (which traditionally means blkio only).
  149. */
  150. void __sched wait_for_completion_io(struct completion *x)
  151. {
  152. wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE);
  153. }
  154. EXPORT_SYMBOL(wait_for_completion_io);
  155. /**
  156. * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout)
  157. * @x: holds the state of this particular completion
  158. * @timeout: timeout value in jiffies
  159. *
  160. * This waits for either a completion of a specific task to be signaled or for a
  161. * specified timeout to expire. The timeout is in jiffies. It is not
  162. * interruptible. The caller is accounted as waiting for IO (which traditionally
  163. * means blkio only).
  164. *
  165. * Return: 0 if timed out, and positive (at least 1, or number of jiffies left
  166. * till timeout) if completed.
  167. */
  168. unsigned long __sched
  169. wait_for_completion_io_timeout(struct completion *x, unsigned long timeout)
  170. {
  171. return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE);
  172. }
  173. EXPORT_SYMBOL(wait_for_completion_io_timeout);
  174. /**
  175. * wait_for_completion_interruptible: - waits for completion of a task (w/intr)
  176. * @x: holds the state of this particular completion
  177. *
  178. * This waits for completion of a specific task to be signaled. It is
  179. * interruptible.
  180. *
  181. * Return: -ERESTARTSYS if interrupted, 0 if completed.
  182. */
  183. int __sched wait_for_completion_interruptible(struct completion *x)
  184. {
  185. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE);
  186. if (t == -ERESTARTSYS)
  187. return t;
  188. return 0;
  189. }
  190. EXPORT_SYMBOL(wait_for_completion_interruptible);
  191. /**
  192. * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr))
  193. * @x: holds the state of this particular completion
  194. * @timeout: timeout value in jiffies
  195. *
  196. * This waits for either a completion of a specific task to be signaled or for a
  197. * specified timeout to expire. It is interruptible. The timeout is in jiffies.
  198. *
  199. * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
  200. * or number of jiffies left till timeout) if completed.
  201. */
  202. long __sched
  203. wait_for_completion_interruptible_timeout(struct completion *x,
  204. unsigned long timeout)
  205. {
  206. return wait_for_common(x, timeout, TASK_INTERRUPTIBLE);
  207. }
  208. EXPORT_SYMBOL(wait_for_completion_interruptible_timeout);
  209. /**
  210. * wait_for_completion_killable: - waits for completion of a task (killable)
  211. * @x: holds the state of this particular completion
  212. *
  213. * This waits to be signaled for completion of a specific task. It can be
  214. * interrupted by a kill signal.
  215. *
  216. * Return: -ERESTARTSYS if interrupted, 0 if completed.
  217. */
  218. int __sched wait_for_completion_killable(struct completion *x)
  219. {
  220. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE);
  221. if (t == -ERESTARTSYS)
  222. return t;
  223. return 0;
  224. }
  225. EXPORT_SYMBOL(wait_for_completion_killable);
  226. int __sched wait_for_completion_state(struct completion *x, unsigned int state)
  227. {
  228. long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state);
  229. if (t == -ERESTARTSYS)
  230. return t;
  231. return 0;
  232. }
  233. EXPORT_SYMBOL(wait_for_completion_state);
  234. /**
  235. * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable))
  236. * @x: holds the state of this particular completion
  237. * @timeout: timeout value in jiffies
  238. *
  239. * This waits for either a completion of a specific task to be
  240. * signaled or for a specified timeout to expire. It can be
  241. * interrupted by a kill signal. The timeout is in jiffies.
  242. *
  243. * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1,
  244. * or number of jiffies left till timeout) if completed.
  245. */
  246. long __sched
  247. wait_for_completion_killable_timeout(struct completion *x,
  248. unsigned long timeout)
  249. {
  250. return wait_for_common(x, timeout, TASK_KILLABLE);
  251. }
  252. EXPORT_SYMBOL(wait_for_completion_killable_timeout);
  253. /**
  254. * try_wait_for_completion - try to decrement a completion without blocking
  255. * @x: completion structure
  256. *
  257. * Return: 0 if a decrement cannot be done without blocking
  258. * 1 if a decrement succeeded.
  259. *
  260. * If a completion is being used as a counting completion,
  261. * attempt to decrement the counter without blocking. This
  262. * enables us to avoid waiting if the resource the completion
  263. * is protecting is not available.
  264. */
  265. bool try_wait_for_completion(struct completion *x)
  266. {
  267. unsigned long flags;
  268. bool ret = true;
  269. /*
  270. * Since x->done will need to be locked only
  271. * in the non-blocking case, we check x->done
  272. * first without taking the lock so we can
  273. * return early in the blocking case.
  274. */
  275. if (!READ_ONCE(x->done))
  276. return false;
  277. raw_spin_lock_irqsave(&x->wait.lock, flags);
  278. if (!x->done)
  279. ret = false;
  280. else if (x->done != UINT_MAX)
  281. x->done--;
  282. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  283. return ret;
  284. }
  285. EXPORT_SYMBOL(try_wait_for_completion);
  286. /**
  287. * completion_done - Test to see if a completion has any waiters
  288. * @x: completion structure
  289. *
  290. * Return: 0 if there are waiters (wait_for_completion() in progress)
  291. * 1 if there are no waiters.
  292. *
  293. * Note, this will always return true if complete_all() was called on @X.
  294. */
  295. bool completion_done(struct completion *x)
  296. {
  297. unsigned long flags;
  298. if (!READ_ONCE(x->done))
  299. return false;
  300. /*
  301. * If ->done, we need to wait for complete() to release ->wait.lock
  302. * otherwise we can end up freeing the completion before complete()
  303. * is done referencing it.
  304. */
  305. raw_spin_lock_irqsave(&x->wait.lock, flags);
  306. raw_spin_unlock_irqrestore(&x->wait.lock, flags);
  307. return true;
  308. }
  309. EXPORT_SYMBOL(completion_done);