waitwake.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. #include <linux/sched/task.h>
  3. #include <linux/sched/signal.h>
  4. #include <linux/freezer.h>
  5. #include "futex.h"
  6. #include <trace/hooks/futex.h>
  7. /*
  8. * READ this before attempting to hack on futexes!
  9. *
  10. * Basic futex operation and ordering guarantees
  11. * =============================================
  12. *
  13. * The waiter reads the futex value in user space and calls
  14. * futex_wait(). This function computes the hash bucket and acquires
  15. * the hash bucket lock. After that it reads the futex user space value
  16. * again and verifies that the data has not changed. If it has not changed
  17. * it enqueues itself into the hash bucket, releases the hash bucket lock
  18. * and schedules.
  19. *
  20. * The waker side modifies the user space value of the futex and calls
  21. * futex_wake(). This function computes the hash bucket and acquires the
  22. * hash bucket lock. Then it looks for waiters on that futex in the hash
  23. * bucket and wakes them.
  24. *
  25. * In futex wake up scenarios where no tasks are blocked on a futex, taking
  26. * the hb spinlock can be avoided and simply return. In order for this
  27. * optimization to work, ordering guarantees must exist so that the waiter
  28. * being added to the list is acknowledged when the list is concurrently being
  29. * checked by the waker, avoiding scenarios like the following:
  30. *
  31. * CPU 0 CPU 1
  32. * val = *futex;
  33. * sys_futex(WAIT, futex, val);
  34. * futex_wait(futex, val);
  35. * uval = *futex;
  36. * *futex = newval;
  37. * sys_futex(WAKE, futex);
  38. * futex_wake(futex);
  39. * if (queue_empty())
  40. * return;
  41. * if (uval == val)
  42. * lock(hash_bucket(futex));
  43. * queue();
  44. * unlock(hash_bucket(futex));
  45. * schedule();
  46. *
  47. * This would cause the waiter on CPU 0 to wait forever because it
  48. * missed the transition of the user space value from val to newval
  49. * and the waker did not find the waiter in the hash bucket queue.
  50. *
  51. * The correct serialization ensures that a waiter either observes
  52. * the changed user space value before blocking or is woken by a
  53. * concurrent waker:
  54. *
  55. * CPU 0 CPU 1
  56. * val = *futex;
  57. * sys_futex(WAIT, futex, val);
  58. * futex_wait(futex, val);
  59. *
  60. * waiters++; (a)
  61. * smp_mb(); (A) <-- paired with -.
  62. * |
  63. * lock(hash_bucket(futex)); |
  64. * |
  65. * uval = *futex; |
  66. * | *futex = newval;
  67. * | sys_futex(WAKE, futex);
  68. * | futex_wake(futex);
  69. * |
  70. * `--------> smp_mb(); (B)
  71. * if (uval == val)
  72. * queue();
  73. * unlock(hash_bucket(futex));
  74. * schedule(); if (waiters)
  75. * lock(hash_bucket(futex));
  76. * else wake_waiters(futex);
  77. * waiters--; (b) unlock(hash_bucket(futex));
  78. *
  79. * Where (A) orders the waiters increment and the futex value read through
  80. * atomic operations (see futex_hb_waiters_inc) and where (B) orders the write
  81. * to futex and the waiters read (see futex_hb_waiters_pending()).
  82. *
  83. * This yields the following case (where X:=waiters, Y:=futex):
  84. *
  85. * X = Y = 0
  86. *
  87. * w[X]=1 w[Y]=1
  88. * MB MB
  89. * r[Y]=y r[X]=x
  90. *
  91. * Which guarantees that x==0 && y==0 is impossible; which translates back into
  92. * the guarantee that we cannot both miss the futex variable change and the
  93. * enqueue.
  94. *
  95. * Note that a new waiter is accounted for in (a) even when it is possible that
  96. * the wait call can return error, in which case we backtrack from it in (b).
  97. * Refer to the comment in futex_q_lock().
  98. *
  99. * Similarly, in order to account for waiters being requeued on another
  100. * address we always increment the waiters for the destination bucket before
  101. * acquiring the lock. It then decrements them again after releasing it -
  102. * the code that actually moves the futex(es) between hash buckets (requeue_futex)
  103. * will do the additional required waiter count housekeeping. This is done for
  104. * double_lock_hb() and double_unlock_hb(), respectively.
  105. */
  106. /*
  107. * The hash bucket lock must be held when this is called.
  108. * Afterwards, the futex_q must not be accessed. Callers
  109. * must ensure to later call wake_up_q() for the actual
  110. * wakeups to occur.
  111. */
  112. void futex_wake_mark(struct wake_q_head *wake_q, struct futex_q *q)
  113. {
  114. struct task_struct *p = q->task;
  115. if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
  116. return;
  117. get_task_struct(p);
  118. __futex_unqueue(q);
  119. /*
  120. * The waiting task can free the futex_q as soon as q->lock_ptr = NULL
  121. * is written, without taking any locks. This is possible in the event
  122. * of a spurious wakeup, for example. A memory barrier is required here
  123. * to prevent the following store to lock_ptr from getting ahead of the
  124. * plist_del in __futex_unqueue().
  125. */
  126. smp_store_release(&q->lock_ptr, NULL);
  127. /*
  128. * Queue the task for later wakeup for after we've released
  129. * the hb->lock.
  130. */
  131. wake_q_add_safe(wake_q, p);
  132. }
  133. /*
  134. * Wake up waiters matching bitset queued on this futex (uaddr).
  135. */
  136. int futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
  137. {
  138. struct futex_hash_bucket *hb;
  139. struct futex_q *this, *next;
  140. union futex_key key = FUTEX_KEY_INIT;
  141. int ret;
  142. int target_nr;
  143. DEFINE_WAKE_Q(wake_q);
  144. if (!bitset)
  145. return -EINVAL;
  146. ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
  147. if (unlikely(ret != 0))
  148. return ret;
  149. hb = futex_hash(&key);
  150. /* Make sure we really have tasks to wakeup */
  151. if (!futex_hb_waiters_pending(hb))
  152. return ret;
  153. spin_lock(&hb->lock);
  154. trace_android_vh_futex_wake_traverse_plist(&hb->chain, &target_nr, key, bitset);
  155. plist_for_each_entry_safe(this, next, &hb->chain, list) {
  156. if (futex_match (&this->key, &key)) {
  157. if (this->pi_state || this->rt_waiter) {
  158. ret = -EINVAL;
  159. break;
  160. }
  161. /* Check if one of the bits is set in both bitsets */
  162. if (!(this->bitset & bitset))
  163. continue;
  164. trace_android_vh_futex_wake_this(ret, nr_wake, target_nr, this->task);
  165. futex_wake_mark(&wake_q, this);
  166. if (++ret >= nr_wake)
  167. break;
  168. }
  169. }
  170. spin_unlock(&hb->lock);
  171. wake_up_q(&wake_q);
  172. trace_android_vh_futex_wake_up_q_finish(nr_wake, target_nr);
  173. return ret;
  174. }
  175. static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)
  176. {
  177. unsigned int op = (encoded_op & 0x70000000) >> 28;
  178. unsigned int cmp = (encoded_op & 0x0f000000) >> 24;
  179. int oparg = sign_extend32((encoded_op & 0x00fff000) >> 12, 11);
  180. int cmparg = sign_extend32(encoded_op & 0x00000fff, 11);
  181. int oldval, ret;
  182. if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) {
  183. if (oparg < 0 || oparg > 31) {
  184. char comm[sizeof(current->comm)];
  185. /*
  186. * kill this print and return -EINVAL when userspace
  187. * is sane again
  188. */
  189. pr_info_ratelimited("futex_wake_op: %s tries to shift op by %d; fix this program\n",
  190. get_task_comm(comm, current), oparg);
  191. oparg &= 31;
  192. }
  193. oparg = 1 << oparg;
  194. }
  195. pagefault_disable();
  196. ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
  197. pagefault_enable();
  198. if (ret)
  199. return ret;
  200. switch (cmp) {
  201. case FUTEX_OP_CMP_EQ:
  202. return oldval == cmparg;
  203. case FUTEX_OP_CMP_NE:
  204. return oldval != cmparg;
  205. case FUTEX_OP_CMP_LT:
  206. return oldval < cmparg;
  207. case FUTEX_OP_CMP_GE:
  208. return oldval >= cmparg;
  209. case FUTEX_OP_CMP_LE:
  210. return oldval <= cmparg;
  211. case FUTEX_OP_CMP_GT:
  212. return oldval > cmparg;
  213. default:
  214. return -ENOSYS;
  215. }
  216. }
  217. /*
  218. * Wake up all waiters hashed on the physical page that is mapped
  219. * to this virtual address:
  220. */
  221. int futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
  222. int nr_wake, int nr_wake2, int op)
  223. {
  224. union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
  225. struct futex_hash_bucket *hb1, *hb2;
  226. struct futex_q *this, *next;
  227. int ret, op_ret;
  228. DEFINE_WAKE_Q(wake_q);
  229. retry:
  230. ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
  231. if (unlikely(ret != 0))
  232. return ret;
  233. ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
  234. if (unlikely(ret != 0))
  235. return ret;
  236. hb1 = futex_hash(&key1);
  237. hb2 = futex_hash(&key2);
  238. retry_private:
  239. double_lock_hb(hb1, hb2);
  240. op_ret = futex_atomic_op_inuser(op, uaddr2);
  241. if (unlikely(op_ret < 0)) {
  242. double_unlock_hb(hb1, hb2);
  243. if (!IS_ENABLED(CONFIG_MMU) ||
  244. unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
  245. /*
  246. * we don't get EFAULT from MMU faults if we don't have
  247. * an MMU, but we might get them from range checking
  248. */
  249. ret = op_ret;
  250. return ret;
  251. }
  252. if (op_ret == -EFAULT) {
  253. ret = fault_in_user_writeable(uaddr2);
  254. if (ret)
  255. return ret;
  256. }
  257. cond_resched();
  258. if (!(flags & FLAGS_SHARED))
  259. goto retry_private;
  260. goto retry;
  261. }
  262. plist_for_each_entry_safe(this, next, &hb1->chain, list) {
  263. if (futex_match (&this->key, &key1)) {
  264. if (this->pi_state || this->rt_waiter) {
  265. ret = -EINVAL;
  266. goto out_unlock;
  267. }
  268. futex_wake_mark(&wake_q, this);
  269. if (++ret >= nr_wake)
  270. break;
  271. }
  272. }
  273. if (op_ret > 0) {
  274. op_ret = 0;
  275. plist_for_each_entry_safe(this, next, &hb2->chain, list) {
  276. if (futex_match (&this->key, &key2)) {
  277. if (this->pi_state || this->rt_waiter) {
  278. ret = -EINVAL;
  279. goto out_unlock;
  280. }
  281. futex_wake_mark(&wake_q, this);
  282. if (++op_ret >= nr_wake2)
  283. break;
  284. }
  285. }
  286. ret += op_ret;
  287. }
  288. out_unlock:
  289. double_unlock_hb(hb1, hb2);
  290. wake_up_q(&wake_q);
  291. return ret;
  292. }
  293. static long futex_wait_restart(struct restart_block *restart);
  294. /**
  295. * futex_wait_queue() - futex_queue() and wait for wakeup, timeout, or signal
  296. * @hb: the futex hash bucket, must be locked by the caller
  297. * @q: the futex_q to queue up on
  298. * @timeout: the prepared hrtimer_sleeper, or null for no timeout
  299. */
  300. void futex_wait_queue(struct futex_hash_bucket *hb, struct futex_q *q,
  301. struct hrtimer_sleeper *timeout)
  302. {
  303. /*
  304. * The task state is guaranteed to be set before another task can
  305. * wake it. set_current_state() is implemented using smp_store_mb() and
  306. * futex_queue() calls spin_unlock() upon completion, both serializing
  307. * access to the hash list and forcing another memory barrier.
  308. */
  309. set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
  310. futex_queue(q, hb);
  311. /* Arm the timer */
  312. if (timeout)
  313. hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
  314. /*
  315. * If we have been removed from the hash list, then another task
  316. * has tried to wake us, and we can skip the call to schedule().
  317. */
  318. if (likely(!plist_node_empty(&q->list))) {
  319. /*
  320. * If the timer has already expired, current will already be
  321. * flagged for rescheduling. Only call schedule if there
  322. * is no timeout, or if it has yet to expire.
  323. */
  324. if (!timeout || timeout->task) {
  325. trace_android_vh_futex_sleep_start(current);
  326. schedule();
  327. }
  328. }
  329. __set_current_state(TASK_RUNNING);
  330. }
  331. /**
  332. * unqueue_multiple - Remove various futexes from their hash bucket
  333. * @v: The list of futexes to unqueue
  334. * @count: Number of futexes in the list
  335. *
  336. * Helper to unqueue a list of futexes. This can't fail.
  337. *
  338. * Return:
  339. * - >=0 - Index of the last futex that was awoken;
  340. * - -1 - No futex was awoken
  341. */
  342. static int unqueue_multiple(struct futex_vector *v, int count)
  343. {
  344. int ret = -1, i;
  345. for (i = 0; i < count; i++) {
  346. if (!futex_unqueue(&v[i].q))
  347. ret = i;
  348. }
  349. return ret;
  350. }
  351. /**
  352. * futex_wait_multiple_setup - Prepare to wait and enqueue multiple futexes
  353. * @vs: The futex list to wait on
  354. * @count: The size of the list
  355. * @woken: Index of the last woken futex, if any. Used to notify the
  356. * caller that it can return this index to userspace (return parameter)
  357. *
  358. * Prepare multiple futexes in a single step and enqueue them. This may fail if
  359. * the futex list is invalid or if any futex was already awoken. On success the
  360. * task is ready to interruptible sleep.
  361. *
  362. * Return:
  363. * - 1 - One of the futexes was woken by another thread
  364. * - 0 - Success
  365. * - <0 - -EFAULT, -EWOULDBLOCK or -EINVAL
  366. */
  367. static int futex_wait_multiple_setup(struct futex_vector *vs, int count, int *woken)
  368. {
  369. struct futex_hash_bucket *hb;
  370. bool retry = false;
  371. int ret, i;
  372. u32 uval;
  373. /*
  374. * Enqueuing multiple futexes is tricky, because we need to enqueue
  375. * each futex on the list before dealing with the next one to avoid
  376. * deadlocking on the hash bucket. But, before enqueuing, we need to
  377. * make sure that current->state is TASK_INTERRUPTIBLE, so we don't
  378. * lose any wake events, which cannot be done before the get_futex_key
  379. * of the next key, because it calls get_user_pages, which can sleep.
  380. * Thus, we fetch the list of futexes keys in two steps, by first
  381. * pinning all the memory keys in the futex key, and only then we read
  382. * each key and queue the corresponding futex.
  383. *
  384. * Private futexes doesn't need to recalculate hash in retry, so skip
  385. * get_futex_key() when retrying.
  386. */
  387. retry:
  388. for (i = 0; i < count; i++) {
  389. if ((vs[i].w.flags & FUTEX_PRIVATE_FLAG) && retry)
  390. continue;
  391. ret = get_futex_key(u64_to_user_ptr(vs[i].w.uaddr),
  392. !(vs[i].w.flags & FUTEX_PRIVATE_FLAG),
  393. &vs[i].q.key, FUTEX_READ);
  394. if (unlikely(ret))
  395. return ret;
  396. }
  397. set_current_state(TASK_INTERRUPTIBLE|TASK_FREEZABLE);
  398. for (i = 0; i < count; i++) {
  399. u32 __user *uaddr = (u32 __user *)(unsigned long)vs[i].w.uaddr;
  400. struct futex_q *q = &vs[i].q;
  401. u32 val = (u32)vs[i].w.val;
  402. hb = futex_q_lock(q);
  403. ret = futex_get_value_locked(&uval, uaddr);
  404. if (!ret && uval == val) {
  405. /*
  406. * The bucket lock can't be held while dealing with the
  407. * next futex. Queue each futex at this moment so hb can
  408. * be unlocked.
  409. */
  410. futex_queue(q, hb);
  411. continue;
  412. }
  413. futex_q_unlock(hb);
  414. __set_current_state(TASK_RUNNING);
  415. /*
  416. * Even if something went wrong, if we find out that a futex
  417. * was woken, we don't return error and return this index to
  418. * userspace
  419. */
  420. *woken = unqueue_multiple(vs, i);
  421. if (*woken >= 0)
  422. return 1;
  423. if (ret) {
  424. /*
  425. * If we need to handle a page fault, we need to do so
  426. * without any lock and any enqueued futex (otherwise
  427. * we could lose some wakeup). So we do it here, after
  428. * undoing all the work done so far. In success, we
  429. * retry all the work.
  430. */
  431. if (get_user(uval, uaddr))
  432. return -EFAULT;
  433. retry = true;
  434. goto retry;
  435. }
  436. if (uval != val)
  437. return -EWOULDBLOCK;
  438. }
  439. return 0;
  440. }
  441. /**
  442. * futex_sleep_multiple - Check sleeping conditions and sleep
  443. * @vs: List of futexes to wait for
  444. * @count: Length of vs
  445. * @to: Timeout
  446. *
  447. * Sleep if and only if the timeout hasn't expired and no futex on the list has
  448. * been woken up.
  449. */
  450. static void futex_sleep_multiple(struct futex_vector *vs, unsigned int count,
  451. struct hrtimer_sleeper *to)
  452. {
  453. if (to && !to->task)
  454. return;
  455. for (; count; count--, vs++) {
  456. if (!READ_ONCE(vs->q.lock_ptr))
  457. return;
  458. }
  459. schedule();
  460. }
  461. /**
  462. * futex_wait_multiple - Prepare to wait on and enqueue several futexes
  463. * @vs: The list of futexes to wait on
  464. * @count: The number of objects
  465. * @to: Timeout before giving up and returning to userspace
  466. *
  467. * Entry point for the FUTEX_WAIT_MULTIPLE futex operation, this function
  468. * sleeps on a group of futexes and returns on the first futex that is
  469. * wake, or after the timeout has elapsed.
  470. *
  471. * Return:
  472. * - >=0 - Hint to the futex that was awoken
  473. * - <0 - On error
  474. */
  475. int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
  476. struct hrtimer_sleeper *to)
  477. {
  478. int ret, hint = 0;
  479. if (to)
  480. hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
  481. while (1) {
  482. ret = futex_wait_multiple_setup(vs, count, &hint);
  483. if (ret) {
  484. if (ret > 0) {
  485. /* A futex was woken during setup */
  486. ret = hint;
  487. }
  488. return ret;
  489. }
  490. futex_sleep_multiple(vs, count, to);
  491. __set_current_state(TASK_RUNNING);
  492. ret = unqueue_multiple(vs, count);
  493. if (ret >= 0)
  494. return ret;
  495. if (to && !to->task)
  496. return -ETIMEDOUT;
  497. else if (signal_pending(current))
  498. return -ERESTARTSYS;
  499. /*
  500. * The final case is a spurious wakeup, for
  501. * which just retry.
  502. */
  503. }
  504. }
  505. /**
  506. * futex_wait_setup() - Prepare to wait on a futex
  507. * @uaddr: the futex userspace address
  508. * @val: the expected value
  509. * @flags: futex flags (FLAGS_SHARED, etc.)
  510. * @q: the associated futex_q
  511. * @hb: storage for hash_bucket pointer to be returned to caller
  512. *
  513. * Setup the futex_q and locate the hash_bucket. Get the futex value and
  514. * compare it with the expected value. Handle atomic faults internally.
  515. * Return with the hb lock held on success, and unlocked on failure.
  516. *
  517. * Return:
  518. * - 0 - uaddr contains val and hb has been locked;
  519. * - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
  520. */
  521. int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
  522. struct futex_q *q, struct futex_hash_bucket **hb)
  523. {
  524. u32 uval;
  525. int ret;
  526. /*
  527. * Access the page AFTER the hash-bucket is locked.
  528. * Order is important:
  529. *
  530. * Userspace waiter: val = var; if (cond(val)) futex_wait(&var, val);
  531. * Userspace waker: if (cond(var)) { var = new; futex_wake(&var); }
  532. *
  533. * The basic logical guarantee of a futex is that it blocks ONLY
  534. * if cond(var) is known to be true at the time of blocking, for
  535. * any cond. If we locked the hash-bucket after testing *uaddr, that
  536. * would open a race condition where we could block indefinitely with
  537. * cond(var) false, which would violate the guarantee.
  538. *
  539. * On the other hand, we insert q and release the hash-bucket only
  540. * after testing *uaddr. This guarantees that futex_wait() will NOT
  541. * absorb a wakeup if *uaddr does not match the desired values
  542. * while the syscall executes.
  543. */
  544. retry:
  545. ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
  546. if (unlikely(ret != 0))
  547. return ret;
  548. retry_private:
  549. *hb = futex_q_lock(q);
  550. ret = futex_get_value_locked(&uval, uaddr);
  551. if (ret) {
  552. futex_q_unlock(*hb);
  553. ret = get_user(uval, uaddr);
  554. if (ret)
  555. return ret;
  556. if (!(flags & FLAGS_SHARED))
  557. goto retry_private;
  558. goto retry;
  559. }
  560. if (uval != val) {
  561. futex_q_unlock(*hb);
  562. ret = -EWOULDBLOCK;
  563. }
  564. return ret;
  565. }
  566. int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val, ktime_t *abs_time, u32 bitset)
  567. {
  568. struct hrtimer_sleeper timeout, *to;
  569. struct restart_block *restart;
  570. struct futex_hash_bucket *hb;
  571. struct futex_q q = futex_q_init;
  572. int ret;
  573. if (!bitset)
  574. return -EINVAL;
  575. q.bitset = bitset;
  576. trace_android_vh_futex_wait_start(flags, bitset);
  577. to = futex_setup_timer(abs_time, &timeout, flags,
  578. current->timer_slack_ns);
  579. retry:
  580. /*
  581. * Prepare to wait on uaddr. On success, it holds hb->lock and q
  582. * is initialized.
  583. */
  584. ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
  585. if (ret)
  586. goto out;
  587. /* futex_queue and wait for wakeup, timeout, or a signal. */
  588. futex_wait_queue(hb, &q, to);
  589. /* If we were woken (and unqueued), we succeeded, whatever. */
  590. ret = 0;
  591. if (!futex_unqueue(&q))
  592. goto out;
  593. ret = -ETIMEDOUT;
  594. if (to && !to->task)
  595. goto out;
  596. /*
  597. * We expect signal_pending(current), but we might be the
  598. * victim of a spurious wakeup as well.
  599. */
  600. if (!signal_pending(current))
  601. goto retry;
  602. ret = -ERESTARTSYS;
  603. if (!abs_time)
  604. goto out;
  605. restart = &current->restart_block;
  606. restart->futex.uaddr = uaddr;
  607. restart->futex.val = val;
  608. restart->futex.time = *abs_time;
  609. restart->futex.bitset = bitset;
  610. restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
  611. ret = set_restart_fn(restart, futex_wait_restart);
  612. out:
  613. if (to) {
  614. hrtimer_cancel(&to->timer);
  615. destroy_hrtimer_on_stack(&to->timer);
  616. }
  617. trace_android_vh_futex_wait_end(flags, bitset);
  618. return ret;
  619. }
  620. static long futex_wait_restart(struct restart_block *restart)
  621. {
  622. u32 __user *uaddr = restart->futex.uaddr;
  623. ktime_t t, *tp = NULL;
  624. if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {
  625. t = restart->futex.time;
  626. tp = &t;
  627. }
  628. restart->fn = do_no_restart_syscall;
  629. return (long)futex_wait(uaddr, restart->futex.flags,
  630. restart->futex.val, tp, restart->futex.bitset);
  631. }