eventpoll.c 66 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * fs/eventpoll.c (Efficient event retrieval implementation)
  4. * Copyright (C) 2001,...,2009 Davide Libenzi
  5. *
  6. * Davide Libenzi <[email protected]>
  7. */
  8. #include <linux/init.h>
  9. #include <linux/kernel.h>
  10. #include <linux/sched/signal.h>
  11. #include <linux/fs.h>
  12. #include <linux/file.h>
  13. #include <linux/signal.h>
  14. #include <linux/errno.h>
  15. #include <linux/mm.h>
  16. #include <linux/slab.h>
  17. #include <linux/poll.h>
  18. #include <linux/string.h>
  19. #include <linux/list.h>
  20. #include <linux/hash.h>
  21. #include <linux/spinlock.h>
  22. #include <linux/syscalls.h>
  23. #include <linux/rbtree.h>
  24. #include <linux/wait.h>
  25. #include <linux/eventpoll.h>
  26. #include <linux/mount.h>
  27. #include <linux/bitops.h>
  28. #include <linux/mutex.h>
  29. #include <linux/anon_inodes.h>
  30. #include <linux/device.h>
  31. #include <linux/uaccess.h>
  32. #include <asm/io.h>
  33. #include <asm/mman.h>
  34. #include <linux/atomic.h>
  35. #include <linux/proc_fs.h>
  36. #include <linux/seq_file.h>
  37. #include <linux/compat.h>
  38. #include <linux/rculist.h>
  39. #include <net/busy_poll.h>
  40. #include <trace/hooks/fs.h>
  41. /*
  42. * LOCKING:
  43. * There are three level of locking required by epoll :
  44. *
  45. * 1) epmutex (mutex)
  46. * 2) ep->mtx (mutex)
  47. * 3) ep->lock (rwlock)
  48. *
  49. * The acquire order is the one listed above, from 1 to 3.
  50. * We need a rwlock (ep->lock) because we manipulate objects
  51. * from inside the poll callback, that might be triggered from
  52. * a wake_up() that in turn might be called from IRQ context.
  53. * So we can't sleep inside the poll callback and hence we need
  54. * a spinlock. During the event transfer loop (from kernel to
  55. * user space) we could end up sleeping due a copy_to_user(), so
  56. * we need a lock that will allow us to sleep. This lock is a
  57. * mutex (ep->mtx). It is acquired during the event transfer loop,
  58. * during epoll_ctl(EPOLL_CTL_DEL) and during eventpoll_release_file().
  59. * The epmutex is acquired when inserting an epoll fd onto another epoll
  60. * fd. We do this so that we walk the epoll tree and ensure that this
  61. * insertion does not create a cycle of epoll file descriptors, which
  62. * could lead to deadlock. We need a global mutex to prevent two
  63. * simultaneous inserts (A into B and B into A) from racing and
  64. * constructing a cycle without either insert observing that it is
  65. * going to.
  66. * It is necessary to acquire multiple "ep->mtx"es at once in the
  67. * case when one epoll fd is added to another. In this case, we
  68. * always acquire the locks in the order of nesting (i.e. after
  69. * epoll_ctl(e1, EPOLL_CTL_ADD, e2), e1->mtx will always be acquired
  70. * before e2->mtx). Since we disallow cycles of epoll file
  71. * descriptors, this ensures that the mutexes are well-ordered. In
  72. * order to communicate this nesting to lockdep, when walking a tree
  73. * of epoll file descriptors, we use the current recursion depth as
  74. * the lockdep subkey.
  75. * It is possible to drop the "ep->mtx" and to use the global
  76. * mutex "epmutex" (together with "ep->lock") to have it working,
  77. * but having "ep->mtx" will make the interface more scalable.
  78. * Events that require holding "epmutex" are very rare, while for
  79. * normal operations the epoll private "ep->mtx" will guarantee
  80. * a better scalability.
  81. */
  82. /* Epoll private bits inside the event mask */
  83. #define EP_PRIVATE_BITS (EPOLLWAKEUP | EPOLLONESHOT | EPOLLET | EPOLLEXCLUSIVE)
  84. #define EPOLLINOUT_BITS (EPOLLIN | EPOLLOUT)
  85. #define EPOLLEXCLUSIVE_OK_BITS (EPOLLINOUT_BITS | EPOLLERR | EPOLLHUP | \
  86. EPOLLWAKEUP | EPOLLET | EPOLLEXCLUSIVE)
  87. /* Maximum number of nesting allowed inside epoll sets */
  88. #define EP_MAX_NESTS 4
  89. #define EP_MAX_EVENTS (INT_MAX / sizeof(struct epoll_event))
  90. #define EP_UNACTIVE_PTR ((void *) -1L)
  91. #define EP_ITEM_COST (sizeof(struct epitem) + sizeof(struct eppoll_entry))
  92. struct epoll_filefd {
  93. struct file *file;
  94. int fd;
  95. } __packed;
  96. /* Wait structure used by the poll hooks */
  97. struct eppoll_entry {
  98. /* List header used to link this structure to the "struct epitem" */
  99. struct eppoll_entry *next;
  100. /* The "base" pointer is set to the container "struct epitem" */
  101. struct epitem *base;
  102. /*
  103. * Wait queue item that will be linked to the target file wait
  104. * queue head.
  105. */
  106. wait_queue_entry_t wait;
  107. /* The wait queue head that linked the "wait" wait queue item */
  108. wait_queue_head_t *whead;
  109. };
  110. /*
  111. * Each file descriptor added to the eventpoll interface will
  112. * have an entry of this type linked to the "rbr" RB tree.
  113. * Avoid increasing the size of this struct, there can be many thousands
  114. * of these on a server and we do not want this to take another cache line.
  115. */
  116. struct epitem {
  117. union {
  118. /* RB tree node links this structure to the eventpoll RB tree */
  119. struct rb_node rbn;
  120. /* Used to free the struct epitem */
  121. struct rcu_head rcu;
  122. };
  123. /* List header used to link this structure to the eventpoll ready list */
  124. struct list_head rdllink;
  125. /*
  126. * Works together "struct eventpoll"->ovflist in keeping the
  127. * single linked chain of items.
  128. */
  129. struct epitem *next;
  130. /* The file descriptor information this item refers to */
  131. struct epoll_filefd ffd;
  132. /*
  133. * Protected by file->f_lock, true for to-be-released epitem already
  134. * removed from the "struct file" items list; together with
  135. * eventpoll->refcount orchestrates "struct eventpoll" disposal
  136. */
  137. bool dying;
  138. /* List containing poll wait queues */
  139. struct eppoll_entry *pwqlist;
  140. /* The "container" of this item */
  141. struct eventpoll *ep;
  142. /* List header used to link this item to the "struct file" items list */
  143. struct hlist_node fllink;
  144. /* wakeup_source used when EPOLLWAKEUP is set */
  145. struct wakeup_source __rcu *ws;
  146. /* The structure that describe the interested events and the source fd */
  147. struct epoll_event event;
  148. };
  149. /*
  150. * This structure is stored inside the "private_data" member of the file
  151. * structure and represents the main data structure for the eventpoll
  152. * interface.
  153. */
  154. struct eventpoll {
  155. /*
  156. * This mutex is used to ensure that files are not removed
  157. * while epoll is using them. This is held during the event
  158. * collection loop, the file cleanup path, the epoll file exit
  159. * code and the ctl operations.
  160. */
  161. struct mutex mtx;
  162. /* Wait queue used by sys_epoll_wait() */
  163. wait_queue_head_t wq;
  164. /* Wait queue used by file->poll() */
  165. wait_queue_head_t poll_wait;
  166. /* List of ready file descriptors */
  167. struct list_head rdllist;
  168. /* Lock which protects rdllist and ovflist */
  169. rwlock_t lock;
  170. /* RB tree root used to store monitored fd structs */
  171. struct rb_root_cached rbr;
  172. /*
  173. * This is a single linked list that chains all the "struct epitem" that
  174. * happened while transferring ready events to userspace w/out
  175. * holding ->lock.
  176. */
  177. struct epitem *ovflist;
  178. /* wakeup_source used when ep_scan_ready_list is running */
  179. struct wakeup_source *ws;
  180. /* The user that created the eventpoll descriptor */
  181. struct user_struct *user;
  182. struct file *file;
  183. /* used to optimize loop detection check */
  184. u64 gen;
  185. struct hlist_head refs;
  186. /*
  187. * usage count, used together with epitem->dying to
  188. * orchestrate the disposal of this struct
  189. */
  190. refcount_t refcount;
  191. #ifdef CONFIG_NET_RX_BUSY_POLL
  192. /* used to track busy poll napi_id */
  193. unsigned int napi_id;
  194. #endif
  195. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  196. /* tracks wakeup nests for lockdep validation */
  197. u8 nests;
  198. #endif
  199. };
  200. /* Wrapper struct used by poll queueing */
  201. struct ep_pqueue {
  202. poll_table pt;
  203. struct epitem *epi;
  204. };
  205. /*
  206. * Configuration options available inside /proc/sys/fs/epoll/
  207. */
  208. /* Maximum number of epoll watched descriptors, per user */
  209. static long max_user_watches __read_mostly;
  210. /* Used for cycles detection */
  211. static DEFINE_MUTEX(epmutex);
  212. static u64 loop_check_gen = 0;
  213. /* Used to check for epoll file descriptor inclusion loops */
  214. static struct eventpoll *inserting_into;
  215. /* Slab cache used to allocate "struct epitem" */
  216. static struct kmem_cache *epi_cache __read_mostly;
  217. /* Slab cache used to allocate "struct eppoll_entry" */
  218. static struct kmem_cache *pwq_cache __read_mostly;
  219. /*
  220. * List of files with newly added links, where we may need to limit the number
  221. * of emanating paths. Protected by the epmutex.
  222. */
  223. struct epitems_head {
  224. struct hlist_head epitems;
  225. struct epitems_head *next;
  226. };
  227. static struct epitems_head *tfile_check_list = EP_UNACTIVE_PTR;
  228. static struct kmem_cache *ephead_cache __read_mostly;
  229. static inline void free_ephead(struct epitems_head *head)
  230. {
  231. if (head)
  232. kmem_cache_free(ephead_cache, head);
  233. }
  234. static void list_file(struct file *file)
  235. {
  236. struct epitems_head *head;
  237. head = container_of(file->f_ep, struct epitems_head, epitems);
  238. if (!head->next) {
  239. head->next = tfile_check_list;
  240. tfile_check_list = head;
  241. }
  242. }
  243. static void unlist_file(struct epitems_head *head)
  244. {
  245. struct epitems_head *to_free = head;
  246. struct hlist_node *p = rcu_dereference(hlist_first_rcu(&head->epitems));
  247. if (p) {
  248. struct epitem *epi= container_of(p, struct epitem, fllink);
  249. spin_lock(&epi->ffd.file->f_lock);
  250. if (!hlist_empty(&head->epitems))
  251. to_free = NULL;
  252. head->next = NULL;
  253. spin_unlock(&epi->ffd.file->f_lock);
  254. }
  255. free_ephead(to_free);
  256. }
  257. #ifdef CONFIG_SYSCTL
  258. #include <linux/sysctl.h>
  259. static long long_zero;
  260. static long long_max = LONG_MAX;
  261. static struct ctl_table epoll_table[] = {
  262. {
  263. .procname = "max_user_watches",
  264. .data = &max_user_watches,
  265. .maxlen = sizeof(max_user_watches),
  266. .mode = 0644,
  267. .proc_handler = proc_doulongvec_minmax,
  268. .extra1 = &long_zero,
  269. .extra2 = &long_max,
  270. },
  271. { }
  272. };
  273. static void __init epoll_sysctls_init(void)
  274. {
  275. register_sysctl("fs/epoll", epoll_table);
  276. }
  277. #else
  278. #define epoll_sysctls_init() do { } while (0)
  279. #endif /* CONFIG_SYSCTL */
  280. static const struct file_operations eventpoll_fops;
  281. static inline int is_file_epoll(struct file *f)
  282. {
  283. return f->f_op == &eventpoll_fops;
  284. }
  285. /* Setup the structure that is used as key for the RB tree */
  286. static inline void ep_set_ffd(struct epoll_filefd *ffd,
  287. struct file *file, int fd)
  288. {
  289. ffd->file = file;
  290. ffd->fd = fd;
  291. }
  292. /* Compare RB tree keys */
  293. static inline int ep_cmp_ffd(struct epoll_filefd *p1,
  294. struct epoll_filefd *p2)
  295. {
  296. return (p1->file > p2->file ? +1:
  297. (p1->file < p2->file ? -1 : p1->fd - p2->fd));
  298. }
  299. /* Tells us if the item is currently linked */
  300. static inline int ep_is_linked(struct epitem *epi)
  301. {
  302. return !list_empty(&epi->rdllink);
  303. }
  304. static inline struct eppoll_entry *ep_pwq_from_wait(wait_queue_entry_t *p)
  305. {
  306. return container_of(p, struct eppoll_entry, wait);
  307. }
  308. /* Get the "struct epitem" from a wait queue pointer */
  309. static inline struct epitem *ep_item_from_wait(wait_queue_entry_t *p)
  310. {
  311. return container_of(p, struct eppoll_entry, wait)->base;
  312. }
  313. /**
  314. * ep_events_available - Checks if ready events might be available.
  315. *
  316. * @ep: Pointer to the eventpoll context.
  317. *
  318. * Return: a value different than %zero if ready events are available,
  319. * or %zero otherwise.
  320. */
  321. static inline int ep_events_available(struct eventpoll *ep)
  322. {
  323. return !list_empty_careful(&ep->rdllist) ||
  324. READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR;
  325. }
  326. #ifdef CONFIG_NET_RX_BUSY_POLL
  327. static bool ep_busy_loop_end(void *p, unsigned long start_time)
  328. {
  329. struct eventpoll *ep = p;
  330. return ep_events_available(ep) || busy_loop_timeout(start_time);
  331. }
  332. /*
  333. * Busy poll if globally on and supporting sockets found && no events,
  334. * busy loop will return if need_resched or ep_events_available.
  335. *
  336. * we must do our busy polling with irqs enabled
  337. */
  338. static bool ep_busy_loop(struct eventpoll *ep, int nonblock)
  339. {
  340. unsigned int napi_id = READ_ONCE(ep->napi_id);
  341. if ((napi_id >= MIN_NAPI_ID) && net_busy_loop_on()) {
  342. napi_busy_loop(napi_id, nonblock ? NULL : ep_busy_loop_end, ep, false,
  343. BUSY_POLL_BUDGET);
  344. if (ep_events_available(ep))
  345. return true;
  346. /*
  347. * Busy poll timed out. Drop NAPI ID for now, we can add
  348. * it back in when we have moved a socket with a valid NAPI
  349. * ID onto the ready list.
  350. */
  351. ep->napi_id = 0;
  352. return false;
  353. }
  354. return false;
  355. }
  356. /*
  357. * Set epoll busy poll NAPI ID from sk.
  358. */
  359. static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
  360. {
  361. struct eventpoll *ep;
  362. unsigned int napi_id;
  363. struct socket *sock;
  364. struct sock *sk;
  365. if (!net_busy_loop_on())
  366. return;
  367. sock = sock_from_file(epi->ffd.file);
  368. if (!sock)
  369. return;
  370. sk = sock->sk;
  371. if (!sk)
  372. return;
  373. napi_id = READ_ONCE(sk->sk_napi_id);
  374. ep = epi->ep;
  375. /* Non-NAPI IDs can be rejected
  376. * or
  377. * Nothing to do if we already have this ID
  378. */
  379. if (napi_id < MIN_NAPI_ID || napi_id == ep->napi_id)
  380. return;
  381. /* record NAPI ID for use in next busy poll */
  382. ep->napi_id = napi_id;
  383. }
  384. #else
  385. static inline bool ep_busy_loop(struct eventpoll *ep, int nonblock)
  386. {
  387. return false;
  388. }
  389. static inline void ep_set_busy_poll_napi_id(struct epitem *epi)
  390. {
  391. }
  392. #endif /* CONFIG_NET_RX_BUSY_POLL */
  393. /*
  394. * As described in commit 0ccf831cb lockdep: annotate epoll
  395. * the use of wait queues used by epoll is done in a very controlled
  396. * manner. Wake ups can nest inside each other, but are never done
  397. * with the same locking. For example:
  398. *
  399. * dfd = socket(...);
  400. * efd1 = epoll_create();
  401. * efd2 = epoll_create();
  402. * epoll_ctl(efd1, EPOLL_CTL_ADD, dfd, ...);
  403. * epoll_ctl(efd2, EPOLL_CTL_ADD, efd1, ...);
  404. *
  405. * When a packet arrives to the device underneath "dfd", the net code will
  406. * issue a wake_up() on its poll wake list. Epoll (efd1) has installed a
  407. * callback wakeup entry on that queue, and the wake_up() performed by the
  408. * "dfd" net code will end up in ep_poll_callback(). At this point epoll
  409. * (efd1) notices that it may have some event ready, so it needs to wake up
  410. * the waiters on its poll wait list (efd2). So it calls ep_poll_safewake()
  411. * that ends up in another wake_up(), after having checked about the
  412. * recursion constraints. That are, no more than EP_MAX_POLLWAKE_NESTS, to
  413. * avoid stack blasting.
  414. *
  415. * When CONFIG_DEBUG_LOCK_ALLOC is enabled, make sure lockdep can handle
  416. * this special case of epoll.
  417. */
  418. #ifdef CONFIG_DEBUG_LOCK_ALLOC
  419. static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,
  420. unsigned pollflags)
  421. {
  422. struct eventpoll *ep_src;
  423. unsigned long flags;
  424. u8 nests = 0;
  425. /*
  426. * To set the subclass or nesting level for spin_lock_irqsave_nested()
  427. * it might be natural to create a per-cpu nest count. However, since
  428. * we can recurse on ep->poll_wait.lock, and a non-raw spinlock can
  429. * schedule() in the -rt kernel, the per-cpu variable are no longer
  430. * protected. Thus, we are introducing a per eventpoll nest field.
  431. * If we are not being call from ep_poll_callback(), epi is NULL and
  432. * we are at the first level of nesting, 0. Otherwise, we are being
  433. * called from ep_poll_callback() and if a previous wakeup source is
  434. * not an epoll file itself, we are at depth 1 since the wakeup source
  435. * is depth 0. If the wakeup source is a previous epoll file in the
  436. * wakeup chain then we use its nests value and record ours as
  437. * nests + 1. The previous epoll file nests value is stable since its
  438. * already holding its own poll_wait.lock.
  439. */
  440. if (epi) {
  441. if ((is_file_epoll(epi->ffd.file))) {
  442. ep_src = epi->ffd.file->private_data;
  443. nests = ep_src->nests;
  444. } else {
  445. nests = 1;
  446. }
  447. }
  448. spin_lock_irqsave_nested(&ep->poll_wait.lock, flags, nests);
  449. ep->nests = nests + 1;
  450. wake_up_locked_poll(&ep->poll_wait, EPOLLIN | pollflags);
  451. ep->nests = 0;
  452. spin_unlock_irqrestore(&ep->poll_wait.lock, flags);
  453. }
  454. #else
  455. static void ep_poll_safewake(struct eventpoll *ep, struct epitem *epi,
  456. unsigned pollflags)
  457. {
  458. wake_up_poll(&ep->poll_wait, EPOLLIN | pollflags);
  459. }
  460. #endif
  461. static void ep_remove_wait_queue(struct eppoll_entry *pwq)
  462. {
  463. wait_queue_head_t *whead;
  464. rcu_read_lock();
  465. /*
  466. * If it is cleared by POLLFREE, it should be rcu-safe.
  467. * If we read NULL we need a barrier paired with
  468. * smp_store_release() in ep_poll_callback(), otherwise
  469. * we rely on whead->lock.
  470. */
  471. whead = smp_load_acquire(&pwq->whead);
  472. if (whead)
  473. remove_wait_queue(whead, &pwq->wait);
  474. rcu_read_unlock();
  475. }
  476. /*
  477. * This function unregisters poll callbacks from the associated file
  478. * descriptor. Must be called with "mtx" held.
  479. */
  480. static void ep_unregister_pollwait(struct eventpoll *ep, struct epitem *epi)
  481. {
  482. struct eppoll_entry **p = &epi->pwqlist;
  483. struct eppoll_entry *pwq;
  484. while ((pwq = *p) != NULL) {
  485. *p = pwq->next;
  486. ep_remove_wait_queue(pwq);
  487. kmem_cache_free(pwq_cache, pwq);
  488. }
  489. }
  490. /* call only when ep->mtx is held */
  491. static inline struct wakeup_source *ep_wakeup_source(struct epitem *epi)
  492. {
  493. return rcu_dereference_check(epi->ws, lockdep_is_held(&epi->ep->mtx));
  494. }
  495. /* call only when ep->mtx is held */
  496. static inline void ep_pm_stay_awake(struct epitem *epi)
  497. {
  498. struct wakeup_source *ws = ep_wakeup_source(epi);
  499. if (ws)
  500. __pm_stay_awake(ws);
  501. }
  502. static inline bool ep_has_wakeup_source(struct epitem *epi)
  503. {
  504. return rcu_access_pointer(epi->ws) ? true : false;
  505. }
  506. /* call when ep->mtx cannot be held (ep_poll_callback) */
  507. static inline void ep_pm_stay_awake_rcu(struct epitem *epi)
  508. {
  509. struct wakeup_source *ws;
  510. rcu_read_lock();
  511. ws = rcu_dereference(epi->ws);
  512. if (ws)
  513. __pm_stay_awake(ws);
  514. rcu_read_unlock();
  515. }
  516. /*
  517. * ep->mutex needs to be held because we could be hit by
  518. * eventpoll_release_file() and epoll_ctl().
  519. */
  520. static void ep_start_scan(struct eventpoll *ep, struct list_head *txlist)
  521. {
  522. /*
  523. * Steal the ready list, and re-init the original one to the
  524. * empty list. Also, set ep->ovflist to NULL so that events
  525. * happening while looping w/out locks, are not lost. We cannot
  526. * have the poll callback to queue directly on ep->rdllist,
  527. * because we want the "sproc" callback to be able to do it
  528. * in a lockless way.
  529. */
  530. lockdep_assert_irqs_enabled();
  531. write_lock_irq(&ep->lock);
  532. list_splice_init(&ep->rdllist, txlist);
  533. WRITE_ONCE(ep->ovflist, NULL);
  534. write_unlock_irq(&ep->lock);
  535. }
  536. static void ep_done_scan(struct eventpoll *ep,
  537. struct list_head *txlist)
  538. {
  539. struct epitem *epi, *nepi;
  540. write_lock_irq(&ep->lock);
  541. /*
  542. * During the time we spent inside the "sproc" callback, some
  543. * other events might have been queued by the poll callback.
  544. * We re-insert them inside the main ready-list here.
  545. */
  546. for (nepi = READ_ONCE(ep->ovflist); (epi = nepi) != NULL;
  547. nepi = epi->next, epi->next = EP_UNACTIVE_PTR) {
  548. /*
  549. * We need to check if the item is already in the list.
  550. * During the "sproc" callback execution time, items are
  551. * queued into ->ovflist but the "txlist" might already
  552. * contain them, and the list_splice() below takes care of them.
  553. */
  554. if (!ep_is_linked(epi)) {
  555. /*
  556. * ->ovflist is LIFO, so we have to reverse it in order
  557. * to keep in FIFO.
  558. */
  559. list_add(&epi->rdllink, &ep->rdllist);
  560. ep_pm_stay_awake(epi);
  561. }
  562. }
  563. /*
  564. * We need to set back ep->ovflist to EP_UNACTIVE_PTR, so that after
  565. * releasing the lock, events will be queued in the normal way inside
  566. * ep->rdllist.
  567. */
  568. WRITE_ONCE(ep->ovflist, EP_UNACTIVE_PTR);
  569. /*
  570. * Quickly re-inject items left on "txlist".
  571. */
  572. list_splice(txlist, &ep->rdllist);
  573. __pm_relax(ep->ws);
  574. if (!list_empty(&ep->rdllist)) {
  575. if (waitqueue_active(&ep->wq))
  576. wake_up(&ep->wq);
  577. }
  578. write_unlock_irq(&ep->lock);
  579. }
  580. static void epi_rcu_free(struct rcu_head *head)
  581. {
  582. struct epitem *epi = container_of(head, struct epitem, rcu);
  583. kmem_cache_free(epi_cache, epi);
  584. }
  585. static void ep_get(struct eventpoll *ep)
  586. {
  587. refcount_inc(&ep->refcount);
  588. }
  589. /*
  590. * Returns true if the event poll can be disposed
  591. */
  592. static bool ep_refcount_dec_and_test(struct eventpoll *ep)
  593. {
  594. if (!refcount_dec_and_test(&ep->refcount))
  595. return false;
  596. WARN_ON_ONCE(!RB_EMPTY_ROOT(&ep->rbr.rb_root));
  597. return true;
  598. }
  599. static void ep_free(struct eventpoll *ep)
  600. {
  601. mutex_destroy(&ep->mtx);
  602. free_uid(ep->user);
  603. wakeup_source_unregister(ep->ws);
  604. kfree(ep);
  605. }
  606. /*
  607. * Removes a "struct epitem" from the eventpoll RB tree and deallocates
  608. * all the associated resources. Must be called with "mtx" held.
  609. * If the dying flag is set, do the removal only if force is true.
  610. * This prevents ep_clear_and_put() from dropping all the ep references
  611. * while running concurrently with eventpoll_release_file().
  612. * Returns true if the eventpoll can be disposed.
  613. */
  614. static bool __ep_remove(struct eventpoll *ep, struct epitem *epi, bool force)
  615. {
  616. struct file *file = epi->ffd.file;
  617. struct epitems_head *to_free;
  618. struct hlist_head *head;
  619. lockdep_assert_irqs_enabled();
  620. /*
  621. * Removes poll wait queue hooks.
  622. */
  623. ep_unregister_pollwait(ep, epi);
  624. /* Remove the current item from the list of epoll hooks */
  625. spin_lock(&file->f_lock);
  626. if (epi->dying && !force) {
  627. spin_unlock(&file->f_lock);
  628. return false;
  629. }
  630. to_free = NULL;
  631. head = file->f_ep;
  632. if (head->first == &epi->fllink && !epi->fllink.next) {
  633. file->f_ep = NULL;
  634. if (!is_file_epoll(file)) {
  635. struct epitems_head *v;
  636. v = container_of(head, struct epitems_head, epitems);
  637. if (!smp_load_acquire(&v->next))
  638. to_free = v;
  639. }
  640. }
  641. hlist_del_rcu(&epi->fllink);
  642. spin_unlock(&file->f_lock);
  643. free_ephead(to_free);
  644. rb_erase_cached(&epi->rbn, &ep->rbr);
  645. write_lock_irq(&ep->lock);
  646. if (ep_is_linked(epi))
  647. list_del_init(&epi->rdllink);
  648. write_unlock_irq(&ep->lock);
  649. wakeup_source_unregister(ep_wakeup_source(epi));
  650. /*
  651. * At this point it is safe to free the eventpoll item. Use the union
  652. * field epi->rcu, since we are trying to minimize the size of
  653. * 'struct epitem'. The 'rbn' field is no longer in use. Protected by
  654. * ep->mtx. The rcu read side, reverse_path_check_proc(), does not make
  655. * use of the rbn field.
  656. */
  657. call_rcu(&epi->rcu, epi_rcu_free);
  658. percpu_counter_dec(&ep->user->epoll_watches);
  659. return ep_refcount_dec_and_test(ep);
  660. }
  661. /*
  662. * ep_remove variant for callers owing an additional reference to the ep
  663. */
  664. static void ep_remove_safe(struct eventpoll *ep, struct epitem *epi)
  665. {
  666. WARN_ON_ONCE(__ep_remove(ep, epi, false));
  667. }
  668. static void ep_clear_and_put(struct eventpoll *ep)
  669. {
  670. struct rb_node *rbp, *next;
  671. struct epitem *epi;
  672. bool dispose;
  673. /* We need to release all tasks waiting for these file */
  674. if (waitqueue_active(&ep->poll_wait))
  675. ep_poll_safewake(ep, NULL, 0);
  676. mutex_lock(&ep->mtx);
  677. /*
  678. * Walks through the whole tree by unregistering poll callbacks.
  679. */
  680. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  681. epi = rb_entry(rbp, struct epitem, rbn);
  682. ep_unregister_pollwait(ep, epi);
  683. cond_resched();
  684. }
  685. /*
  686. * Walks through the whole tree and try to free each "struct epitem".
  687. * Note that ep_remove_safe() will not remove the epitem in case of a
  688. * racing eventpoll_release_file(); the latter will do the removal.
  689. * At this point we are sure no poll callbacks will be lingering around.
  690. * Since we still own a reference to the eventpoll struct, the loop can't
  691. * dispose it.
  692. */
  693. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = next) {
  694. next = rb_next(rbp);
  695. epi = rb_entry(rbp, struct epitem, rbn);
  696. ep_remove_safe(ep, epi);
  697. cond_resched();
  698. }
  699. dispose = ep_refcount_dec_and_test(ep);
  700. mutex_unlock(&ep->mtx);
  701. if (dispose)
  702. ep_free(ep);
  703. }
  704. static int ep_eventpoll_release(struct inode *inode, struct file *file)
  705. {
  706. struct eventpoll *ep = file->private_data;
  707. if (ep)
  708. ep_clear_and_put(ep);
  709. return 0;
  710. }
  711. static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt, int depth);
  712. static __poll_t __ep_eventpoll_poll(struct file *file, poll_table *wait, int depth)
  713. {
  714. struct eventpoll *ep = file->private_data;
  715. LIST_HEAD(txlist);
  716. struct epitem *epi, *tmp;
  717. poll_table pt;
  718. __poll_t res = 0;
  719. init_poll_funcptr(&pt, NULL);
  720. /* Insert inside our poll wait queue */
  721. poll_wait(file, &ep->poll_wait, wait);
  722. /*
  723. * Proceed to find out if wanted events are really available inside
  724. * the ready list.
  725. */
  726. mutex_lock_nested(&ep->mtx, depth);
  727. ep_start_scan(ep, &txlist);
  728. list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {
  729. if (ep_item_poll(epi, &pt, depth + 1)) {
  730. res = EPOLLIN | EPOLLRDNORM;
  731. break;
  732. } else {
  733. /*
  734. * Item has been dropped into the ready list by the poll
  735. * callback, but it's not actually ready, as far as
  736. * caller requested events goes. We can remove it here.
  737. */
  738. __pm_relax(ep_wakeup_source(epi));
  739. list_del_init(&epi->rdllink);
  740. }
  741. }
  742. ep_done_scan(ep, &txlist);
  743. mutex_unlock(&ep->mtx);
  744. return res;
  745. }
  746. /*
  747. * Differs from ep_eventpoll_poll() in that internal callers already have
  748. * the ep->mtx so we need to start from depth=1, such that mutex_lock_nested()
  749. * is correctly annotated.
  750. */
  751. static __poll_t ep_item_poll(const struct epitem *epi, poll_table *pt,
  752. int depth)
  753. {
  754. struct file *file = epi->ffd.file;
  755. __poll_t res;
  756. pt->_key = epi->event.events;
  757. if (!is_file_epoll(file))
  758. res = vfs_poll(file, pt);
  759. else
  760. res = __ep_eventpoll_poll(file, pt, depth);
  761. return res & epi->event.events;
  762. }
  763. static __poll_t ep_eventpoll_poll(struct file *file, poll_table *wait)
  764. {
  765. return __ep_eventpoll_poll(file, wait, 0);
  766. }
  767. #ifdef CONFIG_PROC_FS
  768. static void ep_show_fdinfo(struct seq_file *m, struct file *f)
  769. {
  770. struct eventpoll *ep = f->private_data;
  771. struct rb_node *rbp;
  772. mutex_lock(&ep->mtx);
  773. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  774. struct epitem *epi = rb_entry(rbp, struct epitem, rbn);
  775. struct inode *inode = file_inode(epi->ffd.file);
  776. seq_printf(m, "tfd: %8d events: %8x data: %16llx "
  777. " pos:%lli ino:%lx sdev:%x\n",
  778. epi->ffd.fd, epi->event.events,
  779. (long long)epi->event.data,
  780. (long long)epi->ffd.file->f_pos,
  781. inode->i_ino, inode->i_sb->s_dev);
  782. if (seq_has_overflowed(m))
  783. break;
  784. }
  785. mutex_unlock(&ep->mtx);
  786. }
  787. #endif
  788. /* File callbacks that implement the eventpoll file behaviour */
  789. static const struct file_operations eventpoll_fops = {
  790. #ifdef CONFIG_PROC_FS
  791. .show_fdinfo = ep_show_fdinfo,
  792. #endif
  793. .release = ep_eventpoll_release,
  794. .poll = ep_eventpoll_poll,
  795. .llseek = noop_llseek,
  796. };
  797. /*
  798. * This is called from eventpoll_release() to unlink files from the eventpoll
  799. * interface. We need to have this facility to cleanup correctly files that are
  800. * closed without being removed from the eventpoll interface.
  801. */
  802. void eventpoll_release_file(struct file *file)
  803. {
  804. struct eventpoll *ep;
  805. struct epitem *epi;
  806. bool dispose;
  807. /*
  808. * Use the 'dying' flag to prevent a concurrent ep_clear_and_put() from
  809. * touching the epitems list before eventpoll_release_file() can access
  810. * the ep->mtx.
  811. */
  812. again:
  813. spin_lock(&file->f_lock);
  814. if (file->f_ep && file->f_ep->first) {
  815. epi = hlist_entry(file->f_ep->first, struct epitem, fllink);
  816. epi->dying = true;
  817. spin_unlock(&file->f_lock);
  818. /*
  819. * ep access is safe as we still own a reference to the ep
  820. * struct
  821. */
  822. ep = epi->ep;
  823. mutex_lock(&ep->mtx);
  824. dispose = __ep_remove(ep, epi, true);
  825. mutex_unlock(&ep->mtx);
  826. if (dispose)
  827. ep_free(ep);
  828. goto again;
  829. }
  830. spin_unlock(&file->f_lock);
  831. }
  832. static int ep_alloc(struct eventpoll **pep)
  833. {
  834. int error;
  835. struct user_struct *user;
  836. struct eventpoll *ep;
  837. user = get_current_user();
  838. error = -ENOMEM;
  839. ep = kzalloc(sizeof(*ep), GFP_KERNEL);
  840. if (unlikely(!ep))
  841. goto free_uid;
  842. mutex_init(&ep->mtx);
  843. rwlock_init(&ep->lock);
  844. init_waitqueue_head(&ep->wq);
  845. init_waitqueue_head(&ep->poll_wait);
  846. INIT_LIST_HEAD(&ep->rdllist);
  847. ep->rbr = RB_ROOT_CACHED;
  848. ep->ovflist = EP_UNACTIVE_PTR;
  849. ep->user = user;
  850. refcount_set(&ep->refcount, 1);
  851. *pep = ep;
  852. return 0;
  853. free_uid:
  854. free_uid(user);
  855. return error;
  856. }
  857. /*
  858. * Search the file inside the eventpoll tree. The RB tree operations
  859. * are protected by the "mtx" mutex, and ep_find() must be called with
  860. * "mtx" held.
  861. */
  862. static struct epitem *ep_find(struct eventpoll *ep, struct file *file, int fd)
  863. {
  864. int kcmp;
  865. struct rb_node *rbp;
  866. struct epitem *epi, *epir = NULL;
  867. struct epoll_filefd ffd;
  868. ep_set_ffd(&ffd, file, fd);
  869. for (rbp = ep->rbr.rb_root.rb_node; rbp; ) {
  870. epi = rb_entry(rbp, struct epitem, rbn);
  871. kcmp = ep_cmp_ffd(&ffd, &epi->ffd);
  872. if (kcmp > 0)
  873. rbp = rbp->rb_right;
  874. else if (kcmp < 0)
  875. rbp = rbp->rb_left;
  876. else {
  877. epir = epi;
  878. break;
  879. }
  880. }
  881. return epir;
  882. }
  883. #ifdef CONFIG_KCMP
  884. static struct epitem *ep_find_tfd(struct eventpoll *ep, int tfd, unsigned long toff)
  885. {
  886. struct rb_node *rbp;
  887. struct epitem *epi;
  888. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  889. epi = rb_entry(rbp, struct epitem, rbn);
  890. if (epi->ffd.fd == tfd) {
  891. if (toff == 0)
  892. return epi;
  893. else
  894. toff--;
  895. }
  896. cond_resched();
  897. }
  898. return NULL;
  899. }
  900. struct file *get_epoll_tfile_raw_ptr(struct file *file, int tfd,
  901. unsigned long toff)
  902. {
  903. struct file *file_raw;
  904. struct eventpoll *ep;
  905. struct epitem *epi;
  906. if (!is_file_epoll(file))
  907. return ERR_PTR(-EINVAL);
  908. ep = file->private_data;
  909. mutex_lock(&ep->mtx);
  910. epi = ep_find_tfd(ep, tfd, toff);
  911. if (epi)
  912. file_raw = epi->ffd.file;
  913. else
  914. file_raw = ERR_PTR(-ENOENT);
  915. mutex_unlock(&ep->mtx);
  916. return file_raw;
  917. }
  918. #endif /* CONFIG_KCMP */
  919. /*
  920. * Adds a new entry to the tail of the list in a lockless way, i.e.
  921. * multiple CPUs are allowed to call this function concurrently.
  922. *
  923. * Beware: it is necessary to prevent any other modifications of the
  924. * existing list until all changes are completed, in other words
  925. * concurrent list_add_tail_lockless() calls should be protected
  926. * with a read lock, where write lock acts as a barrier which
  927. * makes sure all list_add_tail_lockless() calls are fully
  928. * completed.
  929. *
  930. * Also an element can be locklessly added to the list only in one
  931. * direction i.e. either to the tail or to the head, otherwise
  932. * concurrent access will corrupt the list.
  933. *
  934. * Return: %false if element has been already added to the list, %true
  935. * otherwise.
  936. */
  937. static inline bool list_add_tail_lockless(struct list_head *new,
  938. struct list_head *head)
  939. {
  940. struct list_head *prev;
  941. /*
  942. * This is simple 'new->next = head' operation, but cmpxchg()
  943. * is used in order to detect that same element has been just
  944. * added to the list from another CPU: the winner observes
  945. * new->next == new.
  946. */
  947. if (!try_cmpxchg(&new->next, &new, head))
  948. return false;
  949. /*
  950. * Initially ->next of a new element must be updated with the head
  951. * (we are inserting to the tail) and only then pointers are atomically
  952. * exchanged. XCHG guarantees memory ordering, thus ->next should be
  953. * updated before pointers are actually swapped and pointers are
  954. * swapped before prev->next is updated.
  955. */
  956. prev = xchg(&head->prev, new);
  957. /*
  958. * It is safe to modify prev->next and new->prev, because a new element
  959. * is added only to the tail and new->next is updated before XCHG.
  960. */
  961. prev->next = new;
  962. new->prev = prev;
  963. return true;
  964. }
  965. /*
  966. * Chains a new epi entry to the tail of the ep->ovflist in a lockless way,
  967. * i.e. multiple CPUs are allowed to call this function concurrently.
  968. *
  969. * Return: %false if epi element has been already chained, %true otherwise.
  970. */
  971. static inline bool chain_epi_lockless(struct epitem *epi)
  972. {
  973. struct eventpoll *ep = epi->ep;
  974. /* Fast preliminary check */
  975. if (epi->next != EP_UNACTIVE_PTR)
  976. return false;
  977. /* Check that the same epi has not been just chained from another CPU */
  978. if (cmpxchg(&epi->next, EP_UNACTIVE_PTR, NULL) != EP_UNACTIVE_PTR)
  979. return false;
  980. /* Atomically exchange tail */
  981. epi->next = xchg(&ep->ovflist, epi);
  982. return true;
  983. }
  984. /*
  985. * This is the callback that is passed to the wait queue wakeup
  986. * mechanism. It is called by the stored file descriptors when they
  987. * have events to report.
  988. *
  989. * This callback takes a read lock in order not to contend with concurrent
  990. * events from another file descriptor, thus all modifications to ->rdllist
  991. * or ->ovflist are lockless. Read lock is paired with the write lock from
  992. * ep_scan_ready_list(), which stops all list modifications and guarantees
  993. * that lists state is seen correctly.
  994. *
  995. * Another thing worth to mention is that ep_poll_callback() can be called
  996. * concurrently for the same @epi from different CPUs if poll table was inited
  997. * with several wait queues entries. Plural wakeup from different CPUs of a
  998. * single wait queue is serialized by wq.lock, but the case when multiple wait
  999. * queues are used should be detected accordingly. This is detected using
  1000. * cmpxchg() operation.
  1001. */
  1002. static int ep_poll_callback(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
  1003. {
  1004. int pwake = 0;
  1005. struct epitem *epi = ep_item_from_wait(wait);
  1006. struct eventpoll *ep = epi->ep;
  1007. __poll_t pollflags = key_to_poll(key);
  1008. unsigned long flags;
  1009. int ewake = 0;
  1010. read_lock_irqsave(&ep->lock, flags);
  1011. ep_set_busy_poll_napi_id(epi);
  1012. /*
  1013. * If the event mask does not contain any poll(2) event, we consider the
  1014. * descriptor to be disabled. This condition is likely the effect of the
  1015. * EPOLLONESHOT bit that disables the descriptor when an event is received,
  1016. * until the next EPOLL_CTL_MOD will be issued.
  1017. */
  1018. if (!(epi->event.events & ~EP_PRIVATE_BITS))
  1019. goto out_unlock;
  1020. /*
  1021. * Check the events coming with the callback. At this stage, not
  1022. * every device reports the events in the "key" parameter of the
  1023. * callback. We need to be able to handle both cases here, hence the
  1024. * test for "key" != NULL before the event match test.
  1025. */
  1026. if (pollflags && !(pollflags & epi->event.events))
  1027. goto out_unlock;
  1028. /*
  1029. * If we are transferring events to userspace, we can hold no locks
  1030. * (because we're accessing user memory, and because of linux f_op->poll()
  1031. * semantics). All the events that happen during that period of time are
  1032. * chained in ep->ovflist and requeued later on.
  1033. */
  1034. if (READ_ONCE(ep->ovflist) != EP_UNACTIVE_PTR) {
  1035. if (chain_epi_lockless(epi))
  1036. ep_pm_stay_awake_rcu(epi);
  1037. } else if (!ep_is_linked(epi)) {
  1038. /* In the usual case, add event to ready list. */
  1039. if (list_add_tail_lockless(&epi->rdllink, &ep->rdllist))
  1040. ep_pm_stay_awake_rcu(epi);
  1041. }
  1042. /*
  1043. * Wake up ( if active ) both the eventpoll wait list and the ->poll()
  1044. * wait list.
  1045. */
  1046. if (waitqueue_active(&ep->wq)) {
  1047. if ((epi->event.events & EPOLLEXCLUSIVE) &&
  1048. !(pollflags & POLLFREE)) {
  1049. switch (pollflags & EPOLLINOUT_BITS) {
  1050. case EPOLLIN:
  1051. if (epi->event.events & EPOLLIN)
  1052. ewake = 1;
  1053. break;
  1054. case EPOLLOUT:
  1055. if (epi->event.events & EPOLLOUT)
  1056. ewake = 1;
  1057. break;
  1058. case 0:
  1059. ewake = 1;
  1060. break;
  1061. }
  1062. }
  1063. wake_up(&ep->wq);
  1064. }
  1065. if (waitqueue_active(&ep->poll_wait))
  1066. pwake++;
  1067. out_unlock:
  1068. read_unlock_irqrestore(&ep->lock, flags);
  1069. /* We have to call this outside the lock */
  1070. if (pwake)
  1071. ep_poll_safewake(ep, epi, pollflags & EPOLL_URING_WAKE);
  1072. if (!(epi->event.events & EPOLLEXCLUSIVE))
  1073. ewake = 1;
  1074. if (pollflags & POLLFREE) {
  1075. /*
  1076. * If we race with ep_remove_wait_queue() it can miss
  1077. * ->whead = NULL and do another remove_wait_queue() after
  1078. * us, so we can't use __remove_wait_queue().
  1079. */
  1080. list_del_init(&wait->entry);
  1081. /*
  1082. * ->whead != NULL protects us from the race with
  1083. * ep_clear_and_put() or ep_remove(), ep_remove_wait_queue()
  1084. * takes whead->lock held by the caller. Once we nullify it,
  1085. * nothing protects ep/epi or even wait.
  1086. */
  1087. smp_store_release(&ep_pwq_from_wait(wait)->whead, NULL);
  1088. }
  1089. return ewake;
  1090. }
  1091. /*
  1092. * This is the callback that is used to add our wait queue to the
  1093. * target file wakeup lists.
  1094. */
  1095. static void ep_ptable_queue_proc(struct file *file, wait_queue_head_t *whead,
  1096. poll_table *pt)
  1097. {
  1098. struct ep_pqueue *epq = container_of(pt, struct ep_pqueue, pt);
  1099. struct epitem *epi = epq->epi;
  1100. struct eppoll_entry *pwq;
  1101. if (unlikely(!epi)) // an earlier allocation has failed
  1102. return;
  1103. pwq = kmem_cache_alloc(pwq_cache, GFP_KERNEL);
  1104. if (unlikely(!pwq)) {
  1105. epq->epi = NULL;
  1106. return;
  1107. }
  1108. init_waitqueue_func_entry(&pwq->wait, ep_poll_callback);
  1109. pwq->whead = whead;
  1110. pwq->base = epi;
  1111. if (epi->event.events & EPOLLEXCLUSIVE)
  1112. add_wait_queue_exclusive(whead, &pwq->wait);
  1113. else
  1114. add_wait_queue(whead, &pwq->wait);
  1115. pwq->next = epi->pwqlist;
  1116. epi->pwqlist = pwq;
  1117. }
  1118. static void ep_rbtree_insert(struct eventpoll *ep, struct epitem *epi)
  1119. {
  1120. int kcmp;
  1121. struct rb_node **p = &ep->rbr.rb_root.rb_node, *parent = NULL;
  1122. struct epitem *epic;
  1123. bool leftmost = true;
  1124. while (*p) {
  1125. parent = *p;
  1126. epic = rb_entry(parent, struct epitem, rbn);
  1127. kcmp = ep_cmp_ffd(&epi->ffd, &epic->ffd);
  1128. if (kcmp > 0) {
  1129. p = &parent->rb_right;
  1130. leftmost = false;
  1131. } else
  1132. p = &parent->rb_left;
  1133. }
  1134. rb_link_node(&epi->rbn, parent, p);
  1135. rb_insert_color_cached(&epi->rbn, &ep->rbr, leftmost);
  1136. }
  1137. #define PATH_ARR_SIZE 5
  1138. /*
  1139. * These are the number paths of length 1 to 5, that we are allowing to emanate
  1140. * from a single file of interest. For example, we allow 1000 paths of length
  1141. * 1, to emanate from each file of interest. This essentially represents the
  1142. * potential wakeup paths, which need to be limited in order to avoid massive
  1143. * uncontrolled wakeup storms. The common use case should be a single ep which
  1144. * is connected to n file sources. In this case each file source has 1 path
  1145. * of length 1. Thus, the numbers below should be more than sufficient. These
  1146. * path limits are enforced during an EPOLL_CTL_ADD operation, since a modify
  1147. * and delete can't add additional paths. Protected by the epmutex.
  1148. */
  1149. static const int path_limits[PATH_ARR_SIZE] = { 1000, 500, 100, 50, 10 };
  1150. static int path_count[PATH_ARR_SIZE];
  1151. static int path_count_inc(int nests)
  1152. {
  1153. /* Allow an arbitrary number of depth 1 paths */
  1154. if (nests == 0)
  1155. return 0;
  1156. if (++path_count[nests] > path_limits[nests])
  1157. return -1;
  1158. return 0;
  1159. }
  1160. static void path_count_init(void)
  1161. {
  1162. int i;
  1163. for (i = 0; i < PATH_ARR_SIZE; i++)
  1164. path_count[i] = 0;
  1165. }
  1166. static int reverse_path_check_proc(struct hlist_head *refs, int depth)
  1167. {
  1168. int error = 0;
  1169. struct epitem *epi;
  1170. if (depth > EP_MAX_NESTS) /* too deep nesting */
  1171. return -1;
  1172. /* CTL_DEL can remove links here, but that can't increase our count */
  1173. hlist_for_each_entry_rcu(epi, refs, fllink) {
  1174. struct hlist_head *refs = &epi->ep->refs;
  1175. if (hlist_empty(refs))
  1176. error = path_count_inc(depth);
  1177. else
  1178. error = reverse_path_check_proc(refs, depth + 1);
  1179. if (error != 0)
  1180. break;
  1181. }
  1182. return error;
  1183. }
  1184. /**
  1185. * reverse_path_check - The tfile_check_list is list of epitem_head, which have
  1186. * links that are proposed to be newly added. We need to
  1187. * make sure that those added links don't add too many
  1188. * paths such that we will spend all our time waking up
  1189. * eventpoll objects.
  1190. *
  1191. * Return: %zero if the proposed links don't create too many paths,
  1192. * %-1 otherwise.
  1193. */
  1194. static int reverse_path_check(void)
  1195. {
  1196. struct epitems_head *p;
  1197. for (p = tfile_check_list; p != EP_UNACTIVE_PTR; p = p->next) {
  1198. int error;
  1199. path_count_init();
  1200. rcu_read_lock();
  1201. error = reverse_path_check_proc(&p->epitems, 0);
  1202. rcu_read_unlock();
  1203. if (error)
  1204. return error;
  1205. }
  1206. return 0;
  1207. }
  1208. static int ep_create_wakeup_source(struct epitem *epi)
  1209. {
  1210. struct name_snapshot n;
  1211. struct wakeup_source *ws;
  1212. char ws_name[64];
  1213. strlcpy(ws_name, "eventpoll", sizeof(ws_name));
  1214. trace_android_vh_ep_create_wakeup_source(ws_name, sizeof(ws_name));
  1215. if (!epi->ep->ws) {
  1216. epi->ep->ws = wakeup_source_register(NULL, ws_name);
  1217. if (!epi->ep->ws)
  1218. return -ENOMEM;
  1219. }
  1220. take_dentry_name_snapshot(&n, epi->ffd.file->f_path.dentry);
  1221. strlcpy(ws_name, n.name.name, sizeof(ws_name));
  1222. trace_android_vh_ep_create_wakeup_source(ws_name, sizeof(ws_name));
  1223. ws = wakeup_source_register(NULL, ws_name);
  1224. release_dentry_name_snapshot(&n);
  1225. if (!ws)
  1226. return -ENOMEM;
  1227. rcu_assign_pointer(epi->ws, ws);
  1228. return 0;
  1229. }
  1230. /* rare code path, only used when EPOLL_CTL_MOD removes a wakeup source */
  1231. static noinline void ep_destroy_wakeup_source(struct epitem *epi)
  1232. {
  1233. struct wakeup_source *ws = ep_wakeup_source(epi);
  1234. RCU_INIT_POINTER(epi->ws, NULL);
  1235. /*
  1236. * wait for ep_pm_stay_awake_rcu to finish, synchronize_rcu is
  1237. * used internally by wakeup_source_remove, too (called by
  1238. * wakeup_source_unregister), so we cannot use call_rcu
  1239. */
  1240. synchronize_rcu();
  1241. wakeup_source_unregister(ws);
  1242. }
  1243. static int attach_epitem(struct file *file, struct epitem *epi)
  1244. {
  1245. struct epitems_head *to_free = NULL;
  1246. struct hlist_head *head = NULL;
  1247. struct eventpoll *ep = NULL;
  1248. if (is_file_epoll(file))
  1249. ep = file->private_data;
  1250. if (ep) {
  1251. head = &ep->refs;
  1252. } else if (!READ_ONCE(file->f_ep)) {
  1253. allocate:
  1254. to_free = kmem_cache_zalloc(ephead_cache, GFP_KERNEL);
  1255. if (!to_free)
  1256. return -ENOMEM;
  1257. head = &to_free->epitems;
  1258. }
  1259. spin_lock(&file->f_lock);
  1260. if (!file->f_ep) {
  1261. if (unlikely(!head)) {
  1262. spin_unlock(&file->f_lock);
  1263. goto allocate;
  1264. }
  1265. file->f_ep = head;
  1266. to_free = NULL;
  1267. }
  1268. hlist_add_head_rcu(&epi->fllink, file->f_ep);
  1269. spin_unlock(&file->f_lock);
  1270. free_ephead(to_free);
  1271. return 0;
  1272. }
  1273. /*
  1274. * Must be called with "mtx" held.
  1275. */
  1276. static int ep_insert(struct eventpoll *ep, const struct epoll_event *event,
  1277. struct file *tfile, int fd, int full_check)
  1278. {
  1279. int error, pwake = 0;
  1280. __poll_t revents;
  1281. struct epitem *epi;
  1282. struct ep_pqueue epq;
  1283. struct eventpoll *tep = NULL;
  1284. if (is_file_epoll(tfile))
  1285. tep = tfile->private_data;
  1286. lockdep_assert_irqs_enabled();
  1287. if (unlikely(percpu_counter_compare(&ep->user->epoll_watches,
  1288. max_user_watches) >= 0))
  1289. return -ENOSPC;
  1290. percpu_counter_inc(&ep->user->epoll_watches);
  1291. if (!(epi = kmem_cache_zalloc(epi_cache, GFP_KERNEL))) {
  1292. percpu_counter_dec(&ep->user->epoll_watches);
  1293. return -ENOMEM;
  1294. }
  1295. /* Item initialization follow here ... */
  1296. INIT_LIST_HEAD(&epi->rdllink);
  1297. epi->ep = ep;
  1298. ep_set_ffd(&epi->ffd, tfile, fd);
  1299. epi->event = *event;
  1300. epi->next = EP_UNACTIVE_PTR;
  1301. if (tep)
  1302. mutex_lock_nested(&tep->mtx, 1);
  1303. /* Add the current item to the list of active epoll hook for this file */
  1304. if (unlikely(attach_epitem(tfile, epi) < 0)) {
  1305. if (tep)
  1306. mutex_unlock(&tep->mtx);
  1307. kmem_cache_free(epi_cache, epi);
  1308. percpu_counter_dec(&ep->user->epoll_watches);
  1309. return -ENOMEM;
  1310. }
  1311. if (full_check && !tep)
  1312. list_file(tfile);
  1313. /*
  1314. * Add the current item to the RB tree. All RB tree operations are
  1315. * protected by "mtx", and ep_insert() is called with "mtx" held.
  1316. */
  1317. ep_rbtree_insert(ep, epi);
  1318. if (tep)
  1319. mutex_unlock(&tep->mtx);
  1320. /*
  1321. * ep_remove_safe() calls in the later error paths can't lead to
  1322. * ep_free() as the ep file itself still holds an ep reference.
  1323. */
  1324. ep_get(ep);
  1325. /* now check if we've created too many backpaths */
  1326. if (unlikely(full_check && reverse_path_check())) {
  1327. ep_remove_safe(ep, epi);
  1328. return -EINVAL;
  1329. }
  1330. if (epi->event.events & EPOLLWAKEUP) {
  1331. error = ep_create_wakeup_source(epi);
  1332. if (error) {
  1333. ep_remove_safe(ep, epi);
  1334. return error;
  1335. }
  1336. }
  1337. /* Initialize the poll table using the queue callback */
  1338. epq.epi = epi;
  1339. init_poll_funcptr(&epq.pt, ep_ptable_queue_proc);
  1340. /*
  1341. * Attach the item to the poll hooks and get current event bits.
  1342. * We can safely use the file* here because its usage count has
  1343. * been increased by the caller of this function. Note that after
  1344. * this operation completes, the poll callback can start hitting
  1345. * the new item.
  1346. */
  1347. revents = ep_item_poll(epi, &epq.pt, 1);
  1348. /*
  1349. * We have to check if something went wrong during the poll wait queue
  1350. * install process. Namely an allocation for a wait queue failed due
  1351. * high memory pressure.
  1352. */
  1353. if (unlikely(!epq.epi)) {
  1354. ep_remove_safe(ep, epi);
  1355. return -ENOMEM;
  1356. }
  1357. /* We have to drop the new item inside our item list to keep track of it */
  1358. write_lock_irq(&ep->lock);
  1359. /* record NAPI ID of new item if present */
  1360. ep_set_busy_poll_napi_id(epi);
  1361. /* If the file is already "ready" we drop it inside the ready list */
  1362. if (revents && !ep_is_linked(epi)) {
  1363. list_add_tail(&epi->rdllink, &ep->rdllist);
  1364. ep_pm_stay_awake(epi);
  1365. /* Notify waiting tasks that events are available */
  1366. if (waitqueue_active(&ep->wq))
  1367. wake_up(&ep->wq);
  1368. if (waitqueue_active(&ep->poll_wait))
  1369. pwake++;
  1370. }
  1371. write_unlock_irq(&ep->lock);
  1372. /* We have to call this outside the lock */
  1373. if (pwake)
  1374. ep_poll_safewake(ep, NULL, 0);
  1375. return 0;
  1376. }
  1377. /*
  1378. * Modify the interest event mask by dropping an event if the new mask
  1379. * has a match in the current file status. Must be called with "mtx" held.
  1380. */
  1381. static int ep_modify(struct eventpoll *ep, struct epitem *epi,
  1382. const struct epoll_event *event)
  1383. {
  1384. int pwake = 0;
  1385. poll_table pt;
  1386. lockdep_assert_irqs_enabled();
  1387. init_poll_funcptr(&pt, NULL);
  1388. /*
  1389. * Set the new event interest mask before calling f_op->poll();
  1390. * otherwise we might miss an event that happens between the
  1391. * f_op->poll() call and the new event set registering.
  1392. */
  1393. epi->event.events = event->events; /* need barrier below */
  1394. epi->event.data = event->data; /* protected by mtx */
  1395. if (epi->event.events & EPOLLWAKEUP) {
  1396. if (!ep_has_wakeup_source(epi))
  1397. ep_create_wakeup_source(epi);
  1398. } else if (ep_has_wakeup_source(epi)) {
  1399. ep_destroy_wakeup_source(epi);
  1400. }
  1401. /*
  1402. * The following barrier has two effects:
  1403. *
  1404. * 1) Flush epi changes above to other CPUs. This ensures
  1405. * we do not miss events from ep_poll_callback if an
  1406. * event occurs immediately after we call f_op->poll().
  1407. * We need this because we did not take ep->lock while
  1408. * changing epi above (but ep_poll_callback does take
  1409. * ep->lock).
  1410. *
  1411. * 2) We also need to ensure we do not miss _past_ events
  1412. * when calling f_op->poll(). This barrier also
  1413. * pairs with the barrier in wq_has_sleeper (see
  1414. * comments for wq_has_sleeper).
  1415. *
  1416. * This barrier will now guarantee ep_poll_callback or f_op->poll
  1417. * (or both) will notice the readiness of an item.
  1418. */
  1419. smp_mb();
  1420. /*
  1421. * Get current event bits. We can safely use the file* here because
  1422. * its usage count has been increased by the caller of this function.
  1423. * If the item is "hot" and it is not registered inside the ready
  1424. * list, push it inside.
  1425. */
  1426. if (ep_item_poll(epi, &pt, 1)) {
  1427. write_lock_irq(&ep->lock);
  1428. if (!ep_is_linked(epi)) {
  1429. list_add_tail(&epi->rdllink, &ep->rdllist);
  1430. ep_pm_stay_awake(epi);
  1431. /* Notify waiting tasks that events are available */
  1432. if (waitqueue_active(&ep->wq))
  1433. wake_up(&ep->wq);
  1434. if (waitqueue_active(&ep->poll_wait))
  1435. pwake++;
  1436. }
  1437. write_unlock_irq(&ep->lock);
  1438. }
  1439. /* We have to call this outside the lock */
  1440. if (pwake)
  1441. ep_poll_safewake(ep, NULL, 0);
  1442. return 0;
  1443. }
  1444. static int ep_send_events(struct eventpoll *ep,
  1445. struct epoll_event __user *events, int maxevents)
  1446. {
  1447. struct epitem *epi, *tmp;
  1448. LIST_HEAD(txlist);
  1449. poll_table pt;
  1450. int res = 0;
  1451. /*
  1452. * Always short-circuit for fatal signals to allow threads to make a
  1453. * timely exit without the chance of finding more events available and
  1454. * fetching repeatedly.
  1455. */
  1456. if (fatal_signal_pending(current))
  1457. return -EINTR;
  1458. init_poll_funcptr(&pt, NULL);
  1459. mutex_lock(&ep->mtx);
  1460. ep_start_scan(ep, &txlist);
  1461. /*
  1462. * We can loop without lock because we are passed a task private list.
  1463. * Items cannot vanish during the loop we are holding ep->mtx.
  1464. */
  1465. list_for_each_entry_safe(epi, tmp, &txlist, rdllink) {
  1466. struct wakeup_source *ws;
  1467. __poll_t revents;
  1468. if (res >= maxevents)
  1469. break;
  1470. /*
  1471. * Activate ep->ws before deactivating epi->ws to prevent
  1472. * triggering auto-suspend here (in case we reactive epi->ws
  1473. * below).
  1474. *
  1475. * This could be rearranged to delay the deactivation of epi->ws
  1476. * instead, but then epi->ws would temporarily be out of sync
  1477. * with ep_is_linked().
  1478. */
  1479. ws = ep_wakeup_source(epi);
  1480. if (ws) {
  1481. if (ws->active)
  1482. __pm_stay_awake(ep->ws);
  1483. __pm_relax(ws);
  1484. }
  1485. list_del_init(&epi->rdllink);
  1486. /*
  1487. * If the event mask intersect the caller-requested one,
  1488. * deliver the event to userspace. Again, we are holding ep->mtx,
  1489. * so no operations coming from userspace can change the item.
  1490. */
  1491. revents = ep_item_poll(epi, &pt, 1);
  1492. if (!revents)
  1493. continue;
  1494. events = epoll_put_uevent(revents, epi->event.data, events);
  1495. if (!events) {
  1496. list_add(&epi->rdllink, &txlist);
  1497. ep_pm_stay_awake(epi);
  1498. if (!res)
  1499. res = -EFAULT;
  1500. break;
  1501. }
  1502. res++;
  1503. if (epi->event.events & EPOLLONESHOT)
  1504. epi->event.events &= EP_PRIVATE_BITS;
  1505. else if (!(epi->event.events & EPOLLET)) {
  1506. /*
  1507. * If this file has been added with Level
  1508. * Trigger mode, we need to insert back inside
  1509. * the ready list, so that the next call to
  1510. * epoll_wait() will check again the events
  1511. * availability. At this point, no one can insert
  1512. * into ep->rdllist besides us. The epoll_ctl()
  1513. * callers are locked out by
  1514. * ep_scan_ready_list() holding "mtx" and the
  1515. * poll callback will queue them in ep->ovflist.
  1516. */
  1517. list_add_tail(&epi->rdllink, &ep->rdllist);
  1518. ep_pm_stay_awake(epi);
  1519. }
  1520. }
  1521. ep_done_scan(ep, &txlist);
  1522. mutex_unlock(&ep->mtx);
  1523. return res;
  1524. }
  1525. static struct timespec64 *ep_timeout_to_timespec(struct timespec64 *to, long ms)
  1526. {
  1527. struct timespec64 now;
  1528. if (ms < 0)
  1529. return NULL;
  1530. if (!ms) {
  1531. to->tv_sec = 0;
  1532. to->tv_nsec = 0;
  1533. return to;
  1534. }
  1535. to->tv_sec = ms / MSEC_PER_SEC;
  1536. to->tv_nsec = NSEC_PER_MSEC * (ms % MSEC_PER_SEC);
  1537. ktime_get_ts64(&now);
  1538. *to = timespec64_add_safe(now, *to);
  1539. return to;
  1540. }
  1541. /*
  1542. * autoremove_wake_function, but remove even on failure to wake up, because we
  1543. * know that default_wake_function/ttwu will only fail if the thread is already
  1544. * woken, and in that case the ep_poll loop will remove the entry anyways, not
  1545. * try to reuse it.
  1546. */
  1547. static int ep_autoremove_wake_function(struct wait_queue_entry *wq_entry,
  1548. unsigned int mode, int sync, void *key)
  1549. {
  1550. int ret = default_wake_function(wq_entry, mode, sync, key);
  1551. /*
  1552. * Pairs with list_empty_careful in ep_poll, and ensures future loop
  1553. * iterations see the cause of this wakeup.
  1554. */
  1555. list_del_init_careful(&wq_entry->entry);
  1556. return ret;
  1557. }
  1558. /**
  1559. * ep_poll - Retrieves ready events, and delivers them to the caller-supplied
  1560. * event buffer.
  1561. *
  1562. * @ep: Pointer to the eventpoll context.
  1563. * @events: Pointer to the userspace buffer where the ready events should be
  1564. * stored.
  1565. * @maxevents: Size (in terms of number of events) of the caller event buffer.
  1566. * @timeout: Maximum timeout for the ready events fetch operation, in
  1567. * timespec. If the timeout is zero, the function will not block,
  1568. * while if the @timeout ptr is NULL, the function will block
  1569. * until at least one event has been retrieved (or an error
  1570. * occurred).
  1571. *
  1572. * Return: the number of ready events which have been fetched, or an
  1573. * error code, in case of error.
  1574. */
  1575. static int ep_poll(struct eventpoll *ep, struct epoll_event __user *events,
  1576. int maxevents, struct timespec64 *timeout)
  1577. {
  1578. int res, eavail, timed_out = 0;
  1579. u64 slack = 0;
  1580. wait_queue_entry_t wait;
  1581. ktime_t expires, *to = NULL;
  1582. lockdep_assert_irqs_enabled();
  1583. if (timeout && (timeout->tv_sec | timeout->tv_nsec)) {
  1584. slack = select_estimate_accuracy(timeout);
  1585. to = &expires;
  1586. *to = timespec64_to_ktime(*timeout);
  1587. } else if (timeout) {
  1588. /*
  1589. * Avoid the unnecessary trip to the wait queue loop, if the
  1590. * caller specified a non blocking operation.
  1591. */
  1592. timed_out = 1;
  1593. }
  1594. /*
  1595. * This call is racy: We may or may not see events that are being added
  1596. * to the ready list under the lock (e.g., in IRQ callbacks). For cases
  1597. * with a non-zero timeout, this thread will check the ready list under
  1598. * lock and will add to the wait queue. For cases with a zero
  1599. * timeout, the user by definition should not care and will have to
  1600. * recheck again.
  1601. */
  1602. eavail = ep_events_available(ep);
  1603. while (1) {
  1604. if (eavail) {
  1605. /*
  1606. * Try to transfer events to user space. In case we get
  1607. * 0 events and there's still timeout left over, we go
  1608. * trying again in search of more luck.
  1609. */
  1610. res = ep_send_events(ep, events, maxevents);
  1611. if (res)
  1612. return res;
  1613. }
  1614. if (timed_out)
  1615. return 0;
  1616. eavail = ep_busy_loop(ep, timed_out);
  1617. if (eavail)
  1618. continue;
  1619. if (signal_pending(current))
  1620. return -EINTR;
  1621. /*
  1622. * Internally init_wait() uses autoremove_wake_function(),
  1623. * thus wait entry is removed from the wait queue on each
  1624. * wakeup. Why it is important? In case of several waiters
  1625. * each new wakeup will hit the next waiter, giving it the
  1626. * chance to harvest new event. Otherwise wakeup can be
  1627. * lost. This is also good performance-wise, because on
  1628. * normal wakeup path no need to call __remove_wait_queue()
  1629. * explicitly, thus ep->lock is not taken, which halts the
  1630. * event delivery.
  1631. *
  1632. * In fact, we now use an even more aggressive function that
  1633. * unconditionally removes, because we don't reuse the wait
  1634. * entry between loop iterations. This lets us also avoid the
  1635. * performance issue if a process is killed, causing all of its
  1636. * threads to wake up without being removed normally.
  1637. */
  1638. init_wait(&wait);
  1639. wait.func = ep_autoremove_wake_function;
  1640. write_lock_irq(&ep->lock);
  1641. /*
  1642. * Barrierless variant, waitqueue_active() is called under
  1643. * the same lock on wakeup ep_poll_callback() side, so it
  1644. * is safe to avoid an explicit barrier.
  1645. */
  1646. __set_current_state(TASK_INTERRUPTIBLE);
  1647. /*
  1648. * Do the final check under the lock. ep_scan_ready_list()
  1649. * plays with two lists (->rdllist and ->ovflist) and there
  1650. * is always a race when both lists are empty for short
  1651. * period of time although events are pending, so lock is
  1652. * important.
  1653. */
  1654. eavail = ep_events_available(ep);
  1655. if (!eavail)
  1656. __add_wait_queue_exclusive(&ep->wq, &wait);
  1657. write_unlock_irq(&ep->lock);
  1658. if (!eavail)
  1659. timed_out = !schedule_hrtimeout_range(to, slack,
  1660. HRTIMER_MODE_ABS);
  1661. __set_current_state(TASK_RUNNING);
  1662. /*
  1663. * We were woken up, thus go and try to harvest some events.
  1664. * If timed out and still on the wait queue, recheck eavail
  1665. * carefully under lock, below.
  1666. */
  1667. eavail = 1;
  1668. if (!list_empty_careful(&wait.entry)) {
  1669. write_lock_irq(&ep->lock);
  1670. /*
  1671. * If the thread timed out and is not on the wait queue,
  1672. * it means that the thread was woken up after its
  1673. * timeout expired before it could reacquire the lock.
  1674. * Thus, when wait.entry is empty, it needs to harvest
  1675. * events.
  1676. */
  1677. if (timed_out)
  1678. eavail = list_empty(&wait.entry);
  1679. __remove_wait_queue(&ep->wq, &wait);
  1680. write_unlock_irq(&ep->lock);
  1681. }
  1682. }
  1683. }
  1684. /**
  1685. * ep_loop_check_proc - verify that adding an epoll file inside another
  1686. * epoll structure does not violate the constraints, in
  1687. * terms of closed loops, or too deep chains (which can
  1688. * result in excessive stack usage).
  1689. *
  1690. * @ep: the &struct eventpoll to be currently checked.
  1691. * @depth: Current depth of the path being checked.
  1692. *
  1693. * Return: %zero if adding the epoll @file inside current epoll
  1694. * structure @ep does not violate the constraints, or %-1 otherwise.
  1695. */
  1696. static int ep_loop_check_proc(struct eventpoll *ep, int depth)
  1697. {
  1698. int error = 0;
  1699. struct rb_node *rbp;
  1700. struct epitem *epi;
  1701. mutex_lock_nested(&ep->mtx, depth + 1);
  1702. ep->gen = loop_check_gen;
  1703. for (rbp = rb_first_cached(&ep->rbr); rbp; rbp = rb_next(rbp)) {
  1704. epi = rb_entry(rbp, struct epitem, rbn);
  1705. if (unlikely(is_file_epoll(epi->ffd.file))) {
  1706. struct eventpoll *ep_tovisit;
  1707. ep_tovisit = epi->ffd.file->private_data;
  1708. if (ep_tovisit->gen == loop_check_gen)
  1709. continue;
  1710. if (ep_tovisit == inserting_into || depth > EP_MAX_NESTS)
  1711. error = -1;
  1712. else
  1713. error = ep_loop_check_proc(ep_tovisit, depth + 1);
  1714. if (error != 0)
  1715. break;
  1716. } else {
  1717. /*
  1718. * If we've reached a file that is not associated with
  1719. * an ep, then we need to check if the newly added
  1720. * links are going to add too many wakeup paths. We do
  1721. * this by adding it to the tfile_check_list, if it's
  1722. * not already there, and calling reverse_path_check()
  1723. * during ep_insert().
  1724. */
  1725. list_file(epi->ffd.file);
  1726. }
  1727. }
  1728. mutex_unlock(&ep->mtx);
  1729. return error;
  1730. }
  1731. /**
  1732. * ep_loop_check - Performs a check to verify that adding an epoll file (@to)
  1733. * into another epoll file (represented by @ep) does not create
  1734. * closed loops or too deep chains.
  1735. *
  1736. * @ep: Pointer to the epoll we are inserting into.
  1737. * @to: Pointer to the epoll to be inserted.
  1738. *
  1739. * Return: %zero if adding the epoll @to inside the epoll @from
  1740. * does not violate the constraints, or %-1 otherwise.
  1741. */
  1742. static int ep_loop_check(struct eventpoll *ep, struct eventpoll *to)
  1743. {
  1744. inserting_into = ep;
  1745. return ep_loop_check_proc(to, 0);
  1746. }
  1747. static void clear_tfile_check_list(void)
  1748. {
  1749. rcu_read_lock();
  1750. while (tfile_check_list != EP_UNACTIVE_PTR) {
  1751. struct epitems_head *head = tfile_check_list;
  1752. tfile_check_list = head->next;
  1753. unlist_file(head);
  1754. }
  1755. rcu_read_unlock();
  1756. }
  1757. /*
  1758. * Open an eventpoll file descriptor.
  1759. */
  1760. static int do_epoll_create(int flags)
  1761. {
  1762. int error, fd;
  1763. struct eventpoll *ep = NULL;
  1764. struct file *file;
  1765. /* Check the EPOLL_* constant for consistency. */
  1766. BUILD_BUG_ON(EPOLL_CLOEXEC != O_CLOEXEC);
  1767. if (flags & ~EPOLL_CLOEXEC)
  1768. return -EINVAL;
  1769. /*
  1770. * Create the internal data structure ("struct eventpoll").
  1771. */
  1772. error = ep_alloc(&ep);
  1773. if (error < 0)
  1774. return error;
  1775. /*
  1776. * Creates all the items needed to setup an eventpoll file. That is,
  1777. * a file structure and a free file descriptor.
  1778. */
  1779. fd = get_unused_fd_flags(O_RDWR | (flags & O_CLOEXEC));
  1780. if (fd < 0) {
  1781. error = fd;
  1782. goto out_free_ep;
  1783. }
  1784. file = anon_inode_getfile("[eventpoll]", &eventpoll_fops, ep,
  1785. O_RDWR | (flags & O_CLOEXEC));
  1786. if (IS_ERR(file)) {
  1787. error = PTR_ERR(file);
  1788. goto out_free_fd;
  1789. }
  1790. ep->file = file;
  1791. fd_install(fd, file);
  1792. return fd;
  1793. out_free_fd:
  1794. put_unused_fd(fd);
  1795. out_free_ep:
  1796. ep_clear_and_put(ep);
  1797. return error;
  1798. }
  1799. SYSCALL_DEFINE1(epoll_create1, int, flags)
  1800. {
  1801. return do_epoll_create(flags);
  1802. }
  1803. SYSCALL_DEFINE1(epoll_create, int, size)
  1804. {
  1805. if (size <= 0)
  1806. return -EINVAL;
  1807. return do_epoll_create(0);
  1808. }
  1809. static inline int epoll_mutex_lock(struct mutex *mutex, int depth,
  1810. bool nonblock)
  1811. {
  1812. if (!nonblock) {
  1813. mutex_lock_nested(mutex, depth);
  1814. return 0;
  1815. }
  1816. if (mutex_trylock(mutex))
  1817. return 0;
  1818. return -EAGAIN;
  1819. }
  1820. int do_epoll_ctl(int epfd, int op, int fd, struct epoll_event *epds,
  1821. bool nonblock)
  1822. {
  1823. int error;
  1824. int full_check = 0;
  1825. struct fd f, tf;
  1826. struct eventpoll *ep;
  1827. struct epitem *epi;
  1828. struct eventpoll *tep = NULL;
  1829. error = -EBADF;
  1830. f = fdget(epfd);
  1831. if (!f.file)
  1832. goto error_return;
  1833. /* Get the "struct file *" for the target file */
  1834. tf = fdget(fd);
  1835. if (!tf.file)
  1836. goto error_fput;
  1837. /* The target file descriptor must support poll */
  1838. error = -EPERM;
  1839. if (!file_can_poll(tf.file))
  1840. goto error_tgt_fput;
  1841. /* Check if EPOLLWAKEUP is allowed */
  1842. if (ep_op_has_event(op))
  1843. ep_take_care_of_epollwakeup(epds);
  1844. /*
  1845. * We have to check that the file structure underneath the file descriptor
  1846. * the user passed to us _is_ an eventpoll file. And also we do not permit
  1847. * adding an epoll file descriptor inside itself.
  1848. */
  1849. error = -EINVAL;
  1850. if (f.file == tf.file || !is_file_epoll(f.file))
  1851. goto error_tgt_fput;
  1852. /*
  1853. * epoll adds to the wakeup queue at EPOLL_CTL_ADD time only,
  1854. * so EPOLLEXCLUSIVE is not allowed for a EPOLL_CTL_MOD operation.
  1855. * Also, we do not currently supported nested exclusive wakeups.
  1856. */
  1857. if (ep_op_has_event(op) && (epds->events & EPOLLEXCLUSIVE)) {
  1858. if (op == EPOLL_CTL_MOD)
  1859. goto error_tgt_fput;
  1860. if (op == EPOLL_CTL_ADD && (is_file_epoll(tf.file) ||
  1861. (epds->events & ~EPOLLEXCLUSIVE_OK_BITS)))
  1862. goto error_tgt_fput;
  1863. }
  1864. /*
  1865. * At this point it is safe to assume that the "private_data" contains
  1866. * our own data structure.
  1867. */
  1868. ep = f.file->private_data;
  1869. /*
  1870. * When we insert an epoll file descriptor inside another epoll file
  1871. * descriptor, there is the chance of creating closed loops, which are
  1872. * better be handled here, than in more critical paths. While we are
  1873. * checking for loops we also determine the list of files reachable
  1874. * and hang them on the tfile_check_list, so we can check that we
  1875. * haven't created too many possible wakeup paths.
  1876. *
  1877. * We do not need to take the global 'epumutex' on EPOLL_CTL_ADD when
  1878. * the epoll file descriptor is attaching directly to a wakeup source,
  1879. * unless the epoll file descriptor is nested. The purpose of taking the
  1880. * 'epmutex' on add is to prevent complex toplogies such as loops and
  1881. * deep wakeup paths from forming in parallel through multiple
  1882. * EPOLL_CTL_ADD operations.
  1883. */
  1884. error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
  1885. if (error)
  1886. goto error_tgt_fput;
  1887. if (op == EPOLL_CTL_ADD) {
  1888. if (READ_ONCE(f.file->f_ep) || ep->gen == loop_check_gen ||
  1889. is_file_epoll(tf.file)) {
  1890. mutex_unlock(&ep->mtx);
  1891. error = epoll_mutex_lock(&epmutex, 0, nonblock);
  1892. if (error)
  1893. goto error_tgt_fput;
  1894. loop_check_gen++;
  1895. full_check = 1;
  1896. if (is_file_epoll(tf.file)) {
  1897. tep = tf.file->private_data;
  1898. error = -ELOOP;
  1899. if (ep_loop_check(ep, tep) != 0)
  1900. goto error_tgt_fput;
  1901. }
  1902. error = epoll_mutex_lock(&ep->mtx, 0, nonblock);
  1903. if (error)
  1904. goto error_tgt_fput;
  1905. }
  1906. }
  1907. /*
  1908. * Try to lookup the file inside our RB tree. Since we grabbed "mtx"
  1909. * above, we can be sure to be able to use the item looked up by
  1910. * ep_find() till we release the mutex.
  1911. */
  1912. epi = ep_find(ep, tf.file, fd);
  1913. error = -EINVAL;
  1914. switch (op) {
  1915. case EPOLL_CTL_ADD:
  1916. if (!epi) {
  1917. epds->events |= EPOLLERR | EPOLLHUP;
  1918. error = ep_insert(ep, epds, tf.file, fd, full_check);
  1919. } else
  1920. error = -EEXIST;
  1921. break;
  1922. case EPOLL_CTL_DEL:
  1923. if (epi) {
  1924. /*
  1925. * The eventpoll itself is still alive: the refcount
  1926. * can't go to zero here.
  1927. */
  1928. ep_remove_safe(ep, epi);
  1929. error = 0;
  1930. } else {
  1931. error = -ENOENT;
  1932. }
  1933. break;
  1934. case EPOLL_CTL_MOD:
  1935. if (epi) {
  1936. if (!(epi->event.events & EPOLLEXCLUSIVE)) {
  1937. epds->events |= EPOLLERR | EPOLLHUP;
  1938. error = ep_modify(ep, epi, epds);
  1939. }
  1940. } else
  1941. error = -ENOENT;
  1942. break;
  1943. }
  1944. mutex_unlock(&ep->mtx);
  1945. error_tgt_fput:
  1946. if (full_check) {
  1947. clear_tfile_check_list();
  1948. loop_check_gen++;
  1949. mutex_unlock(&epmutex);
  1950. }
  1951. fdput(tf);
  1952. error_fput:
  1953. fdput(f);
  1954. error_return:
  1955. return error;
  1956. }
  1957. /*
  1958. * The following function implements the controller interface for
  1959. * the eventpoll file that enables the insertion/removal/change of
  1960. * file descriptors inside the interest set.
  1961. */
  1962. SYSCALL_DEFINE4(epoll_ctl, int, epfd, int, op, int, fd,
  1963. struct epoll_event __user *, event)
  1964. {
  1965. struct epoll_event epds;
  1966. if (ep_op_has_event(op) &&
  1967. copy_from_user(&epds, event, sizeof(struct epoll_event)))
  1968. return -EFAULT;
  1969. return do_epoll_ctl(epfd, op, fd, &epds, false);
  1970. }
  1971. /*
  1972. * Implement the event wait interface for the eventpoll file. It is the kernel
  1973. * part of the user space epoll_wait(2).
  1974. */
  1975. static int do_epoll_wait(int epfd, struct epoll_event __user *events,
  1976. int maxevents, struct timespec64 *to)
  1977. {
  1978. int error;
  1979. struct fd f;
  1980. struct eventpoll *ep;
  1981. /* The maximum number of event must be greater than zero */
  1982. if (maxevents <= 0 || maxevents > EP_MAX_EVENTS)
  1983. return -EINVAL;
  1984. /* Verify that the area passed by the user is writeable */
  1985. if (!access_ok(events, maxevents * sizeof(struct epoll_event)))
  1986. return -EFAULT;
  1987. /* Get the "struct file *" for the eventpoll file */
  1988. f = fdget(epfd);
  1989. if (!f.file)
  1990. return -EBADF;
  1991. /*
  1992. * We have to check that the file structure underneath the fd
  1993. * the user passed to us _is_ an eventpoll file.
  1994. */
  1995. error = -EINVAL;
  1996. if (!is_file_epoll(f.file))
  1997. goto error_fput;
  1998. /*
  1999. * At this point it is safe to assume that the "private_data" contains
  2000. * our own data structure.
  2001. */
  2002. ep = f.file->private_data;
  2003. /* Time to fish for events ... */
  2004. error = ep_poll(ep, events, maxevents, to);
  2005. error_fput:
  2006. fdput(f);
  2007. return error;
  2008. }
  2009. SYSCALL_DEFINE4(epoll_wait, int, epfd, struct epoll_event __user *, events,
  2010. int, maxevents, int, timeout)
  2011. {
  2012. struct timespec64 to;
  2013. return do_epoll_wait(epfd, events, maxevents,
  2014. ep_timeout_to_timespec(&to, timeout));
  2015. }
  2016. /*
  2017. * Implement the event wait interface for the eventpoll file. It is the kernel
  2018. * part of the user space epoll_pwait(2).
  2019. */
  2020. static int do_epoll_pwait(int epfd, struct epoll_event __user *events,
  2021. int maxevents, struct timespec64 *to,
  2022. const sigset_t __user *sigmask, size_t sigsetsize)
  2023. {
  2024. int error;
  2025. /*
  2026. * If the caller wants a certain signal mask to be set during the wait,
  2027. * we apply it here.
  2028. */
  2029. error = set_user_sigmask(sigmask, sigsetsize);
  2030. if (error)
  2031. return error;
  2032. error = do_epoll_wait(epfd, events, maxevents, to);
  2033. restore_saved_sigmask_unless(error == -EINTR);
  2034. return error;
  2035. }
  2036. SYSCALL_DEFINE6(epoll_pwait, int, epfd, struct epoll_event __user *, events,
  2037. int, maxevents, int, timeout, const sigset_t __user *, sigmask,
  2038. size_t, sigsetsize)
  2039. {
  2040. struct timespec64 to;
  2041. return do_epoll_pwait(epfd, events, maxevents,
  2042. ep_timeout_to_timespec(&to, timeout),
  2043. sigmask, sigsetsize);
  2044. }
  2045. SYSCALL_DEFINE6(epoll_pwait2, int, epfd, struct epoll_event __user *, events,
  2046. int, maxevents, const struct __kernel_timespec __user *, timeout,
  2047. const sigset_t __user *, sigmask, size_t, sigsetsize)
  2048. {
  2049. struct timespec64 ts, *to = NULL;
  2050. if (timeout) {
  2051. if (get_timespec64(&ts, timeout))
  2052. return -EFAULT;
  2053. to = &ts;
  2054. if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
  2055. return -EINVAL;
  2056. }
  2057. return do_epoll_pwait(epfd, events, maxevents, to,
  2058. sigmask, sigsetsize);
  2059. }
  2060. #ifdef CONFIG_COMPAT
  2061. static int do_compat_epoll_pwait(int epfd, struct epoll_event __user *events,
  2062. int maxevents, struct timespec64 *timeout,
  2063. const compat_sigset_t __user *sigmask,
  2064. compat_size_t sigsetsize)
  2065. {
  2066. long err;
  2067. /*
  2068. * If the caller wants a certain signal mask to be set during the wait,
  2069. * we apply it here.
  2070. */
  2071. err = set_compat_user_sigmask(sigmask, sigsetsize);
  2072. if (err)
  2073. return err;
  2074. err = do_epoll_wait(epfd, events, maxevents, timeout);
  2075. restore_saved_sigmask_unless(err == -EINTR);
  2076. return err;
  2077. }
  2078. COMPAT_SYSCALL_DEFINE6(epoll_pwait, int, epfd,
  2079. struct epoll_event __user *, events,
  2080. int, maxevents, int, timeout,
  2081. const compat_sigset_t __user *, sigmask,
  2082. compat_size_t, sigsetsize)
  2083. {
  2084. struct timespec64 to;
  2085. return do_compat_epoll_pwait(epfd, events, maxevents,
  2086. ep_timeout_to_timespec(&to, timeout),
  2087. sigmask, sigsetsize);
  2088. }
  2089. COMPAT_SYSCALL_DEFINE6(epoll_pwait2, int, epfd,
  2090. struct epoll_event __user *, events,
  2091. int, maxevents,
  2092. const struct __kernel_timespec __user *, timeout,
  2093. const compat_sigset_t __user *, sigmask,
  2094. compat_size_t, sigsetsize)
  2095. {
  2096. struct timespec64 ts, *to = NULL;
  2097. if (timeout) {
  2098. if (get_timespec64(&ts, timeout))
  2099. return -EFAULT;
  2100. to = &ts;
  2101. if (poll_select_set_timeout(to, ts.tv_sec, ts.tv_nsec))
  2102. return -EINVAL;
  2103. }
  2104. return do_compat_epoll_pwait(epfd, events, maxevents, to,
  2105. sigmask, sigsetsize);
  2106. }
  2107. #endif
  2108. static int __init eventpoll_init(void)
  2109. {
  2110. struct sysinfo si;
  2111. si_meminfo(&si);
  2112. /*
  2113. * Allows top 4% of lomem to be allocated for epoll watches (per user).
  2114. */
  2115. max_user_watches = (((si.totalram - si.totalhigh) / 25) << PAGE_SHIFT) /
  2116. EP_ITEM_COST;
  2117. BUG_ON(max_user_watches < 0);
  2118. /*
  2119. * We can have many thousands of epitems, so prevent this from
  2120. * using an extra cache line on 64-bit (and smaller) CPUs
  2121. */
  2122. BUILD_BUG_ON(sizeof(void *) <= 8 && sizeof(struct epitem) > 128);
  2123. /* Allocates slab cache used to allocate "struct epitem" items */
  2124. epi_cache = kmem_cache_create("eventpoll_epi", sizeof(struct epitem),
  2125. 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC|SLAB_ACCOUNT, NULL);
  2126. /* Allocates slab cache used to allocate "struct eppoll_entry" */
  2127. pwq_cache = kmem_cache_create("eventpoll_pwq",
  2128. sizeof(struct eppoll_entry), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);
  2129. epoll_sysctls_init();
  2130. ephead_cache = kmem_cache_create("ep_head",
  2131. sizeof(struct epitems_head), 0, SLAB_PANIC|SLAB_ACCOUNT, NULL);
  2132. return 0;
  2133. }
  2134. fs_initcall(eventpoll_init);