exit.c 48 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/kernel/exit.c
  4. *
  5. * Copyright (C) 1991, 1992 Linus Torvalds
  6. */
  7. #include <linux/mm.h>
  8. #include <linux/slab.h>
  9. #include <linux/sched/autogroup.h>
  10. #include <linux/sched/mm.h>
  11. #include <linux/sched/stat.h>
  12. #include <linux/sched/task.h>
  13. #include <linux/sched/task_stack.h>
  14. #include <linux/sched/cputime.h>
  15. #include <linux/interrupt.h>
  16. #include <linux/module.h>
  17. #include <linux/capability.h>
  18. #include <linux/completion.h>
  19. #include <linux/personality.h>
  20. #include <linux/tty.h>
  21. #include <linux/iocontext.h>
  22. #include <linux/key.h>
  23. #include <linux/cpu.h>
  24. #include <linux/acct.h>
  25. #include <linux/tsacct_kern.h>
  26. #include <linux/file.h>
  27. #include <linux/fdtable.h>
  28. #include <linux/freezer.h>
  29. #include <linux/binfmts.h>
  30. #include <linux/nsproxy.h>
  31. #include <linux/pid_namespace.h>
  32. #include <linux/ptrace.h>
  33. #include <linux/profile.h>
  34. #include <linux/mount.h>
  35. #include <linux/proc_fs.h>
  36. #include <linux/kthread.h>
  37. #include <linux/mempolicy.h>
  38. #include <linux/taskstats_kern.h>
  39. #include <linux/delayacct.h>
  40. #include <linux/cgroup.h>
  41. #include <linux/syscalls.h>
  42. #include <linux/signal.h>
  43. #include <linux/posix-timers.h>
  44. #include <linux/cn_proc.h>
  45. #include <linux/mutex.h>
  46. #include <linux/futex.h>
  47. #include <linux/pipe_fs_i.h>
  48. #include <linux/audit.h> /* for audit_free() */
  49. #include <linux/resource.h>
  50. #include <linux/task_io_accounting_ops.h>
  51. #include <linux/blkdev.h>
  52. #include <linux/task_work.h>
  53. #include <linux/fs_struct.h>
  54. #include <linux/init_task.h>
  55. #include <linux/perf_event.h>
  56. #include <trace/events/sched.h>
  57. #include <linux/hw_breakpoint.h>
  58. #include <linux/oom.h>
  59. #include <linux/writeback.h>
  60. #include <linux/shm.h>
  61. #include <linux/kcov.h>
  62. #include <linux/kmsan.h>
  63. #include <linux/random.h>
  64. #include <linux/rcuwait.h>
  65. #include <linux/compat.h>
  66. #include <linux/io_uring.h>
  67. #include <linux/kprobes.h>
  68. #include <linux/rethook.h>
  69. #include <linux/sysfs.h>
  70. #include <linux/uaccess.h>
  71. #include <asm/unistd.h>
  72. #include <asm/mmu_context.h>
  73. #include <trace/hooks/mm.h>
  74. #include <trace/hooks/dtask.h>
  75. /*
  76. * The default value should be high enough to not crash a system that randomly
  77. * crashes its kernel from time to time, but low enough to at least not permit
  78. * overflowing 32-bit refcounts or the ldsem writer count.
  79. */
  80. static unsigned int oops_limit = 10000;
  81. #ifdef CONFIG_SYSCTL
  82. static struct ctl_table kern_exit_table[] = {
  83. {
  84. .procname = "oops_limit",
  85. .data = &oops_limit,
  86. .maxlen = sizeof(oops_limit),
  87. .mode = 0644,
  88. .proc_handler = proc_douintvec,
  89. },
  90. { }
  91. };
  92. static __init int kernel_exit_sysctls_init(void)
  93. {
  94. register_sysctl_init("kernel", kern_exit_table);
  95. return 0;
  96. }
  97. late_initcall(kernel_exit_sysctls_init);
  98. #endif
  99. static atomic_t oops_count = ATOMIC_INIT(0);
  100. #ifdef CONFIG_SYSFS
  101. static ssize_t oops_count_show(struct kobject *kobj, struct kobj_attribute *attr,
  102. char *page)
  103. {
  104. return sysfs_emit(page, "%d\n", atomic_read(&oops_count));
  105. }
  106. static struct kobj_attribute oops_count_attr = __ATTR_RO(oops_count);
  107. static __init int kernel_exit_sysfs_init(void)
  108. {
  109. sysfs_add_file_to_group(kernel_kobj, &oops_count_attr.attr, NULL);
  110. return 0;
  111. }
  112. late_initcall(kernel_exit_sysfs_init);
  113. #endif
  114. static void __unhash_process(struct task_struct *p, bool group_dead)
  115. {
  116. nr_threads--;
  117. detach_pid(p, PIDTYPE_PID);
  118. if (group_dead) {
  119. detach_pid(p, PIDTYPE_TGID);
  120. detach_pid(p, PIDTYPE_PGID);
  121. detach_pid(p, PIDTYPE_SID);
  122. list_del_rcu(&p->tasks);
  123. list_del_init(&p->sibling);
  124. __this_cpu_dec(process_counts);
  125. }
  126. list_del_rcu(&p->thread_group);
  127. list_del_rcu(&p->thread_node);
  128. }
  129. /*
  130. * This function expects the tasklist_lock write-locked.
  131. */
  132. static void __exit_signal(struct task_struct *tsk)
  133. {
  134. struct signal_struct *sig = tsk->signal;
  135. bool group_dead = thread_group_leader(tsk);
  136. struct sighand_struct *sighand;
  137. struct tty_struct *tty;
  138. u64 utime, stime;
  139. sighand = rcu_dereference_check(tsk->sighand,
  140. lockdep_tasklist_lock_is_held());
  141. spin_lock(&sighand->siglock);
  142. #ifdef CONFIG_POSIX_TIMERS
  143. posix_cpu_timers_exit(tsk);
  144. if (group_dead)
  145. posix_cpu_timers_exit_group(tsk);
  146. #endif
  147. if (group_dead) {
  148. tty = sig->tty;
  149. sig->tty = NULL;
  150. } else {
  151. /*
  152. * If there is any task waiting for the group exit
  153. * then notify it:
  154. */
  155. if (sig->notify_count > 0 && !--sig->notify_count)
  156. wake_up_process(sig->group_exec_task);
  157. if (tsk == sig->curr_target)
  158. sig->curr_target = next_thread(tsk);
  159. }
  160. add_device_randomness((const void*) &tsk->se.sum_exec_runtime,
  161. sizeof(unsigned long long));
  162. /*
  163. * Accumulate here the counters for all threads as they die. We could
  164. * skip the group leader because it is the last user of signal_struct,
  165. * but we want to avoid the race with thread_group_cputime() which can
  166. * see the empty ->thread_head list.
  167. */
  168. task_cputime(tsk, &utime, &stime);
  169. write_seqlock(&sig->stats_lock);
  170. sig->utime += utime;
  171. sig->stime += stime;
  172. sig->gtime += task_gtime(tsk);
  173. sig->min_flt += tsk->min_flt;
  174. sig->maj_flt += tsk->maj_flt;
  175. sig->nvcsw += tsk->nvcsw;
  176. sig->nivcsw += tsk->nivcsw;
  177. sig->inblock += task_io_get_inblock(tsk);
  178. sig->oublock += task_io_get_oublock(tsk);
  179. task_io_accounting_add(&sig->ioac, &tsk->ioac);
  180. sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
  181. sig->nr_threads--;
  182. __unhash_process(tsk, group_dead);
  183. write_sequnlock(&sig->stats_lock);
  184. /*
  185. * Do this under ->siglock, we can race with another thread
  186. * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals.
  187. */
  188. flush_sigqueue(&tsk->pending);
  189. tsk->sighand = NULL;
  190. spin_unlock(&sighand->siglock);
  191. __cleanup_sighand(sighand);
  192. clear_tsk_thread_flag(tsk, TIF_SIGPENDING);
  193. if (group_dead) {
  194. flush_sigqueue(&sig->shared_pending);
  195. tty_kref_put(tty);
  196. }
  197. }
  198. static void delayed_put_task_struct(struct rcu_head *rhp)
  199. {
  200. struct task_struct *tsk = container_of(rhp, struct task_struct, rcu);
  201. kprobe_flush_task(tsk);
  202. rethook_flush_task(tsk);
  203. perf_event_delayed_put(tsk);
  204. trace_sched_process_free(tsk);
  205. put_task_struct(tsk);
  206. }
  207. void put_task_struct_rcu_user(struct task_struct *task)
  208. {
  209. if (refcount_dec_and_test(&task->rcu_users))
  210. call_rcu(&task->rcu, delayed_put_task_struct);
  211. }
  212. void __weak release_thread(struct task_struct *dead_task)
  213. {
  214. }
  215. void release_task(struct task_struct *p)
  216. {
  217. struct task_struct *leader;
  218. struct pid *thread_pid;
  219. int zap_leader;
  220. repeat:
  221. /* don't need to get the RCU readlock here - the process is dead and
  222. * can't be modifying its own credentials. But shut RCU-lockdep up */
  223. rcu_read_lock();
  224. dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1);
  225. rcu_read_unlock();
  226. cgroup_release(p);
  227. write_lock_irq(&tasklist_lock);
  228. ptrace_release_task(p);
  229. thread_pid = get_pid(p->thread_pid);
  230. __exit_signal(p);
  231. /*
  232. * If we are the last non-leader member of the thread
  233. * group, and the leader is zombie, then notify the
  234. * group leader's parent process. (if it wants notification.)
  235. */
  236. zap_leader = 0;
  237. leader = p->group_leader;
  238. if (leader != p && thread_group_empty(leader)
  239. && leader->exit_state == EXIT_ZOMBIE) {
  240. /*
  241. * If we were the last child thread and the leader has
  242. * exited already, and the leader's parent ignores SIGCHLD,
  243. * then we are the one who should release the leader.
  244. */
  245. zap_leader = do_notify_parent(leader, leader->exit_signal);
  246. if (zap_leader)
  247. leader->exit_state = EXIT_DEAD;
  248. }
  249. write_unlock_irq(&tasklist_lock);
  250. seccomp_filter_release(p);
  251. proc_flush_pid(thread_pid);
  252. put_pid(thread_pid);
  253. release_thread(p);
  254. put_task_struct_rcu_user(p);
  255. p = leader;
  256. if (unlikely(zap_leader))
  257. goto repeat;
  258. }
  259. int rcuwait_wake_up(struct rcuwait *w)
  260. {
  261. int ret = 0;
  262. struct task_struct *task;
  263. rcu_read_lock();
  264. /*
  265. * Order condition vs @task, such that everything prior to the load
  266. * of @task is visible. This is the condition as to why the user called
  267. * rcuwait_wake() in the first place. Pairs with set_current_state()
  268. * barrier (A) in rcuwait_wait_event().
  269. *
  270. * WAIT WAKE
  271. * [S] tsk = current [S] cond = true
  272. * MB (A) MB (B)
  273. * [L] cond [L] tsk
  274. */
  275. smp_mb(); /* (B) */
  276. task = rcu_dereference(w->task);
  277. if (task)
  278. ret = wake_up_process(task);
  279. rcu_read_unlock();
  280. return ret;
  281. }
  282. EXPORT_SYMBOL_GPL(rcuwait_wake_up);
  283. /*
  284. * Determine if a process group is "orphaned", according to the POSIX
  285. * definition in 2.2.2.52. Orphaned process groups are not to be affected
  286. * by terminal-generated stop signals. Newly orphaned process groups are
  287. * to receive a SIGHUP and a SIGCONT.
  288. *
  289. * "I ask you, have you ever known what it is to be an orphan?"
  290. */
  291. static int will_become_orphaned_pgrp(struct pid *pgrp,
  292. struct task_struct *ignored_task)
  293. {
  294. struct task_struct *p;
  295. do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
  296. if ((p == ignored_task) ||
  297. (p->exit_state && thread_group_empty(p)) ||
  298. is_global_init(p->real_parent))
  299. continue;
  300. if (task_pgrp(p->real_parent) != pgrp &&
  301. task_session(p->real_parent) == task_session(p))
  302. return 0;
  303. } while_each_pid_task(pgrp, PIDTYPE_PGID, p);
  304. return 1;
  305. }
  306. int is_current_pgrp_orphaned(void)
  307. {
  308. int retval;
  309. read_lock(&tasklist_lock);
  310. retval = will_become_orphaned_pgrp(task_pgrp(current), NULL);
  311. read_unlock(&tasklist_lock);
  312. return retval;
  313. }
  314. static bool has_stopped_jobs(struct pid *pgrp)
  315. {
  316. struct task_struct *p;
  317. do_each_pid_task(pgrp, PIDTYPE_PGID, p) {
  318. if (p->signal->flags & SIGNAL_STOP_STOPPED)
  319. return true;
  320. } while_each_pid_task(pgrp, PIDTYPE_PGID, p);
  321. return false;
  322. }
  323. /*
  324. * Check to see if any process groups have become orphaned as
  325. * a result of our exiting, and if they have any stopped jobs,
  326. * send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
  327. */
  328. static void
  329. kill_orphaned_pgrp(struct task_struct *tsk, struct task_struct *parent)
  330. {
  331. struct pid *pgrp = task_pgrp(tsk);
  332. struct task_struct *ignored_task = tsk;
  333. if (!parent)
  334. /* exit: our father is in a different pgrp than
  335. * we are and we were the only connection outside.
  336. */
  337. parent = tsk->real_parent;
  338. else
  339. /* reparent: our child is in a different pgrp than
  340. * we are, and it was the only connection outside.
  341. */
  342. ignored_task = NULL;
  343. if (task_pgrp(parent) != pgrp &&
  344. task_session(parent) == task_session(tsk) &&
  345. will_become_orphaned_pgrp(pgrp, ignored_task) &&
  346. has_stopped_jobs(pgrp)) {
  347. __kill_pgrp_info(SIGHUP, SEND_SIG_PRIV, pgrp);
  348. __kill_pgrp_info(SIGCONT, SEND_SIG_PRIV, pgrp);
  349. }
  350. }
  351. static void coredump_task_exit(struct task_struct *tsk)
  352. {
  353. struct core_state *core_state;
  354. /*
  355. * Serialize with any possible pending coredump.
  356. * We must hold siglock around checking core_state
  357. * and setting PF_POSTCOREDUMP. The core-inducing thread
  358. * will increment ->nr_threads for each thread in the
  359. * group without PF_POSTCOREDUMP set.
  360. */
  361. spin_lock_irq(&tsk->sighand->siglock);
  362. tsk->flags |= PF_POSTCOREDUMP;
  363. core_state = tsk->signal->core_state;
  364. spin_unlock_irq(&tsk->sighand->siglock);
  365. if (core_state) {
  366. struct core_thread self;
  367. self.task = current;
  368. if (self.task->flags & PF_SIGNALED)
  369. self.next = xchg(&core_state->dumper.next, &self);
  370. else
  371. self.task = NULL;
  372. /*
  373. * Implies mb(), the result of xchg() must be visible
  374. * to core_state->dumper.
  375. */
  376. if (atomic_dec_and_test(&core_state->nr_threads))
  377. complete(&core_state->startup);
  378. for (;;) {
  379. set_current_state(TASK_UNINTERRUPTIBLE|TASK_FREEZABLE);
  380. if (!self.task) /* see coredump_finish() */
  381. break;
  382. schedule();
  383. }
  384. __set_current_state(TASK_RUNNING);
  385. }
  386. }
  387. #ifdef CONFIG_MEMCG
  388. /*
  389. * A task is exiting. If it owned this mm, find a new owner for the mm.
  390. */
  391. void mm_update_next_owner(struct mm_struct *mm)
  392. {
  393. struct task_struct *c, *g, *p = current;
  394. retry:
  395. /*
  396. * If the exiting or execing task is not the owner, it's
  397. * someone else's problem.
  398. */
  399. if (mm->owner != p)
  400. return;
  401. /*
  402. * The current owner is exiting/execing and there are no other
  403. * candidates. Do not leave the mm pointing to a possibly
  404. * freed task structure.
  405. */
  406. if (atomic_read(&mm->mm_users) <= 1) {
  407. WRITE_ONCE(mm->owner, NULL);
  408. return;
  409. }
  410. read_lock(&tasklist_lock);
  411. /*
  412. * Search in the children
  413. */
  414. list_for_each_entry(c, &p->children, sibling) {
  415. if (c->mm == mm)
  416. goto assign_new_owner;
  417. }
  418. /*
  419. * Search in the siblings
  420. */
  421. list_for_each_entry(c, &p->real_parent->children, sibling) {
  422. if (c->mm == mm)
  423. goto assign_new_owner;
  424. }
  425. /*
  426. * Search through everything else, we should not get here often.
  427. */
  428. for_each_process(g) {
  429. if (g->flags & PF_KTHREAD)
  430. continue;
  431. for_each_thread(g, c) {
  432. if (c->mm == mm)
  433. goto assign_new_owner;
  434. if (c->mm)
  435. break;
  436. }
  437. }
  438. read_unlock(&tasklist_lock);
  439. /*
  440. * We found no owner yet mm_users > 1: this implies that we are
  441. * most likely racing with swapoff (try_to_unuse()) or /proc or
  442. * ptrace or page migration (get_task_mm()). Mark owner as NULL.
  443. */
  444. WRITE_ONCE(mm->owner, NULL);
  445. return;
  446. assign_new_owner:
  447. BUG_ON(c == p);
  448. get_task_struct(c);
  449. /*
  450. * The task_lock protects c->mm from changing.
  451. * We always want mm->owner->mm == mm
  452. */
  453. task_lock(c);
  454. /*
  455. * Delay read_unlock() till we have the task_lock()
  456. * to ensure that c does not slip away underneath us
  457. */
  458. read_unlock(&tasklist_lock);
  459. if (c->mm != mm) {
  460. task_unlock(c);
  461. put_task_struct(c);
  462. goto retry;
  463. }
  464. WRITE_ONCE(mm->owner, c);
  465. lru_gen_migrate_mm(mm);
  466. task_unlock(c);
  467. put_task_struct(c);
  468. }
  469. #endif /* CONFIG_MEMCG */
  470. /*
  471. * Turn us into a lazy TLB process if we
  472. * aren't already..
  473. */
  474. static void exit_mm(void)
  475. {
  476. struct mm_struct *mm = current->mm;
  477. exit_mm_release(current, mm);
  478. if (!mm)
  479. return;
  480. sync_mm_rss(mm);
  481. mmap_read_lock(mm);
  482. mmgrab(mm);
  483. BUG_ON(mm != current->active_mm);
  484. /* more a memory barrier than a real lock */
  485. task_lock(current);
  486. /*
  487. * When a thread stops operating on an address space, the loop
  488. * in membarrier_private_expedited() may not observe that
  489. * tsk->mm, and the loop in membarrier_global_expedited() may
  490. * not observe a MEMBARRIER_STATE_GLOBAL_EXPEDITED
  491. * rq->membarrier_state, so those would not issue an IPI.
  492. * Membarrier requires a memory barrier after accessing
  493. * user-space memory, before clearing tsk->mm or the
  494. * rq->membarrier_state.
  495. */
  496. smp_mb__after_spinlock();
  497. local_irq_disable();
  498. current->mm = NULL;
  499. membarrier_update_current_mm(NULL);
  500. enter_lazy_tlb(mm, current);
  501. local_irq_enable();
  502. task_unlock(current);
  503. mmap_read_unlock(mm);
  504. mm_update_next_owner(mm);
  505. trace_android_vh_exit_mm(mm);
  506. mmput(mm);
  507. if (test_thread_flag(TIF_MEMDIE))
  508. exit_oom_victim();
  509. }
  510. static struct task_struct *find_alive_thread(struct task_struct *p)
  511. {
  512. struct task_struct *t;
  513. for_each_thread(p, t) {
  514. if (!(t->flags & PF_EXITING))
  515. return t;
  516. }
  517. return NULL;
  518. }
  519. static struct task_struct *find_child_reaper(struct task_struct *father,
  520. struct list_head *dead)
  521. __releases(&tasklist_lock)
  522. __acquires(&tasklist_lock)
  523. {
  524. struct pid_namespace *pid_ns = task_active_pid_ns(father);
  525. struct task_struct *reaper = pid_ns->child_reaper;
  526. struct task_struct *p, *n;
  527. if (likely(reaper != father))
  528. return reaper;
  529. reaper = find_alive_thread(father);
  530. if (reaper) {
  531. pid_ns->child_reaper = reaper;
  532. return reaper;
  533. }
  534. write_unlock_irq(&tasklist_lock);
  535. list_for_each_entry_safe(p, n, dead, ptrace_entry) {
  536. list_del_init(&p->ptrace_entry);
  537. release_task(p);
  538. }
  539. zap_pid_ns_processes(pid_ns);
  540. write_lock_irq(&tasklist_lock);
  541. return father;
  542. }
  543. /*
  544. * When we die, we re-parent all our children, and try to:
  545. * 1. give them to another thread in our thread group, if such a member exists
  546. * 2. give it to the first ancestor process which prctl'd itself as a
  547. * child_subreaper for its children (like a service manager)
  548. * 3. give it to the init process (PID 1) in our pid namespace
  549. */
  550. static struct task_struct *find_new_reaper(struct task_struct *father,
  551. struct task_struct *child_reaper)
  552. {
  553. struct task_struct *thread, *reaper;
  554. thread = find_alive_thread(father);
  555. if (thread)
  556. return thread;
  557. if (father->signal->has_child_subreaper) {
  558. unsigned int ns_level = task_pid(father)->level;
  559. /*
  560. * Find the first ->is_child_subreaper ancestor in our pid_ns.
  561. * We can't check reaper != child_reaper to ensure we do not
  562. * cross the namespaces, the exiting parent could be injected
  563. * by setns() + fork().
  564. * We check pid->level, this is slightly more efficient than
  565. * task_active_pid_ns(reaper) != task_active_pid_ns(father).
  566. */
  567. for (reaper = father->real_parent;
  568. task_pid(reaper)->level == ns_level;
  569. reaper = reaper->real_parent) {
  570. if (reaper == &init_task)
  571. break;
  572. if (!reaper->signal->is_child_subreaper)
  573. continue;
  574. thread = find_alive_thread(reaper);
  575. if (thread)
  576. return thread;
  577. }
  578. }
  579. return child_reaper;
  580. }
  581. /*
  582. * Any that need to be release_task'd are put on the @dead list.
  583. */
  584. static void reparent_leader(struct task_struct *father, struct task_struct *p,
  585. struct list_head *dead)
  586. {
  587. if (unlikely(p->exit_state == EXIT_DEAD))
  588. return;
  589. /* We don't want people slaying init. */
  590. p->exit_signal = SIGCHLD;
  591. /* If it has exited notify the new parent about this child's death. */
  592. if (!p->ptrace &&
  593. p->exit_state == EXIT_ZOMBIE && thread_group_empty(p)) {
  594. if (do_notify_parent(p, p->exit_signal)) {
  595. p->exit_state = EXIT_DEAD;
  596. list_add(&p->ptrace_entry, dead);
  597. }
  598. }
  599. kill_orphaned_pgrp(p, father);
  600. }
  601. /*
  602. * This does two things:
  603. *
  604. * A. Make init inherit all the child processes
  605. * B. Check to see if any process groups have become orphaned
  606. * as a result of our exiting, and if they have any stopped
  607. * jobs, send them a SIGHUP and then a SIGCONT. (POSIX 3.2.2.2)
  608. */
  609. static void forget_original_parent(struct task_struct *father,
  610. struct list_head *dead)
  611. {
  612. struct task_struct *p, *t, *reaper;
  613. if (unlikely(!list_empty(&father->ptraced)))
  614. exit_ptrace(father, dead);
  615. /* Can drop and reacquire tasklist_lock */
  616. reaper = find_child_reaper(father, dead);
  617. if (list_empty(&father->children))
  618. return;
  619. reaper = find_new_reaper(father, reaper);
  620. list_for_each_entry(p, &father->children, sibling) {
  621. for_each_thread(p, t) {
  622. RCU_INIT_POINTER(t->real_parent, reaper);
  623. BUG_ON((!t->ptrace) != (rcu_access_pointer(t->parent) == father));
  624. if (likely(!t->ptrace))
  625. t->parent = t->real_parent;
  626. if (t->pdeath_signal)
  627. group_send_sig_info(t->pdeath_signal,
  628. SEND_SIG_NOINFO, t,
  629. PIDTYPE_TGID);
  630. }
  631. /*
  632. * If this is a threaded reparent there is no need to
  633. * notify anyone anything has happened.
  634. */
  635. if (!same_thread_group(reaper, father))
  636. reparent_leader(father, p, dead);
  637. }
  638. list_splice_tail_init(&father->children, &reaper->children);
  639. }
  640. /*
  641. * Send signals to all our closest relatives so that they know
  642. * to properly mourn us..
  643. */
  644. static void exit_notify(struct task_struct *tsk, int group_dead)
  645. {
  646. bool autoreap;
  647. struct task_struct *p, *n;
  648. LIST_HEAD(dead);
  649. write_lock_irq(&tasklist_lock);
  650. forget_original_parent(tsk, &dead);
  651. if (group_dead)
  652. kill_orphaned_pgrp(tsk->group_leader, NULL);
  653. tsk->exit_state = EXIT_ZOMBIE;
  654. if (unlikely(tsk->ptrace)) {
  655. int sig = thread_group_leader(tsk) &&
  656. thread_group_empty(tsk) &&
  657. !ptrace_reparented(tsk) ?
  658. tsk->exit_signal : SIGCHLD;
  659. autoreap = do_notify_parent(tsk, sig);
  660. } else if (thread_group_leader(tsk)) {
  661. autoreap = thread_group_empty(tsk) &&
  662. do_notify_parent(tsk, tsk->exit_signal);
  663. } else {
  664. autoreap = true;
  665. }
  666. if (autoreap) {
  667. tsk->exit_state = EXIT_DEAD;
  668. list_add(&tsk->ptrace_entry, &dead);
  669. }
  670. /* mt-exec, de_thread() is waiting for group leader */
  671. if (unlikely(tsk->signal->notify_count < 0))
  672. wake_up_process(tsk->signal->group_exec_task);
  673. write_unlock_irq(&tasklist_lock);
  674. list_for_each_entry_safe(p, n, &dead, ptrace_entry) {
  675. list_del_init(&p->ptrace_entry);
  676. release_task(p);
  677. }
  678. }
  679. #ifdef CONFIG_DEBUG_STACK_USAGE
  680. static void check_stack_usage(void)
  681. {
  682. static DEFINE_SPINLOCK(low_water_lock);
  683. static int lowest_to_date = THREAD_SIZE;
  684. unsigned long free;
  685. free = stack_not_used(current);
  686. if (free >= lowest_to_date)
  687. return;
  688. spin_lock(&low_water_lock);
  689. if (free < lowest_to_date) {
  690. pr_info("%s (%d) used greatest stack depth: %lu bytes left\n",
  691. current->comm, task_pid_nr(current), free);
  692. lowest_to_date = free;
  693. }
  694. spin_unlock(&low_water_lock);
  695. }
  696. #else
  697. static inline void check_stack_usage(void) {}
  698. #endif
  699. static void synchronize_group_exit(struct task_struct *tsk, long code)
  700. {
  701. struct sighand_struct *sighand = tsk->sighand;
  702. struct signal_struct *signal = tsk->signal;
  703. spin_lock_irq(&sighand->siglock);
  704. signal->quick_threads--;
  705. if ((signal->quick_threads == 0) &&
  706. !(signal->flags & SIGNAL_GROUP_EXIT)) {
  707. signal->flags = SIGNAL_GROUP_EXIT;
  708. signal->group_exit_code = code;
  709. signal->group_stop_count = 0;
  710. }
  711. spin_unlock_irq(&sighand->siglock);
  712. }
  713. void __noreturn do_exit(long code)
  714. {
  715. struct task_struct *tsk = current;
  716. int group_dead;
  717. WARN_ON(irqs_disabled());
  718. synchronize_group_exit(tsk, code);
  719. WARN_ON(tsk->plug);
  720. profile_task_exit(tsk);
  721. kcov_task_exit(tsk);
  722. kmsan_task_exit(tsk);
  723. coredump_task_exit(tsk);
  724. ptrace_event(PTRACE_EVENT_EXIT, code);
  725. validate_creds_for_do_exit(tsk);
  726. io_uring_files_cancel();
  727. exit_signals(tsk); /* sets PF_EXITING */
  728. trace_android_vh_exit_check(current);
  729. /* sync mm's RSS info before statistics gathering */
  730. if (tsk->mm)
  731. sync_mm_rss(tsk->mm);
  732. acct_update_integrals(tsk);
  733. group_dead = atomic_dec_and_test(&tsk->signal->live);
  734. if (group_dead) {
  735. /*
  736. * If the last thread of global init has exited, panic
  737. * immediately to get a useable coredump.
  738. */
  739. if (unlikely(is_global_init(tsk)))
  740. panic("Attempted to kill init! exitcode=0x%08x\n",
  741. tsk->signal->group_exit_code ?: (int)code);
  742. #ifdef CONFIG_POSIX_TIMERS
  743. hrtimer_cancel(&tsk->signal->real_timer);
  744. exit_itimers(tsk);
  745. #endif
  746. if (tsk->mm)
  747. setmax_mm_hiwater_rss(&tsk->signal->maxrss, tsk->mm);
  748. }
  749. acct_collect(code, group_dead);
  750. if (group_dead)
  751. tty_audit_exit();
  752. audit_free(tsk);
  753. tsk->exit_code = code;
  754. taskstats_exit(tsk, group_dead);
  755. exit_mm();
  756. if (group_dead)
  757. acct_process();
  758. trace_sched_process_exit(tsk);
  759. exit_sem(tsk);
  760. exit_shm(tsk);
  761. exit_files(tsk);
  762. exit_fs(tsk);
  763. if (group_dead)
  764. disassociate_ctty(1);
  765. exit_task_namespaces(tsk);
  766. exit_task_work(tsk);
  767. exit_thread(tsk);
  768. /*
  769. * Flush inherited counters to the parent - before the parent
  770. * gets woken up by child-exit notifications.
  771. *
  772. * because of cgroup mode, must be called before cgroup_exit()
  773. */
  774. perf_event_exit_task(tsk);
  775. sched_autogroup_exit_task(tsk);
  776. cgroup_exit(tsk);
  777. /*
  778. * FIXME: do that only when needed, using sched_exit tracepoint
  779. */
  780. flush_ptrace_hw_breakpoint(tsk);
  781. exit_tasks_rcu_start();
  782. exit_notify(tsk, group_dead);
  783. proc_exit_connector(tsk);
  784. mpol_put_task_policy(tsk);
  785. #ifdef CONFIG_FUTEX
  786. if (unlikely(current->pi_state_cache))
  787. kfree(current->pi_state_cache);
  788. #endif
  789. /*
  790. * Make sure we are holding no locks:
  791. */
  792. debug_check_no_locks_held();
  793. if (tsk->io_context)
  794. exit_io_context(tsk);
  795. if (tsk->splice_pipe)
  796. free_pipe_info(tsk->splice_pipe);
  797. if (tsk->task_frag.page)
  798. put_page(tsk->task_frag.page);
  799. validate_creds_for_do_exit(tsk);
  800. exit_task_stack_account(tsk);
  801. check_stack_usage();
  802. preempt_disable();
  803. if (tsk->nr_dirtied)
  804. __this_cpu_add(dirty_throttle_leaks, tsk->nr_dirtied);
  805. exit_rcu();
  806. exit_tasks_rcu_finish();
  807. lockdep_free_task(tsk);
  808. do_task_dead();
  809. }
  810. void __noreturn make_task_dead(int signr)
  811. {
  812. /*
  813. * Take the task off the cpu after something catastrophic has
  814. * happened.
  815. *
  816. * We can get here from a kernel oops, sometimes with preemption off.
  817. * Start by checking for critical errors.
  818. * Then fix up important state like USER_DS and preemption.
  819. * Then do everything else.
  820. */
  821. struct task_struct *tsk = current;
  822. unsigned int limit;
  823. if (unlikely(in_interrupt()))
  824. panic("Aiee, killing interrupt handler!");
  825. if (unlikely(!tsk->pid))
  826. panic("Attempted to kill the idle task!");
  827. if (unlikely(irqs_disabled())) {
  828. pr_info("note: %s[%d] exited with irqs disabled\n",
  829. current->comm, task_pid_nr(current));
  830. local_irq_enable();
  831. }
  832. if (unlikely(in_atomic())) {
  833. pr_info("note: %s[%d] exited with preempt_count %d\n",
  834. current->comm, task_pid_nr(current),
  835. preempt_count());
  836. preempt_count_set(PREEMPT_ENABLED);
  837. }
  838. /*
  839. * Every time the system oopses, if the oops happens while a reference
  840. * to an object was held, the reference leaks.
  841. * If the oops doesn't also leak memory, repeated oopsing can cause
  842. * reference counters to wrap around (if they're not using refcount_t).
  843. * This means that repeated oopsing can make unexploitable-looking bugs
  844. * exploitable through repeated oopsing.
  845. * To make sure this can't happen, place an upper bound on how often the
  846. * kernel may oops without panic().
  847. */
  848. limit = READ_ONCE(oops_limit);
  849. if (atomic_inc_return(&oops_count) >= limit && limit)
  850. panic("Oopsed too often (kernel.oops_limit is %d)", limit);
  851. /*
  852. * We're taking recursive faults here in make_task_dead. Safest is to just
  853. * leave this task alone and wait for reboot.
  854. */
  855. if (unlikely(tsk->flags & PF_EXITING)) {
  856. pr_alert("Fixing recursive fault but reboot is needed!\n");
  857. futex_exit_recursive(tsk);
  858. tsk->exit_state = EXIT_DEAD;
  859. refcount_inc(&tsk->rcu_users);
  860. do_task_dead();
  861. }
  862. do_exit(signr);
  863. }
  864. SYSCALL_DEFINE1(exit, int, error_code)
  865. {
  866. do_exit((error_code&0xff)<<8);
  867. }
  868. /*
  869. * Take down every thread in the group. This is called by fatal signals
  870. * as well as by sys_exit_group (below).
  871. */
  872. void __noreturn
  873. do_group_exit(int exit_code)
  874. {
  875. struct signal_struct *sig = current->signal;
  876. if (sig->flags & SIGNAL_GROUP_EXIT)
  877. exit_code = sig->group_exit_code;
  878. else if (sig->group_exec_task)
  879. exit_code = 0;
  880. else {
  881. struct sighand_struct *const sighand = current->sighand;
  882. spin_lock_irq(&sighand->siglock);
  883. if (sig->flags & SIGNAL_GROUP_EXIT)
  884. /* Another thread got here before we took the lock. */
  885. exit_code = sig->group_exit_code;
  886. else if (sig->group_exec_task)
  887. exit_code = 0;
  888. else {
  889. sig->group_exit_code = exit_code;
  890. sig->flags = SIGNAL_GROUP_EXIT;
  891. zap_other_threads(current);
  892. }
  893. spin_unlock_irq(&sighand->siglock);
  894. }
  895. do_exit(exit_code);
  896. /* NOTREACHED */
  897. }
  898. /*
  899. * this kills every thread in the thread group. Note that any externally
  900. * wait4()-ing process will get the correct exit code - even if this
  901. * thread is not the thread group leader.
  902. */
  903. SYSCALL_DEFINE1(exit_group, int, error_code)
  904. {
  905. do_group_exit((error_code & 0xff) << 8);
  906. /* NOTREACHED */
  907. return 0;
  908. }
  909. struct waitid_info {
  910. pid_t pid;
  911. uid_t uid;
  912. int status;
  913. int cause;
  914. };
  915. struct wait_opts {
  916. enum pid_type wo_type;
  917. int wo_flags;
  918. struct pid *wo_pid;
  919. struct waitid_info *wo_info;
  920. int wo_stat;
  921. struct rusage *wo_rusage;
  922. wait_queue_entry_t child_wait;
  923. int notask_error;
  924. };
  925. static int eligible_pid(struct wait_opts *wo, struct task_struct *p)
  926. {
  927. return wo->wo_type == PIDTYPE_MAX ||
  928. task_pid_type(p, wo->wo_type) == wo->wo_pid;
  929. }
  930. static int
  931. eligible_child(struct wait_opts *wo, bool ptrace, struct task_struct *p)
  932. {
  933. if (!eligible_pid(wo, p))
  934. return 0;
  935. /*
  936. * Wait for all children (clone and not) if __WALL is set or
  937. * if it is traced by us.
  938. */
  939. if (ptrace || (wo->wo_flags & __WALL))
  940. return 1;
  941. /*
  942. * Otherwise, wait for clone children *only* if __WCLONE is set;
  943. * otherwise, wait for non-clone children *only*.
  944. *
  945. * Note: a "clone" child here is one that reports to its parent
  946. * using a signal other than SIGCHLD, or a non-leader thread which
  947. * we can only see if it is traced by us.
  948. */
  949. if ((p->exit_signal != SIGCHLD) ^ !!(wo->wo_flags & __WCLONE))
  950. return 0;
  951. return 1;
  952. }
  953. /*
  954. * Handle sys_wait4 work for one task in state EXIT_ZOMBIE. We hold
  955. * read_lock(&tasklist_lock) on entry. If we return zero, we still hold
  956. * the lock and this task is uninteresting. If we return nonzero, we have
  957. * released the lock and the system call should return.
  958. */
  959. static int wait_task_zombie(struct wait_opts *wo, struct task_struct *p)
  960. {
  961. int state, status;
  962. pid_t pid = task_pid_vnr(p);
  963. uid_t uid = from_kuid_munged(current_user_ns(), task_uid(p));
  964. struct waitid_info *infop;
  965. if (!likely(wo->wo_flags & WEXITED))
  966. return 0;
  967. if (unlikely(wo->wo_flags & WNOWAIT)) {
  968. status = (p->signal->flags & SIGNAL_GROUP_EXIT)
  969. ? p->signal->group_exit_code : p->exit_code;
  970. get_task_struct(p);
  971. read_unlock(&tasklist_lock);
  972. sched_annotate_sleep();
  973. if (wo->wo_rusage)
  974. getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
  975. put_task_struct(p);
  976. goto out_info;
  977. }
  978. /*
  979. * Move the task's state to DEAD/TRACE, only one thread can do this.
  980. */
  981. state = (ptrace_reparented(p) && thread_group_leader(p)) ?
  982. EXIT_TRACE : EXIT_DEAD;
  983. if (cmpxchg(&p->exit_state, EXIT_ZOMBIE, state) != EXIT_ZOMBIE)
  984. return 0;
  985. /*
  986. * We own this thread, nobody else can reap it.
  987. */
  988. read_unlock(&tasklist_lock);
  989. sched_annotate_sleep();
  990. /*
  991. * Check thread_group_leader() to exclude the traced sub-threads.
  992. */
  993. if (state == EXIT_DEAD && thread_group_leader(p)) {
  994. struct signal_struct *sig = p->signal;
  995. struct signal_struct *psig = current->signal;
  996. unsigned long maxrss;
  997. u64 tgutime, tgstime;
  998. /*
  999. * The resource counters for the group leader are in its
  1000. * own task_struct. Those for dead threads in the group
  1001. * are in its signal_struct, as are those for the child
  1002. * processes it has previously reaped. All these
  1003. * accumulate in the parent's signal_struct c* fields.
  1004. *
  1005. * We don't bother to take a lock here to protect these
  1006. * p->signal fields because the whole thread group is dead
  1007. * and nobody can change them.
  1008. *
  1009. * psig->stats_lock also protects us from our sub-threads
  1010. * which can reap other children at the same time. Until
  1011. * we change k_getrusage()-like users to rely on this lock
  1012. * we have to take ->siglock as well.
  1013. *
  1014. * We use thread_group_cputime_adjusted() to get times for
  1015. * the thread group, which consolidates times for all threads
  1016. * in the group including the group leader.
  1017. */
  1018. thread_group_cputime_adjusted(p, &tgutime, &tgstime);
  1019. spin_lock_irq(&current->sighand->siglock);
  1020. write_seqlock(&psig->stats_lock);
  1021. psig->cutime += tgutime + sig->cutime;
  1022. psig->cstime += tgstime + sig->cstime;
  1023. psig->cgtime += task_gtime(p) + sig->gtime + sig->cgtime;
  1024. psig->cmin_flt +=
  1025. p->min_flt + sig->min_flt + sig->cmin_flt;
  1026. psig->cmaj_flt +=
  1027. p->maj_flt + sig->maj_flt + sig->cmaj_flt;
  1028. psig->cnvcsw +=
  1029. p->nvcsw + sig->nvcsw + sig->cnvcsw;
  1030. psig->cnivcsw +=
  1031. p->nivcsw + sig->nivcsw + sig->cnivcsw;
  1032. psig->cinblock +=
  1033. task_io_get_inblock(p) +
  1034. sig->inblock + sig->cinblock;
  1035. psig->coublock +=
  1036. task_io_get_oublock(p) +
  1037. sig->oublock + sig->coublock;
  1038. maxrss = max(sig->maxrss, sig->cmaxrss);
  1039. if (psig->cmaxrss < maxrss)
  1040. psig->cmaxrss = maxrss;
  1041. task_io_accounting_add(&psig->ioac, &p->ioac);
  1042. task_io_accounting_add(&psig->ioac, &sig->ioac);
  1043. write_sequnlock(&psig->stats_lock);
  1044. spin_unlock_irq(&current->sighand->siglock);
  1045. }
  1046. if (wo->wo_rusage)
  1047. getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
  1048. status = (p->signal->flags & SIGNAL_GROUP_EXIT)
  1049. ? p->signal->group_exit_code : p->exit_code;
  1050. wo->wo_stat = status;
  1051. if (state == EXIT_TRACE) {
  1052. write_lock_irq(&tasklist_lock);
  1053. /* We dropped tasklist, ptracer could die and untrace */
  1054. ptrace_unlink(p);
  1055. /* If parent wants a zombie, don't release it now */
  1056. state = EXIT_ZOMBIE;
  1057. if (do_notify_parent(p, p->exit_signal))
  1058. state = EXIT_DEAD;
  1059. p->exit_state = state;
  1060. write_unlock_irq(&tasklist_lock);
  1061. }
  1062. if (state == EXIT_DEAD)
  1063. release_task(p);
  1064. out_info:
  1065. infop = wo->wo_info;
  1066. if (infop) {
  1067. if ((status & 0x7f) == 0) {
  1068. infop->cause = CLD_EXITED;
  1069. infop->status = status >> 8;
  1070. } else {
  1071. infop->cause = (status & 0x80) ? CLD_DUMPED : CLD_KILLED;
  1072. infop->status = status & 0x7f;
  1073. }
  1074. infop->pid = pid;
  1075. infop->uid = uid;
  1076. }
  1077. return pid;
  1078. }
  1079. static int *task_stopped_code(struct task_struct *p, bool ptrace)
  1080. {
  1081. if (ptrace) {
  1082. if (task_is_traced(p) && !(p->jobctl & JOBCTL_LISTENING))
  1083. return &p->exit_code;
  1084. } else {
  1085. if (p->signal->flags & SIGNAL_STOP_STOPPED)
  1086. return &p->signal->group_exit_code;
  1087. }
  1088. return NULL;
  1089. }
  1090. /**
  1091. * wait_task_stopped - Wait for %TASK_STOPPED or %TASK_TRACED
  1092. * @wo: wait options
  1093. * @ptrace: is the wait for ptrace
  1094. * @p: task to wait for
  1095. *
  1096. * Handle sys_wait4() work for %p in state %TASK_STOPPED or %TASK_TRACED.
  1097. *
  1098. * CONTEXT:
  1099. * read_lock(&tasklist_lock), which is released if return value is
  1100. * non-zero. Also, grabs and releases @p->sighand->siglock.
  1101. *
  1102. * RETURNS:
  1103. * 0 if wait condition didn't exist and search for other wait conditions
  1104. * should continue. Non-zero return, -errno on failure and @p's pid on
  1105. * success, implies that tasklist_lock is released and wait condition
  1106. * search should terminate.
  1107. */
  1108. static int wait_task_stopped(struct wait_opts *wo,
  1109. int ptrace, struct task_struct *p)
  1110. {
  1111. struct waitid_info *infop;
  1112. int exit_code, *p_code, why;
  1113. uid_t uid = 0; /* unneeded, required by compiler */
  1114. pid_t pid;
  1115. /*
  1116. * Traditionally we see ptrace'd stopped tasks regardless of options.
  1117. */
  1118. if (!ptrace && !(wo->wo_flags & WUNTRACED))
  1119. return 0;
  1120. if (!task_stopped_code(p, ptrace))
  1121. return 0;
  1122. exit_code = 0;
  1123. spin_lock_irq(&p->sighand->siglock);
  1124. p_code = task_stopped_code(p, ptrace);
  1125. if (unlikely(!p_code))
  1126. goto unlock_sig;
  1127. exit_code = *p_code;
  1128. if (!exit_code)
  1129. goto unlock_sig;
  1130. if (!unlikely(wo->wo_flags & WNOWAIT))
  1131. *p_code = 0;
  1132. uid = from_kuid_munged(current_user_ns(), task_uid(p));
  1133. unlock_sig:
  1134. spin_unlock_irq(&p->sighand->siglock);
  1135. if (!exit_code)
  1136. return 0;
  1137. /*
  1138. * Now we are pretty sure this task is interesting.
  1139. * Make sure it doesn't get reaped out from under us while we
  1140. * give up the lock and then examine it below. We don't want to
  1141. * keep holding onto the tasklist_lock while we call getrusage and
  1142. * possibly take page faults for user memory.
  1143. */
  1144. get_task_struct(p);
  1145. pid = task_pid_vnr(p);
  1146. why = ptrace ? CLD_TRAPPED : CLD_STOPPED;
  1147. read_unlock(&tasklist_lock);
  1148. sched_annotate_sleep();
  1149. if (wo->wo_rusage)
  1150. getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
  1151. put_task_struct(p);
  1152. if (likely(!(wo->wo_flags & WNOWAIT)))
  1153. wo->wo_stat = (exit_code << 8) | 0x7f;
  1154. infop = wo->wo_info;
  1155. if (infop) {
  1156. infop->cause = why;
  1157. infop->status = exit_code;
  1158. infop->pid = pid;
  1159. infop->uid = uid;
  1160. }
  1161. return pid;
  1162. }
  1163. /*
  1164. * Handle do_wait work for one task in a live, non-stopped state.
  1165. * read_lock(&tasklist_lock) on entry. If we return zero, we still hold
  1166. * the lock and this task is uninteresting. If we return nonzero, we have
  1167. * released the lock and the system call should return.
  1168. */
  1169. static int wait_task_continued(struct wait_opts *wo, struct task_struct *p)
  1170. {
  1171. struct waitid_info *infop;
  1172. pid_t pid;
  1173. uid_t uid;
  1174. if (!unlikely(wo->wo_flags & WCONTINUED))
  1175. return 0;
  1176. if (!(p->signal->flags & SIGNAL_STOP_CONTINUED))
  1177. return 0;
  1178. spin_lock_irq(&p->sighand->siglock);
  1179. /* Re-check with the lock held. */
  1180. if (!(p->signal->flags & SIGNAL_STOP_CONTINUED)) {
  1181. spin_unlock_irq(&p->sighand->siglock);
  1182. return 0;
  1183. }
  1184. if (!unlikely(wo->wo_flags & WNOWAIT))
  1185. p->signal->flags &= ~SIGNAL_STOP_CONTINUED;
  1186. uid = from_kuid_munged(current_user_ns(), task_uid(p));
  1187. spin_unlock_irq(&p->sighand->siglock);
  1188. pid = task_pid_vnr(p);
  1189. get_task_struct(p);
  1190. read_unlock(&tasklist_lock);
  1191. sched_annotate_sleep();
  1192. if (wo->wo_rusage)
  1193. getrusage(p, RUSAGE_BOTH, wo->wo_rusage);
  1194. put_task_struct(p);
  1195. infop = wo->wo_info;
  1196. if (!infop) {
  1197. wo->wo_stat = 0xffff;
  1198. } else {
  1199. infop->cause = CLD_CONTINUED;
  1200. infop->pid = pid;
  1201. infop->uid = uid;
  1202. infop->status = SIGCONT;
  1203. }
  1204. return pid;
  1205. }
  1206. /*
  1207. * Consider @p for a wait by @parent.
  1208. *
  1209. * -ECHILD should be in ->notask_error before the first call.
  1210. * Returns nonzero for a final return, when we have unlocked tasklist_lock.
  1211. * Returns zero if the search for a child should continue;
  1212. * then ->notask_error is 0 if @p is an eligible child,
  1213. * or still -ECHILD.
  1214. */
  1215. static int wait_consider_task(struct wait_opts *wo, int ptrace,
  1216. struct task_struct *p)
  1217. {
  1218. /*
  1219. * We can race with wait_task_zombie() from another thread.
  1220. * Ensure that EXIT_ZOMBIE -> EXIT_DEAD/EXIT_TRACE transition
  1221. * can't confuse the checks below.
  1222. */
  1223. int exit_state = READ_ONCE(p->exit_state);
  1224. int ret;
  1225. if (unlikely(exit_state == EXIT_DEAD))
  1226. return 0;
  1227. ret = eligible_child(wo, ptrace, p);
  1228. if (!ret)
  1229. return ret;
  1230. if (unlikely(exit_state == EXIT_TRACE)) {
  1231. /*
  1232. * ptrace == 0 means we are the natural parent. In this case
  1233. * we should clear notask_error, debugger will notify us.
  1234. */
  1235. if (likely(!ptrace))
  1236. wo->notask_error = 0;
  1237. return 0;
  1238. }
  1239. if (likely(!ptrace) && unlikely(p->ptrace)) {
  1240. /*
  1241. * If it is traced by its real parent's group, just pretend
  1242. * the caller is ptrace_do_wait() and reap this child if it
  1243. * is zombie.
  1244. *
  1245. * This also hides group stop state from real parent; otherwise
  1246. * a single stop can be reported twice as group and ptrace stop.
  1247. * If a ptracer wants to distinguish these two events for its
  1248. * own children it should create a separate process which takes
  1249. * the role of real parent.
  1250. */
  1251. if (!ptrace_reparented(p))
  1252. ptrace = 1;
  1253. }
  1254. /* slay zombie? */
  1255. if (exit_state == EXIT_ZOMBIE) {
  1256. /* we don't reap group leaders with subthreads */
  1257. if (!delay_group_leader(p)) {
  1258. /*
  1259. * A zombie ptracee is only visible to its ptracer.
  1260. * Notification and reaping will be cascaded to the
  1261. * real parent when the ptracer detaches.
  1262. */
  1263. if (unlikely(ptrace) || likely(!p->ptrace))
  1264. return wait_task_zombie(wo, p);
  1265. }
  1266. /*
  1267. * Allow access to stopped/continued state via zombie by
  1268. * falling through. Clearing of notask_error is complex.
  1269. *
  1270. * When !@ptrace:
  1271. *
  1272. * If WEXITED is set, notask_error should naturally be
  1273. * cleared. If not, subset of WSTOPPED|WCONTINUED is set,
  1274. * so, if there are live subthreads, there are events to
  1275. * wait for. If all subthreads are dead, it's still safe
  1276. * to clear - this function will be called again in finite
  1277. * amount time once all the subthreads are released and
  1278. * will then return without clearing.
  1279. *
  1280. * When @ptrace:
  1281. *
  1282. * Stopped state is per-task and thus can't change once the
  1283. * target task dies. Only continued and exited can happen.
  1284. * Clear notask_error if WCONTINUED | WEXITED.
  1285. */
  1286. if (likely(!ptrace) || (wo->wo_flags & (WCONTINUED | WEXITED)))
  1287. wo->notask_error = 0;
  1288. } else {
  1289. /*
  1290. * @p is alive and it's gonna stop, continue or exit, so
  1291. * there always is something to wait for.
  1292. */
  1293. wo->notask_error = 0;
  1294. }
  1295. /*
  1296. * Wait for stopped. Depending on @ptrace, different stopped state
  1297. * is used and the two don't interact with each other.
  1298. */
  1299. ret = wait_task_stopped(wo, ptrace, p);
  1300. if (ret)
  1301. return ret;
  1302. /*
  1303. * Wait for continued. There's only one continued state and the
  1304. * ptracer can consume it which can confuse the real parent. Don't
  1305. * use WCONTINUED from ptracer. You don't need or want it.
  1306. */
  1307. return wait_task_continued(wo, p);
  1308. }
  1309. /*
  1310. * Do the work of do_wait() for one thread in the group, @tsk.
  1311. *
  1312. * -ECHILD should be in ->notask_error before the first call.
  1313. * Returns nonzero for a final return, when we have unlocked tasklist_lock.
  1314. * Returns zero if the search for a child should continue; then
  1315. * ->notask_error is 0 if there were any eligible children,
  1316. * or still -ECHILD.
  1317. */
  1318. static int do_wait_thread(struct wait_opts *wo, struct task_struct *tsk)
  1319. {
  1320. struct task_struct *p;
  1321. list_for_each_entry(p, &tsk->children, sibling) {
  1322. int ret = wait_consider_task(wo, 0, p);
  1323. if (ret)
  1324. return ret;
  1325. }
  1326. return 0;
  1327. }
  1328. static int ptrace_do_wait(struct wait_opts *wo, struct task_struct *tsk)
  1329. {
  1330. struct task_struct *p;
  1331. list_for_each_entry(p, &tsk->ptraced, ptrace_entry) {
  1332. int ret = wait_consider_task(wo, 1, p);
  1333. if (ret)
  1334. return ret;
  1335. }
  1336. return 0;
  1337. }
  1338. static int child_wait_callback(wait_queue_entry_t *wait, unsigned mode,
  1339. int sync, void *key)
  1340. {
  1341. struct wait_opts *wo = container_of(wait, struct wait_opts,
  1342. child_wait);
  1343. struct task_struct *p = key;
  1344. if (!eligible_pid(wo, p))
  1345. return 0;
  1346. if ((wo->wo_flags & __WNOTHREAD) && wait->private != p->parent)
  1347. return 0;
  1348. return default_wake_function(wait, mode, sync, key);
  1349. }
  1350. void __wake_up_parent(struct task_struct *p, struct task_struct *parent)
  1351. {
  1352. __wake_up_sync_key(&parent->signal->wait_chldexit,
  1353. TASK_INTERRUPTIBLE, p);
  1354. }
  1355. static bool is_effectively_child(struct wait_opts *wo, bool ptrace,
  1356. struct task_struct *target)
  1357. {
  1358. struct task_struct *parent =
  1359. !ptrace ? target->real_parent : target->parent;
  1360. return current == parent || (!(wo->wo_flags & __WNOTHREAD) &&
  1361. same_thread_group(current, parent));
  1362. }
  1363. /*
  1364. * Optimization for waiting on PIDTYPE_PID. No need to iterate through child
  1365. * and tracee lists to find the target task.
  1366. */
  1367. static int do_wait_pid(struct wait_opts *wo)
  1368. {
  1369. bool ptrace;
  1370. struct task_struct *target;
  1371. int retval;
  1372. ptrace = false;
  1373. target = pid_task(wo->wo_pid, PIDTYPE_TGID);
  1374. if (target && is_effectively_child(wo, ptrace, target)) {
  1375. retval = wait_consider_task(wo, ptrace, target);
  1376. if (retval)
  1377. return retval;
  1378. }
  1379. ptrace = true;
  1380. target = pid_task(wo->wo_pid, PIDTYPE_PID);
  1381. if (target && target->ptrace &&
  1382. is_effectively_child(wo, ptrace, target)) {
  1383. retval = wait_consider_task(wo, ptrace, target);
  1384. if (retval)
  1385. return retval;
  1386. }
  1387. return 0;
  1388. }
  1389. static long do_wait(struct wait_opts *wo)
  1390. {
  1391. int retval;
  1392. trace_sched_process_wait(wo->wo_pid);
  1393. init_waitqueue_func_entry(&wo->child_wait, child_wait_callback);
  1394. wo->child_wait.private = current;
  1395. add_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
  1396. repeat:
  1397. /*
  1398. * If there is nothing that can match our criteria, just get out.
  1399. * We will clear ->notask_error to zero if we see any child that
  1400. * might later match our criteria, even if we are not able to reap
  1401. * it yet.
  1402. */
  1403. wo->notask_error = -ECHILD;
  1404. if ((wo->wo_type < PIDTYPE_MAX) &&
  1405. (!wo->wo_pid || !pid_has_task(wo->wo_pid, wo->wo_type)))
  1406. goto notask;
  1407. set_current_state(TASK_INTERRUPTIBLE);
  1408. read_lock(&tasklist_lock);
  1409. if (wo->wo_type == PIDTYPE_PID) {
  1410. retval = do_wait_pid(wo);
  1411. if (retval)
  1412. goto end;
  1413. } else {
  1414. struct task_struct *tsk = current;
  1415. do {
  1416. retval = do_wait_thread(wo, tsk);
  1417. if (retval)
  1418. goto end;
  1419. retval = ptrace_do_wait(wo, tsk);
  1420. if (retval)
  1421. goto end;
  1422. if (wo->wo_flags & __WNOTHREAD)
  1423. break;
  1424. } while_each_thread(current, tsk);
  1425. }
  1426. read_unlock(&tasklist_lock);
  1427. notask:
  1428. retval = wo->notask_error;
  1429. if (!retval && !(wo->wo_flags & WNOHANG)) {
  1430. retval = -ERESTARTSYS;
  1431. if (!signal_pending(current)) {
  1432. schedule();
  1433. goto repeat;
  1434. }
  1435. }
  1436. end:
  1437. __set_current_state(TASK_RUNNING);
  1438. remove_wait_queue(&current->signal->wait_chldexit, &wo->child_wait);
  1439. return retval;
  1440. }
  1441. static long kernel_waitid(int which, pid_t upid, struct waitid_info *infop,
  1442. int options, struct rusage *ru)
  1443. {
  1444. struct wait_opts wo;
  1445. struct pid *pid = NULL;
  1446. enum pid_type type;
  1447. long ret;
  1448. unsigned int f_flags = 0;
  1449. if (options & ~(WNOHANG|WNOWAIT|WEXITED|WSTOPPED|WCONTINUED|
  1450. __WNOTHREAD|__WCLONE|__WALL))
  1451. return -EINVAL;
  1452. if (!(options & (WEXITED|WSTOPPED|WCONTINUED)))
  1453. return -EINVAL;
  1454. switch (which) {
  1455. case P_ALL:
  1456. type = PIDTYPE_MAX;
  1457. break;
  1458. case P_PID:
  1459. type = PIDTYPE_PID;
  1460. if (upid <= 0)
  1461. return -EINVAL;
  1462. pid = find_get_pid(upid);
  1463. break;
  1464. case P_PGID:
  1465. type = PIDTYPE_PGID;
  1466. if (upid < 0)
  1467. return -EINVAL;
  1468. if (upid)
  1469. pid = find_get_pid(upid);
  1470. else
  1471. pid = get_task_pid(current, PIDTYPE_PGID);
  1472. break;
  1473. case P_PIDFD:
  1474. type = PIDTYPE_PID;
  1475. if (upid < 0)
  1476. return -EINVAL;
  1477. pid = pidfd_get_pid(upid, &f_flags);
  1478. if (IS_ERR(pid))
  1479. return PTR_ERR(pid);
  1480. break;
  1481. default:
  1482. return -EINVAL;
  1483. }
  1484. wo.wo_type = type;
  1485. wo.wo_pid = pid;
  1486. wo.wo_flags = options;
  1487. wo.wo_info = infop;
  1488. wo.wo_rusage = ru;
  1489. if (f_flags & O_NONBLOCK)
  1490. wo.wo_flags |= WNOHANG;
  1491. ret = do_wait(&wo);
  1492. if (!ret && !(options & WNOHANG) && (f_flags & O_NONBLOCK))
  1493. ret = -EAGAIN;
  1494. put_pid(pid);
  1495. return ret;
  1496. }
  1497. SYSCALL_DEFINE5(waitid, int, which, pid_t, upid, struct siginfo __user *,
  1498. infop, int, options, struct rusage __user *, ru)
  1499. {
  1500. struct rusage r;
  1501. struct waitid_info info = {.status = 0};
  1502. long err = kernel_waitid(which, upid, &info, options, ru ? &r : NULL);
  1503. int signo = 0;
  1504. if (err > 0) {
  1505. signo = SIGCHLD;
  1506. err = 0;
  1507. if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
  1508. return -EFAULT;
  1509. }
  1510. if (!infop)
  1511. return err;
  1512. if (!user_write_access_begin(infop, sizeof(*infop)))
  1513. return -EFAULT;
  1514. unsafe_put_user(signo, &infop->si_signo, Efault);
  1515. unsafe_put_user(0, &infop->si_errno, Efault);
  1516. unsafe_put_user(info.cause, &infop->si_code, Efault);
  1517. unsafe_put_user(info.pid, &infop->si_pid, Efault);
  1518. unsafe_put_user(info.uid, &infop->si_uid, Efault);
  1519. unsafe_put_user(info.status, &infop->si_status, Efault);
  1520. user_write_access_end();
  1521. return err;
  1522. Efault:
  1523. user_write_access_end();
  1524. return -EFAULT;
  1525. }
  1526. long kernel_wait4(pid_t upid, int __user *stat_addr, int options,
  1527. struct rusage *ru)
  1528. {
  1529. struct wait_opts wo;
  1530. struct pid *pid = NULL;
  1531. enum pid_type type;
  1532. long ret;
  1533. if (options & ~(WNOHANG|WUNTRACED|WCONTINUED|
  1534. __WNOTHREAD|__WCLONE|__WALL))
  1535. return -EINVAL;
  1536. /* -INT_MIN is not defined */
  1537. if (upid == INT_MIN)
  1538. return -ESRCH;
  1539. if (upid == -1)
  1540. type = PIDTYPE_MAX;
  1541. else if (upid < 0) {
  1542. type = PIDTYPE_PGID;
  1543. pid = find_get_pid(-upid);
  1544. } else if (upid == 0) {
  1545. type = PIDTYPE_PGID;
  1546. pid = get_task_pid(current, PIDTYPE_PGID);
  1547. } else /* upid > 0 */ {
  1548. type = PIDTYPE_PID;
  1549. pid = find_get_pid(upid);
  1550. }
  1551. wo.wo_type = type;
  1552. wo.wo_pid = pid;
  1553. wo.wo_flags = options | WEXITED;
  1554. wo.wo_info = NULL;
  1555. wo.wo_stat = 0;
  1556. wo.wo_rusage = ru;
  1557. ret = do_wait(&wo);
  1558. put_pid(pid);
  1559. if (ret > 0 && stat_addr && put_user(wo.wo_stat, stat_addr))
  1560. ret = -EFAULT;
  1561. return ret;
  1562. }
  1563. int kernel_wait(pid_t pid, int *stat)
  1564. {
  1565. struct wait_opts wo = {
  1566. .wo_type = PIDTYPE_PID,
  1567. .wo_pid = find_get_pid(pid),
  1568. .wo_flags = WEXITED,
  1569. };
  1570. int ret;
  1571. ret = do_wait(&wo);
  1572. if (ret > 0 && wo.wo_stat)
  1573. *stat = wo.wo_stat;
  1574. put_pid(wo.wo_pid);
  1575. return ret;
  1576. }
  1577. SYSCALL_DEFINE4(wait4, pid_t, upid, int __user *, stat_addr,
  1578. int, options, struct rusage __user *, ru)
  1579. {
  1580. struct rusage r;
  1581. long err = kernel_wait4(upid, stat_addr, options, ru ? &r : NULL);
  1582. if (err > 0) {
  1583. if (ru && copy_to_user(ru, &r, sizeof(struct rusage)))
  1584. return -EFAULT;
  1585. }
  1586. return err;
  1587. }
  1588. #ifdef __ARCH_WANT_SYS_WAITPID
  1589. /*
  1590. * sys_waitpid() remains for compatibility. waitpid() should be
  1591. * implemented by calling sys_wait4() from libc.a.
  1592. */
  1593. SYSCALL_DEFINE3(waitpid, pid_t, pid, int __user *, stat_addr, int, options)
  1594. {
  1595. return kernel_wait4(pid, stat_addr, options, NULL);
  1596. }
  1597. #endif
  1598. #ifdef CONFIG_COMPAT
  1599. COMPAT_SYSCALL_DEFINE4(wait4,
  1600. compat_pid_t, pid,
  1601. compat_uint_t __user *, stat_addr,
  1602. int, options,
  1603. struct compat_rusage __user *, ru)
  1604. {
  1605. struct rusage r;
  1606. long err = kernel_wait4(pid, stat_addr, options, ru ? &r : NULL);
  1607. if (err > 0) {
  1608. if (ru && put_compat_rusage(&r, ru))
  1609. return -EFAULT;
  1610. }
  1611. return err;
  1612. }
  1613. COMPAT_SYSCALL_DEFINE5(waitid,
  1614. int, which, compat_pid_t, pid,
  1615. struct compat_siginfo __user *, infop, int, options,
  1616. struct compat_rusage __user *, uru)
  1617. {
  1618. struct rusage ru;
  1619. struct waitid_info info = {.status = 0};
  1620. long err = kernel_waitid(which, pid, &info, options, uru ? &ru : NULL);
  1621. int signo = 0;
  1622. if (err > 0) {
  1623. signo = SIGCHLD;
  1624. err = 0;
  1625. if (uru) {
  1626. /* kernel_waitid() overwrites everything in ru */
  1627. if (COMPAT_USE_64BIT_TIME)
  1628. err = copy_to_user(uru, &ru, sizeof(ru));
  1629. else
  1630. err = put_compat_rusage(&ru, uru);
  1631. if (err)
  1632. return -EFAULT;
  1633. }
  1634. }
  1635. if (!infop)
  1636. return err;
  1637. if (!user_write_access_begin(infop, sizeof(*infop)))
  1638. return -EFAULT;
  1639. unsafe_put_user(signo, &infop->si_signo, Efault);
  1640. unsafe_put_user(0, &infop->si_errno, Efault);
  1641. unsafe_put_user(info.cause, &infop->si_code, Efault);
  1642. unsafe_put_user(info.pid, &infop->si_pid, Efault);
  1643. unsafe_put_user(info.uid, &infop->si_uid, Efault);
  1644. unsafe_put_user(info.status, &infop->si_status, Efault);
  1645. user_write_access_end();
  1646. return err;
  1647. Efault:
  1648. user_write_access_end();
  1649. return -EFAULT;
  1650. }
  1651. #endif
  1652. /**
  1653. * thread_group_exited - check that a thread group has exited
  1654. * @pid: tgid of thread group to be checked.
  1655. *
  1656. * Test if the thread group represented by tgid has exited (all
  1657. * threads are zombies, dead or completely gone).
  1658. *
  1659. * Return: true if the thread group has exited. false otherwise.
  1660. */
  1661. bool thread_group_exited(struct pid *pid)
  1662. {
  1663. struct task_struct *task;
  1664. bool exited;
  1665. rcu_read_lock();
  1666. task = pid_task(pid, PIDTYPE_PID);
  1667. exited = !task ||
  1668. (READ_ONCE(task->exit_state) && thread_group_empty(task));
  1669. rcu_read_unlock();
  1670. return exited;
  1671. }
  1672. EXPORT_SYMBOL(thread_group_exited);
  1673. __weak void abort(void)
  1674. {
  1675. BUG();
  1676. /* if that doesn't kill us, halt */
  1677. panic("Oops failed to kill thread");
  1678. }
  1679. EXPORT_SYMBOL(abort);