sched.h 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _LINUX_SCHED_H
  3. #define _LINUX_SCHED_H
  4. /*
  5. * Define 'struct task_struct' and provide the main scheduler
  6. * APIs (schedule(), wakeup variants, etc.)
  7. */
  8. #include <uapi/linux/sched.h>
  9. #include <asm/current.h>
  10. #include <linux/pid.h>
  11. #include <linux/sem.h>
  12. #include <linux/shm.h>
  13. #include <linux/kmsan_types.h>
  14. #include <linux/mutex.h>
  15. #include <linux/plist.h>
  16. #include <linux/hrtimer.h>
  17. #include <linux/irqflags.h>
  18. #include <linux/seccomp.h>
  19. #include <linux/nodemask.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/refcount.h>
  22. #include <linux/resource.h>
  23. #include <linux/latencytop.h>
  24. #include <linux/sched/prio.h>
  25. #include <linux/sched/types.h>
  26. #include <linux/signal_types.h>
  27. #include <linux/syscall_user_dispatch.h>
  28. #include <linux/mm_types_task.h>
  29. #include <linux/task_io_accounting.h>
  30. #include <linux/posix-timers.h>
  31. #include <linux/rseq.h>
  32. #include <linux/seqlock.h>
  33. #include <linux/kcsan.h>
  34. #include <linux/rv.h>
  35. #include <linux/android_vendor.h>
  36. #include <asm/kmap_size.h>
  37. #include <linux/android_kabi.h>
  38. /* task_struct member predeclarations (sorted alphabetically): */
  39. struct audit_context;
  40. struct backing_dev_info;
  41. struct bio_list;
  42. struct blk_plug;
  43. struct bpf_local_storage;
  44. struct bpf_run_ctx;
  45. struct capture_control;
  46. struct cfs_rq;
  47. struct fs_struct;
  48. struct futex_pi_state;
  49. struct io_context;
  50. struct io_uring_task;
  51. struct mempolicy;
  52. struct nameidata;
  53. struct nsproxy;
  54. struct perf_event_context;
  55. struct pid_namespace;
  56. struct pipe_inode_info;
  57. struct rcu_node;
  58. struct reclaim_state;
  59. struct robust_list_head;
  60. struct root_domain;
  61. struct rq;
  62. struct sched_attr;
  63. struct sched_param;
  64. struct seq_file;
  65. struct sighand_struct;
  66. struct signal_struct;
  67. struct task_delay_info;
  68. struct task_group;
  69. /*
  70. * Task state bitmask. NOTE! These bits are also
  71. * encoded in fs/proc/array.c: get_task_state().
  72. *
  73. * We have two separate sets of flags: task->state
  74. * is about runnability, while task->exit_state are
  75. * about the task exiting. Confusing, but this way
  76. * modifying one set can't modify the other one by
  77. * mistake.
  78. */
  79. /* Used in tsk->state: */
  80. #define TASK_RUNNING 0x00000000
  81. #define TASK_INTERRUPTIBLE 0x00000001
  82. #define TASK_UNINTERRUPTIBLE 0x00000002
  83. #define __TASK_STOPPED 0x00000004
  84. #define __TASK_TRACED 0x00000008
  85. /* Used in tsk->exit_state: */
  86. #define EXIT_DEAD 0x00000010
  87. #define EXIT_ZOMBIE 0x00000020
  88. #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD)
  89. /* Used in tsk->state again: */
  90. #define TASK_PARKED 0x00000040
  91. #define TASK_DEAD 0x00000080
  92. #define TASK_WAKEKILL 0x00000100
  93. #define TASK_WAKING 0x00000200
  94. #define TASK_NOLOAD 0x00000400
  95. #define TASK_NEW 0x00000800
  96. #define TASK_RTLOCK_WAIT 0x00001000
  97. #define TASK_FREEZABLE 0x00002000
  98. #define __TASK_FREEZABLE_UNSAFE (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP))
  99. #define TASK_FROZEN 0x00008000
  100. #define TASK_STATE_MAX 0x00010000
  101. #define TASK_ANY (TASK_STATE_MAX-1)
  102. /*
  103. * DO NOT ADD ANY NEW USERS !
  104. */
  105. #define TASK_FREEZABLE_UNSAFE (TASK_FREEZABLE | __TASK_FREEZABLE_UNSAFE)
  106. /* Convenience macros for the sake of set_current_state: */
  107. #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
  108. #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED)
  109. #define TASK_TRACED __TASK_TRACED
  110. #define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD)
  111. /* Convenience macros for the sake of wake_up(): */
  112. #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE)
  113. /* get_task_state(): */
  114. #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \
  115. TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \
  116. __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \
  117. TASK_PARKED)
  118. #define task_is_running(task) (READ_ONCE((task)->__state) == TASK_RUNNING)
  119. #define task_is_traced(task) ((READ_ONCE(task->jobctl) & JOBCTL_TRACED) != 0)
  120. #define task_is_stopped(task) ((READ_ONCE(task->jobctl) & JOBCTL_STOPPED) != 0)
  121. #define task_is_stopped_or_traced(task) ((READ_ONCE(task->jobctl) & (JOBCTL_STOPPED | JOBCTL_TRACED)) != 0)
  122. /*
  123. * Special states are those that do not use the normal wait-loop pattern. See
  124. * the comment with set_special_state().
  125. */
  126. #define is_special_task_state(state) \
  127. ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | TASK_DEAD))
  128. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  129. # define debug_normal_state_change(state_value) \
  130. do { \
  131. WARN_ON_ONCE(is_special_task_state(state_value)); \
  132. current->task_state_change = _THIS_IP_; \
  133. } while (0)
  134. # define debug_special_state_change(state_value) \
  135. do { \
  136. WARN_ON_ONCE(!is_special_task_state(state_value)); \
  137. current->task_state_change = _THIS_IP_; \
  138. } while (0)
  139. # define debug_rtlock_wait_set_state() \
  140. do { \
  141. current->saved_state_change = current->task_state_change;\
  142. current->task_state_change = _THIS_IP_; \
  143. } while (0)
  144. # define debug_rtlock_wait_restore_state() \
  145. do { \
  146. current->task_state_change = current->saved_state_change;\
  147. } while (0)
  148. #else
  149. # define debug_normal_state_change(cond) do { } while (0)
  150. # define debug_special_state_change(cond) do { } while (0)
  151. # define debug_rtlock_wait_set_state() do { } while (0)
  152. # define debug_rtlock_wait_restore_state() do { } while (0)
  153. #endif
  154. /*
  155. * set_current_state() includes a barrier so that the write of current->state
  156. * is correctly serialised wrt the caller's subsequent test of whether to
  157. * actually sleep:
  158. *
  159. * for (;;) {
  160. * set_current_state(TASK_UNINTERRUPTIBLE);
  161. * if (CONDITION)
  162. * break;
  163. *
  164. * schedule();
  165. * }
  166. * __set_current_state(TASK_RUNNING);
  167. *
  168. * If the caller does not need such serialisation (because, for instance, the
  169. * CONDITION test and condition change and wakeup are under the same lock) then
  170. * use __set_current_state().
  171. *
  172. * The above is typically ordered against the wakeup, which does:
  173. *
  174. * CONDITION = 1;
  175. * wake_up_state(p, TASK_UNINTERRUPTIBLE);
  176. *
  177. * where wake_up_state()/try_to_wake_up() executes a full memory barrier before
  178. * accessing p->state.
  179. *
  180. * Wakeup will do: if (@state & p->state) p->state = TASK_RUNNING, that is,
  181. * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a
  182. * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING).
  183. *
  184. * However, with slightly different timing the wakeup TASK_RUNNING store can
  185. * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not
  186. * a problem either because that will result in one extra go around the loop
  187. * and our @cond test will save the day.
  188. *
  189. * Also see the comments of try_to_wake_up().
  190. */
  191. #define __set_current_state(state_value) \
  192. do { \
  193. debug_normal_state_change((state_value)); \
  194. WRITE_ONCE(current->__state, (state_value)); \
  195. } while (0)
  196. #define set_current_state(state_value) \
  197. do { \
  198. debug_normal_state_change((state_value)); \
  199. smp_store_mb(current->__state, (state_value)); \
  200. } while (0)
  201. /*
  202. * set_special_state() should be used for those states when the blocking task
  203. * can not use the regular condition based wait-loop. In that case we must
  204. * serialize against wakeups such that any possible in-flight TASK_RUNNING
  205. * stores will not collide with our state change.
  206. */
  207. #define set_special_state(state_value) \
  208. do { \
  209. unsigned long flags; /* may shadow */ \
  210. \
  211. raw_spin_lock_irqsave(&current->pi_lock, flags); \
  212. debug_special_state_change((state_value)); \
  213. WRITE_ONCE(current->__state, (state_value)); \
  214. raw_spin_unlock_irqrestore(&current->pi_lock, flags); \
  215. } while (0)
  216. /*
  217. * PREEMPT_RT specific variants for "sleeping" spin/rwlocks
  218. *
  219. * RT's spin/rwlock substitutions are state preserving. The state of the
  220. * task when blocking on the lock is saved in task_struct::saved_state and
  221. * restored after the lock has been acquired. These operations are
  222. * serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT
  223. * lock related wakeups while the task is blocked on the lock are
  224. * redirected to operate on task_struct::saved_state to ensure that these
  225. * are not dropped. On restore task_struct::saved_state is set to
  226. * TASK_RUNNING so any wakeup attempt redirected to saved_state will fail.
  227. *
  228. * The lock operation looks like this:
  229. *
  230. * current_save_and_set_rtlock_wait_state();
  231. * for (;;) {
  232. * if (try_lock())
  233. * break;
  234. * raw_spin_unlock_irq(&lock->wait_lock);
  235. * schedule_rtlock();
  236. * raw_spin_lock_irq(&lock->wait_lock);
  237. * set_current_state(TASK_RTLOCK_WAIT);
  238. * }
  239. * current_restore_rtlock_saved_state();
  240. */
  241. #define current_save_and_set_rtlock_wait_state() \
  242. do { \
  243. lockdep_assert_irqs_disabled(); \
  244. raw_spin_lock(&current->pi_lock); \
  245. current->saved_state = current->__state; \
  246. debug_rtlock_wait_set_state(); \
  247. WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT); \
  248. raw_spin_unlock(&current->pi_lock); \
  249. } while (0);
  250. #define current_restore_rtlock_saved_state() \
  251. do { \
  252. lockdep_assert_irqs_disabled(); \
  253. raw_spin_lock(&current->pi_lock); \
  254. debug_rtlock_wait_restore_state(); \
  255. WRITE_ONCE(current->__state, current->saved_state); \
  256. current->saved_state = TASK_RUNNING; \
  257. raw_spin_unlock(&current->pi_lock); \
  258. } while (0);
  259. #define get_current_state() READ_ONCE(current->__state)
  260. /*
  261. * Define the task command name length as enum, then it can be visible to
  262. * BPF programs.
  263. */
  264. enum {
  265. TASK_COMM_LEN = 16,
  266. };
  267. extern void scheduler_tick(void);
  268. #define MAX_SCHEDULE_TIMEOUT LONG_MAX
  269. extern long schedule_timeout(long timeout);
  270. extern long schedule_timeout_interruptible(long timeout);
  271. extern long schedule_timeout_killable(long timeout);
  272. extern long schedule_timeout_uninterruptible(long timeout);
  273. extern long schedule_timeout_idle(long timeout);
  274. asmlinkage void schedule(void);
  275. extern void schedule_preempt_disabled(void);
  276. asmlinkage void preempt_schedule_irq(void);
  277. #ifdef CONFIG_PREEMPT_RT
  278. extern void schedule_rtlock(void);
  279. #endif
  280. extern int __must_check io_schedule_prepare(void);
  281. extern void io_schedule_finish(int token);
  282. extern long io_schedule_timeout(long timeout);
  283. extern void io_schedule(void);
  284. extern struct task_struct *pick_migrate_task(struct rq *rq);
  285. extern int select_fallback_rq(int cpu, struct task_struct *p);
  286. /**
  287. * struct prev_cputime - snapshot of system and user cputime
  288. * @utime: time spent in user mode
  289. * @stime: time spent in system mode
  290. * @lock: protects the above two fields
  291. *
  292. * Stores previous user/system time values such that we can guarantee
  293. * monotonicity.
  294. */
  295. struct prev_cputime {
  296. #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
  297. u64 utime;
  298. u64 stime;
  299. raw_spinlock_t lock;
  300. #endif
  301. };
  302. enum vtime_state {
  303. /* Task is sleeping or running in a CPU with VTIME inactive: */
  304. VTIME_INACTIVE = 0,
  305. /* Task is idle */
  306. VTIME_IDLE,
  307. /* Task runs in kernelspace in a CPU with VTIME active: */
  308. VTIME_SYS,
  309. /* Task runs in userspace in a CPU with VTIME active: */
  310. VTIME_USER,
  311. /* Task runs as guests in a CPU with VTIME active: */
  312. VTIME_GUEST,
  313. };
  314. struct vtime {
  315. seqcount_t seqcount;
  316. unsigned long long starttime;
  317. enum vtime_state state;
  318. unsigned int cpu;
  319. u64 utime;
  320. u64 stime;
  321. u64 gtime;
  322. };
  323. /*
  324. * Utilization clamp constraints.
  325. * @UCLAMP_MIN: Minimum utilization
  326. * @UCLAMP_MAX: Maximum utilization
  327. * @UCLAMP_CNT: Utilization clamp constraints count
  328. */
  329. enum uclamp_id {
  330. UCLAMP_MIN = 0,
  331. UCLAMP_MAX,
  332. UCLAMP_CNT
  333. };
  334. #ifdef CONFIG_SMP
  335. extern struct root_domain def_root_domain;
  336. extern struct mutex sched_domains_mutex;
  337. #endif
  338. struct sched_info {
  339. #ifdef CONFIG_SCHED_INFO
  340. /* Cumulative counters: */
  341. /* # of times we have run on this CPU: */
  342. unsigned long pcount;
  343. /* Time spent waiting on a runqueue: */
  344. unsigned long long run_delay;
  345. /* Timestamps: */
  346. /* When did we last run on a CPU? */
  347. unsigned long long last_arrival;
  348. /* When were we last queued to run? */
  349. unsigned long long last_queued;
  350. #endif /* CONFIG_SCHED_INFO */
  351. };
  352. /*
  353. * Integer metrics need fixed point arithmetic, e.g., sched/fair
  354. * has a few: load, load_avg, util_avg, freq, and capacity.
  355. *
  356. * We define a basic fixed point arithmetic range, and then formalize
  357. * all these metrics based on that basic range.
  358. */
  359. # define SCHED_FIXEDPOINT_SHIFT 10
  360. # define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT)
  361. /* Increase resolution of cpu_capacity calculations */
  362. # define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT
  363. # define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT)
  364. struct load_weight {
  365. unsigned long weight;
  366. u32 inv_weight;
  367. };
  368. /**
  369. * struct util_est - Estimation utilization of FAIR tasks
  370. * @enqueued: instantaneous estimated utilization of a task/cpu
  371. * @ewma: the Exponential Weighted Moving Average (EWMA)
  372. * utilization of a task
  373. *
  374. * Support data structure to track an Exponential Weighted Moving Average
  375. * (EWMA) of a FAIR task's utilization. New samples are added to the moving
  376. * average each time a task completes an activation. Sample's weight is chosen
  377. * so that the EWMA will be relatively insensitive to transient changes to the
  378. * task's workload.
  379. *
  380. * The enqueued attribute has a slightly different meaning for tasks and cpus:
  381. * - task: the task's util_avg at last task dequeue time
  382. * - cfs_rq: the sum of util_est.enqueued for each RUNNABLE task on that CPU
  383. * Thus, the util_est.enqueued of a task represents the contribution on the
  384. * estimated utilization of the CPU where that task is currently enqueued.
  385. *
  386. * Only for tasks we track a moving average of the past instantaneous
  387. * estimated utilization. This allows to absorb sporadic drops in utilization
  388. * of an otherwise almost periodic task.
  389. *
  390. * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg
  391. * updates. When a task is dequeued, its util_est should not be updated if its
  392. * util_avg has not been updated in the meantime.
  393. * This information is mapped into the MSB bit of util_est.enqueued at dequeue
  394. * time. Since max value of util_est.enqueued for a task is 1024 (PELT util_avg
  395. * for a task) it is safe to use MSB.
  396. */
  397. struct util_est {
  398. unsigned int enqueued;
  399. unsigned int ewma;
  400. #define UTIL_EST_WEIGHT_SHIFT 2
  401. #define UTIL_AVG_UNCHANGED 0x80000000
  402. } __attribute__((__aligned__(sizeof(u64))));
  403. /*
  404. * The load/runnable/util_avg accumulates an infinite geometric series
  405. * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c).
  406. *
  407. * [load_avg definition]
  408. *
  409. * load_avg = runnable% * scale_load_down(load)
  410. *
  411. * [runnable_avg definition]
  412. *
  413. * runnable_avg = runnable% * SCHED_CAPACITY_SCALE
  414. *
  415. * [util_avg definition]
  416. *
  417. * util_avg = running% * SCHED_CAPACITY_SCALE
  418. *
  419. * where runnable% is the time ratio that a sched_entity is runnable and
  420. * running% the time ratio that a sched_entity is running.
  421. *
  422. * For cfs_rq, they are the aggregated values of all runnable and blocked
  423. * sched_entities.
  424. *
  425. * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU
  426. * capacity scaling. The scaling is done through the rq_clock_pelt that is used
  427. * for computing those signals (see update_rq_clock_pelt())
  428. *
  429. * N.B., the above ratios (runnable% and running%) themselves are in the
  430. * range of [0, 1]. To do fixed point arithmetics, we therefore scale them
  431. * to as large a range as necessary. This is for example reflected by
  432. * util_avg's SCHED_CAPACITY_SCALE.
  433. *
  434. * [Overflow issue]
  435. *
  436. * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities
  437. * with the highest load (=88761), always runnable on a single cfs_rq,
  438. * and should not overflow as the number already hits PID_MAX_LIMIT.
  439. *
  440. * For all other cases (including 32-bit kernels), struct load_weight's
  441. * weight will overflow first before we do, because:
  442. *
  443. * Max(load_avg) <= Max(load.weight)
  444. *
  445. * Then it is the load_weight's responsibility to consider overflow
  446. * issues.
  447. */
  448. struct sched_avg {
  449. u64 last_update_time;
  450. u64 load_sum;
  451. u64 runnable_sum;
  452. u32 util_sum;
  453. u32 period_contrib;
  454. unsigned long load_avg;
  455. unsigned long runnable_avg;
  456. unsigned long util_avg;
  457. struct util_est util_est;
  458. } ____cacheline_aligned;
  459. struct sched_statistics {
  460. #ifdef CONFIG_SCHEDSTATS
  461. u64 wait_start;
  462. u64 wait_max;
  463. u64 wait_count;
  464. u64 wait_sum;
  465. u64 iowait_count;
  466. u64 iowait_sum;
  467. u64 sleep_start;
  468. u64 sleep_max;
  469. s64 sum_sleep_runtime;
  470. u64 block_start;
  471. u64 block_max;
  472. s64 sum_block_runtime;
  473. u64 exec_max;
  474. u64 slice_max;
  475. u64 nr_migrations_cold;
  476. u64 nr_failed_migrations_affine;
  477. u64 nr_failed_migrations_running;
  478. u64 nr_failed_migrations_hot;
  479. u64 nr_forced_migrations;
  480. u64 nr_wakeups;
  481. u64 nr_wakeups_sync;
  482. u64 nr_wakeups_migrate;
  483. u64 nr_wakeups_local;
  484. u64 nr_wakeups_remote;
  485. u64 nr_wakeups_affine;
  486. u64 nr_wakeups_affine_attempts;
  487. u64 nr_wakeups_passive;
  488. u64 nr_wakeups_idle;
  489. #ifdef CONFIG_SCHED_CORE
  490. u64 core_forceidle_sum;
  491. #endif
  492. #endif /* CONFIG_SCHEDSTATS */
  493. } ____cacheline_aligned;
  494. struct sched_entity {
  495. /* For load-balancing: */
  496. struct load_weight load;
  497. struct rb_node run_node;
  498. struct list_head group_node;
  499. unsigned int on_rq;
  500. u64 exec_start;
  501. u64 sum_exec_runtime;
  502. u64 vruntime;
  503. u64 prev_sum_exec_runtime;
  504. u64 nr_migrations;
  505. #ifdef CONFIG_FAIR_GROUP_SCHED
  506. int depth;
  507. struct sched_entity *parent;
  508. /* rq on which this entity is (to be) queued: */
  509. struct cfs_rq *cfs_rq;
  510. /* rq "owned" by this entity/group: */
  511. struct cfs_rq *my_q;
  512. /* cached value of my_q->h_nr_running */
  513. unsigned long runnable_weight;
  514. #endif
  515. #ifdef CONFIG_SMP
  516. /*
  517. * Per entity load average tracking.
  518. *
  519. * Put into separate cache line so it does not
  520. * collide with read-mostly values above.
  521. */
  522. struct sched_avg avg;
  523. #endif
  524. ANDROID_KABI_RESERVE(1);
  525. ANDROID_KABI_RESERVE(2);
  526. ANDROID_KABI_RESERVE(3);
  527. ANDROID_KABI_RESERVE(4);
  528. };
  529. struct sched_rt_entity {
  530. struct list_head run_list;
  531. unsigned long timeout;
  532. unsigned long watchdog_stamp;
  533. unsigned int time_slice;
  534. unsigned short on_rq;
  535. unsigned short on_list;
  536. struct sched_rt_entity *back;
  537. #ifdef CONFIG_RT_GROUP_SCHED
  538. struct sched_rt_entity *parent;
  539. /* rq on which this entity is (to be) queued: */
  540. struct rt_rq *rt_rq;
  541. /* rq "owned" by this entity/group: */
  542. struct rt_rq *my_q;
  543. #endif
  544. ANDROID_KABI_RESERVE(1);
  545. ANDROID_KABI_RESERVE(2);
  546. ANDROID_KABI_RESERVE(3);
  547. ANDROID_KABI_RESERVE(4);
  548. } __randomize_layout;
  549. struct sched_dl_entity {
  550. struct rb_node rb_node;
  551. /*
  552. * Original scheduling parameters. Copied here from sched_attr
  553. * during sched_setattr(), they will remain the same until
  554. * the next sched_setattr().
  555. */
  556. u64 dl_runtime; /* Maximum runtime for each instance */
  557. u64 dl_deadline; /* Relative deadline of each instance */
  558. u64 dl_period; /* Separation of two instances (period) */
  559. u64 dl_bw; /* dl_runtime / dl_period */
  560. u64 dl_density; /* dl_runtime / dl_deadline */
  561. /*
  562. * Actual scheduling parameters. Initialized with the values above,
  563. * they are continuously updated during task execution. Note that
  564. * the remaining runtime could be < 0 in case we are in overrun.
  565. */
  566. s64 runtime; /* Remaining runtime for this instance */
  567. u64 deadline; /* Absolute deadline for this instance */
  568. unsigned int flags; /* Specifying the scheduler behaviour */
  569. /*
  570. * Some bool flags:
  571. *
  572. * @dl_throttled tells if we exhausted the runtime. If so, the
  573. * task has to wait for a replenishment to be performed at the
  574. * next firing of dl_timer.
  575. *
  576. * @dl_yielded tells if task gave up the CPU before consuming
  577. * all its available runtime during the last job.
  578. *
  579. * @dl_non_contending tells if the task is inactive while still
  580. * contributing to the active utilization. In other words, it
  581. * indicates if the inactive timer has been armed and its handler
  582. * has not been executed yet. This flag is useful to avoid race
  583. * conditions between the inactive timer handler and the wakeup
  584. * code.
  585. *
  586. * @dl_overrun tells if the task asked to be informed about runtime
  587. * overruns.
  588. */
  589. unsigned int dl_throttled : 1;
  590. unsigned int dl_yielded : 1;
  591. unsigned int dl_non_contending : 1;
  592. unsigned int dl_overrun : 1;
  593. /*
  594. * Bandwidth enforcement timer. Each -deadline task has its
  595. * own bandwidth to be enforced, thus we need one timer per task.
  596. */
  597. struct hrtimer dl_timer;
  598. /*
  599. * Inactive timer, responsible for decreasing the active utilization
  600. * at the "0-lag time". When a -deadline task blocks, it contributes
  601. * to GRUB's active utilization until the "0-lag time", hence a
  602. * timer is needed to decrease the active utilization at the correct
  603. * time.
  604. */
  605. struct hrtimer inactive_timer;
  606. #ifdef CONFIG_RT_MUTEXES
  607. /*
  608. * Priority Inheritance. When a DEADLINE scheduling entity is boosted
  609. * pi_se points to the donor, otherwise points to the dl_se it belongs
  610. * to (the original one/itself).
  611. */
  612. struct sched_dl_entity *pi_se;
  613. #endif
  614. };
  615. #ifdef CONFIG_UCLAMP_TASK
  616. /* Number of utilization clamp buckets (shorter alias) */
  617. #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT
  618. /*
  619. * Utilization clamp for a scheduling entity
  620. * @value: clamp value "assigned" to a se
  621. * @bucket_id: bucket index corresponding to the "assigned" value
  622. * @active: the se is currently refcounted in a rq's bucket
  623. * @user_defined: the requested clamp value comes from user-space
  624. *
  625. * The bucket_id is the index of the clamp bucket matching the clamp value
  626. * which is pre-computed and stored to avoid expensive integer divisions from
  627. * the fast path.
  628. *
  629. * The active bit is set whenever a task has got an "effective" value assigned,
  630. * which can be different from the clamp value "requested" from user-space.
  631. * This allows to know a task is refcounted in the rq's bucket corresponding
  632. * to the "effective" bucket_id.
  633. *
  634. * The user_defined bit is set whenever a task has got a task-specific clamp
  635. * value requested from userspace, i.e. the system defaults apply to this task
  636. * just as a restriction. This allows to relax default clamps when a less
  637. * restrictive task-specific value has been requested, thus allowing to
  638. * implement a "nice" semantic. For example, a task running with a 20%
  639. * default boost can still drop its own boosting to 0%.
  640. */
  641. struct uclamp_se {
  642. unsigned int value : bits_per(SCHED_CAPACITY_SCALE);
  643. unsigned int bucket_id : bits_per(UCLAMP_BUCKETS);
  644. unsigned int active : 1;
  645. unsigned int user_defined : 1;
  646. };
  647. #endif /* CONFIG_UCLAMP_TASK */
  648. union rcu_special {
  649. struct {
  650. u8 blocked;
  651. u8 need_qs;
  652. u8 exp_hint; /* Hint for performance. */
  653. u8 need_mb; /* Readers need smp_mb(). */
  654. } b; /* Bits. */
  655. u32 s; /* Set of bits. */
  656. };
  657. enum perf_event_task_context {
  658. perf_invalid_context = -1,
  659. perf_hw_context = 0,
  660. perf_sw_context,
  661. perf_nr_task_contexts,
  662. };
  663. struct wake_q_node {
  664. struct wake_q_node *next;
  665. };
  666. struct kmap_ctrl {
  667. #ifdef CONFIG_KMAP_LOCAL
  668. int idx;
  669. pte_t pteval[KM_MAX_IDX];
  670. #endif
  671. };
  672. struct task_struct {
  673. #ifdef CONFIG_THREAD_INFO_IN_TASK
  674. /*
  675. * For reasons of header soup (see current_thread_info()), this
  676. * must be the first element of task_struct.
  677. */
  678. struct thread_info thread_info;
  679. #endif
  680. unsigned int __state;
  681. /* saved state for "spinlock sleepers" */
  682. /* moved to ANDROID_KABI_USE(1, unsigned int saved_state) */
  683. /*
  684. * This begins the randomizable portion of task_struct. Only
  685. * scheduling-critical items should be added above here.
  686. */
  687. randomized_struct_fields_start
  688. void *stack;
  689. refcount_t usage;
  690. /* Per task flags (PF_*), defined further below: */
  691. unsigned int flags;
  692. unsigned int ptrace;
  693. #ifdef CONFIG_SMP
  694. int on_cpu;
  695. struct __call_single_node wake_entry;
  696. unsigned int wakee_flips;
  697. unsigned long wakee_flip_decay_ts;
  698. struct task_struct *last_wakee;
  699. /*
  700. * recent_used_cpu is initially set as the last CPU used by a task
  701. * that wakes affine another task. Waker/wakee relationships can
  702. * push tasks around a CPU where each wakeup moves to the next one.
  703. * Tracking a recently used CPU allows a quick search for a recently
  704. * used CPU that may be idle.
  705. */
  706. int recent_used_cpu;
  707. int wake_cpu;
  708. #endif
  709. int on_rq;
  710. int prio;
  711. int static_prio;
  712. int normal_prio;
  713. unsigned int rt_priority;
  714. struct sched_entity se;
  715. struct sched_rt_entity rt;
  716. struct sched_dl_entity dl;
  717. const struct sched_class *sched_class;
  718. #ifdef CONFIG_SCHED_CORE
  719. struct rb_node core_node;
  720. unsigned long core_cookie;
  721. unsigned int core_occupation;
  722. #endif
  723. #ifdef CONFIG_CGROUP_SCHED
  724. struct task_group *sched_task_group;
  725. #endif
  726. #ifdef CONFIG_UCLAMP_TASK
  727. /*
  728. * Clamp values requested for a scheduling entity.
  729. * Must be updated with task_rq_lock() held.
  730. */
  731. struct uclamp_se uclamp_req[UCLAMP_CNT];
  732. /*
  733. * Effective clamp values used for a scheduling entity.
  734. * Must be updated with task_rq_lock() held.
  735. */
  736. struct uclamp_se uclamp[UCLAMP_CNT];
  737. #endif
  738. struct sched_statistics stats;
  739. #ifdef CONFIG_PREEMPT_NOTIFIERS
  740. /* List of struct preempt_notifier: */
  741. struct hlist_head preempt_notifiers;
  742. #endif
  743. #ifdef CONFIG_BLK_DEV_IO_TRACE
  744. unsigned int btrace_seq;
  745. #endif
  746. unsigned int policy;
  747. int nr_cpus_allowed;
  748. const cpumask_t *cpus_ptr;
  749. cpumask_t *user_cpus_ptr;
  750. cpumask_t cpus_mask;
  751. void *migration_pending;
  752. #ifdef CONFIG_SMP
  753. unsigned short migration_disabled;
  754. #endif
  755. unsigned short migration_flags;
  756. #ifdef CONFIG_PREEMPT_RCU
  757. int rcu_read_lock_nesting;
  758. union rcu_special rcu_read_unlock_special;
  759. struct list_head rcu_node_entry;
  760. struct rcu_node *rcu_blocked_node;
  761. #endif /* #ifdef CONFIG_PREEMPT_RCU */
  762. #ifdef CONFIG_TASKS_RCU
  763. unsigned long rcu_tasks_nvcsw;
  764. u8 rcu_tasks_holdout;
  765. u8 rcu_tasks_idx;
  766. int rcu_tasks_idle_cpu;
  767. struct list_head rcu_tasks_holdout_list;
  768. #endif /* #ifdef CONFIG_TASKS_RCU */
  769. #ifdef CONFIG_TASKS_TRACE_RCU
  770. int trc_reader_nesting;
  771. int trc_ipi_to_cpu;
  772. union rcu_special trc_reader_special;
  773. struct list_head trc_holdout_list;
  774. struct list_head trc_blkd_node;
  775. int trc_blkd_cpu;
  776. #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
  777. struct sched_info sched_info;
  778. struct list_head tasks;
  779. #ifdef CONFIG_SMP
  780. struct plist_node pushable_tasks;
  781. struct rb_node pushable_dl_tasks;
  782. #endif
  783. struct mm_struct *mm;
  784. struct mm_struct *active_mm;
  785. #ifdef SPLIT_RSS_COUNTING
  786. struct task_rss_stat rss_stat;
  787. #endif
  788. int exit_state;
  789. int exit_code;
  790. int exit_signal;
  791. /* The signal sent when the parent dies: */
  792. int pdeath_signal;
  793. /* JOBCTL_*, siglock protected: */
  794. unsigned long jobctl;
  795. /* Used for emulating ABI behavior of previous Linux versions: */
  796. unsigned int personality;
  797. /* Scheduler bits, serialized by scheduler locks: */
  798. unsigned sched_reset_on_fork:1;
  799. unsigned sched_contributes_to_load:1;
  800. unsigned sched_migrated:1;
  801. #ifdef CONFIG_PSI
  802. unsigned sched_psi_wake_requeue:1;
  803. #endif
  804. /* Force alignment to the next boundary: */
  805. unsigned :0;
  806. /* Unserialized, strictly 'current' */
  807. /*
  808. * This field must not be in the scheduler word above due to wakelist
  809. * queueing no longer being serialized by p->on_cpu. However:
  810. *
  811. * p->XXX = X; ttwu()
  812. * schedule() if (p->on_rq && ..) // false
  813. * smp_mb__after_spinlock(); if (smp_load_acquire(&p->on_cpu) && //true
  814. * deactivate_task() ttwu_queue_wakelist())
  815. * p->on_rq = 0; p->sched_remote_wakeup = Y;
  816. *
  817. * guarantees all stores of 'current' are visible before
  818. * ->sched_remote_wakeup gets used, so it can be in this word.
  819. */
  820. unsigned sched_remote_wakeup:1;
  821. /* Bit to tell LSMs we're in execve(): */
  822. unsigned in_execve:1;
  823. unsigned in_iowait:1;
  824. #ifndef TIF_RESTORE_SIGMASK
  825. unsigned restore_sigmask:1;
  826. #endif
  827. #ifdef CONFIG_MEMCG
  828. unsigned in_user_fault:1;
  829. #endif
  830. #ifdef CONFIG_LRU_GEN
  831. /* whether the LRU algorithm may apply to this access */
  832. unsigned in_lru_fault:1;
  833. #endif
  834. #ifdef CONFIG_COMPAT_BRK
  835. unsigned brk_randomized:1;
  836. #endif
  837. #ifdef CONFIG_CGROUPS
  838. /* disallow userland-initiated cgroup migration */
  839. unsigned no_cgroup_migration:1;
  840. /* task is frozen/stopped (used by the cgroup freezer) */
  841. unsigned frozen:1;
  842. #endif
  843. #ifdef CONFIG_BLK_CGROUP
  844. unsigned use_memdelay:1;
  845. #endif
  846. #ifdef CONFIG_PSI
  847. /* Stalled due to lack of memory */
  848. unsigned in_memstall:1;
  849. #endif
  850. #ifdef CONFIG_PAGE_OWNER
  851. /* Used by page_owner=on to detect recursion in page tracking. */
  852. unsigned in_page_owner:1;
  853. #endif
  854. #ifdef CONFIG_EVENTFD
  855. /* Recursion prevention for eventfd_signal() */
  856. unsigned in_eventfd:1;
  857. #endif
  858. #ifdef CONFIG_IOMMU_SVA
  859. unsigned pasid_activated:1;
  860. #endif
  861. #ifdef CONFIG_CPU_SUP_INTEL
  862. unsigned reported_split_lock:1;
  863. #endif
  864. #ifdef CONFIG_TASK_DELAY_ACCT
  865. /* delay due to memory thrashing */
  866. unsigned in_thrashing:1;
  867. #endif
  868. unsigned long atomic_flags; /* Flags requiring atomic access. */
  869. struct restart_block restart_block;
  870. pid_t pid;
  871. pid_t tgid;
  872. #ifdef CONFIG_STACKPROTECTOR
  873. /* Canary value for the -fstack-protector GCC feature: */
  874. unsigned long stack_canary;
  875. #endif
  876. /*
  877. * Pointers to the (original) parent process, youngest child, younger sibling,
  878. * older sibling, respectively. (p->father can be replaced with
  879. * p->real_parent->pid)
  880. */
  881. /* Real parent process: */
  882. struct task_struct __rcu *real_parent;
  883. /* Recipient of SIGCHLD, wait4() reports: */
  884. struct task_struct __rcu *parent;
  885. /*
  886. * Children/sibling form the list of natural children:
  887. */
  888. struct list_head children;
  889. struct list_head sibling;
  890. struct task_struct *group_leader;
  891. /*
  892. * 'ptraced' is the list of tasks this task is using ptrace() on.
  893. *
  894. * This includes both natural children and PTRACE_ATTACH targets.
  895. * 'ptrace_entry' is this task's link on the p->parent->ptraced list.
  896. */
  897. struct list_head ptraced;
  898. struct list_head ptrace_entry;
  899. /* PID/PID hash table linkage. */
  900. struct pid *thread_pid;
  901. struct hlist_node pid_links[PIDTYPE_MAX];
  902. struct list_head thread_group;
  903. struct list_head thread_node;
  904. struct completion *vfork_done;
  905. /* CLONE_CHILD_SETTID: */
  906. int __user *set_child_tid;
  907. /* CLONE_CHILD_CLEARTID: */
  908. int __user *clear_child_tid;
  909. /* PF_KTHREAD | PF_IO_WORKER */
  910. void *worker_private;
  911. u64 utime;
  912. u64 stime;
  913. #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME
  914. u64 utimescaled;
  915. u64 stimescaled;
  916. #endif
  917. u64 gtime;
  918. #ifdef CONFIG_CPU_FREQ_TIMES
  919. u64 *time_in_state;
  920. unsigned int max_state;
  921. #endif
  922. struct prev_cputime prev_cputime;
  923. #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
  924. struct vtime vtime;
  925. #endif
  926. #ifdef CONFIG_NO_HZ_FULL
  927. atomic_t tick_dep_mask;
  928. #endif
  929. /* Context switch counts: */
  930. unsigned long nvcsw;
  931. unsigned long nivcsw;
  932. /* Monotonic time in nsecs: */
  933. u64 start_time;
  934. /* Boot based time in nsecs: */
  935. u64 start_boottime;
  936. /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */
  937. unsigned long min_flt;
  938. unsigned long maj_flt;
  939. /* Empty if CONFIG_POSIX_CPUTIMERS=n */
  940. struct posix_cputimers posix_cputimers;
  941. #ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK
  942. struct posix_cputimers_work posix_cputimers_work;
  943. #endif
  944. /* Process credentials: */
  945. /* Tracer's credentials at attach: */
  946. const struct cred __rcu *ptracer_cred;
  947. /* Objective and real subjective task credentials (COW): */
  948. const struct cred __rcu *real_cred;
  949. /* Effective (overridable) subjective task credentials (COW): */
  950. const struct cred __rcu *cred;
  951. #ifdef CONFIG_KEYS
  952. /* Cached requested key. */
  953. struct key *cached_requested_key;
  954. #endif
  955. /*
  956. * executable name, excluding path.
  957. *
  958. * - normally initialized setup_new_exec()
  959. * - access it with [gs]et_task_comm()
  960. * - lock it with task_lock()
  961. */
  962. char comm[TASK_COMM_LEN];
  963. struct nameidata *nameidata;
  964. #ifdef CONFIG_SYSVIPC
  965. struct sysv_sem sysvsem;
  966. struct sysv_shm sysvshm;
  967. #endif
  968. #ifdef CONFIG_DETECT_HUNG_TASK
  969. unsigned long last_switch_count;
  970. unsigned long last_switch_time;
  971. #endif
  972. /* Filesystem information: */
  973. struct fs_struct *fs;
  974. /* Open file information: */
  975. struct files_struct *files;
  976. #ifdef CONFIG_IO_URING
  977. struct io_uring_task *io_uring;
  978. #endif
  979. /* Namespaces: */
  980. struct nsproxy *nsproxy;
  981. /* Signal handlers: */
  982. struct signal_struct *signal;
  983. struct sighand_struct __rcu *sighand;
  984. sigset_t blocked;
  985. sigset_t real_blocked;
  986. /* Restored if set_restore_sigmask() was used: */
  987. sigset_t saved_sigmask;
  988. struct sigpending pending;
  989. unsigned long sas_ss_sp;
  990. size_t sas_ss_size;
  991. unsigned int sas_ss_flags;
  992. struct callback_head *task_works;
  993. #ifdef CONFIG_AUDIT
  994. #ifdef CONFIG_AUDITSYSCALL
  995. struct audit_context *audit_context;
  996. #endif
  997. kuid_t loginuid;
  998. unsigned int sessionid;
  999. #endif
  1000. struct seccomp seccomp;
  1001. struct syscall_user_dispatch syscall_dispatch;
  1002. /* Thread group tracking: */
  1003. u64 parent_exec_id;
  1004. u64 self_exec_id;
  1005. /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */
  1006. spinlock_t alloc_lock;
  1007. /* Protection of the PI data structures: */
  1008. raw_spinlock_t pi_lock;
  1009. struct wake_q_node wake_q;
  1010. int wake_q_count;
  1011. #ifdef CONFIG_RT_MUTEXES
  1012. /* PI waiters blocked on a rt_mutex held by this task: */
  1013. struct rb_root_cached pi_waiters;
  1014. /* Updated under owner's pi_lock and rq lock */
  1015. struct task_struct *pi_top_task;
  1016. /* Deadlock detection and priority inheritance handling: */
  1017. struct rt_mutex_waiter *pi_blocked_on;
  1018. #endif
  1019. #ifdef CONFIG_DEBUG_MUTEXES
  1020. /* Mutex deadlock detection: */
  1021. struct mutex_waiter *blocked_on;
  1022. #endif
  1023. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  1024. int non_block_count;
  1025. #endif
  1026. #ifdef CONFIG_TRACE_IRQFLAGS
  1027. struct irqtrace_events irqtrace;
  1028. unsigned int hardirq_threaded;
  1029. u64 hardirq_chain_key;
  1030. int softirqs_enabled;
  1031. int softirq_context;
  1032. int irq_config;
  1033. #endif
  1034. #ifdef CONFIG_PREEMPT_RT
  1035. int softirq_disable_cnt;
  1036. #endif
  1037. #ifdef CONFIG_LOCKDEP
  1038. # define MAX_LOCK_DEPTH 48UL
  1039. u64 curr_chain_key;
  1040. int lockdep_depth;
  1041. unsigned int lockdep_recursion;
  1042. struct held_lock held_locks[MAX_LOCK_DEPTH];
  1043. #endif
  1044. #if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP)
  1045. unsigned int in_ubsan;
  1046. #endif
  1047. /* Journalling filesystem info: */
  1048. void *journal_info;
  1049. /* Stacked block device info: */
  1050. struct bio_list *bio_list;
  1051. /* Stack plugging: */
  1052. struct blk_plug *plug;
  1053. /* VM state: */
  1054. struct reclaim_state *reclaim_state;
  1055. struct backing_dev_info *backing_dev_info;
  1056. struct io_context *io_context;
  1057. #ifdef CONFIG_COMPACTION
  1058. struct capture_control *capture_control;
  1059. #endif
  1060. /* Ptrace state: */
  1061. unsigned long ptrace_message;
  1062. kernel_siginfo_t *last_siginfo;
  1063. struct task_io_accounting ioac;
  1064. #ifdef CONFIG_PSI
  1065. /* Pressure stall state */
  1066. unsigned int psi_flags;
  1067. #endif
  1068. #ifdef CONFIG_TASK_XACCT
  1069. /* Accumulated RSS usage: */
  1070. u64 acct_rss_mem1;
  1071. /* Accumulated virtual memory usage: */
  1072. u64 acct_vm_mem1;
  1073. /* stime + utime since last update: */
  1074. u64 acct_timexpd;
  1075. #endif
  1076. #ifdef CONFIG_CPUSETS
  1077. /* Protected by ->alloc_lock: */
  1078. nodemask_t mems_allowed;
  1079. /* Sequence number to catch updates: */
  1080. seqcount_spinlock_t mems_allowed_seq;
  1081. int cpuset_mem_spread_rotor;
  1082. int cpuset_slab_spread_rotor;
  1083. #endif
  1084. #ifdef CONFIG_CGROUPS
  1085. /* Control Group info protected by css_set_lock: */
  1086. struct css_set __rcu *cgroups;
  1087. /* cg_list protected by css_set_lock and tsk->alloc_lock: */
  1088. struct list_head cg_list;
  1089. #endif
  1090. #ifdef CONFIG_X86_CPU_RESCTRL
  1091. u32 closid;
  1092. u32 rmid;
  1093. #endif
  1094. #ifdef CONFIG_FUTEX
  1095. struct robust_list_head __user *robust_list;
  1096. #ifdef CONFIG_COMPAT
  1097. struct compat_robust_list_head __user *compat_robust_list;
  1098. #endif
  1099. struct list_head pi_state_list;
  1100. struct futex_pi_state *pi_state_cache;
  1101. struct mutex futex_exit_mutex;
  1102. unsigned int futex_state;
  1103. #endif
  1104. #ifdef CONFIG_PERF_EVENTS
  1105. struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
  1106. struct mutex perf_event_mutex;
  1107. struct list_head perf_event_list;
  1108. #endif
  1109. #ifdef CONFIG_DEBUG_PREEMPT
  1110. unsigned long preempt_disable_ip;
  1111. #endif
  1112. #ifdef CONFIG_NUMA
  1113. /* Protected by alloc_lock: */
  1114. struct mempolicy *mempolicy;
  1115. short il_prev;
  1116. short pref_node_fork;
  1117. #endif
  1118. #ifdef CONFIG_NUMA_BALANCING
  1119. int numa_scan_seq;
  1120. unsigned int numa_scan_period;
  1121. unsigned int numa_scan_period_max;
  1122. int numa_preferred_nid;
  1123. unsigned long numa_migrate_retry;
  1124. /* Migration stamp: */
  1125. u64 node_stamp;
  1126. u64 last_task_numa_placement;
  1127. u64 last_sum_exec_runtime;
  1128. struct callback_head numa_work;
  1129. /*
  1130. * This pointer is only modified for current in syscall and
  1131. * pagefault context (and for tasks being destroyed), so it can be read
  1132. * from any of the following contexts:
  1133. * - RCU read-side critical section
  1134. * - current->numa_group from everywhere
  1135. * - task's runqueue locked, task not running
  1136. */
  1137. struct numa_group __rcu *numa_group;
  1138. /*
  1139. * numa_faults is an array split into four regions:
  1140. * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
  1141. * in this precise order.
  1142. *
  1143. * faults_memory: Exponential decaying average of faults on a per-node
  1144. * basis. Scheduling placement decisions are made based on these
  1145. * counts. The values remain static for the duration of a PTE scan.
  1146. * faults_cpu: Track the nodes the process was running on when a NUMA
  1147. * hinting fault was incurred.
  1148. * faults_memory_buffer and faults_cpu_buffer: Record faults per node
  1149. * during the current scan window. When the scan completes, the counts
  1150. * in faults_memory and faults_cpu decay and these values are copied.
  1151. */
  1152. unsigned long *numa_faults;
  1153. unsigned long total_numa_faults;
  1154. /*
  1155. * numa_faults_locality tracks if faults recorded during the last
  1156. * scan window were remote/local or failed to migrate. The task scan
  1157. * period is adapted based on the locality of the faults with different
  1158. * weights depending on whether they were shared or private faults
  1159. */
  1160. unsigned long numa_faults_locality[3];
  1161. unsigned long numa_pages_migrated;
  1162. #endif /* CONFIG_NUMA_BALANCING */
  1163. #ifdef CONFIG_RSEQ
  1164. struct rseq __user *rseq;
  1165. u32 rseq_sig;
  1166. /*
  1167. * RmW on rseq_event_mask must be performed atomically
  1168. * with respect to preemption.
  1169. */
  1170. unsigned long rseq_event_mask;
  1171. #endif
  1172. struct tlbflush_unmap_batch tlb_ubc;
  1173. union {
  1174. refcount_t rcu_users;
  1175. struct rcu_head rcu;
  1176. };
  1177. /* Cache last used pipe for splice(): */
  1178. struct pipe_inode_info *splice_pipe;
  1179. struct page_frag task_frag;
  1180. #ifdef CONFIG_TASK_DELAY_ACCT
  1181. struct task_delay_info *delays;
  1182. #endif
  1183. #ifdef CONFIG_FAULT_INJECTION
  1184. int make_it_fail;
  1185. unsigned int fail_nth;
  1186. #endif
  1187. /*
  1188. * When (nr_dirtied >= nr_dirtied_pause), it's time to call
  1189. * balance_dirty_pages() for a dirty throttling pause:
  1190. */
  1191. int nr_dirtied;
  1192. int nr_dirtied_pause;
  1193. /* Start of a write-and-pause period: */
  1194. unsigned long dirty_paused_when;
  1195. #ifdef CONFIG_LATENCYTOP
  1196. int latency_record_count;
  1197. struct latency_record latency_record[LT_SAVECOUNT];
  1198. #endif
  1199. /*
  1200. * Time slack values; these are used to round up poll() and
  1201. * select() etc timeout values. These are in nanoseconds.
  1202. */
  1203. u64 timer_slack_ns;
  1204. u64 default_timer_slack_ns;
  1205. #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
  1206. unsigned int kasan_depth;
  1207. #endif
  1208. #ifdef CONFIG_KCSAN
  1209. struct kcsan_ctx kcsan_ctx;
  1210. #ifdef CONFIG_TRACE_IRQFLAGS
  1211. struct irqtrace_events kcsan_save_irqtrace;
  1212. #endif
  1213. #ifdef CONFIG_KCSAN_WEAK_MEMORY
  1214. int kcsan_stack_depth;
  1215. #endif
  1216. #endif
  1217. #ifdef CONFIG_KMSAN
  1218. struct kmsan_ctx kmsan_ctx;
  1219. #endif
  1220. #if IS_ENABLED(CONFIG_KUNIT)
  1221. struct kunit *kunit_test;
  1222. #endif
  1223. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  1224. /* Index of current stored address in ret_stack: */
  1225. int curr_ret_stack;
  1226. int curr_ret_depth;
  1227. /* Stack of return addresses for return function tracing: */
  1228. struct ftrace_ret_stack *ret_stack;
  1229. /* Timestamp for last schedule: */
  1230. unsigned long long ftrace_timestamp;
  1231. /*
  1232. * Number of functions that haven't been traced
  1233. * because of depth overrun:
  1234. */
  1235. atomic_t trace_overrun;
  1236. /* Pause tracing: */
  1237. atomic_t tracing_graph_pause;
  1238. #endif
  1239. #ifdef CONFIG_TRACING
  1240. /* Bitmask and counter of trace recursion: */
  1241. unsigned long trace_recursion;
  1242. #endif /* CONFIG_TRACING */
  1243. #ifdef CONFIG_KCOV
  1244. /* See kernel/kcov.c for more details. */
  1245. /* Coverage collection mode enabled for this task (0 if disabled): */
  1246. unsigned int kcov_mode;
  1247. /* Size of the kcov_area: */
  1248. unsigned int kcov_size;
  1249. /* Buffer for coverage collection: */
  1250. void *kcov_area;
  1251. /* KCOV descriptor wired with this task or NULL: */
  1252. struct kcov *kcov;
  1253. /* KCOV common handle for remote coverage collection: */
  1254. u64 kcov_handle;
  1255. /* KCOV sequence number: */
  1256. int kcov_sequence;
  1257. /* Collect coverage from softirq context: */
  1258. unsigned int kcov_softirq;
  1259. #endif
  1260. #ifdef CONFIG_MEMCG
  1261. struct mem_cgroup *memcg_in_oom;
  1262. gfp_t memcg_oom_gfp_mask;
  1263. int memcg_oom_order;
  1264. /* Number of pages to reclaim on returning to userland: */
  1265. unsigned int memcg_nr_pages_over_high;
  1266. /* Used by memcontrol for targeted memcg charge: */
  1267. struct mem_cgroup *active_memcg;
  1268. #endif
  1269. #ifdef CONFIG_BLK_CGROUP
  1270. struct request_queue *throttle_queue;
  1271. #endif
  1272. #ifdef CONFIG_UPROBES
  1273. struct uprobe_task *utask;
  1274. #endif
  1275. #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
  1276. unsigned int sequential_io;
  1277. unsigned int sequential_io_avg;
  1278. #endif
  1279. struct kmap_ctrl kmap_ctrl;
  1280. #ifdef CONFIG_DEBUG_ATOMIC_SLEEP
  1281. unsigned long task_state_change;
  1282. # ifdef CONFIG_PREEMPT_RT
  1283. unsigned long saved_state_change;
  1284. # endif
  1285. #endif
  1286. int pagefault_disabled;
  1287. #ifdef CONFIG_MMU
  1288. struct task_struct *oom_reaper_list;
  1289. struct timer_list oom_reaper_timer;
  1290. #endif
  1291. #ifdef CONFIG_VMAP_STACK
  1292. struct vm_struct *stack_vm_area;
  1293. #endif
  1294. #ifdef CONFIG_THREAD_INFO_IN_TASK
  1295. /* A live task holds one reference: */
  1296. refcount_t stack_refcount;
  1297. #endif
  1298. #ifdef CONFIG_LIVEPATCH
  1299. int patch_state;
  1300. #endif
  1301. #ifdef CONFIG_SECURITY
  1302. /* Used by LSM modules for access restriction: */
  1303. void *security;
  1304. #endif
  1305. #ifdef CONFIG_BPF_SYSCALL
  1306. /* Used by BPF task local storage */
  1307. struct bpf_local_storage __rcu *bpf_storage;
  1308. /* Used for BPF run context */
  1309. struct bpf_run_ctx *bpf_ctx;
  1310. #endif
  1311. #ifdef CONFIG_GCC_PLUGIN_STACKLEAK
  1312. unsigned long lowest_stack;
  1313. unsigned long prev_lowest_stack;
  1314. #endif
  1315. #ifdef CONFIG_X86_MCE
  1316. void __user *mce_vaddr;
  1317. __u64 mce_kflags;
  1318. u64 mce_addr;
  1319. __u64 mce_ripv : 1,
  1320. mce_whole_page : 1,
  1321. __mce_reserved : 62;
  1322. struct callback_head mce_kill_me;
  1323. int mce_count;
  1324. #endif
  1325. ANDROID_VENDOR_DATA_ARRAY(1, 64);
  1326. ANDROID_OEM_DATA_ARRAY(1, 6);
  1327. #ifdef CONFIG_KRETPROBES
  1328. struct llist_head kretprobe_instances;
  1329. #endif
  1330. #ifdef CONFIG_RETHOOK
  1331. struct llist_head rethooks;
  1332. #endif
  1333. #ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH
  1334. /*
  1335. * If L1D flush is supported on mm context switch
  1336. * then we use this callback head to queue kill work
  1337. * to kill tasks that are not running on SMT disabled
  1338. * cores
  1339. */
  1340. struct callback_head l1d_flush_kill;
  1341. #endif
  1342. #ifdef CONFIG_RV
  1343. /*
  1344. * Per-task RV monitor. Nowadays fixed in RV_PER_TASK_MONITORS.
  1345. * If we find justification for more monitors, we can think
  1346. * about adding more or developing a dynamic method. So far,
  1347. * none of these are justified.
  1348. */
  1349. union rv_task_monitor rv[RV_PER_TASK_MONITORS];
  1350. #endif
  1351. ANDROID_KABI_USE(1, unsigned int saved_state);
  1352. ANDROID_KABI_RESERVE(2);
  1353. ANDROID_KABI_RESERVE(3);
  1354. ANDROID_KABI_RESERVE(4);
  1355. ANDROID_KABI_RESERVE(5);
  1356. ANDROID_KABI_RESERVE(6);
  1357. ANDROID_KABI_RESERVE(7);
  1358. ANDROID_KABI_RESERVE(8);
  1359. /*
  1360. * New fields for task_struct should be added above here, so that
  1361. * they are included in the randomized portion of task_struct.
  1362. */
  1363. randomized_struct_fields_end
  1364. /* CPU-specific state of this task: */
  1365. struct thread_struct thread;
  1366. /*
  1367. * WARNING: on x86, 'thread_struct' contains a variable-sized
  1368. * structure. It *MUST* be at the end of 'task_struct'.
  1369. *
  1370. * Do not put anything below here!
  1371. */
  1372. };
  1373. static inline struct pid *task_pid(struct task_struct *task)
  1374. {
  1375. return task->thread_pid;
  1376. }
  1377. /*
  1378. * the helpers to get the task's different pids as they are seen
  1379. * from various namespaces
  1380. *
  1381. * task_xid_nr() : global id, i.e. the id seen from the init namespace;
  1382. * task_xid_vnr() : virtual id, i.e. the id seen from the pid namespace of
  1383. * current.
  1384. * task_xid_nr_ns() : id seen from the ns specified;
  1385. *
  1386. * see also pid_nr() etc in include/linux/pid.h
  1387. */
  1388. pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns);
  1389. static inline pid_t task_pid_nr(struct task_struct *tsk)
  1390. {
  1391. return tsk->pid;
  1392. }
  1393. static inline pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
  1394. {
  1395. return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns);
  1396. }
  1397. static inline pid_t task_pid_vnr(struct task_struct *tsk)
  1398. {
  1399. return __task_pid_nr_ns(tsk, PIDTYPE_PID, NULL);
  1400. }
  1401. static inline pid_t task_tgid_nr(struct task_struct *tsk)
  1402. {
  1403. return tsk->tgid;
  1404. }
  1405. /**
  1406. * pid_alive - check that a task structure is not stale
  1407. * @p: Task structure to be checked.
  1408. *
  1409. * Test if a process is not yet dead (at most zombie state)
  1410. * If pid_alive fails, then pointers within the task structure
  1411. * can be stale and must not be dereferenced.
  1412. *
  1413. * Return: 1 if the process is alive. 0 otherwise.
  1414. */
  1415. static inline int pid_alive(const struct task_struct *p)
  1416. {
  1417. return p->thread_pid != NULL;
  1418. }
  1419. static inline pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
  1420. {
  1421. return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns);
  1422. }
  1423. static inline pid_t task_pgrp_vnr(struct task_struct *tsk)
  1424. {
  1425. return __task_pid_nr_ns(tsk, PIDTYPE_PGID, NULL);
  1426. }
  1427. static inline pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
  1428. {
  1429. return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns);
  1430. }
  1431. static inline pid_t task_session_vnr(struct task_struct *tsk)
  1432. {
  1433. return __task_pid_nr_ns(tsk, PIDTYPE_SID, NULL);
  1434. }
  1435. static inline pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns)
  1436. {
  1437. return __task_pid_nr_ns(tsk, PIDTYPE_TGID, ns);
  1438. }
  1439. static inline pid_t task_tgid_vnr(struct task_struct *tsk)
  1440. {
  1441. return __task_pid_nr_ns(tsk, PIDTYPE_TGID, NULL);
  1442. }
  1443. static inline pid_t task_ppid_nr_ns(const struct task_struct *tsk, struct pid_namespace *ns)
  1444. {
  1445. pid_t pid = 0;
  1446. rcu_read_lock();
  1447. if (pid_alive(tsk))
  1448. pid = task_tgid_nr_ns(rcu_dereference(tsk->real_parent), ns);
  1449. rcu_read_unlock();
  1450. return pid;
  1451. }
  1452. static inline pid_t task_ppid_nr(const struct task_struct *tsk)
  1453. {
  1454. return task_ppid_nr_ns(tsk, &init_pid_ns);
  1455. }
  1456. /* Obsolete, do not use: */
  1457. static inline pid_t task_pgrp_nr(struct task_struct *tsk)
  1458. {
  1459. return task_pgrp_nr_ns(tsk, &init_pid_ns);
  1460. }
  1461. #define TASK_REPORT_IDLE (TASK_REPORT + 1)
  1462. #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1)
  1463. static inline unsigned int __task_state_index(unsigned int tsk_state,
  1464. unsigned int tsk_exit_state)
  1465. {
  1466. unsigned int state = (tsk_state | tsk_exit_state) & TASK_REPORT;
  1467. BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX);
  1468. if (tsk_state == TASK_IDLE)
  1469. state = TASK_REPORT_IDLE;
  1470. /*
  1471. * We're lying here, but rather than expose a completely new task state
  1472. * to userspace, we can make this appear as if the task has gone through
  1473. * a regular rt_mutex_lock() call.
  1474. */
  1475. if (tsk_state == TASK_RTLOCK_WAIT)
  1476. state = TASK_UNINTERRUPTIBLE;
  1477. return fls(state);
  1478. }
  1479. static inline unsigned int task_state_index(struct task_struct *tsk)
  1480. {
  1481. return __task_state_index(READ_ONCE(tsk->__state), tsk->exit_state);
  1482. }
  1483. static inline char task_index_to_char(unsigned int state)
  1484. {
  1485. static const char state_char[] = "RSDTtXZPI";
  1486. BUILD_BUG_ON(1 + ilog2(TASK_REPORT_MAX) != sizeof(state_char) - 1);
  1487. return state_char[state];
  1488. }
  1489. static inline char task_state_to_char(struct task_struct *tsk)
  1490. {
  1491. return task_index_to_char(task_state_index(tsk));
  1492. }
  1493. /**
  1494. * is_global_init - check if a task structure is init. Since init
  1495. * is free to have sub-threads we need to check tgid.
  1496. * @tsk: Task structure to be checked.
  1497. *
  1498. * Check if a task structure is the first user space task the kernel created.
  1499. *
  1500. * Return: 1 if the task structure is init. 0 otherwise.
  1501. */
  1502. static inline int is_global_init(struct task_struct *tsk)
  1503. {
  1504. return task_tgid_nr(tsk) == 1;
  1505. }
  1506. extern struct pid *cad_pid;
  1507. /*
  1508. * Per process flags
  1509. */
  1510. #define PF_VCPU 0x00000001 /* I'm a virtual CPU */
  1511. #define PF_IDLE 0x00000002 /* I am an IDLE thread */
  1512. #define PF_EXITING 0x00000004 /* Getting shut down */
  1513. #define PF_POSTCOREDUMP 0x00000008 /* Coredumps should ignore this task */
  1514. #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */
  1515. #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
  1516. #define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */
  1517. #define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */
  1518. #define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */
  1519. #define PF_DUMPCORE 0x00000200 /* Dumped core */
  1520. #define PF_SIGNALED 0x00000400 /* Killed by a signal */
  1521. #define PF_MEMALLOC 0x00000800 /* Allocating memory */
  1522. #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */
  1523. #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */
  1524. #define PF__HOLE__00004000 0x00004000
  1525. #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */
  1526. #define PF__HOLE__00010000 0x00010000
  1527. #define PF_KSWAPD 0x00020000 /* I am kswapd */
  1528. #define PF_MEMALLOC_NOFS 0x00040000 /* All allocation requests will inherit GFP_NOFS */
  1529. #define PF_MEMALLOC_NOIO 0x00080000 /* All allocation requests will inherit GFP_NOIO */
  1530. #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to,
  1531. * I am cleaning dirty pages from some other bdi. */
  1532. #define PF_KTHREAD 0x00200000 /* I am a kernel thread */
  1533. #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */
  1534. #define PF__HOLE__00800000 0x00800000
  1535. #define PF__HOLE__01000000 0x01000000
  1536. #define PF__HOLE__02000000 0x02000000
  1537. #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */
  1538. #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */
  1539. #define PF_MEMALLOC_PIN 0x10000000 /* Allocation context constrained to zones which allow long term pinning. */
  1540. #define PF__HOLE__20000000 0x20000000
  1541. #define PF__HOLE__40000000 0x40000000
  1542. #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */
  1543. /*
  1544. * Only the _current_ task can read/write to tsk->flags, but other
  1545. * tasks can access tsk->flags in readonly mode for example
  1546. * with tsk_used_math (like during threaded core dumping).
  1547. * There is however an exception to this rule during ptrace
  1548. * or during fork: the ptracer task is allowed to write to the
  1549. * child->flags of its traced child (same goes for fork, the parent
  1550. * can write to the child->flags), because we're guaranteed the
  1551. * child is not running and in turn not changing child->flags
  1552. * at the same time the parent does it.
  1553. */
  1554. #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0)
  1555. #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0)
  1556. #define clear_used_math() clear_stopped_child_used_math(current)
  1557. #define set_used_math() set_stopped_child_used_math(current)
  1558. #define conditional_stopped_child_used_math(condition, child) \
  1559. do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0)
  1560. #define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current)
  1561. #define copy_to_stopped_child_used_math(child) \
  1562. do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0)
  1563. /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */
  1564. #define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
  1565. #define used_math() tsk_used_math(current)
  1566. static __always_inline bool is_percpu_thread(void)
  1567. {
  1568. #ifdef CONFIG_SMP
  1569. return (current->flags & PF_NO_SETAFFINITY) &&
  1570. (current->nr_cpus_allowed == 1);
  1571. #else
  1572. return true;
  1573. #endif
  1574. }
  1575. /* Per-process atomic flags. */
  1576. #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */
  1577. #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */
  1578. #define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */
  1579. #define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */
  1580. #define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/
  1581. #define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */
  1582. #define PFA_SPEC_IB_FORCE_DISABLE 6 /* Indirect branch speculation permanently restricted */
  1583. #define PFA_SPEC_SSB_NOEXEC 7 /* Speculative Store Bypass clear on execve() */
  1584. #define TASK_PFA_TEST(name, func) \
  1585. static inline bool task_##func(struct task_struct *p) \
  1586. { return test_bit(PFA_##name, &p->atomic_flags); }
  1587. #define TASK_PFA_SET(name, func) \
  1588. static inline void task_set_##func(struct task_struct *p) \
  1589. { set_bit(PFA_##name, &p->atomic_flags); }
  1590. #define TASK_PFA_CLEAR(name, func) \
  1591. static inline void task_clear_##func(struct task_struct *p) \
  1592. { clear_bit(PFA_##name, &p->atomic_flags); }
  1593. TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs)
  1594. TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs)
  1595. TASK_PFA_TEST(SPREAD_PAGE, spread_page)
  1596. TASK_PFA_SET(SPREAD_PAGE, spread_page)
  1597. TASK_PFA_CLEAR(SPREAD_PAGE, spread_page)
  1598. TASK_PFA_TEST(SPREAD_SLAB, spread_slab)
  1599. TASK_PFA_SET(SPREAD_SLAB, spread_slab)
  1600. TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab)
  1601. TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable)
  1602. TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable)
  1603. TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable)
  1604. TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec)
  1605. TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec)
  1606. TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec)
  1607. TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
  1608. TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable)
  1609. TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable)
  1610. TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable)
  1611. TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable)
  1612. TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
  1613. TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable)
  1614. static inline void
  1615. current_restore_flags(unsigned long orig_flags, unsigned long flags)
  1616. {
  1617. current->flags &= ~flags;
  1618. current->flags |= orig_flags & flags;
  1619. }
  1620. extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial);
  1621. extern int task_can_attach(struct task_struct *p);
  1622. extern int dl_bw_alloc(int cpu, u64 dl_bw);
  1623. extern void dl_bw_free(int cpu, u64 dl_bw);
  1624. #ifdef CONFIG_SMP
  1625. extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask);
  1626. extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask);
  1627. extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node);
  1628. extern void release_user_cpus_ptr(struct task_struct *p);
  1629. extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask);
  1630. extern void force_compatible_cpus_allowed_ptr(struct task_struct *p);
  1631. extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p);
  1632. #else
  1633. static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask)
  1634. {
  1635. }
  1636. static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
  1637. {
  1638. if (!cpumask_test_cpu(0, new_mask))
  1639. return -EINVAL;
  1640. return 0;
  1641. }
  1642. static inline int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node)
  1643. {
  1644. if (src->user_cpus_ptr)
  1645. return -EINVAL;
  1646. return 0;
  1647. }
  1648. static inline void release_user_cpus_ptr(struct task_struct *p)
  1649. {
  1650. WARN_ON(p->user_cpus_ptr);
  1651. }
  1652. static inline int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask)
  1653. {
  1654. return 0;
  1655. }
  1656. #endif
  1657. extern int yield_to(struct task_struct *p, bool preempt);
  1658. extern void set_user_nice(struct task_struct *p, long nice);
  1659. extern int task_prio(const struct task_struct *p);
  1660. /**
  1661. * task_nice - return the nice value of a given task.
  1662. * @p: the task in question.
  1663. *
  1664. * Return: The nice value [ -20 ... 0 ... 19 ].
  1665. */
  1666. static inline int task_nice(const struct task_struct *p)
  1667. {
  1668. return PRIO_TO_NICE((p)->static_prio);
  1669. }
  1670. extern int can_nice(const struct task_struct *p, const int nice);
  1671. extern int task_curr(const struct task_struct *p);
  1672. extern int idle_cpu(int cpu);
  1673. extern int available_idle_cpu(int cpu);
  1674. extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *);
  1675. extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *);
  1676. extern void sched_set_fifo(struct task_struct *p);
  1677. extern void sched_set_fifo_low(struct task_struct *p);
  1678. extern void sched_set_normal(struct task_struct *p, int nice);
  1679. extern int sched_setattr(struct task_struct *, const struct sched_attr *);
  1680. extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *);
  1681. extern struct task_struct *idle_task(int cpu);
  1682. /**
  1683. * is_idle_task - is the specified task an idle task?
  1684. * @p: the task in question.
  1685. *
  1686. * Return: 1 if @p is an idle task. 0 otherwise.
  1687. */
  1688. static __always_inline bool is_idle_task(const struct task_struct *p)
  1689. {
  1690. return !!(p->flags & PF_IDLE);
  1691. }
  1692. extern struct task_struct *curr_task(int cpu);
  1693. extern void ia64_set_curr_task(int cpu, struct task_struct *p);
  1694. void yield(void);
  1695. union thread_union {
  1696. #ifndef CONFIG_ARCH_TASK_STRUCT_ON_STACK
  1697. struct task_struct task;
  1698. #endif
  1699. #ifndef CONFIG_THREAD_INFO_IN_TASK
  1700. struct thread_info thread_info;
  1701. #endif
  1702. unsigned long stack[THREAD_SIZE/sizeof(long)];
  1703. };
  1704. #ifndef CONFIG_THREAD_INFO_IN_TASK
  1705. extern struct thread_info init_thread_info;
  1706. #endif
  1707. extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)];
  1708. #ifdef CONFIG_THREAD_INFO_IN_TASK
  1709. # define task_thread_info(task) (&(task)->thread_info)
  1710. #elif !defined(__HAVE_THREAD_FUNCTIONS)
  1711. # define task_thread_info(task) ((struct thread_info *)(task)->stack)
  1712. #endif
  1713. /*
  1714. * find a task by one of its numerical ids
  1715. *
  1716. * find_task_by_pid_ns():
  1717. * finds a task by its pid in the specified namespace
  1718. * find_task_by_vpid():
  1719. * finds a task by its virtual pid
  1720. *
  1721. * see also find_vpid() etc in include/linux/pid.h
  1722. */
  1723. extern struct task_struct *find_task_by_vpid(pid_t nr);
  1724. extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns);
  1725. /*
  1726. * find a task by its virtual pid and get the task struct
  1727. */
  1728. extern struct task_struct *find_get_task_by_vpid(pid_t nr);
  1729. extern int wake_up_state(struct task_struct *tsk, unsigned int state);
  1730. extern int wake_up_process(struct task_struct *tsk);
  1731. extern void wake_up_new_task(struct task_struct *tsk);
  1732. #ifdef CONFIG_SMP
  1733. extern void kick_process(struct task_struct *tsk);
  1734. #else
  1735. static inline void kick_process(struct task_struct *tsk) { }
  1736. #endif
  1737. extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec);
  1738. static inline void set_task_comm(struct task_struct *tsk, const char *from)
  1739. {
  1740. __set_task_comm(tsk, from, false);
  1741. }
  1742. extern char *__get_task_comm(char *to, size_t len, struct task_struct *tsk);
  1743. #define get_task_comm(buf, tsk) ({ \
  1744. BUILD_BUG_ON(sizeof(buf) != TASK_COMM_LEN); \
  1745. __get_task_comm(buf, sizeof(buf), tsk); \
  1746. })
  1747. #ifdef CONFIG_SMP
  1748. static __always_inline void scheduler_ipi(void)
  1749. {
  1750. /*
  1751. * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
  1752. * TIF_NEED_RESCHED remotely (for the first time) will also send
  1753. * this IPI.
  1754. */
  1755. preempt_fold_need_resched();
  1756. }
  1757. #else
  1758. static inline void scheduler_ipi(void) { }
  1759. #endif
  1760. extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state);
  1761. /*
  1762. * Set thread flags in other task's structures.
  1763. * See asm/thread_info.h for TIF_xxxx flags available:
  1764. */
  1765. static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag)
  1766. {
  1767. set_ti_thread_flag(task_thread_info(tsk), flag);
  1768. }
  1769. static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag)
  1770. {
  1771. clear_ti_thread_flag(task_thread_info(tsk), flag);
  1772. }
  1773. static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag,
  1774. bool value)
  1775. {
  1776. update_ti_thread_flag(task_thread_info(tsk), flag, value);
  1777. }
  1778. static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag)
  1779. {
  1780. return test_and_set_ti_thread_flag(task_thread_info(tsk), flag);
  1781. }
  1782. static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag)
  1783. {
  1784. return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag);
  1785. }
  1786. static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag)
  1787. {
  1788. return test_ti_thread_flag(task_thread_info(tsk), flag);
  1789. }
  1790. static inline void set_tsk_need_resched(struct task_struct *tsk)
  1791. {
  1792. set_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
  1793. }
  1794. static inline void clear_tsk_need_resched(struct task_struct *tsk)
  1795. {
  1796. clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED);
  1797. }
  1798. static inline int test_tsk_need_resched(struct task_struct *tsk)
  1799. {
  1800. return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
  1801. }
  1802. /*
  1803. * cond_resched() and cond_resched_lock(): latency reduction via
  1804. * explicit rescheduling in places that are safe. The return
  1805. * value indicates whether a reschedule was done in fact.
  1806. * cond_resched_lock() will drop the spinlock before scheduling,
  1807. */
  1808. #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
  1809. extern int __cond_resched(void);
  1810. #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
  1811. DECLARE_STATIC_CALL(cond_resched, __cond_resched);
  1812. static __always_inline int _cond_resched(void)
  1813. {
  1814. return static_call_mod(cond_resched)();
  1815. }
  1816. #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
  1817. extern int dynamic_cond_resched(void);
  1818. static __always_inline int _cond_resched(void)
  1819. {
  1820. return dynamic_cond_resched();
  1821. }
  1822. #else
  1823. static inline int _cond_resched(void)
  1824. {
  1825. return __cond_resched();
  1826. }
  1827. #endif /* CONFIG_PREEMPT_DYNAMIC */
  1828. #else
  1829. static inline int _cond_resched(void) { return 0; }
  1830. #endif /* !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) */
  1831. #define cond_resched() ({ \
  1832. __might_resched(__FILE__, __LINE__, 0); \
  1833. _cond_resched(); \
  1834. })
  1835. extern int __cond_resched_lock(spinlock_t *lock);
  1836. extern int __cond_resched_rwlock_read(rwlock_t *lock);
  1837. extern int __cond_resched_rwlock_write(rwlock_t *lock);
  1838. #define MIGHT_RESCHED_RCU_SHIFT 8
  1839. #define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1)
  1840. #ifndef CONFIG_PREEMPT_RT
  1841. /*
  1842. * Non RT kernels have an elevated preempt count due to the held lock,
  1843. * but are not allowed to be inside a RCU read side critical section
  1844. */
  1845. # define PREEMPT_LOCK_RESCHED_OFFSETS PREEMPT_LOCK_OFFSET
  1846. #else
  1847. /*
  1848. * spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in
  1849. * cond_resched*lock() has to take that into account because it checks for
  1850. * preempt_count() and rcu_preempt_depth().
  1851. */
  1852. # define PREEMPT_LOCK_RESCHED_OFFSETS \
  1853. (PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT))
  1854. #endif
  1855. #define cond_resched_lock(lock) ({ \
  1856. __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
  1857. __cond_resched_lock(lock); \
  1858. })
  1859. #define cond_resched_rwlock_read(lock) ({ \
  1860. __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
  1861. __cond_resched_rwlock_read(lock); \
  1862. })
  1863. #define cond_resched_rwlock_write(lock) ({ \
  1864. __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \
  1865. __cond_resched_rwlock_write(lock); \
  1866. })
  1867. static inline void cond_resched_rcu(void)
  1868. {
  1869. #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU)
  1870. rcu_read_unlock();
  1871. cond_resched();
  1872. rcu_read_lock();
  1873. #endif
  1874. }
  1875. #ifdef CONFIG_PREEMPT_DYNAMIC
  1876. extern bool preempt_model_none(void);
  1877. extern bool preempt_model_voluntary(void);
  1878. extern bool preempt_model_full(void);
  1879. #else
  1880. static inline bool preempt_model_none(void)
  1881. {
  1882. return IS_ENABLED(CONFIG_PREEMPT_NONE);
  1883. }
  1884. static inline bool preempt_model_voluntary(void)
  1885. {
  1886. return IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY);
  1887. }
  1888. static inline bool preempt_model_full(void)
  1889. {
  1890. return IS_ENABLED(CONFIG_PREEMPT);
  1891. }
  1892. #endif
  1893. static inline bool preempt_model_rt(void)
  1894. {
  1895. return IS_ENABLED(CONFIG_PREEMPT_RT);
  1896. }
  1897. /*
  1898. * Does the preemption model allow non-cooperative preemption?
  1899. *
  1900. * For !CONFIG_PREEMPT_DYNAMIC kernels this is an exact match with
  1901. * CONFIG_PREEMPTION; for CONFIG_PREEMPT_DYNAMIC this doesn't work as the
  1902. * kernel is *built* with CONFIG_PREEMPTION=y but may run with e.g. the
  1903. * PREEMPT_NONE model.
  1904. */
  1905. static inline bool preempt_model_preemptible(void)
  1906. {
  1907. return preempt_model_full() || preempt_model_rt();
  1908. }
  1909. /*
  1910. * Does a critical section need to be broken due to another
  1911. * task waiting?: (technically does not depend on CONFIG_PREEMPTION,
  1912. * but a general need for low latency)
  1913. */
  1914. static inline int spin_needbreak(spinlock_t *lock)
  1915. {
  1916. #ifdef CONFIG_PREEMPTION
  1917. return spin_is_contended(lock);
  1918. #else
  1919. return 0;
  1920. #endif
  1921. }
  1922. /*
  1923. * Check if a rwlock is contended.
  1924. * Returns non-zero if there is another task waiting on the rwlock.
  1925. * Returns zero if the lock is not contended or the system / underlying
  1926. * rwlock implementation does not support contention detection.
  1927. * Technically does not depend on CONFIG_PREEMPTION, but a general need
  1928. * for low latency.
  1929. */
  1930. static inline int rwlock_needbreak(rwlock_t *lock)
  1931. {
  1932. #ifdef CONFIG_PREEMPTION
  1933. return rwlock_is_contended(lock);
  1934. #else
  1935. return 0;
  1936. #endif
  1937. }
  1938. static __always_inline bool need_resched(void)
  1939. {
  1940. return unlikely(tif_need_resched());
  1941. }
  1942. /*
  1943. * Wrappers for p->thread_info->cpu access. No-op on UP.
  1944. */
  1945. #ifdef CONFIG_SMP
  1946. static inline unsigned int task_cpu(const struct task_struct *p)
  1947. {
  1948. return READ_ONCE(task_thread_info(p)->cpu);
  1949. }
  1950. extern void set_task_cpu(struct task_struct *p, unsigned int cpu);
  1951. #else
  1952. static inline unsigned int task_cpu(const struct task_struct *p)
  1953. {
  1954. return 0;
  1955. }
  1956. static inline void set_task_cpu(struct task_struct *p, unsigned int cpu)
  1957. {
  1958. }
  1959. #endif /* CONFIG_SMP */
  1960. extern bool sched_task_on_rq(struct task_struct *p);
  1961. extern unsigned long get_wchan(struct task_struct *p);
  1962. extern struct task_struct *cpu_curr_snapshot(int cpu);
  1963. /*
  1964. * In order to reduce various lock holder preemption latencies provide an
  1965. * interface to see if a vCPU is currently running or not.
  1966. *
  1967. * This allows us to terminate optimistic spin loops and block, analogous to
  1968. * the native optimistic spin heuristic of testing if the lock owner task is
  1969. * running or not.
  1970. */
  1971. #ifndef vcpu_is_preempted
  1972. static inline bool vcpu_is_preempted(int cpu)
  1973. {
  1974. return false;
  1975. }
  1976. #endif
  1977. extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask);
  1978. extern long sched_getaffinity(pid_t pid, struct cpumask *mask);
  1979. #ifndef TASK_SIZE_OF
  1980. #define TASK_SIZE_OF(tsk) TASK_SIZE
  1981. #endif
  1982. #ifdef CONFIG_SMP
  1983. static inline bool owner_on_cpu(struct task_struct *owner)
  1984. {
  1985. /*
  1986. * As lock holder preemption issue, we both skip spinning if
  1987. * task is not on cpu or its cpu is preempted
  1988. */
  1989. return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(task_cpu(owner));
  1990. }
  1991. /* Returns effective CPU energy utilization, as seen by the scheduler */
  1992. unsigned long sched_cpu_util(int cpu);
  1993. #endif /* CONFIG_SMP */
  1994. #ifdef CONFIG_RSEQ
  1995. /*
  1996. * Map the event mask on the user-space ABI enum rseq_cs_flags
  1997. * for direct mask checks.
  1998. */
  1999. enum rseq_event_mask_bits {
  2000. RSEQ_EVENT_PREEMPT_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT,
  2001. RSEQ_EVENT_SIGNAL_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT,
  2002. RSEQ_EVENT_MIGRATE_BIT = RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT,
  2003. };
  2004. enum rseq_event_mask {
  2005. RSEQ_EVENT_PREEMPT = (1U << RSEQ_EVENT_PREEMPT_BIT),
  2006. RSEQ_EVENT_SIGNAL = (1U << RSEQ_EVENT_SIGNAL_BIT),
  2007. RSEQ_EVENT_MIGRATE = (1U << RSEQ_EVENT_MIGRATE_BIT),
  2008. };
  2009. static inline void rseq_set_notify_resume(struct task_struct *t)
  2010. {
  2011. if (t->rseq)
  2012. set_tsk_thread_flag(t, TIF_NOTIFY_RESUME);
  2013. }
  2014. void __rseq_handle_notify_resume(struct ksignal *sig, struct pt_regs *regs);
  2015. static inline void rseq_handle_notify_resume(struct ksignal *ksig,
  2016. struct pt_regs *regs)
  2017. {
  2018. if (current->rseq)
  2019. __rseq_handle_notify_resume(ksig, regs);
  2020. }
  2021. static inline void rseq_signal_deliver(struct ksignal *ksig,
  2022. struct pt_regs *regs)
  2023. {
  2024. preempt_disable();
  2025. __set_bit(RSEQ_EVENT_SIGNAL_BIT, &current->rseq_event_mask);
  2026. preempt_enable();
  2027. rseq_handle_notify_resume(ksig, regs);
  2028. }
  2029. /* rseq_preempt() requires preemption to be disabled. */
  2030. static inline void rseq_preempt(struct task_struct *t)
  2031. {
  2032. __set_bit(RSEQ_EVENT_PREEMPT_BIT, &t->rseq_event_mask);
  2033. rseq_set_notify_resume(t);
  2034. }
  2035. /* rseq_migrate() requires preemption to be disabled. */
  2036. static inline void rseq_migrate(struct task_struct *t)
  2037. {
  2038. __set_bit(RSEQ_EVENT_MIGRATE_BIT, &t->rseq_event_mask);
  2039. rseq_set_notify_resume(t);
  2040. }
  2041. /*
  2042. * If parent process has a registered restartable sequences area, the
  2043. * child inherits. Unregister rseq for a clone with CLONE_VM set.
  2044. */
  2045. static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
  2046. {
  2047. if (clone_flags & CLONE_VM) {
  2048. t->rseq = NULL;
  2049. t->rseq_sig = 0;
  2050. t->rseq_event_mask = 0;
  2051. } else {
  2052. t->rseq = current->rseq;
  2053. t->rseq_sig = current->rseq_sig;
  2054. t->rseq_event_mask = current->rseq_event_mask;
  2055. }
  2056. }
  2057. static inline void rseq_execve(struct task_struct *t)
  2058. {
  2059. t->rseq = NULL;
  2060. t->rseq_sig = 0;
  2061. t->rseq_event_mask = 0;
  2062. }
  2063. #else
  2064. static inline void rseq_set_notify_resume(struct task_struct *t)
  2065. {
  2066. }
  2067. static inline void rseq_handle_notify_resume(struct ksignal *ksig,
  2068. struct pt_regs *regs)
  2069. {
  2070. }
  2071. static inline void rseq_signal_deliver(struct ksignal *ksig,
  2072. struct pt_regs *regs)
  2073. {
  2074. }
  2075. static inline void rseq_preempt(struct task_struct *t)
  2076. {
  2077. }
  2078. static inline void rseq_migrate(struct task_struct *t)
  2079. {
  2080. }
  2081. static inline void rseq_fork(struct task_struct *t, unsigned long clone_flags)
  2082. {
  2083. }
  2084. static inline void rseq_execve(struct task_struct *t)
  2085. {
  2086. }
  2087. #endif
  2088. #ifdef CONFIG_DEBUG_RSEQ
  2089. void rseq_syscall(struct pt_regs *regs);
  2090. #else
  2091. static inline void rseq_syscall(struct pt_regs *regs)
  2092. {
  2093. }
  2094. #endif
  2095. #ifdef CONFIG_SCHED_CORE
  2096. extern void sched_core_free(struct task_struct *tsk);
  2097. extern void sched_core_fork(struct task_struct *p);
  2098. extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
  2099. unsigned long uaddr);
  2100. #else
  2101. static inline void sched_core_free(struct task_struct *tsk) { }
  2102. static inline void sched_core_fork(struct task_struct *p) { }
  2103. #endif
  2104. extern void sched_set_stop_task(int cpu, struct task_struct *stop);
  2105. #endif