cpuset.c 117 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272
  1. /*
  2. * kernel/cpuset.c
  3. *
  4. * Processor and Memory placement constraints for sets of tasks.
  5. *
  6. * Copyright (C) 2003 BULL SA.
  7. * Copyright (C) 2004-2007 Silicon Graphics, Inc.
  8. * Copyright (C) 2006 Google, Inc
  9. *
  10. * Portions derived from Patrick Mochel's sysfs code.
  11. * sysfs is Copyright (c) 2001-3 Patrick Mochel
  12. *
  13. * 2003-10-10 Written by Simon Derr.
  14. * 2003-10-22 Updates by Stephen Hemminger.
  15. * 2004 May-July Rework by Paul Jackson.
  16. * 2006 Rework by Paul Menage to use generic cgroups
  17. * 2008 Rework of the scheduler domains and CPU hotplug handling
  18. * by Max Krasnyansky
  19. *
  20. * This file is subject to the terms and conditions of the GNU General Public
  21. * License. See the file COPYING in the main directory of the Linux
  22. * distribution for more details.
  23. */
  24. #include <linux/cpu.h>
  25. #include <linux/cpumask.h>
  26. #include <linux/cpuset.h>
  27. #include <linux/err.h>
  28. #include <linux/errno.h>
  29. #include <linux/file.h>
  30. #include <linux/fs.h>
  31. #include <linux/init.h>
  32. #include <linux/interrupt.h>
  33. #include <linux/kernel.h>
  34. #include <linux/kmod.h>
  35. #include <linux/kthread.h>
  36. #include <linux/list.h>
  37. #include <linux/mempolicy.h>
  38. #include <linux/mm.h>
  39. #include <linux/memory.h>
  40. #include <linux/export.h>
  41. #include <linux/mount.h>
  42. #include <linux/fs_context.h>
  43. #include <linux/namei.h>
  44. #include <linux/pagemap.h>
  45. #include <linux/proc_fs.h>
  46. #include <linux/rcupdate.h>
  47. #include <linux/sched.h>
  48. #include <linux/sched/deadline.h>
  49. #include <linux/sched/mm.h>
  50. #include <linux/sched/task.h>
  51. #include <linux/seq_file.h>
  52. #include <linux/security.h>
  53. #include <linux/slab.h>
  54. #include <linux/spinlock.h>
  55. #include <linux/stat.h>
  56. #include <linux/string.h>
  57. #include <linux/time.h>
  58. #include <linux/time64.h>
  59. #include <linux/backing-dev.h>
  60. #include <linux/sort.h>
  61. #include <linux/oom.h>
  62. #include <linux/sched/isolation.h>
  63. #include <linux/uaccess.h>
  64. #include <linux/atomic.h>
  65. #include <linux/mutex.h>
  66. #include <linux/cgroup.h>
  67. #include <linux/wait.h>
  68. #include <trace/hooks/cgroup.h>
  69. #include <trace/hooks/sched.h>
  70. DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key);
  71. DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key);
  72. /*
  73. * There could be abnormal cpuset configurations for cpu or memory
  74. * node binding, add this key to provide a quick low-cost judgment
  75. * of the situation.
  76. */
  77. DEFINE_STATIC_KEY_FALSE(cpusets_insane_config_key);
  78. /* See "Frequency meter" comments, below. */
  79. struct fmeter {
  80. int cnt; /* unprocessed events count */
  81. int val; /* most recent output value */
  82. time64_t time; /* clock (secs) when val computed */
  83. spinlock_t lock; /* guards read or write of above */
  84. };
  85. /*
  86. * Invalid partition error code
  87. */
  88. enum prs_errcode {
  89. PERR_NONE = 0,
  90. PERR_INVCPUS,
  91. PERR_INVPARENT,
  92. PERR_NOTPART,
  93. PERR_NOTEXCL,
  94. PERR_NOCPUS,
  95. PERR_HOTPLUG,
  96. PERR_CPUSEMPTY,
  97. };
  98. static const char * const perr_strings[] = {
  99. [PERR_INVCPUS] = "Invalid cpu list in cpuset.cpus",
  100. [PERR_INVPARENT] = "Parent is an invalid partition root",
  101. [PERR_NOTPART] = "Parent is not a partition root",
  102. [PERR_NOTEXCL] = "Cpu list in cpuset.cpus not exclusive",
  103. [PERR_NOCPUS] = "Parent unable to distribute cpu downstream",
  104. [PERR_HOTPLUG] = "No cpu available due to hotplug",
  105. [PERR_CPUSEMPTY] = "cpuset.cpus is empty",
  106. };
  107. struct cpuset {
  108. struct cgroup_subsys_state css;
  109. unsigned long flags; /* "unsigned long" so bitops work */
  110. /*
  111. * On default hierarchy:
  112. *
  113. * The user-configured masks can only be changed by writing to
  114. * cpuset.cpus and cpuset.mems, and won't be limited by the
  115. * parent masks.
  116. *
  117. * The effective masks is the real masks that apply to the tasks
  118. * in the cpuset. They may be changed if the configured masks are
  119. * changed or hotplug happens.
  120. *
  121. * effective_mask == configured_mask & parent's effective_mask,
  122. * and if it ends up empty, it will inherit the parent's mask.
  123. *
  124. *
  125. * On legacy hierarchy:
  126. *
  127. * The user-configured masks are always the same with effective masks.
  128. */
  129. /* user-configured CPUs and Memory Nodes allow to tasks */
  130. cpumask_var_t cpus_allowed;
  131. cpumask_var_t cpus_requested;
  132. nodemask_t mems_allowed;
  133. /* effective CPUs and Memory Nodes allow to tasks */
  134. cpumask_var_t effective_cpus;
  135. nodemask_t effective_mems;
  136. /*
  137. * CPUs allocated to child sub-partitions (default hierarchy only)
  138. * - CPUs granted by the parent = effective_cpus U subparts_cpus
  139. * - effective_cpus and subparts_cpus are mutually exclusive.
  140. *
  141. * effective_cpus contains only onlined CPUs, but subparts_cpus
  142. * may have offlined ones.
  143. */
  144. cpumask_var_t subparts_cpus;
  145. /*
  146. * This is old Memory Nodes tasks took on.
  147. *
  148. * - top_cpuset.old_mems_allowed is initialized to mems_allowed.
  149. * - A new cpuset's old_mems_allowed is initialized when some
  150. * task is moved into it.
  151. * - old_mems_allowed is used in cpuset_migrate_mm() when we change
  152. * cpuset.mems_allowed and have tasks' nodemask updated, and
  153. * then old_mems_allowed is updated to mems_allowed.
  154. */
  155. nodemask_t old_mems_allowed;
  156. struct fmeter fmeter; /* memory_pressure filter */
  157. /*
  158. * Tasks are being attached to this cpuset. Used to prevent
  159. * zeroing cpus/mems_allowed between ->can_attach() and ->attach().
  160. */
  161. int attach_in_progress;
  162. /* partition number for rebuild_sched_domains() */
  163. int pn;
  164. /* for custom sched domain */
  165. int relax_domain_level;
  166. /* number of CPUs in subparts_cpus */
  167. int nr_subparts_cpus;
  168. /* partition root state */
  169. int partition_root_state;
  170. /*
  171. * Default hierarchy only:
  172. * use_parent_ecpus - set if using parent's effective_cpus
  173. * child_ecpus_count - # of children with use_parent_ecpus set
  174. */
  175. int use_parent_ecpus;
  176. int child_ecpus_count;
  177. /*
  178. * number of SCHED_DEADLINE tasks attached to this cpuset, so that we
  179. * know when to rebuild associated root domain bandwidth information.
  180. */
  181. int nr_deadline_tasks;
  182. int nr_migrate_dl_tasks;
  183. u64 sum_migrate_dl_bw;
  184. /* Invalid partition error code, not lock protected */
  185. enum prs_errcode prs_err;
  186. /* Handle for cpuset.cpus.partition */
  187. struct cgroup_file partition_file;
  188. };
  189. /*
  190. * Partition root states:
  191. *
  192. * 0 - member (not a partition root)
  193. * 1 - partition root
  194. * 2 - partition root without load balancing (isolated)
  195. * -1 - invalid partition root
  196. * -2 - invalid isolated partition root
  197. */
  198. #define PRS_MEMBER 0
  199. #define PRS_ROOT 1
  200. #define PRS_ISOLATED 2
  201. #define PRS_INVALID_ROOT -1
  202. #define PRS_INVALID_ISOLATED -2
  203. static inline bool is_prs_invalid(int prs_state)
  204. {
  205. return prs_state < 0;
  206. }
  207. /*
  208. * Temporary cpumasks for working with partitions that are passed among
  209. * functions to avoid memory allocation in inner functions.
  210. */
  211. struct tmpmasks {
  212. cpumask_var_t addmask, delmask; /* For partition root */
  213. cpumask_var_t new_cpus; /* For update_cpumasks_hier() */
  214. };
  215. static inline struct cpuset *css_cs(struct cgroup_subsys_state *css)
  216. {
  217. return css ? container_of(css, struct cpuset, css) : NULL;
  218. }
  219. /* Retrieve the cpuset for a task */
  220. static inline struct cpuset *task_cs(struct task_struct *task)
  221. {
  222. return css_cs(task_css(task, cpuset_cgrp_id));
  223. }
  224. static inline struct cpuset *parent_cs(struct cpuset *cs)
  225. {
  226. return css_cs(cs->css.parent);
  227. }
  228. void inc_dl_tasks_cs(struct task_struct *p)
  229. {
  230. struct cpuset *cs = task_cs(p);
  231. cs->nr_deadline_tasks++;
  232. }
  233. void dec_dl_tasks_cs(struct task_struct *p)
  234. {
  235. struct cpuset *cs = task_cs(p);
  236. cs->nr_deadline_tasks--;
  237. }
  238. /* bits in struct cpuset flags field */
  239. typedef enum {
  240. CS_ONLINE,
  241. CS_CPU_EXCLUSIVE,
  242. CS_MEM_EXCLUSIVE,
  243. CS_MEM_HARDWALL,
  244. CS_MEMORY_MIGRATE,
  245. CS_SCHED_LOAD_BALANCE,
  246. CS_SPREAD_PAGE,
  247. CS_SPREAD_SLAB,
  248. } cpuset_flagbits_t;
  249. /* convenient tests for these bits */
  250. static inline bool is_cpuset_online(struct cpuset *cs)
  251. {
  252. return test_bit(CS_ONLINE, &cs->flags) && !css_is_dying(&cs->css);
  253. }
  254. static inline int is_cpu_exclusive(const struct cpuset *cs)
  255. {
  256. return test_bit(CS_CPU_EXCLUSIVE, &cs->flags);
  257. }
  258. static inline int is_mem_exclusive(const struct cpuset *cs)
  259. {
  260. return test_bit(CS_MEM_EXCLUSIVE, &cs->flags);
  261. }
  262. static inline int is_mem_hardwall(const struct cpuset *cs)
  263. {
  264. return test_bit(CS_MEM_HARDWALL, &cs->flags);
  265. }
  266. static inline int is_sched_load_balance(const struct cpuset *cs)
  267. {
  268. return test_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
  269. }
  270. static inline int is_memory_migrate(const struct cpuset *cs)
  271. {
  272. return test_bit(CS_MEMORY_MIGRATE, &cs->flags);
  273. }
  274. static inline int is_spread_page(const struct cpuset *cs)
  275. {
  276. return test_bit(CS_SPREAD_PAGE, &cs->flags);
  277. }
  278. static inline int is_spread_slab(const struct cpuset *cs)
  279. {
  280. return test_bit(CS_SPREAD_SLAB, &cs->flags);
  281. }
  282. static inline int is_partition_valid(const struct cpuset *cs)
  283. {
  284. return cs->partition_root_state > 0;
  285. }
  286. static inline int is_partition_invalid(const struct cpuset *cs)
  287. {
  288. return cs->partition_root_state < 0;
  289. }
  290. /*
  291. * Callers should hold callback_lock to modify partition_root_state.
  292. */
  293. static inline void make_partition_invalid(struct cpuset *cs)
  294. {
  295. if (is_partition_valid(cs))
  296. cs->partition_root_state = -cs->partition_root_state;
  297. }
  298. /*
  299. * Send notification event of whenever partition_root_state changes.
  300. */
  301. static inline void notify_partition_change(struct cpuset *cs, int old_prs)
  302. {
  303. if (old_prs == cs->partition_root_state)
  304. return;
  305. cgroup_file_notify(&cs->partition_file);
  306. /* Reset prs_err if not invalid */
  307. if (is_partition_valid(cs))
  308. WRITE_ONCE(cs->prs_err, PERR_NONE);
  309. }
  310. static struct cpuset top_cpuset = {
  311. .flags = ((1 << CS_ONLINE) | (1 << CS_CPU_EXCLUSIVE) |
  312. (1 << CS_MEM_EXCLUSIVE)),
  313. .partition_root_state = PRS_ROOT,
  314. };
  315. /**
  316. * cpuset_for_each_child - traverse online children of a cpuset
  317. * @child_cs: loop cursor pointing to the current child
  318. * @pos_css: used for iteration
  319. * @parent_cs: target cpuset to walk children of
  320. *
  321. * Walk @child_cs through the online children of @parent_cs. Must be used
  322. * with RCU read locked.
  323. */
  324. #define cpuset_for_each_child(child_cs, pos_css, parent_cs) \
  325. css_for_each_child((pos_css), &(parent_cs)->css) \
  326. if (is_cpuset_online(((child_cs) = css_cs((pos_css)))))
  327. /**
  328. * cpuset_for_each_descendant_pre - pre-order walk of a cpuset's descendants
  329. * @des_cs: loop cursor pointing to the current descendant
  330. * @pos_css: used for iteration
  331. * @root_cs: target cpuset to walk ancestor of
  332. *
  333. * Walk @des_cs through the online descendants of @root_cs. Must be used
  334. * with RCU read locked. The caller may modify @pos_css by calling
  335. * css_rightmost_descendant() to skip subtree. @root_cs is included in the
  336. * iteration and the first node to be visited.
  337. */
  338. #define cpuset_for_each_descendant_pre(des_cs, pos_css, root_cs) \
  339. css_for_each_descendant_pre((pos_css), &(root_cs)->css) \
  340. if (is_cpuset_online(((des_cs) = css_cs((pos_css)))))
  341. /*
  342. * There are two global locks guarding cpuset structures - cpuset_mutex and
  343. * callback_lock. We also require taking task_lock() when dereferencing a
  344. * task's cpuset pointer. See "The task_lock() exception", at the end of this
  345. * comment. The cpuset code uses only cpuset_mutex. Other kernel subsystems
  346. * can use cpuset_lock()/cpuset_unlock() to prevent change to cpuset
  347. * structures. Note that cpuset_mutex needs to be a mutex as it is used in
  348. * paths that rely on priority inheritance (e.g. scheduler - on RT) for
  349. * correctness.
  350. *
  351. * A task must hold both locks to modify cpusets. If a task holds
  352. * cpuset_mutex, it blocks others, ensuring that it is the only task able to
  353. * also acquire callback_lock and be able to modify cpusets. It can perform
  354. * various checks on the cpuset structure first, knowing nothing will change.
  355. * It can also allocate memory while just holding cpuset_mutex. While it is
  356. * performing these checks, various callback routines can briefly acquire
  357. * callback_lock to query cpusets. Once it is ready to make the changes, it
  358. * takes callback_lock, blocking everyone else.
  359. *
  360. * Calls to the kernel memory allocator can not be made while holding
  361. * callback_lock, as that would risk double tripping on callback_lock
  362. * from one of the callbacks into the cpuset code from within
  363. * __alloc_pages().
  364. *
  365. * If a task is only holding callback_lock, then it has read-only
  366. * access to cpusets.
  367. *
  368. * Now, the task_struct fields mems_allowed and mempolicy may be changed
  369. * by other task, we use alloc_lock in the task_struct fields to protect
  370. * them.
  371. *
  372. * The cpuset_common_file_read() handlers only hold callback_lock across
  373. * small pieces of code, such as when reading out possibly multi-word
  374. * cpumasks and nodemasks.
  375. *
  376. * Accessing a task's cpuset should be done in accordance with the
  377. * guidelines for accessing subsystem state in kernel/cgroup.c
  378. */
  379. DEFINE_STATIC_PERCPU_RWSEM(cpuset_rwsem);
  380. static DEFINE_MUTEX(cpuset_mutex);
  381. void cpuset_lock(void)
  382. {
  383. mutex_lock(&cpuset_mutex);
  384. }
  385. void cpuset_unlock(void)
  386. {
  387. mutex_unlock(&cpuset_mutex);
  388. }
  389. static DEFINE_SPINLOCK(callback_lock);
  390. static struct workqueue_struct *cpuset_migrate_mm_wq;
  391. /*
  392. * CPU / memory hotplug is handled asynchronously.
  393. */
  394. static void cpuset_hotplug_workfn(struct work_struct *work);
  395. static DECLARE_WORK(cpuset_hotplug_work, cpuset_hotplug_workfn);
  396. static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq);
  397. static inline void check_insane_mems_config(nodemask_t *nodes)
  398. {
  399. if (!cpusets_insane_config() &&
  400. movable_only_nodes(nodes)) {
  401. static_branch_enable(&cpusets_insane_config_key);
  402. pr_info("Unsupported (movable nodes only) cpuset configuration detected (nmask=%*pbl)!\n"
  403. "Cpuset allocations might fail even with a lot of memory available.\n",
  404. nodemask_pr_args(nodes));
  405. }
  406. }
  407. /*
  408. * Cgroup v2 behavior is used on the "cpus" and "mems" control files when
  409. * on default hierarchy or when the cpuset_v2_mode flag is set by mounting
  410. * the v1 cpuset cgroup filesystem with the "cpuset_v2_mode" mount option.
  411. * With v2 behavior, "cpus" and "mems" are always what the users have
  412. * requested and won't be changed by hotplug events. Only the effective
  413. * cpus or mems will be affected.
  414. */
  415. static inline bool is_in_v2_mode(void)
  416. {
  417. return cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
  418. (cpuset_cgrp_subsys.root->flags & CGRP_ROOT_CPUSET_V2_MODE);
  419. }
  420. /**
  421. * partition_is_populated - check if partition has tasks
  422. * @cs: partition root to be checked
  423. * @excluded_child: a child cpuset to be excluded in task checking
  424. * Return: true if there are tasks, false otherwise
  425. *
  426. * It is assumed that @cs is a valid partition root. @excluded_child should
  427. * be non-NULL when this cpuset is going to become a partition itself.
  428. */
  429. static inline bool partition_is_populated(struct cpuset *cs,
  430. struct cpuset *excluded_child)
  431. {
  432. struct cgroup_subsys_state *css;
  433. struct cpuset *child;
  434. if (cs->css.cgroup->nr_populated_csets)
  435. return true;
  436. if (!excluded_child && !cs->nr_subparts_cpus)
  437. return cgroup_is_populated(cs->css.cgroup);
  438. rcu_read_lock();
  439. cpuset_for_each_child(child, css, cs) {
  440. if (child == excluded_child)
  441. continue;
  442. if (is_partition_valid(child))
  443. continue;
  444. if (cgroup_is_populated(child->css.cgroup)) {
  445. rcu_read_unlock();
  446. return true;
  447. }
  448. }
  449. rcu_read_unlock();
  450. return false;
  451. }
  452. /*
  453. * Return in pmask the portion of a task's cpusets's cpus_allowed that
  454. * are online and are capable of running the task. If none are found,
  455. * walk up the cpuset hierarchy until we find one that does have some
  456. * appropriate cpus.
  457. *
  458. * One way or another, we guarantee to return some non-empty subset
  459. * of cpu_online_mask.
  460. *
  461. * Call with callback_lock or cpuset_mutex held.
  462. */
  463. static void guarantee_online_cpus(struct task_struct *tsk,
  464. struct cpumask *pmask)
  465. {
  466. const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
  467. struct cpuset *cs;
  468. if (WARN_ON(!cpumask_and(pmask, possible_mask, cpu_online_mask)))
  469. cpumask_copy(pmask, cpu_online_mask);
  470. rcu_read_lock();
  471. cs = task_cs(tsk);
  472. while (!cpumask_intersects(cs->effective_cpus, pmask)) {
  473. cs = parent_cs(cs);
  474. if (unlikely(!cs)) {
  475. /*
  476. * The top cpuset doesn't have any online cpu as a
  477. * consequence of a race between cpuset_hotplug_work
  478. * and cpu hotplug notifier. But we know the top
  479. * cpuset's effective_cpus is on its way to be
  480. * identical to cpu_online_mask.
  481. */
  482. goto out_unlock;
  483. }
  484. }
  485. cpumask_and(pmask, pmask, cs->effective_cpus);
  486. out_unlock:
  487. rcu_read_unlock();
  488. }
  489. /*
  490. * Return in *pmask the portion of a cpusets's mems_allowed that
  491. * are online, with memory. If none are online with memory, walk
  492. * up the cpuset hierarchy until we find one that does have some
  493. * online mems. The top cpuset always has some mems online.
  494. *
  495. * One way or another, we guarantee to return some non-empty subset
  496. * of node_states[N_MEMORY].
  497. *
  498. * Call with callback_lock or cpuset_mutex held.
  499. */
  500. static void guarantee_online_mems(struct cpuset *cs, nodemask_t *pmask)
  501. {
  502. while (!nodes_intersects(cs->effective_mems, node_states[N_MEMORY]))
  503. cs = parent_cs(cs);
  504. nodes_and(*pmask, cs->effective_mems, node_states[N_MEMORY]);
  505. }
  506. /*
  507. * update task's spread flag if cpuset's page/slab spread flag is set
  508. *
  509. * Call with callback_lock or cpuset_mutex held. The check can be skipped
  510. * if on default hierarchy.
  511. */
  512. static void cpuset_update_task_spread_flags(struct cpuset *cs,
  513. struct task_struct *tsk)
  514. {
  515. if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys))
  516. return;
  517. if (is_spread_page(cs))
  518. task_set_spread_page(tsk);
  519. else
  520. task_clear_spread_page(tsk);
  521. if (is_spread_slab(cs))
  522. task_set_spread_slab(tsk);
  523. else
  524. task_clear_spread_slab(tsk);
  525. }
  526. /*
  527. * is_cpuset_subset(p, q) - Is cpuset p a subset of cpuset q?
  528. *
  529. * One cpuset is a subset of another if all its allowed CPUs and
  530. * Memory Nodes are a subset of the other, and its exclusive flags
  531. * are only set if the other's are set. Call holding cpuset_mutex.
  532. */
  533. static int is_cpuset_subset(const struct cpuset *p, const struct cpuset *q)
  534. {
  535. return cpumask_subset(p->cpus_requested, q->cpus_requested) &&
  536. nodes_subset(p->mems_allowed, q->mems_allowed) &&
  537. is_cpu_exclusive(p) <= is_cpu_exclusive(q) &&
  538. is_mem_exclusive(p) <= is_mem_exclusive(q);
  539. }
  540. /**
  541. * alloc_cpumasks - allocate three cpumasks for cpuset
  542. * @cs: the cpuset that have cpumasks to be allocated.
  543. * @tmp: the tmpmasks structure pointer
  544. * Return: 0 if successful, -ENOMEM otherwise.
  545. *
  546. * Only one of the two input arguments should be non-NULL.
  547. */
  548. static inline int alloc_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
  549. {
  550. cpumask_var_t *pmask1, *pmask2, *pmask3;
  551. if (cs) {
  552. pmask1 = &cs->cpus_allowed;
  553. pmask2 = &cs->effective_cpus;
  554. pmask3 = &cs->subparts_cpus;
  555. } else {
  556. pmask1 = &tmp->new_cpus;
  557. pmask2 = &tmp->addmask;
  558. pmask3 = &tmp->delmask;
  559. }
  560. if (!zalloc_cpumask_var(pmask1, GFP_KERNEL))
  561. return -ENOMEM;
  562. if (!zalloc_cpumask_var(pmask2, GFP_KERNEL))
  563. goto free_one;
  564. if (!zalloc_cpumask_var(pmask3, GFP_KERNEL))
  565. goto free_two;
  566. if (cs && !zalloc_cpumask_var(&cs->cpus_requested, GFP_KERNEL))
  567. goto free_three;
  568. return 0;
  569. free_three:
  570. free_cpumask_var(*pmask3);
  571. free_two:
  572. free_cpumask_var(*pmask2);
  573. free_one:
  574. free_cpumask_var(*pmask1);
  575. return -ENOMEM;
  576. }
  577. /**
  578. * free_cpumasks - free cpumasks in a tmpmasks structure
  579. * @cs: the cpuset that have cpumasks to be free.
  580. * @tmp: the tmpmasks structure pointer
  581. */
  582. static inline void free_cpumasks(struct cpuset *cs, struct tmpmasks *tmp)
  583. {
  584. if (cs) {
  585. free_cpumask_var(cs->cpus_allowed);
  586. free_cpumask_var(cs->cpus_requested);
  587. free_cpumask_var(cs->effective_cpus);
  588. free_cpumask_var(cs->subparts_cpus);
  589. }
  590. if (tmp) {
  591. free_cpumask_var(tmp->new_cpus);
  592. free_cpumask_var(tmp->addmask);
  593. free_cpumask_var(tmp->delmask);
  594. }
  595. }
  596. /**
  597. * alloc_trial_cpuset - allocate a trial cpuset
  598. * @cs: the cpuset that the trial cpuset duplicates
  599. */
  600. static struct cpuset *alloc_trial_cpuset(struct cpuset *cs)
  601. {
  602. struct cpuset *trial;
  603. trial = kmemdup(cs, sizeof(*cs), GFP_KERNEL);
  604. if (!trial)
  605. return NULL;
  606. if (alloc_cpumasks(trial, NULL)) {
  607. kfree(trial);
  608. return NULL;
  609. }
  610. cpumask_copy(trial->cpus_allowed, cs->cpus_allowed);
  611. cpumask_copy(trial->cpus_requested, cs->cpus_requested);
  612. cpumask_copy(trial->effective_cpus, cs->effective_cpus);
  613. return trial;
  614. }
  615. /**
  616. * free_cpuset - free the cpuset
  617. * @cs: the cpuset to be freed
  618. */
  619. static inline void free_cpuset(struct cpuset *cs)
  620. {
  621. free_cpumasks(cs, NULL);
  622. kfree(cs);
  623. }
  624. /*
  625. * validate_change_legacy() - Validate conditions specific to legacy (v1)
  626. * behavior.
  627. */
  628. static int validate_change_legacy(struct cpuset *cur, struct cpuset *trial)
  629. {
  630. struct cgroup_subsys_state *css;
  631. struct cpuset *c, *par;
  632. int ret;
  633. WARN_ON_ONCE(!rcu_read_lock_held());
  634. /* Each of our child cpusets must be a subset of us */
  635. ret = -EBUSY;
  636. cpuset_for_each_child(c, css, cur)
  637. if (!is_cpuset_subset(c, trial))
  638. goto out;
  639. /* On legacy hierarchy, we must be a subset of our parent cpuset. */
  640. ret = -EACCES;
  641. par = parent_cs(cur);
  642. if (par && !is_cpuset_subset(trial, par))
  643. goto out;
  644. ret = 0;
  645. out:
  646. return ret;
  647. }
  648. /*
  649. * validate_change() - Used to validate that any proposed cpuset change
  650. * follows the structural rules for cpusets.
  651. *
  652. * If we replaced the flag and mask values of the current cpuset
  653. * (cur) with those values in the trial cpuset (trial), would
  654. * our various subset and exclusive rules still be valid? Presumes
  655. * cpuset_mutex held.
  656. *
  657. * 'cur' is the address of an actual, in-use cpuset. Operations
  658. * such as list traversal that depend on the actual address of the
  659. * cpuset in the list must use cur below, not trial.
  660. *
  661. * 'trial' is the address of bulk structure copy of cur, with
  662. * perhaps one or more of the fields cpus_allowed, mems_allowed,
  663. * or flags changed to new, trial values.
  664. *
  665. * Return 0 if valid, -errno if not.
  666. */
  667. static int validate_change(struct cpuset *cur, struct cpuset *trial)
  668. {
  669. struct cgroup_subsys_state *css;
  670. struct cpuset *c, *par;
  671. int ret = 0;
  672. rcu_read_lock();
  673. if (!is_in_v2_mode())
  674. ret = validate_change_legacy(cur, trial);
  675. if (ret)
  676. goto out;
  677. /* Remaining checks don't apply to root cpuset */
  678. if (cur == &top_cpuset)
  679. goto out;
  680. par = parent_cs(cur);
  681. /*
  682. * Cpusets with tasks - existing or newly being attached - can't
  683. * be changed to have empty cpus_allowed or mems_allowed.
  684. */
  685. ret = -ENOSPC;
  686. if ((cgroup_is_populated(cur->css.cgroup) || cur->attach_in_progress)) {
  687. if (!cpumask_empty(cur->cpus_allowed) &&
  688. cpumask_empty(trial->cpus_allowed))
  689. goto out;
  690. if (!nodes_empty(cur->mems_allowed) &&
  691. nodes_empty(trial->mems_allowed))
  692. goto out;
  693. }
  694. /*
  695. * We can't shrink if we won't have enough room for SCHED_DEADLINE
  696. * tasks.
  697. */
  698. ret = -EBUSY;
  699. if (is_cpu_exclusive(cur) &&
  700. !cpuset_cpumask_can_shrink(cur->cpus_allowed,
  701. trial->cpus_allowed))
  702. goto out;
  703. /*
  704. * If either I or some sibling (!= me) is exclusive, we can't
  705. * overlap
  706. */
  707. ret = -EINVAL;
  708. cpuset_for_each_child(c, css, par) {
  709. if ((is_cpu_exclusive(trial) || is_cpu_exclusive(c)) &&
  710. c != cur &&
  711. cpumask_intersects(trial->cpus_requested, c->cpus_requested))
  712. goto out;
  713. if ((is_mem_exclusive(trial) || is_mem_exclusive(c)) &&
  714. c != cur &&
  715. nodes_intersects(trial->mems_allowed, c->mems_allowed))
  716. goto out;
  717. }
  718. ret = 0;
  719. out:
  720. rcu_read_unlock();
  721. return ret;
  722. }
  723. #ifdef CONFIG_SMP
  724. /*
  725. * Helper routine for generate_sched_domains().
  726. * Do cpusets a, b have overlapping effective cpus_allowed masks?
  727. */
  728. static int cpusets_overlap(struct cpuset *a, struct cpuset *b)
  729. {
  730. return cpumask_intersects(a->effective_cpus, b->effective_cpus);
  731. }
  732. static void
  733. update_domain_attr(struct sched_domain_attr *dattr, struct cpuset *c)
  734. {
  735. if (dattr->relax_domain_level < c->relax_domain_level)
  736. dattr->relax_domain_level = c->relax_domain_level;
  737. return;
  738. }
  739. static void update_domain_attr_tree(struct sched_domain_attr *dattr,
  740. struct cpuset *root_cs)
  741. {
  742. struct cpuset *cp;
  743. struct cgroup_subsys_state *pos_css;
  744. rcu_read_lock();
  745. cpuset_for_each_descendant_pre(cp, pos_css, root_cs) {
  746. /* skip the whole subtree if @cp doesn't have any CPU */
  747. if (cpumask_empty(cp->cpus_allowed)) {
  748. pos_css = css_rightmost_descendant(pos_css);
  749. continue;
  750. }
  751. if (is_sched_load_balance(cp))
  752. update_domain_attr(dattr, cp);
  753. }
  754. rcu_read_unlock();
  755. }
  756. /* Must be called with cpuset_mutex held. */
  757. static inline int nr_cpusets(void)
  758. {
  759. /* jump label reference count + the top-level cpuset */
  760. return static_key_count(&cpusets_enabled_key.key) + 1;
  761. }
  762. /*
  763. * generate_sched_domains()
  764. *
  765. * This function builds a partial partition of the systems CPUs
  766. * A 'partial partition' is a set of non-overlapping subsets whose
  767. * union is a subset of that set.
  768. * The output of this function needs to be passed to kernel/sched/core.c
  769. * partition_sched_domains() routine, which will rebuild the scheduler's
  770. * load balancing domains (sched domains) as specified by that partial
  771. * partition.
  772. *
  773. * See "What is sched_load_balance" in Documentation/admin-guide/cgroup-v1/cpusets.rst
  774. * for a background explanation of this.
  775. *
  776. * Does not return errors, on the theory that the callers of this
  777. * routine would rather not worry about failures to rebuild sched
  778. * domains when operating in the severe memory shortage situations
  779. * that could cause allocation failures below.
  780. *
  781. * Must be called with cpuset_mutex held.
  782. *
  783. * The three key local variables below are:
  784. * cp - cpuset pointer, used (together with pos_css) to perform a
  785. * top-down scan of all cpusets. For our purposes, rebuilding
  786. * the schedulers sched domains, we can ignore !is_sched_load_
  787. * balance cpusets.
  788. * csa - (for CpuSet Array) Array of pointers to all the cpusets
  789. * that need to be load balanced, for convenient iterative
  790. * access by the subsequent code that finds the best partition,
  791. * i.e the set of domains (subsets) of CPUs such that the
  792. * cpus_allowed of every cpuset marked is_sched_load_balance
  793. * is a subset of one of these domains, while there are as
  794. * many such domains as possible, each as small as possible.
  795. * doms - Conversion of 'csa' to an array of cpumasks, for passing to
  796. * the kernel/sched/core.c routine partition_sched_domains() in a
  797. * convenient format, that can be easily compared to the prior
  798. * value to determine what partition elements (sched domains)
  799. * were changed (added or removed.)
  800. *
  801. * Finding the best partition (set of domains):
  802. * The triple nested loops below over i, j, k scan over the
  803. * load balanced cpusets (using the array of cpuset pointers in
  804. * csa[]) looking for pairs of cpusets that have overlapping
  805. * cpus_allowed, but which don't have the same 'pn' partition
  806. * number and gives them in the same partition number. It keeps
  807. * looping on the 'restart' label until it can no longer find
  808. * any such pairs.
  809. *
  810. * The union of the cpus_allowed masks from the set of
  811. * all cpusets having the same 'pn' value then form the one
  812. * element of the partition (one sched domain) to be passed to
  813. * partition_sched_domains().
  814. */
  815. static int generate_sched_domains(cpumask_var_t **domains,
  816. struct sched_domain_attr **attributes)
  817. {
  818. struct cpuset *cp; /* top-down scan of cpusets */
  819. struct cpuset **csa; /* array of all cpuset ptrs */
  820. int csn; /* how many cpuset ptrs in csa so far */
  821. int i, j, k; /* indices for partition finding loops */
  822. cpumask_var_t *doms; /* resulting partition; i.e. sched domains */
  823. struct sched_domain_attr *dattr; /* attributes for custom domains */
  824. int ndoms = 0; /* number of sched domains in result */
  825. int nslot; /* next empty doms[] struct cpumask slot */
  826. struct cgroup_subsys_state *pos_css;
  827. bool root_load_balance = is_sched_load_balance(&top_cpuset);
  828. doms = NULL;
  829. dattr = NULL;
  830. csa = NULL;
  831. /* Special case for the 99% of systems with one, full, sched domain */
  832. if (root_load_balance && !top_cpuset.nr_subparts_cpus) {
  833. ndoms = 1;
  834. doms = alloc_sched_domains(ndoms);
  835. if (!doms)
  836. goto done;
  837. dattr = kmalloc(sizeof(struct sched_domain_attr), GFP_KERNEL);
  838. if (dattr) {
  839. *dattr = SD_ATTR_INIT;
  840. update_domain_attr_tree(dattr, &top_cpuset);
  841. }
  842. cpumask_and(doms[0], top_cpuset.effective_cpus,
  843. housekeeping_cpumask(HK_TYPE_DOMAIN));
  844. goto done;
  845. }
  846. csa = kmalloc_array(nr_cpusets(), sizeof(cp), GFP_KERNEL);
  847. if (!csa)
  848. goto done;
  849. csn = 0;
  850. rcu_read_lock();
  851. if (root_load_balance)
  852. csa[csn++] = &top_cpuset;
  853. cpuset_for_each_descendant_pre(cp, pos_css, &top_cpuset) {
  854. if (cp == &top_cpuset)
  855. continue;
  856. /*
  857. * Continue traversing beyond @cp iff @cp has some CPUs and
  858. * isn't load balancing. The former is obvious. The
  859. * latter: All child cpusets contain a subset of the
  860. * parent's cpus, so just skip them, and then we call
  861. * update_domain_attr_tree() to calc relax_domain_level of
  862. * the corresponding sched domain.
  863. *
  864. * If root is load-balancing, we can skip @cp if it
  865. * is a subset of the root's effective_cpus.
  866. */
  867. if (!cpumask_empty(cp->cpus_allowed) &&
  868. !(is_sched_load_balance(cp) &&
  869. cpumask_intersects(cp->cpus_allowed,
  870. housekeeping_cpumask(HK_TYPE_DOMAIN))))
  871. continue;
  872. if (root_load_balance &&
  873. cpumask_subset(cp->cpus_allowed, top_cpuset.effective_cpus))
  874. continue;
  875. if (is_sched_load_balance(cp) &&
  876. !cpumask_empty(cp->effective_cpus))
  877. csa[csn++] = cp;
  878. /* skip @cp's subtree if not a partition root */
  879. if (!is_partition_valid(cp))
  880. pos_css = css_rightmost_descendant(pos_css);
  881. }
  882. rcu_read_unlock();
  883. for (i = 0; i < csn; i++)
  884. csa[i]->pn = i;
  885. ndoms = csn;
  886. restart:
  887. /* Find the best partition (set of sched domains) */
  888. for (i = 0; i < csn; i++) {
  889. struct cpuset *a = csa[i];
  890. int apn = a->pn;
  891. for (j = 0; j < csn; j++) {
  892. struct cpuset *b = csa[j];
  893. int bpn = b->pn;
  894. if (apn != bpn && cpusets_overlap(a, b)) {
  895. for (k = 0; k < csn; k++) {
  896. struct cpuset *c = csa[k];
  897. if (c->pn == bpn)
  898. c->pn = apn;
  899. }
  900. ndoms--; /* one less element */
  901. goto restart;
  902. }
  903. }
  904. }
  905. /*
  906. * Now we know how many domains to create.
  907. * Convert <csn, csa> to <ndoms, doms> and populate cpu masks.
  908. */
  909. doms = alloc_sched_domains(ndoms);
  910. if (!doms)
  911. goto done;
  912. /*
  913. * The rest of the code, including the scheduler, can deal with
  914. * dattr==NULL case. No need to abort if alloc fails.
  915. */
  916. dattr = kmalloc_array(ndoms, sizeof(struct sched_domain_attr),
  917. GFP_KERNEL);
  918. for (nslot = 0, i = 0; i < csn; i++) {
  919. struct cpuset *a = csa[i];
  920. struct cpumask *dp;
  921. int apn = a->pn;
  922. if (apn < 0) {
  923. /* Skip completed partitions */
  924. continue;
  925. }
  926. dp = doms[nslot];
  927. if (nslot == ndoms) {
  928. static int warnings = 10;
  929. if (warnings) {
  930. pr_warn("rebuild_sched_domains confused: nslot %d, ndoms %d, csn %d, i %d, apn %d\n",
  931. nslot, ndoms, csn, i, apn);
  932. warnings--;
  933. }
  934. continue;
  935. }
  936. cpumask_clear(dp);
  937. if (dattr)
  938. *(dattr + nslot) = SD_ATTR_INIT;
  939. for (j = i; j < csn; j++) {
  940. struct cpuset *b = csa[j];
  941. if (apn == b->pn) {
  942. cpumask_or(dp, dp, b->effective_cpus);
  943. cpumask_and(dp, dp, housekeeping_cpumask(HK_TYPE_DOMAIN));
  944. if (dattr)
  945. update_domain_attr_tree(dattr + nslot, b);
  946. /* Done with this partition */
  947. b->pn = -1;
  948. }
  949. }
  950. nslot++;
  951. }
  952. BUG_ON(nslot != ndoms);
  953. done:
  954. kfree(csa);
  955. /*
  956. * Fallback to the default domain if kmalloc() failed.
  957. * See comments in partition_sched_domains().
  958. */
  959. if (doms == NULL)
  960. ndoms = 1;
  961. *domains = doms;
  962. *attributes = dattr;
  963. return ndoms;
  964. }
  965. static void dl_update_tasks_root_domain(struct cpuset *cs)
  966. {
  967. struct css_task_iter it;
  968. struct task_struct *task;
  969. if (cs->nr_deadline_tasks == 0)
  970. return;
  971. css_task_iter_start(&cs->css, 0, &it);
  972. while ((task = css_task_iter_next(&it)))
  973. dl_add_task_root_domain(task);
  974. css_task_iter_end(&it);
  975. }
  976. static void dl_rebuild_rd_accounting(void)
  977. {
  978. struct cpuset *cs = NULL;
  979. struct cgroup_subsys_state *pos_css;
  980. lockdep_assert_held(&cpuset_mutex);
  981. lockdep_assert_cpus_held();
  982. lockdep_assert_held(&sched_domains_mutex);
  983. rcu_read_lock();
  984. /*
  985. * Clear default root domain DL accounting, it will be computed again
  986. * if a task belongs to it.
  987. */
  988. dl_clear_root_domain(&def_root_domain);
  989. cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
  990. if (cpumask_empty(cs->effective_cpus)) {
  991. pos_css = css_rightmost_descendant(pos_css);
  992. continue;
  993. }
  994. css_get(&cs->css);
  995. rcu_read_unlock();
  996. dl_update_tasks_root_domain(cs);
  997. rcu_read_lock();
  998. css_put(&cs->css);
  999. }
  1000. rcu_read_unlock();
  1001. }
  1002. static void
  1003. partition_and_rebuild_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
  1004. struct sched_domain_attr *dattr_new)
  1005. {
  1006. mutex_lock(&sched_domains_mutex);
  1007. partition_sched_domains_locked(ndoms_new, doms_new, dattr_new);
  1008. dl_rebuild_rd_accounting();
  1009. mutex_unlock(&sched_domains_mutex);
  1010. }
  1011. /*
  1012. * Rebuild scheduler domains.
  1013. *
  1014. * If the flag 'sched_load_balance' of any cpuset with non-empty
  1015. * 'cpus' changes, or if the 'cpus' allowed changes in any cpuset
  1016. * which has that flag enabled, or if any cpuset with a non-empty
  1017. * 'cpus' is removed, then call this routine to rebuild the
  1018. * scheduler's dynamic sched domains.
  1019. *
  1020. * Call with cpuset_mutex held. Takes cpus_read_lock().
  1021. */
  1022. static void rebuild_sched_domains_locked(void)
  1023. {
  1024. struct cgroup_subsys_state *pos_css;
  1025. struct sched_domain_attr *attr;
  1026. cpumask_var_t *doms;
  1027. struct cpuset *cs;
  1028. int ndoms;
  1029. lockdep_assert_cpus_held();
  1030. lockdep_assert_held(&cpuset_mutex);
  1031. /*
  1032. * If we have raced with CPU hotplug, return early to avoid
  1033. * passing doms with offlined cpu to partition_sched_domains().
  1034. * Anyways, cpuset_hotplug_workfn() will rebuild sched domains.
  1035. *
  1036. * With no CPUs in any subpartitions, top_cpuset's effective CPUs
  1037. * should be the same as the active CPUs, so checking only top_cpuset
  1038. * is enough to detect racing CPU offlines.
  1039. */
  1040. if (!top_cpuset.nr_subparts_cpus &&
  1041. !cpumask_equal(top_cpuset.effective_cpus, cpu_active_mask))
  1042. return;
  1043. /*
  1044. * With subpartition CPUs, however, the effective CPUs of a partition
  1045. * root should be only a subset of the active CPUs. Since a CPU in any
  1046. * partition root could be offlined, all must be checked.
  1047. */
  1048. if (top_cpuset.nr_subparts_cpus) {
  1049. rcu_read_lock();
  1050. cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
  1051. if (!is_partition_valid(cs)) {
  1052. pos_css = css_rightmost_descendant(pos_css);
  1053. continue;
  1054. }
  1055. if (!cpumask_subset(cs->effective_cpus,
  1056. cpu_active_mask)) {
  1057. rcu_read_unlock();
  1058. return;
  1059. }
  1060. }
  1061. rcu_read_unlock();
  1062. }
  1063. /* Generate domain masks and attrs */
  1064. ndoms = generate_sched_domains(&doms, &attr);
  1065. /* Have scheduler rebuild the domains */
  1066. partition_and_rebuild_sched_domains(ndoms, doms, attr);
  1067. }
  1068. #else /* !CONFIG_SMP */
  1069. static void rebuild_sched_domains_locked(void)
  1070. {
  1071. }
  1072. #endif /* CONFIG_SMP */
  1073. void rebuild_sched_domains(void)
  1074. {
  1075. cpus_read_lock();
  1076. mutex_lock(&cpuset_mutex);
  1077. rebuild_sched_domains_locked();
  1078. mutex_unlock(&cpuset_mutex);
  1079. cpus_read_unlock();
  1080. }
  1081. EXPORT_SYMBOL_GPL(rebuild_sched_domains);
  1082. static int update_cpus_allowed(struct cpuset *cs, struct task_struct *p,
  1083. const struct cpumask *new_mask)
  1084. {
  1085. int ret = -EINVAL;
  1086. trace_android_rvh_update_cpus_allowed(p, cs->cpus_requested, new_mask, &ret);
  1087. if (!ret)
  1088. return ret;
  1089. return set_cpus_allowed_ptr(p, new_mask);
  1090. }
  1091. /**
  1092. * update_tasks_cpumask - Update the cpumasks of tasks in the cpuset.
  1093. * @cs: the cpuset in which each task's cpus_allowed mask needs to be changed
  1094. * @new_cpus: the temp variable for the new effective_cpus mask
  1095. *
  1096. * Iterate through each task of @cs updating its cpus_allowed to the
  1097. * effective cpuset's. As this function is called with cpuset_mutex held,
  1098. * cpuset membership stays stable.
  1099. */
  1100. static void update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus)
  1101. {
  1102. struct css_task_iter it;
  1103. struct task_struct *task;
  1104. bool top_cs = cs == &top_cpuset;
  1105. css_task_iter_start(&cs->css, 0, &it);
  1106. while ((task = css_task_iter_next(&it))) {
  1107. /*
  1108. * Percpu kthreads in top_cpuset are ignored
  1109. */
  1110. if (top_cs && (task->flags & PF_KTHREAD) &&
  1111. kthread_is_per_cpu(task))
  1112. continue;
  1113. cpumask_and(new_cpus, cs->effective_cpus,
  1114. task_cpu_possible_mask(task));
  1115. update_cpus_allowed(cs, task, new_cpus);
  1116. }
  1117. css_task_iter_end(&it);
  1118. }
  1119. /**
  1120. * compute_effective_cpumask - Compute the effective cpumask of the cpuset
  1121. * @new_cpus: the temp variable for the new effective_cpus mask
  1122. * @cs: the cpuset the need to recompute the new effective_cpus mask
  1123. * @parent: the parent cpuset
  1124. *
  1125. * If the parent has subpartition CPUs, include them in the list of
  1126. * allowable CPUs in computing the new effective_cpus mask. Since offlined
  1127. * CPUs are not removed from subparts_cpus, we have to use cpu_active_mask
  1128. * to mask those out.
  1129. */
  1130. static void compute_effective_cpumask(struct cpumask *new_cpus,
  1131. struct cpuset *cs, struct cpuset *parent)
  1132. {
  1133. if (parent->nr_subparts_cpus) {
  1134. cpumask_or(new_cpus, parent->effective_cpus,
  1135. parent->subparts_cpus);
  1136. cpumask_and(new_cpus, new_cpus, cs->cpus_requested);
  1137. cpumask_and(new_cpus, new_cpus, cpu_active_mask);
  1138. } else {
  1139. cpumask_and(new_cpus, cs->cpus_requested, parent_cs(cs)->effective_cpus);
  1140. }
  1141. }
  1142. /*
  1143. * Commands for update_parent_subparts_cpumask
  1144. */
  1145. enum subparts_cmd {
  1146. partcmd_enable, /* Enable partition root */
  1147. partcmd_disable, /* Disable partition root */
  1148. partcmd_update, /* Update parent's subparts_cpus */
  1149. partcmd_invalidate, /* Make partition invalid */
  1150. };
  1151. static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
  1152. int turning_on);
  1153. /**
  1154. * update_parent_subparts_cpumask - update subparts_cpus mask of parent cpuset
  1155. * @cpuset: The cpuset that requests change in partition root state
  1156. * @cmd: Partition root state change command
  1157. * @newmask: Optional new cpumask for partcmd_update
  1158. * @tmp: Temporary addmask and delmask
  1159. * Return: 0 or a partition root state error code
  1160. *
  1161. * For partcmd_enable, the cpuset is being transformed from a non-partition
  1162. * root to a partition root. The cpus_allowed mask of the given cpuset will
  1163. * be put into parent's subparts_cpus and taken away from parent's
  1164. * effective_cpus. The function will return 0 if all the CPUs listed in
  1165. * cpus_allowed can be granted or an error code will be returned.
  1166. *
  1167. * For partcmd_disable, the cpuset is being transformed from a partition
  1168. * root back to a non-partition root. Any CPUs in cpus_allowed that are in
  1169. * parent's subparts_cpus will be taken away from that cpumask and put back
  1170. * into parent's effective_cpus. 0 will always be returned.
  1171. *
  1172. * For partcmd_update, if the optional newmask is specified, the cpu list is
  1173. * to be changed from cpus_allowed to newmask. Otherwise, cpus_allowed is
  1174. * assumed to remain the same. The cpuset should either be a valid or invalid
  1175. * partition root. The partition root state may change from valid to invalid
  1176. * or vice versa. An error code will only be returned if transitioning from
  1177. * invalid to valid violates the exclusivity rule.
  1178. *
  1179. * For partcmd_invalidate, the current partition will be made invalid.
  1180. *
  1181. * The partcmd_enable and partcmd_disable commands are used by
  1182. * update_prstate(). An error code may be returned and the caller will check
  1183. * for error.
  1184. *
  1185. * The partcmd_update command is used by update_cpumasks_hier() with newmask
  1186. * NULL and update_cpumask() with newmask set. The partcmd_invalidate is used
  1187. * by update_cpumask() with NULL newmask. In both cases, the callers won't
  1188. * check for error and so partition_root_state and prs_error will be updated
  1189. * directly.
  1190. */
  1191. static int update_parent_subparts_cpumask(struct cpuset *cs, int cmd,
  1192. struct cpumask *newmask,
  1193. struct tmpmasks *tmp)
  1194. {
  1195. struct cpuset *parent = parent_cs(cs);
  1196. int adding; /* Moving cpus from effective_cpus to subparts_cpus */
  1197. int deleting; /* Moving cpus from subparts_cpus to effective_cpus */
  1198. int old_prs, new_prs;
  1199. int part_error = PERR_NONE; /* Partition error? */
  1200. lockdep_assert_held(&cpuset_mutex);
  1201. /*
  1202. * The parent must be a partition root.
  1203. * The new cpumask, if present, or the current cpus_allowed must
  1204. * not be empty.
  1205. */
  1206. if (!is_partition_valid(parent)) {
  1207. return is_partition_invalid(parent)
  1208. ? PERR_INVPARENT : PERR_NOTPART;
  1209. }
  1210. if ((newmask && cpumask_empty(newmask)) ||
  1211. (!newmask && cpumask_empty(cs->cpus_allowed)))
  1212. return PERR_CPUSEMPTY;
  1213. /*
  1214. * new_prs will only be changed for the partcmd_update and
  1215. * partcmd_invalidate commands.
  1216. */
  1217. adding = deleting = false;
  1218. old_prs = new_prs = cs->partition_root_state;
  1219. if (cmd == partcmd_enable) {
  1220. /*
  1221. * Enabling partition root is not allowed if cpus_allowed
  1222. * doesn't overlap parent's cpus_allowed.
  1223. */
  1224. if (!cpumask_intersects(cs->cpus_allowed, parent->cpus_allowed))
  1225. return PERR_INVCPUS;
  1226. /*
  1227. * A parent can be left with no CPU as long as there is no
  1228. * task directly associated with the parent partition.
  1229. */
  1230. if (cpumask_subset(parent->effective_cpus, cs->cpus_allowed) &&
  1231. partition_is_populated(parent, cs))
  1232. return PERR_NOCPUS;
  1233. cpumask_copy(tmp->addmask, cs->cpus_allowed);
  1234. adding = true;
  1235. } else if (cmd == partcmd_disable) {
  1236. /*
  1237. * Need to remove cpus from parent's subparts_cpus for valid
  1238. * partition root.
  1239. */
  1240. deleting = !is_prs_invalid(old_prs) &&
  1241. cpumask_and(tmp->delmask, cs->cpus_allowed,
  1242. parent->subparts_cpus);
  1243. } else if (cmd == partcmd_invalidate) {
  1244. if (is_prs_invalid(old_prs))
  1245. return 0;
  1246. /*
  1247. * Make the current partition invalid. It is assumed that
  1248. * invalidation is caused by violating cpu exclusivity rule.
  1249. */
  1250. deleting = cpumask_and(tmp->delmask, cs->cpus_allowed,
  1251. parent->subparts_cpus);
  1252. if (old_prs > 0) {
  1253. new_prs = -old_prs;
  1254. part_error = PERR_NOTEXCL;
  1255. }
  1256. } else if (newmask) {
  1257. /*
  1258. * partcmd_update with newmask:
  1259. *
  1260. * Compute add/delete mask to/from subparts_cpus
  1261. *
  1262. * delmask = cpus_allowed & ~newmask & parent->subparts_cpus
  1263. * addmask = newmask & parent->cpus_allowed
  1264. * & ~parent->subparts_cpus
  1265. */
  1266. cpumask_andnot(tmp->delmask, cs->cpus_allowed, newmask);
  1267. deleting = cpumask_and(tmp->delmask, tmp->delmask,
  1268. parent->subparts_cpus);
  1269. cpumask_and(tmp->addmask, newmask, parent->cpus_allowed);
  1270. adding = cpumask_andnot(tmp->addmask, tmp->addmask,
  1271. parent->subparts_cpus);
  1272. /*
  1273. * Make partition invalid if parent's effective_cpus could
  1274. * become empty and there are tasks in the parent.
  1275. */
  1276. if (adding &&
  1277. cpumask_subset(parent->effective_cpus, tmp->addmask) &&
  1278. !cpumask_intersects(tmp->delmask, cpu_active_mask) &&
  1279. partition_is_populated(parent, cs)) {
  1280. part_error = PERR_NOCPUS;
  1281. adding = false;
  1282. deleting = cpumask_and(tmp->delmask, cs->cpus_allowed,
  1283. parent->subparts_cpus);
  1284. }
  1285. } else {
  1286. /*
  1287. * partcmd_update w/o newmask:
  1288. *
  1289. * delmask = cpus_allowed & parent->subparts_cpus
  1290. * addmask = cpus_allowed & parent->cpus_allowed
  1291. * & ~parent->subparts_cpus
  1292. *
  1293. * This gets invoked either due to a hotplug event or from
  1294. * update_cpumasks_hier(). This can cause the state of a
  1295. * partition root to transition from valid to invalid or vice
  1296. * versa. So we still need to compute the addmask and delmask.
  1297. * A partition error happens when:
  1298. * 1) Cpuset is valid partition, but parent does not distribute
  1299. * out any CPUs.
  1300. * 2) Parent has tasks and all its effective CPUs will have
  1301. * to be distributed out.
  1302. */
  1303. cpumask_and(tmp->addmask, cs->cpus_allowed,
  1304. parent->cpus_allowed);
  1305. adding = cpumask_andnot(tmp->addmask, tmp->addmask,
  1306. parent->subparts_cpus);
  1307. if ((is_partition_valid(cs) && !parent->nr_subparts_cpus) ||
  1308. (adding &&
  1309. cpumask_subset(parent->effective_cpus, tmp->addmask) &&
  1310. partition_is_populated(parent, cs))) {
  1311. part_error = PERR_NOCPUS;
  1312. adding = false;
  1313. }
  1314. if (part_error && is_partition_valid(cs) &&
  1315. parent->nr_subparts_cpus)
  1316. deleting = cpumask_and(tmp->delmask, cs->cpus_allowed,
  1317. parent->subparts_cpus);
  1318. }
  1319. if (part_error)
  1320. WRITE_ONCE(cs->prs_err, part_error);
  1321. if (cmd == partcmd_update) {
  1322. /*
  1323. * Check for possible transition between valid and invalid
  1324. * partition root.
  1325. */
  1326. switch (cs->partition_root_state) {
  1327. case PRS_ROOT:
  1328. case PRS_ISOLATED:
  1329. if (part_error)
  1330. new_prs = -old_prs;
  1331. break;
  1332. case PRS_INVALID_ROOT:
  1333. case PRS_INVALID_ISOLATED:
  1334. if (!part_error)
  1335. new_prs = -old_prs;
  1336. break;
  1337. }
  1338. }
  1339. if (!adding && !deleting && (new_prs == old_prs))
  1340. return 0;
  1341. /*
  1342. * Transitioning between invalid to valid or vice versa may require
  1343. * changing CS_CPU_EXCLUSIVE and CS_SCHED_LOAD_BALANCE.
  1344. */
  1345. if (old_prs != new_prs) {
  1346. if (is_prs_invalid(old_prs) && !is_cpu_exclusive(cs) &&
  1347. (update_flag(CS_CPU_EXCLUSIVE, cs, 1) < 0))
  1348. return PERR_NOTEXCL;
  1349. if (is_prs_invalid(new_prs) && is_cpu_exclusive(cs))
  1350. update_flag(CS_CPU_EXCLUSIVE, cs, 0);
  1351. }
  1352. /*
  1353. * Change the parent's subparts_cpus.
  1354. * Newly added CPUs will be removed from effective_cpus and
  1355. * newly deleted ones will be added back to effective_cpus.
  1356. */
  1357. spin_lock_irq(&callback_lock);
  1358. if (adding) {
  1359. cpumask_or(parent->subparts_cpus,
  1360. parent->subparts_cpus, tmp->addmask);
  1361. cpumask_andnot(parent->effective_cpus,
  1362. parent->effective_cpus, tmp->addmask);
  1363. }
  1364. if (deleting) {
  1365. cpumask_andnot(parent->subparts_cpus,
  1366. parent->subparts_cpus, tmp->delmask);
  1367. /*
  1368. * Some of the CPUs in subparts_cpus might have been offlined.
  1369. */
  1370. cpumask_and(tmp->delmask, tmp->delmask, cpu_active_mask);
  1371. cpumask_or(parent->effective_cpus,
  1372. parent->effective_cpus, tmp->delmask);
  1373. }
  1374. parent->nr_subparts_cpus = cpumask_weight(parent->subparts_cpus);
  1375. if (old_prs != new_prs)
  1376. cs->partition_root_state = new_prs;
  1377. spin_unlock_irq(&callback_lock);
  1378. if (adding || deleting)
  1379. update_tasks_cpumask(parent, tmp->addmask);
  1380. /*
  1381. * Set or clear CS_SCHED_LOAD_BALANCE when partcmd_update, if necessary.
  1382. * rebuild_sched_domains_locked() may be called.
  1383. */
  1384. if (old_prs != new_prs) {
  1385. if (old_prs == PRS_ISOLATED)
  1386. update_flag(CS_SCHED_LOAD_BALANCE, cs, 1);
  1387. else if (new_prs == PRS_ISOLATED)
  1388. update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
  1389. }
  1390. notify_partition_change(cs, old_prs);
  1391. return 0;
  1392. }
  1393. /*
  1394. * update_cpumasks_hier - Update effective cpumasks and tasks in the subtree
  1395. * @cs: the cpuset to consider
  1396. * @tmp: temp variables for calculating effective_cpus & partition setup
  1397. * @force: don't skip any descendant cpusets if set
  1398. *
  1399. * When configured cpumask is changed, the effective cpumasks of this cpuset
  1400. * and all its descendants need to be updated.
  1401. *
  1402. * On legacy hierarchy, effective_cpus will be the same with cpu_allowed.
  1403. *
  1404. * Called with cpuset_mutex held
  1405. */
  1406. static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp,
  1407. bool force)
  1408. {
  1409. struct cpuset *cp;
  1410. struct cgroup_subsys_state *pos_css;
  1411. bool need_rebuild_sched_domains = false;
  1412. int old_prs, new_prs;
  1413. rcu_read_lock();
  1414. cpuset_for_each_descendant_pre(cp, pos_css, cs) {
  1415. struct cpuset *parent = parent_cs(cp);
  1416. bool update_parent = false;
  1417. compute_effective_cpumask(tmp->new_cpus, cp, parent);
  1418. /*
  1419. * If it becomes empty, inherit the effective mask of the
  1420. * parent, which is guaranteed to have some CPUs unless
  1421. * it is a partition root that has explicitly distributed
  1422. * out all its CPUs.
  1423. */
  1424. if (is_in_v2_mode() && cpumask_empty(tmp->new_cpus)) {
  1425. if (is_partition_valid(cp) &&
  1426. cpumask_equal(cp->cpus_allowed, cp->subparts_cpus))
  1427. goto update_parent_subparts;
  1428. cpumask_copy(tmp->new_cpus, parent->effective_cpus);
  1429. if (!cp->use_parent_ecpus) {
  1430. cp->use_parent_ecpus = true;
  1431. parent->child_ecpus_count++;
  1432. }
  1433. } else if (cp->use_parent_ecpus) {
  1434. cp->use_parent_ecpus = false;
  1435. WARN_ON_ONCE(!parent->child_ecpus_count);
  1436. parent->child_ecpus_count--;
  1437. }
  1438. /*
  1439. * Skip the whole subtree if
  1440. * 1) the cpumask remains the same,
  1441. * 2) has no partition root state,
  1442. * 3) force flag not set, and
  1443. * 4) for v2 load balance state same as its parent.
  1444. */
  1445. if (!cp->partition_root_state && !force &&
  1446. cpumask_equal(tmp->new_cpus, cp->effective_cpus) &&
  1447. (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
  1448. (is_sched_load_balance(parent) == is_sched_load_balance(cp)))) {
  1449. pos_css = css_rightmost_descendant(pos_css);
  1450. continue;
  1451. }
  1452. update_parent_subparts:
  1453. /*
  1454. * update_parent_subparts_cpumask() should have been called
  1455. * for cs already in update_cpumask(). We should also call
  1456. * update_tasks_cpumask() again for tasks in the parent
  1457. * cpuset if the parent's subparts_cpus changes.
  1458. */
  1459. old_prs = new_prs = cp->partition_root_state;
  1460. if ((cp != cs) && old_prs) {
  1461. switch (parent->partition_root_state) {
  1462. case PRS_ROOT:
  1463. case PRS_ISOLATED:
  1464. update_parent = true;
  1465. break;
  1466. default:
  1467. /*
  1468. * When parent is not a partition root or is
  1469. * invalid, child partition roots become
  1470. * invalid too.
  1471. */
  1472. if (is_partition_valid(cp))
  1473. new_prs = -cp->partition_root_state;
  1474. WRITE_ONCE(cp->prs_err,
  1475. is_partition_invalid(parent)
  1476. ? PERR_INVPARENT : PERR_NOTPART);
  1477. break;
  1478. }
  1479. }
  1480. if (!css_tryget_online(&cp->css))
  1481. continue;
  1482. rcu_read_unlock();
  1483. if (update_parent) {
  1484. update_parent_subparts_cpumask(cp, partcmd_update, NULL,
  1485. tmp);
  1486. /*
  1487. * The cpuset partition_root_state may become
  1488. * invalid. Capture it.
  1489. */
  1490. new_prs = cp->partition_root_state;
  1491. }
  1492. spin_lock_irq(&callback_lock);
  1493. if (cp->nr_subparts_cpus && !is_partition_valid(cp)) {
  1494. /*
  1495. * Put all active subparts_cpus back to effective_cpus.
  1496. */
  1497. cpumask_or(tmp->new_cpus, tmp->new_cpus,
  1498. cp->subparts_cpus);
  1499. cpumask_and(tmp->new_cpus, tmp->new_cpus,
  1500. cpu_active_mask);
  1501. cp->nr_subparts_cpus = 0;
  1502. cpumask_clear(cp->subparts_cpus);
  1503. }
  1504. cpumask_copy(cp->effective_cpus, tmp->new_cpus);
  1505. if (cp->nr_subparts_cpus) {
  1506. /*
  1507. * Make sure that effective_cpus & subparts_cpus
  1508. * are mutually exclusive.
  1509. */
  1510. cpumask_andnot(cp->effective_cpus, cp->effective_cpus,
  1511. cp->subparts_cpus);
  1512. }
  1513. cp->partition_root_state = new_prs;
  1514. spin_unlock_irq(&callback_lock);
  1515. notify_partition_change(cp, old_prs);
  1516. WARN_ON(!is_in_v2_mode() &&
  1517. !cpumask_equal(cp->cpus_allowed, cp->effective_cpus));
  1518. update_tasks_cpumask(cp, tmp->new_cpus);
  1519. /*
  1520. * On default hierarchy, inherit the CS_SCHED_LOAD_BALANCE
  1521. * from parent if current cpuset isn't a valid partition root
  1522. * and their load balance states differ.
  1523. */
  1524. if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
  1525. !is_partition_valid(cp) &&
  1526. (is_sched_load_balance(parent) != is_sched_load_balance(cp))) {
  1527. if (is_sched_load_balance(parent))
  1528. set_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
  1529. else
  1530. clear_bit(CS_SCHED_LOAD_BALANCE, &cp->flags);
  1531. }
  1532. /*
  1533. * On legacy hierarchy, if the effective cpumask of any non-
  1534. * empty cpuset is changed, we need to rebuild sched domains.
  1535. * On default hierarchy, the cpuset needs to be a partition
  1536. * root as well.
  1537. */
  1538. if (!cpumask_empty(cp->cpus_allowed) &&
  1539. is_sched_load_balance(cp) &&
  1540. (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) ||
  1541. is_partition_valid(cp)))
  1542. need_rebuild_sched_domains = true;
  1543. rcu_read_lock();
  1544. css_put(&cp->css);
  1545. }
  1546. rcu_read_unlock();
  1547. if (need_rebuild_sched_domains)
  1548. rebuild_sched_domains_locked();
  1549. }
  1550. /**
  1551. * update_sibling_cpumasks - Update siblings cpumasks
  1552. * @parent: Parent cpuset
  1553. * @cs: Current cpuset
  1554. * @tmp: Temp variables
  1555. */
  1556. static void update_sibling_cpumasks(struct cpuset *parent, struct cpuset *cs,
  1557. struct tmpmasks *tmp)
  1558. {
  1559. struct cpuset *sibling;
  1560. struct cgroup_subsys_state *pos_css;
  1561. lockdep_assert_held(&cpuset_mutex);
  1562. /*
  1563. * Check all its siblings and call update_cpumasks_hier()
  1564. * if their use_parent_ecpus flag is set in order for them
  1565. * to use the right effective_cpus value.
  1566. *
  1567. * The update_cpumasks_hier() function may sleep. So we have to
  1568. * release the RCU read lock before calling it.
  1569. */
  1570. rcu_read_lock();
  1571. cpuset_for_each_child(sibling, pos_css, parent) {
  1572. if (sibling == cs)
  1573. continue;
  1574. if (!sibling->use_parent_ecpus)
  1575. continue;
  1576. if (!css_tryget_online(&sibling->css))
  1577. continue;
  1578. rcu_read_unlock();
  1579. update_cpumasks_hier(sibling, tmp, false);
  1580. rcu_read_lock();
  1581. css_put(&sibling->css);
  1582. }
  1583. rcu_read_unlock();
  1584. }
  1585. /**
  1586. * update_cpumask - update the cpus_allowed mask of a cpuset and all tasks in it
  1587. * @cs: the cpuset to consider
  1588. * @trialcs: trial cpuset
  1589. * @buf: buffer of cpu numbers written to this cpuset
  1590. */
  1591. static int update_cpumask(struct cpuset *cs, struct cpuset *trialcs,
  1592. const char *buf)
  1593. {
  1594. int retval;
  1595. struct tmpmasks tmp;
  1596. bool invalidate = false;
  1597. /* top_cpuset.cpus_allowed tracks cpu_online_mask; it's read-only */
  1598. if (cs == &top_cpuset)
  1599. return -EACCES;
  1600. /*
  1601. * An empty cpus_requested is ok only if the cpuset has no tasks.
  1602. * Since cpulist_parse() fails on an empty mask, we special case
  1603. * that parsing. The validate_change() call ensures that cpusets
  1604. * with tasks have cpus.
  1605. */
  1606. if (!*buf) {
  1607. cpumask_clear(trialcs->cpus_requested);
  1608. } else {
  1609. retval = cpulist_parse(buf, trialcs->cpus_requested);
  1610. if (retval < 0)
  1611. return retval;
  1612. }
  1613. if (!cpumask_subset(trialcs->cpus_requested, cpu_present_mask))
  1614. return -EINVAL;
  1615. cpumask_and(trialcs->cpus_allowed, trialcs->cpus_requested, cpu_active_mask);
  1616. /* Nothing to do if the cpus didn't change */
  1617. if (cpumask_equal(cs->cpus_requested, trialcs->cpus_requested))
  1618. return 0;
  1619. #ifdef CONFIG_CPUMASK_OFFSTACK
  1620. /*
  1621. * Use the cpumasks in trialcs for tmpmasks when they are pointers
  1622. * to allocated cpumasks.
  1623. *
  1624. * Note that update_parent_subparts_cpumask() uses only addmask &
  1625. * delmask, but not new_cpus.
  1626. */
  1627. tmp.addmask = trialcs->subparts_cpus;
  1628. tmp.delmask = trialcs->effective_cpus;
  1629. tmp.new_cpus = NULL;
  1630. #endif
  1631. retval = validate_change(cs, trialcs);
  1632. if ((retval == -EINVAL) && cgroup_subsys_on_dfl(cpuset_cgrp_subsys)) {
  1633. struct cpuset *cp, *parent;
  1634. struct cgroup_subsys_state *css;
  1635. /*
  1636. * The -EINVAL error code indicates that partition sibling
  1637. * CPU exclusivity rule has been violated. We still allow
  1638. * the cpumask change to proceed while invalidating the
  1639. * partition. However, any conflicting sibling partitions
  1640. * have to be marked as invalid too.
  1641. */
  1642. invalidate = true;
  1643. rcu_read_lock();
  1644. parent = parent_cs(cs);
  1645. cpuset_for_each_child(cp, css, parent)
  1646. if (is_partition_valid(cp) &&
  1647. cpumask_intersects(trialcs->cpus_allowed, cp->cpus_allowed)) {
  1648. rcu_read_unlock();
  1649. update_parent_subparts_cpumask(cp, partcmd_invalidate, NULL, &tmp);
  1650. rcu_read_lock();
  1651. }
  1652. rcu_read_unlock();
  1653. retval = 0;
  1654. }
  1655. if (retval < 0)
  1656. return retval;
  1657. if (cs->partition_root_state) {
  1658. if (invalidate)
  1659. update_parent_subparts_cpumask(cs, partcmd_invalidate,
  1660. NULL, &tmp);
  1661. else
  1662. update_parent_subparts_cpumask(cs, partcmd_update,
  1663. trialcs->cpus_allowed, &tmp);
  1664. }
  1665. compute_effective_cpumask(trialcs->effective_cpus, trialcs,
  1666. parent_cs(cs));
  1667. spin_lock_irq(&callback_lock);
  1668. cpumask_copy(cs->cpus_allowed, trialcs->cpus_allowed);
  1669. cpumask_copy(cs->cpus_requested, trialcs->cpus_requested);
  1670. /*
  1671. * Make sure that subparts_cpus, if not empty, is a subset of
  1672. * cpus_allowed. Clear subparts_cpus if partition not valid or
  1673. * empty effective cpus with tasks.
  1674. */
  1675. if (cs->nr_subparts_cpus) {
  1676. if (!is_partition_valid(cs) ||
  1677. (cpumask_subset(trialcs->effective_cpus, cs->subparts_cpus) &&
  1678. partition_is_populated(cs, NULL))) {
  1679. cs->nr_subparts_cpus = 0;
  1680. cpumask_clear(cs->subparts_cpus);
  1681. } else {
  1682. cpumask_and(cs->subparts_cpus, cs->subparts_cpus,
  1683. cs->cpus_allowed);
  1684. cs->nr_subparts_cpus = cpumask_weight(cs->subparts_cpus);
  1685. }
  1686. }
  1687. spin_unlock_irq(&callback_lock);
  1688. #ifdef CONFIG_CPUMASK_OFFSTACK
  1689. /* Now trialcs->cpus_allowed is available */
  1690. tmp.new_cpus = trialcs->cpus_allowed;
  1691. #endif
  1692. /* effective_cpus will be updated here */
  1693. update_cpumasks_hier(cs, &tmp, false);
  1694. if (cs->partition_root_state) {
  1695. struct cpuset *parent = parent_cs(cs);
  1696. /*
  1697. * For partition root, update the cpumasks of sibling
  1698. * cpusets if they use parent's effective_cpus.
  1699. */
  1700. if (parent->child_ecpus_count)
  1701. update_sibling_cpumasks(parent, cs, &tmp);
  1702. }
  1703. return 0;
  1704. }
  1705. /*
  1706. * Migrate memory region from one set of nodes to another. This is
  1707. * performed asynchronously as it can be called from process migration path
  1708. * holding locks involved in process management. All mm migrations are
  1709. * performed in the queued order and can be waited for by flushing
  1710. * cpuset_migrate_mm_wq.
  1711. */
  1712. struct cpuset_migrate_mm_work {
  1713. struct work_struct work;
  1714. struct mm_struct *mm;
  1715. nodemask_t from;
  1716. nodemask_t to;
  1717. };
  1718. static void cpuset_migrate_mm_workfn(struct work_struct *work)
  1719. {
  1720. struct cpuset_migrate_mm_work *mwork =
  1721. container_of(work, struct cpuset_migrate_mm_work, work);
  1722. /* on a wq worker, no need to worry about %current's mems_allowed */
  1723. do_migrate_pages(mwork->mm, &mwork->from, &mwork->to, MPOL_MF_MOVE_ALL);
  1724. mmput(mwork->mm);
  1725. kfree(mwork);
  1726. }
  1727. static void cpuset_migrate_mm(struct mm_struct *mm, const nodemask_t *from,
  1728. const nodemask_t *to)
  1729. {
  1730. struct cpuset_migrate_mm_work *mwork;
  1731. if (nodes_equal(*from, *to)) {
  1732. mmput(mm);
  1733. return;
  1734. }
  1735. mwork = kzalloc(sizeof(*mwork), GFP_KERNEL);
  1736. if (mwork) {
  1737. mwork->mm = mm;
  1738. mwork->from = *from;
  1739. mwork->to = *to;
  1740. INIT_WORK(&mwork->work, cpuset_migrate_mm_workfn);
  1741. queue_work(cpuset_migrate_mm_wq, &mwork->work);
  1742. } else {
  1743. mmput(mm);
  1744. }
  1745. }
  1746. static void cpuset_post_attach(void)
  1747. {
  1748. flush_workqueue(cpuset_migrate_mm_wq);
  1749. }
  1750. /*
  1751. * cpuset_change_task_nodemask - change task's mems_allowed and mempolicy
  1752. * @tsk: the task to change
  1753. * @newmems: new nodes that the task will be set
  1754. *
  1755. * We use the mems_allowed_seq seqlock to safely update both tsk->mems_allowed
  1756. * and rebind an eventual tasks' mempolicy. If the task is allocating in
  1757. * parallel, it might temporarily see an empty intersection, which results in
  1758. * a seqlock check and retry before OOM or allocation failure.
  1759. */
  1760. static void cpuset_change_task_nodemask(struct task_struct *tsk,
  1761. nodemask_t *newmems)
  1762. {
  1763. task_lock(tsk);
  1764. local_irq_disable();
  1765. write_seqcount_begin(&tsk->mems_allowed_seq);
  1766. nodes_or(tsk->mems_allowed, tsk->mems_allowed, *newmems);
  1767. mpol_rebind_task(tsk, newmems);
  1768. tsk->mems_allowed = *newmems;
  1769. write_seqcount_end(&tsk->mems_allowed_seq);
  1770. local_irq_enable();
  1771. task_unlock(tsk);
  1772. }
  1773. static void *cpuset_being_rebound;
  1774. /**
  1775. * update_tasks_nodemask - Update the nodemasks of tasks in the cpuset.
  1776. * @cs: the cpuset in which each task's mems_allowed mask needs to be changed
  1777. *
  1778. * Iterate through each task of @cs updating its mems_allowed to the
  1779. * effective cpuset's. As this function is called with cpuset_mutex held,
  1780. * cpuset membership stays stable.
  1781. */
  1782. static void update_tasks_nodemask(struct cpuset *cs)
  1783. {
  1784. static nodemask_t newmems; /* protected by cpuset_mutex */
  1785. struct css_task_iter it;
  1786. struct task_struct *task;
  1787. cpuset_being_rebound = cs; /* causes mpol_dup() rebind */
  1788. guarantee_online_mems(cs, &newmems);
  1789. /*
  1790. * The mpol_rebind_mm() call takes mmap_lock, which we couldn't
  1791. * take while holding tasklist_lock. Forks can happen - the
  1792. * mpol_dup() cpuset_being_rebound check will catch such forks,
  1793. * and rebind their vma mempolicies too. Because we still hold
  1794. * the global cpuset_mutex, we know that no other rebind effort
  1795. * will be contending for the global variable cpuset_being_rebound.
  1796. * It's ok if we rebind the same mm twice; mpol_rebind_mm()
  1797. * is idempotent. Also migrate pages in each mm to new nodes.
  1798. */
  1799. css_task_iter_start(&cs->css, 0, &it);
  1800. while ((task = css_task_iter_next(&it))) {
  1801. struct mm_struct *mm;
  1802. bool migrate;
  1803. cpuset_change_task_nodemask(task, &newmems);
  1804. mm = get_task_mm(task);
  1805. if (!mm)
  1806. continue;
  1807. migrate = is_memory_migrate(cs);
  1808. mpol_rebind_mm(mm, &cs->mems_allowed);
  1809. if (migrate)
  1810. cpuset_migrate_mm(mm, &cs->old_mems_allowed, &newmems);
  1811. else
  1812. mmput(mm);
  1813. }
  1814. css_task_iter_end(&it);
  1815. /*
  1816. * All the tasks' nodemasks have been updated, update
  1817. * cs->old_mems_allowed.
  1818. */
  1819. cs->old_mems_allowed = newmems;
  1820. /* We're done rebinding vmas to this cpuset's new mems_allowed. */
  1821. cpuset_being_rebound = NULL;
  1822. }
  1823. /*
  1824. * update_nodemasks_hier - Update effective nodemasks and tasks in the subtree
  1825. * @cs: the cpuset to consider
  1826. * @new_mems: a temp variable for calculating new effective_mems
  1827. *
  1828. * When configured nodemask is changed, the effective nodemasks of this cpuset
  1829. * and all its descendants need to be updated.
  1830. *
  1831. * On legacy hierarchy, effective_mems will be the same with mems_allowed.
  1832. *
  1833. * Called with cpuset_mutex held
  1834. */
  1835. static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems)
  1836. {
  1837. struct cpuset *cp;
  1838. struct cgroup_subsys_state *pos_css;
  1839. rcu_read_lock();
  1840. cpuset_for_each_descendant_pre(cp, pos_css, cs) {
  1841. struct cpuset *parent = parent_cs(cp);
  1842. nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems);
  1843. /*
  1844. * If it becomes empty, inherit the effective mask of the
  1845. * parent, which is guaranteed to have some MEMs.
  1846. */
  1847. if (is_in_v2_mode() && nodes_empty(*new_mems))
  1848. *new_mems = parent->effective_mems;
  1849. /* Skip the whole subtree if the nodemask remains the same. */
  1850. if (nodes_equal(*new_mems, cp->effective_mems)) {
  1851. pos_css = css_rightmost_descendant(pos_css);
  1852. continue;
  1853. }
  1854. if (!css_tryget_online(&cp->css))
  1855. continue;
  1856. rcu_read_unlock();
  1857. spin_lock_irq(&callback_lock);
  1858. cp->effective_mems = *new_mems;
  1859. spin_unlock_irq(&callback_lock);
  1860. WARN_ON(!is_in_v2_mode() &&
  1861. !nodes_equal(cp->mems_allowed, cp->effective_mems));
  1862. update_tasks_nodemask(cp);
  1863. rcu_read_lock();
  1864. css_put(&cp->css);
  1865. }
  1866. rcu_read_unlock();
  1867. }
  1868. /*
  1869. * Handle user request to change the 'mems' memory placement
  1870. * of a cpuset. Needs to validate the request, update the
  1871. * cpusets mems_allowed, and for each task in the cpuset,
  1872. * update mems_allowed and rebind task's mempolicy and any vma
  1873. * mempolicies and if the cpuset is marked 'memory_migrate',
  1874. * migrate the tasks pages to the new memory.
  1875. *
  1876. * Call with cpuset_mutex held. May take callback_lock during call.
  1877. * Will take tasklist_lock, scan tasklist for tasks in cpuset cs,
  1878. * lock each such tasks mm->mmap_lock, scan its vma's and rebind
  1879. * their mempolicies to the cpusets new mems_allowed.
  1880. */
  1881. static int update_nodemask(struct cpuset *cs, struct cpuset *trialcs,
  1882. const char *buf)
  1883. {
  1884. int retval;
  1885. /*
  1886. * top_cpuset.mems_allowed tracks node_stats[N_MEMORY];
  1887. * it's read-only
  1888. */
  1889. if (cs == &top_cpuset) {
  1890. retval = -EACCES;
  1891. goto done;
  1892. }
  1893. /*
  1894. * An empty mems_allowed is ok iff there are no tasks in the cpuset.
  1895. * Since nodelist_parse() fails on an empty mask, we special case
  1896. * that parsing. The validate_change() call ensures that cpusets
  1897. * with tasks have memory.
  1898. */
  1899. if (!*buf) {
  1900. nodes_clear(trialcs->mems_allowed);
  1901. } else {
  1902. retval = nodelist_parse(buf, trialcs->mems_allowed);
  1903. if (retval < 0)
  1904. goto done;
  1905. if (!nodes_subset(trialcs->mems_allowed,
  1906. top_cpuset.mems_allowed)) {
  1907. retval = -EINVAL;
  1908. goto done;
  1909. }
  1910. }
  1911. if (nodes_equal(cs->mems_allowed, trialcs->mems_allowed)) {
  1912. retval = 0; /* Too easy - nothing to do */
  1913. goto done;
  1914. }
  1915. retval = validate_change(cs, trialcs);
  1916. if (retval < 0)
  1917. goto done;
  1918. check_insane_mems_config(&trialcs->mems_allowed);
  1919. spin_lock_irq(&callback_lock);
  1920. cs->mems_allowed = trialcs->mems_allowed;
  1921. spin_unlock_irq(&callback_lock);
  1922. /* use trialcs->mems_allowed as a temp variable */
  1923. update_nodemasks_hier(cs, &trialcs->mems_allowed);
  1924. done:
  1925. return retval;
  1926. }
  1927. bool current_cpuset_is_being_rebound(void)
  1928. {
  1929. bool ret;
  1930. rcu_read_lock();
  1931. ret = task_cs(current) == cpuset_being_rebound;
  1932. rcu_read_unlock();
  1933. return ret;
  1934. }
  1935. static int update_relax_domain_level(struct cpuset *cs, s64 val)
  1936. {
  1937. #ifdef CONFIG_SMP
  1938. if (val < -1 || val >= sched_domain_level_max)
  1939. return -EINVAL;
  1940. #endif
  1941. if (val != cs->relax_domain_level) {
  1942. cs->relax_domain_level = val;
  1943. if (!cpumask_empty(cs->cpus_allowed) &&
  1944. is_sched_load_balance(cs))
  1945. rebuild_sched_domains_locked();
  1946. }
  1947. return 0;
  1948. }
  1949. /**
  1950. * update_tasks_flags - update the spread flags of tasks in the cpuset.
  1951. * @cs: the cpuset in which each task's spread flags needs to be changed
  1952. *
  1953. * Iterate through each task of @cs updating its spread flags. As this
  1954. * function is called with cpuset_mutex held, cpuset membership stays
  1955. * stable.
  1956. */
  1957. static void update_tasks_flags(struct cpuset *cs)
  1958. {
  1959. struct css_task_iter it;
  1960. struct task_struct *task;
  1961. css_task_iter_start(&cs->css, 0, &it);
  1962. while ((task = css_task_iter_next(&it)))
  1963. cpuset_update_task_spread_flags(cs, task);
  1964. css_task_iter_end(&it);
  1965. }
  1966. /*
  1967. * update_flag - read a 0 or a 1 in a file and update associated flag
  1968. * bit: the bit to update (see cpuset_flagbits_t)
  1969. * cs: the cpuset to update
  1970. * turning_on: whether the flag is being set or cleared
  1971. *
  1972. * Call with cpuset_mutex held.
  1973. */
  1974. static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs,
  1975. int turning_on)
  1976. {
  1977. struct cpuset *trialcs;
  1978. int balance_flag_changed;
  1979. int spread_flag_changed;
  1980. int err;
  1981. trialcs = alloc_trial_cpuset(cs);
  1982. if (!trialcs)
  1983. return -ENOMEM;
  1984. if (turning_on)
  1985. set_bit(bit, &trialcs->flags);
  1986. else
  1987. clear_bit(bit, &trialcs->flags);
  1988. err = validate_change(cs, trialcs);
  1989. if (err < 0)
  1990. goto out;
  1991. balance_flag_changed = (is_sched_load_balance(cs) !=
  1992. is_sched_load_balance(trialcs));
  1993. spread_flag_changed = ((is_spread_slab(cs) != is_spread_slab(trialcs))
  1994. || (is_spread_page(cs) != is_spread_page(trialcs)));
  1995. spin_lock_irq(&callback_lock);
  1996. cs->flags = trialcs->flags;
  1997. spin_unlock_irq(&callback_lock);
  1998. if (!cpumask_empty(trialcs->cpus_allowed) && balance_flag_changed)
  1999. rebuild_sched_domains_locked();
  2000. if (spread_flag_changed)
  2001. update_tasks_flags(cs);
  2002. out:
  2003. free_cpuset(trialcs);
  2004. return err;
  2005. }
  2006. /**
  2007. * update_prstate - update partition_root_state
  2008. * @cs: the cpuset to update
  2009. * @new_prs: new partition root state
  2010. * Return: 0 if successful, != 0 if error
  2011. *
  2012. * Call with cpuset_mutex held.
  2013. */
  2014. static int update_prstate(struct cpuset *cs, int new_prs)
  2015. {
  2016. int err = PERR_NONE, old_prs = cs->partition_root_state;
  2017. bool sched_domain_rebuilt = false;
  2018. struct cpuset *parent = parent_cs(cs);
  2019. struct tmpmasks tmpmask;
  2020. if (old_prs == new_prs)
  2021. return 0;
  2022. /*
  2023. * For a previously invalid partition root, leave it at being
  2024. * invalid if new_prs is not "member".
  2025. */
  2026. if (new_prs && is_prs_invalid(old_prs)) {
  2027. cs->partition_root_state = -new_prs;
  2028. return 0;
  2029. }
  2030. if (alloc_cpumasks(NULL, &tmpmask))
  2031. return -ENOMEM;
  2032. if (!old_prs) {
  2033. /*
  2034. * Turning on partition root requires setting the
  2035. * CS_CPU_EXCLUSIVE bit implicitly as well and cpus_allowed
  2036. * cannot be empty.
  2037. */
  2038. if (cpumask_empty(cs->cpus_allowed)) {
  2039. err = PERR_CPUSEMPTY;
  2040. goto out;
  2041. }
  2042. err = update_flag(CS_CPU_EXCLUSIVE, cs, 1);
  2043. if (err) {
  2044. err = PERR_NOTEXCL;
  2045. goto out;
  2046. }
  2047. err = update_parent_subparts_cpumask(cs, partcmd_enable,
  2048. NULL, &tmpmask);
  2049. if (err) {
  2050. update_flag(CS_CPU_EXCLUSIVE, cs, 0);
  2051. goto out;
  2052. }
  2053. if (new_prs == PRS_ISOLATED) {
  2054. /*
  2055. * Disable the load balance flag should not return an
  2056. * error unless the system is running out of memory.
  2057. */
  2058. update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
  2059. sched_domain_rebuilt = true;
  2060. }
  2061. } else if (old_prs && new_prs) {
  2062. /*
  2063. * A change in load balance state only, no change in cpumasks.
  2064. */
  2065. update_flag(CS_SCHED_LOAD_BALANCE, cs, (new_prs != PRS_ISOLATED));
  2066. sched_domain_rebuilt = true;
  2067. goto out; /* Sched domain is rebuilt in update_flag() */
  2068. } else {
  2069. /*
  2070. * Switching back to member is always allowed even if it
  2071. * disables child partitions.
  2072. */
  2073. update_parent_subparts_cpumask(cs, partcmd_disable, NULL,
  2074. &tmpmask);
  2075. /*
  2076. * If there are child partitions, they will all become invalid.
  2077. */
  2078. if (unlikely(cs->nr_subparts_cpus)) {
  2079. spin_lock_irq(&callback_lock);
  2080. cs->nr_subparts_cpus = 0;
  2081. cpumask_clear(cs->subparts_cpus);
  2082. compute_effective_cpumask(cs->effective_cpus, cs, parent);
  2083. spin_unlock_irq(&callback_lock);
  2084. }
  2085. /* Turning off CS_CPU_EXCLUSIVE will not return error */
  2086. update_flag(CS_CPU_EXCLUSIVE, cs, 0);
  2087. if (!is_sched_load_balance(cs)) {
  2088. /* Make sure load balance is on */
  2089. update_flag(CS_SCHED_LOAD_BALANCE, cs, 1);
  2090. sched_domain_rebuilt = true;
  2091. }
  2092. }
  2093. update_tasks_cpumask(parent, tmpmask.new_cpus);
  2094. if (parent->child_ecpus_count)
  2095. update_sibling_cpumasks(parent, cs, &tmpmask);
  2096. if (!sched_domain_rebuilt)
  2097. rebuild_sched_domains_locked();
  2098. out:
  2099. /*
  2100. * Make partition invalid if an error happen
  2101. */
  2102. if (err)
  2103. new_prs = -new_prs;
  2104. spin_lock_irq(&callback_lock);
  2105. cs->partition_root_state = new_prs;
  2106. WRITE_ONCE(cs->prs_err, err);
  2107. spin_unlock_irq(&callback_lock);
  2108. /*
  2109. * Update child cpusets, if present.
  2110. * Force update if switching back to member.
  2111. */
  2112. if (!list_empty(&cs->css.children))
  2113. update_cpumasks_hier(cs, &tmpmask, !new_prs);
  2114. notify_partition_change(cs, old_prs);
  2115. free_cpumasks(NULL, &tmpmask);
  2116. return 0;
  2117. }
  2118. /*
  2119. * Frequency meter - How fast is some event occurring?
  2120. *
  2121. * These routines manage a digitally filtered, constant time based,
  2122. * event frequency meter. There are four routines:
  2123. * fmeter_init() - initialize a frequency meter.
  2124. * fmeter_markevent() - called each time the event happens.
  2125. * fmeter_getrate() - returns the recent rate of such events.
  2126. * fmeter_update() - internal routine used to update fmeter.
  2127. *
  2128. * A common data structure is passed to each of these routines,
  2129. * which is used to keep track of the state required to manage the
  2130. * frequency meter and its digital filter.
  2131. *
  2132. * The filter works on the number of events marked per unit time.
  2133. * The filter is single-pole low-pass recursive (IIR). The time unit
  2134. * is 1 second. Arithmetic is done using 32-bit integers scaled to
  2135. * simulate 3 decimal digits of precision (multiplied by 1000).
  2136. *
  2137. * With an FM_COEF of 933, and a time base of 1 second, the filter
  2138. * has a half-life of 10 seconds, meaning that if the events quit
  2139. * happening, then the rate returned from the fmeter_getrate()
  2140. * will be cut in half each 10 seconds, until it converges to zero.
  2141. *
  2142. * It is not worth doing a real infinitely recursive filter. If more
  2143. * than FM_MAXTICKS ticks have elapsed since the last filter event,
  2144. * just compute FM_MAXTICKS ticks worth, by which point the level
  2145. * will be stable.
  2146. *
  2147. * Limit the count of unprocessed events to FM_MAXCNT, so as to avoid
  2148. * arithmetic overflow in the fmeter_update() routine.
  2149. *
  2150. * Given the simple 32 bit integer arithmetic used, this meter works
  2151. * best for reporting rates between one per millisecond (msec) and
  2152. * one per 32 (approx) seconds. At constant rates faster than one
  2153. * per msec it maxes out at values just under 1,000,000. At constant
  2154. * rates between one per msec, and one per second it will stabilize
  2155. * to a value N*1000, where N is the rate of events per second.
  2156. * At constant rates between one per second and one per 32 seconds,
  2157. * it will be choppy, moving up on the seconds that have an event,
  2158. * and then decaying until the next event. At rates slower than
  2159. * about one in 32 seconds, it decays all the way back to zero between
  2160. * each event.
  2161. */
  2162. #define FM_COEF 933 /* coefficient for half-life of 10 secs */
  2163. #define FM_MAXTICKS ((u32)99) /* useless computing more ticks than this */
  2164. #define FM_MAXCNT 1000000 /* limit cnt to avoid overflow */
  2165. #define FM_SCALE 1000 /* faux fixed point scale */
  2166. /* Initialize a frequency meter */
  2167. static void fmeter_init(struct fmeter *fmp)
  2168. {
  2169. fmp->cnt = 0;
  2170. fmp->val = 0;
  2171. fmp->time = 0;
  2172. spin_lock_init(&fmp->lock);
  2173. }
  2174. /* Internal meter update - process cnt events and update value */
  2175. static void fmeter_update(struct fmeter *fmp)
  2176. {
  2177. time64_t now;
  2178. u32 ticks;
  2179. now = ktime_get_seconds();
  2180. ticks = now - fmp->time;
  2181. if (ticks == 0)
  2182. return;
  2183. ticks = min(FM_MAXTICKS, ticks);
  2184. while (ticks-- > 0)
  2185. fmp->val = (FM_COEF * fmp->val) / FM_SCALE;
  2186. fmp->time = now;
  2187. fmp->val += ((FM_SCALE - FM_COEF) * fmp->cnt) / FM_SCALE;
  2188. fmp->cnt = 0;
  2189. }
  2190. /* Process any previous ticks, then bump cnt by one (times scale). */
  2191. static void fmeter_markevent(struct fmeter *fmp)
  2192. {
  2193. spin_lock(&fmp->lock);
  2194. fmeter_update(fmp);
  2195. fmp->cnt = min(FM_MAXCNT, fmp->cnt + FM_SCALE);
  2196. spin_unlock(&fmp->lock);
  2197. }
  2198. /* Process any previous ticks, then return current value. */
  2199. static int fmeter_getrate(struct fmeter *fmp)
  2200. {
  2201. int val;
  2202. spin_lock(&fmp->lock);
  2203. fmeter_update(fmp);
  2204. val = fmp->val;
  2205. spin_unlock(&fmp->lock);
  2206. return val;
  2207. }
  2208. static struct cpuset *cpuset_attach_old_cs;
  2209. /*
  2210. * Check to see if a cpuset can accept a new task
  2211. * For v1, cpus_allowed and mems_allowed can't be empty.
  2212. * For v2, effective_cpus can't be empty.
  2213. * Note that in v1, effective_cpus = cpus_allowed.
  2214. */
  2215. static int cpuset_can_attach_check(struct cpuset *cs)
  2216. {
  2217. if (cpumask_empty(cs->effective_cpus) ||
  2218. (!is_in_v2_mode() && nodes_empty(cs->mems_allowed)))
  2219. return -ENOSPC;
  2220. return 0;
  2221. }
  2222. static void reset_migrate_dl_data(struct cpuset *cs)
  2223. {
  2224. cs->nr_migrate_dl_tasks = 0;
  2225. cs->sum_migrate_dl_bw = 0;
  2226. }
  2227. /* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */
  2228. static int cpuset_can_attach(struct cgroup_taskset *tset)
  2229. {
  2230. struct cgroup_subsys_state *css;
  2231. struct cpuset *cs, *oldcs;
  2232. struct task_struct *task;
  2233. int ret;
  2234. /* used later by cpuset_attach() */
  2235. cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css));
  2236. oldcs = cpuset_attach_old_cs;
  2237. cs = css_cs(css);
  2238. mutex_lock(&cpuset_mutex);
  2239. /* Check to see if task is allowed in the cpuset */
  2240. ret = cpuset_can_attach_check(cs);
  2241. if (ret)
  2242. goto out_unlock;
  2243. cgroup_taskset_for_each(task, css, tset) {
  2244. ret = task_can_attach(task);
  2245. if (ret)
  2246. goto out_unlock;
  2247. ret = security_task_setscheduler(task);
  2248. if (ret)
  2249. goto out_unlock;
  2250. if (dl_task(task)) {
  2251. cs->nr_migrate_dl_tasks++;
  2252. cs->sum_migrate_dl_bw += task->dl.dl_bw;
  2253. }
  2254. }
  2255. if (!cs->nr_migrate_dl_tasks)
  2256. goto out_success;
  2257. if (!cpumask_intersects(oldcs->effective_cpus, cs->effective_cpus)) {
  2258. int cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus);
  2259. if (unlikely(cpu >= nr_cpu_ids)) {
  2260. reset_migrate_dl_data(cs);
  2261. ret = -EINVAL;
  2262. goto out_unlock;
  2263. }
  2264. ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw);
  2265. if (ret) {
  2266. reset_migrate_dl_data(cs);
  2267. goto out_unlock;
  2268. }
  2269. }
  2270. out_success:
  2271. /*
  2272. * Mark attach is in progress. This makes validate_change() fail
  2273. * changes which zero cpus/mems_allowed.
  2274. */
  2275. cs->attach_in_progress++;
  2276. out_unlock:
  2277. mutex_unlock(&cpuset_mutex);
  2278. return ret;
  2279. }
  2280. static void cpuset_cancel_attach(struct cgroup_taskset *tset)
  2281. {
  2282. struct cgroup_subsys_state *css;
  2283. struct cpuset *cs;
  2284. cgroup_taskset_first(tset, &css);
  2285. cs = css_cs(css);
  2286. mutex_lock(&cpuset_mutex);
  2287. cs->attach_in_progress--;
  2288. if (!cs->attach_in_progress)
  2289. wake_up(&cpuset_attach_wq);
  2290. if (cs->nr_migrate_dl_tasks) {
  2291. int cpu = cpumask_any(cs->effective_cpus);
  2292. dl_bw_free(cpu, cs->sum_migrate_dl_bw);
  2293. reset_migrate_dl_data(cs);
  2294. }
  2295. mutex_unlock(&cpuset_mutex);
  2296. }
  2297. /*
  2298. * Protected by cpuset_mutex. cpus_attach is used only by cpuset_attach_task()
  2299. * but we can't allocate it dynamically there. Define it global and
  2300. * allocate from cpuset_init().
  2301. */
  2302. static cpumask_var_t cpus_attach;
  2303. static nodemask_t cpuset_attach_nodemask_to;
  2304. static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task)
  2305. {
  2306. lockdep_assert_held(&cpuset_mutex);
  2307. if (cs != &top_cpuset)
  2308. guarantee_online_cpus(task, cpus_attach);
  2309. else
  2310. cpumask_copy(cpus_attach, task_cpu_possible_mask(task));
  2311. /*
  2312. * can_attach beforehand should guarantee that this doesn't
  2313. * fail. TODO: have a better way to handle failure here
  2314. */
  2315. WARN_ON_ONCE(update_cpus_allowed(cs, task, cpus_attach));
  2316. cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to);
  2317. cpuset_update_task_spread_flags(cs, task);
  2318. }
  2319. static void cpuset_attach(struct cgroup_taskset *tset)
  2320. {
  2321. struct task_struct *task;
  2322. struct task_struct *leader;
  2323. struct cgroup_subsys_state *css;
  2324. struct cpuset *cs;
  2325. struct cpuset *oldcs = cpuset_attach_old_cs;
  2326. cgroup_taskset_first(tset, &css);
  2327. cs = css_cs(css);
  2328. lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */
  2329. mutex_lock(&cpuset_mutex);
  2330. guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
  2331. cgroup_taskset_for_each(task, css, tset)
  2332. cpuset_attach_task(cs, task);
  2333. /*
  2334. * Change mm for all threadgroup leaders. This is expensive and may
  2335. * sleep and should be moved outside migration path proper.
  2336. */
  2337. cpuset_attach_nodemask_to = cs->effective_mems;
  2338. cgroup_taskset_for_each_leader(leader, css, tset) {
  2339. struct mm_struct *mm = get_task_mm(leader);
  2340. if (mm) {
  2341. mpol_rebind_mm(mm, &cpuset_attach_nodemask_to);
  2342. /*
  2343. * old_mems_allowed is the same with mems_allowed
  2344. * here, except if this task is being moved
  2345. * automatically due to hotplug. In that case
  2346. * @mems_allowed has been updated and is empty, so
  2347. * @old_mems_allowed is the right nodesets that we
  2348. * migrate mm from.
  2349. */
  2350. if (is_memory_migrate(cs))
  2351. cpuset_migrate_mm(mm, &oldcs->old_mems_allowed,
  2352. &cpuset_attach_nodemask_to);
  2353. else
  2354. mmput(mm);
  2355. }
  2356. }
  2357. cs->old_mems_allowed = cpuset_attach_nodemask_to;
  2358. if (cs->nr_migrate_dl_tasks) {
  2359. cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks;
  2360. oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks;
  2361. reset_migrate_dl_data(cs);
  2362. }
  2363. cs->attach_in_progress--;
  2364. if (!cs->attach_in_progress)
  2365. wake_up(&cpuset_attach_wq);
  2366. mutex_unlock(&cpuset_mutex);
  2367. }
  2368. /* The various types of files and directories in a cpuset file system */
  2369. typedef enum {
  2370. FILE_MEMORY_MIGRATE,
  2371. FILE_CPULIST,
  2372. FILE_MEMLIST,
  2373. FILE_EFFECTIVE_CPULIST,
  2374. FILE_EFFECTIVE_MEMLIST,
  2375. FILE_SUBPARTS_CPULIST,
  2376. FILE_CPU_EXCLUSIVE,
  2377. FILE_MEM_EXCLUSIVE,
  2378. FILE_MEM_HARDWALL,
  2379. FILE_SCHED_LOAD_BALANCE,
  2380. FILE_PARTITION_ROOT,
  2381. FILE_SCHED_RELAX_DOMAIN_LEVEL,
  2382. FILE_MEMORY_PRESSURE_ENABLED,
  2383. FILE_MEMORY_PRESSURE,
  2384. FILE_SPREAD_PAGE,
  2385. FILE_SPREAD_SLAB,
  2386. } cpuset_filetype_t;
  2387. static int cpuset_write_u64(struct cgroup_subsys_state *css, struct cftype *cft,
  2388. u64 val)
  2389. {
  2390. struct cpuset *cs = css_cs(css);
  2391. cpuset_filetype_t type = cft->private;
  2392. int retval = 0;
  2393. cpus_read_lock();
  2394. mutex_lock(&cpuset_mutex);
  2395. if (!is_cpuset_online(cs)) {
  2396. retval = -ENODEV;
  2397. goto out_unlock;
  2398. }
  2399. switch (type) {
  2400. case FILE_CPU_EXCLUSIVE:
  2401. retval = update_flag(CS_CPU_EXCLUSIVE, cs, val);
  2402. break;
  2403. case FILE_MEM_EXCLUSIVE:
  2404. retval = update_flag(CS_MEM_EXCLUSIVE, cs, val);
  2405. break;
  2406. case FILE_MEM_HARDWALL:
  2407. retval = update_flag(CS_MEM_HARDWALL, cs, val);
  2408. break;
  2409. case FILE_SCHED_LOAD_BALANCE:
  2410. retval = update_flag(CS_SCHED_LOAD_BALANCE, cs, val);
  2411. break;
  2412. case FILE_MEMORY_MIGRATE:
  2413. retval = update_flag(CS_MEMORY_MIGRATE, cs, val);
  2414. break;
  2415. case FILE_MEMORY_PRESSURE_ENABLED:
  2416. cpuset_memory_pressure_enabled = !!val;
  2417. break;
  2418. case FILE_SPREAD_PAGE:
  2419. retval = update_flag(CS_SPREAD_PAGE, cs, val);
  2420. break;
  2421. case FILE_SPREAD_SLAB:
  2422. retval = update_flag(CS_SPREAD_SLAB, cs, val);
  2423. break;
  2424. default:
  2425. retval = -EINVAL;
  2426. break;
  2427. }
  2428. out_unlock:
  2429. mutex_unlock(&cpuset_mutex);
  2430. cpus_read_unlock();
  2431. return retval;
  2432. }
  2433. static int cpuset_write_s64(struct cgroup_subsys_state *css, struct cftype *cft,
  2434. s64 val)
  2435. {
  2436. struct cpuset *cs = css_cs(css);
  2437. cpuset_filetype_t type = cft->private;
  2438. int retval = -ENODEV;
  2439. cpus_read_lock();
  2440. mutex_lock(&cpuset_mutex);
  2441. if (!is_cpuset_online(cs))
  2442. goto out_unlock;
  2443. switch (type) {
  2444. case FILE_SCHED_RELAX_DOMAIN_LEVEL:
  2445. retval = update_relax_domain_level(cs, val);
  2446. break;
  2447. default:
  2448. retval = -EINVAL;
  2449. break;
  2450. }
  2451. out_unlock:
  2452. mutex_unlock(&cpuset_mutex);
  2453. cpus_read_unlock();
  2454. return retval;
  2455. }
  2456. /*
  2457. * Common handling for a write to a "cpus" or "mems" file.
  2458. */
  2459. static ssize_t cpuset_write_resmask(struct kernfs_open_file *of,
  2460. char *buf, size_t nbytes, loff_t off)
  2461. {
  2462. struct cpuset *cs = css_cs(of_css(of));
  2463. struct cpuset *trialcs;
  2464. int retval = -ENODEV;
  2465. buf = strstrip(buf);
  2466. /*
  2467. * CPU or memory hotunplug may leave @cs w/o any execution
  2468. * resources, in which case the hotplug code asynchronously updates
  2469. * configuration and transfers all tasks to the nearest ancestor
  2470. * which can execute.
  2471. *
  2472. * As writes to "cpus" or "mems" may restore @cs's execution
  2473. * resources, wait for the previously scheduled operations before
  2474. * proceeding, so that we don't end up keep removing tasks added
  2475. * after execution capability is restored.
  2476. *
  2477. * cpuset_hotplug_work calls back into cgroup core via
  2478. * cgroup_transfer_tasks() and waiting for it from a cgroupfs
  2479. * operation like this one can lead to a deadlock through kernfs
  2480. * active_ref protection. Let's break the protection. Losing the
  2481. * protection is okay as we check whether @cs is online after
  2482. * grabbing cpuset_mutex anyway. This only happens on the legacy
  2483. * hierarchies.
  2484. */
  2485. css_get(&cs->css);
  2486. kernfs_break_active_protection(of->kn);
  2487. flush_work(&cpuset_hotplug_work);
  2488. cpus_read_lock();
  2489. mutex_lock(&cpuset_mutex);
  2490. if (!is_cpuset_online(cs))
  2491. goto out_unlock;
  2492. trialcs = alloc_trial_cpuset(cs);
  2493. if (!trialcs) {
  2494. retval = -ENOMEM;
  2495. goto out_unlock;
  2496. }
  2497. switch (of_cft(of)->private) {
  2498. case FILE_CPULIST:
  2499. retval = update_cpumask(cs, trialcs, buf);
  2500. break;
  2501. case FILE_MEMLIST:
  2502. retval = update_nodemask(cs, trialcs, buf);
  2503. break;
  2504. default:
  2505. retval = -EINVAL;
  2506. break;
  2507. }
  2508. free_cpuset(trialcs);
  2509. out_unlock:
  2510. mutex_unlock(&cpuset_mutex);
  2511. cpus_read_unlock();
  2512. kernfs_unbreak_active_protection(of->kn);
  2513. css_put(&cs->css);
  2514. flush_workqueue(cpuset_migrate_mm_wq);
  2515. return retval ?: nbytes;
  2516. }
  2517. /*
  2518. * These ascii lists should be read in a single call, by using a user
  2519. * buffer large enough to hold the entire map. If read in smaller
  2520. * chunks, there is no guarantee of atomicity. Since the display format
  2521. * used, list of ranges of sequential numbers, is variable length,
  2522. * and since these maps can change value dynamically, one could read
  2523. * gibberish by doing partial reads while a list was changing.
  2524. */
  2525. static int cpuset_common_seq_show(struct seq_file *sf, void *v)
  2526. {
  2527. struct cpuset *cs = css_cs(seq_css(sf));
  2528. cpuset_filetype_t type = seq_cft(sf)->private;
  2529. int ret = 0;
  2530. spin_lock_irq(&callback_lock);
  2531. switch (type) {
  2532. case FILE_CPULIST:
  2533. seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->cpus_requested));
  2534. break;
  2535. case FILE_MEMLIST:
  2536. seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->mems_allowed));
  2537. break;
  2538. case FILE_EFFECTIVE_CPULIST:
  2539. seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->effective_cpus));
  2540. break;
  2541. case FILE_EFFECTIVE_MEMLIST:
  2542. seq_printf(sf, "%*pbl\n", nodemask_pr_args(&cs->effective_mems));
  2543. break;
  2544. case FILE_SUBPARTS_CPULIST:
  2545. seq_printf(sf, "%*pbl\n", cpumask_pr_args(cs->subparts_cpus));
  2546. break;
  2547. default:
  2548. ret = -EINVAL;
  2549. }
  2550. spin_unlock_irq(&callback_lock);
  2551. return ret;
  2552. }
  2553. static u64 cpuset_read_u64(struct cgroup_subsys_state *css, struct cftype *cft)
  2554. {
  2555. struct cpuset *cs = css_cs(css);
  2556. cpuset_filetype_t type = cft->private;
  2557. switch (type) {
  2558. case FILE_CPU_EXCLUSIVE:
  2559. return is_cpu_exclusive(cs);
  2560. case FILE_MEM_EXCLUSIVE:
  2561. return is_mem_exclusive(cs);
  2562. case FILE_MEM_HARDWALL:
  2563. return is_mem_hardwall(cs);
  2564. case FILE_SCHED_LOAD_BALANCE:
  2565. return is_sched_load_balance(cs);
  2566. case FILE_MEMORY_MIGRATE:
  2567. return is_memory_migrate(cs);
  2568. case FILE_MEMORY_PRESSURE_ENABLED:
  2569. return cpuset_memory_pressure_enabled;
  2570. case FILE_MEMORY_PRESSURE:
  2571. return fmeter_getrate(&cs->fmeter);
  2572. case FILE_SPREAD_PAGE:
  2573. return is_spread_page(cs);
  2574. case FILE_SPREAD_SLAB:
  2575. return is_spread_slab(cs);
  2576. default:
  2577. BUG();
  2578. }
  2579. /* Unreachable but makes gcc happy */
  2580. return 0;
  2581. }
  2582. static s64 cpuset_read_s64(struct cgroup_subsys_state *css, struct cftype *cft)
  2583. {
  2584. struct cpuset *cs = css_cs(css);
  2585. cpuset_filetype_t type = cft->private;
  2586. switch (type) {
  2587. case FILE_SCHED_RELAX_DOMAIN_LEVEL:
  2588. return cs->relax_domain_level;
  2589. default:
  2590. BUG();
  2591. }
  2592. /* Unreachable but makes gcc happy */
  2593. return 0;
  2594. }
  2595. static int sched_partition_show(struct seq_file *seq, void *v)
  2596. {
  2597. struct cpuset *cs = css_cs(seq_css(seq));
  2598. const char *err, *type = NULL;
  2599. switch (cs->partition_root_state) {
  2600. case PRS_ROOT:
  2601. seq_puts(seq, "root\n");
  2602. break;
  2603. case PRS_ISOLATED:
  2604. seq_puts(seq, "isolated\n");
  2605. break;
  2606. case PRS_MEMBER:
  2607. seq_puts(seq, "member\n");
  2608. break;
  2609. case PRS_INVALID_ROOT:
  2610. type = "root";
  2611. fallthrough;
  2612. case PRS_INVALID_ISOLATED:
  2613. if (!type)
  2614. type = "isolated";
  2615. err = perr_strings[READ_ONCE(cs->prs_err)];
  2616. if (err)
  2617. seq_printf(seq, "%s invalid (%s)\n", type, err);
  2618. else
  2619. seq_printf(seq, "%s invalid\n", type);
  2620. break;
  2621. }
  2622. return 0;
  2623. }
  2624. static ssize_t sched_partition_write(struct kernfs_open_file *of, char *buf,
  2625. size_t nbytes, loff_t off)
  2626. {
  2627. struct cpuset *cs = css_cs(of_css(of));
  2628. int val;
  2629. int retval = -ENODEV;
  2630. buf = strstrip(buf);
  2631. /*
  2632. * Convert "root" to ENABLED, and convert "member" to DISABLED.
  2633. */
  2634. if (!strcmp(buf, "root"))
  2635. val = PRS_ROOT;
  2636. else if (!strcmp(buf, "member"))
  2637. val = PRS_MEMBER;
  2638. else if (!strcmp(buf, "isolated"))
  2639. val = PRS_ISOLATED;
  2640. else
  2641. return -EINVAL;
  2642. css_get(&cs->css);
  2643. cpus_read_lock();
  2644. mutex_lock(&cpuset_mutex);
  2645. if (!is_cpuset_online(cs))
  2646. goto out_unlock;
  2647. retval = update_prstate(cs, val);
  2648. out_unlock:
  2649. mutex_unlock(&cpuset_mutex);
  2650. cpus_read_unlock();
  2651. css_put(&cs->css);
  2652. return retval ?: nbytes;
  2653. }
  2654. /*
  2655. * for the common functions, 'private' gives the type of file
  2656. */
  2657. static struct cftype legacy_files[] = {
  2658. {
  2659. .name = "cpus",
  2660. .seq_show = cpuset_common_seq_show,
  2661. .write = cpuset_write_resmask,
  2662. .max_write_len = (100U + 6 * NR_CPUS),
  2663. .private = FILE_CPULIST,
  2664. },
  2665. {
  2666. .name = "mems",
  2667. .seq_show = cpuset_common_seq_show,
  2668. .write = cpuset_write_resmask,
  2669. .max_write_len = (100U + 6 * MAX_NUMNODES),
  2670. .private = FILE_MEMLIST,
  2671. },
  2672. {
  2673. .name = "effective_cpus",
  2674. .seq_show = cpuset_common_seq_show,
  2675. .private = FILE_EFFECTIVE_CPULIST,
  2676. },
  2677. {
  2678. .name = "effective_mems",
  2679. .seq_show = cpuset_common_seq_show,
  2680. .private = FILE_EFFECTIVE_MEMLIST,
  2681. },
  2682. {
  2683. .name = "cpu_exclusive",
  2684. .read_u64 = cpuset_read_u64,
  2685. .write_u64 = cpuset_write_u64,
  2686. .private = FILE_CPU_EXCLUSIVE,
  2687. },
  2688. {
  2689. .name = "mem_exclusive",
  2690. .read_u64 = cpuset_read_u64,
  2691. .write_u64 = cpuset_write_u64,
  2692. .private = FILE_MEM_EXCLUSIVE,
  2693. },
  2694. {
  2695. .name = "mem_hardwall",
  2696. .read_u64 = cpuset_read_u64,
  2697. .write_u64 = cpuset_write_u64,
  2698. .private = FILE_MEM_HARDWALL,
  2699. },
  2700. {
  2701. .name = "sched_load_balance",
  2702. .read_u64 = cpuset_read_u64,
  2703. .write_u64 = cpuset_write_u64,
  2704. .private = FILE_SCHED_LOAD_BALANCE,
  2705. },
  2706. {
  2707. .name = "sched_relax_domain_level",
  2708. .read_s64 = cpuset_read_s64,
  2709. .write_s64 = cpuset_write_s64,
  2710. .private = FILE_SCHED_RELAX_DOMAIN_LEVEL,
  2711. },
  2712. {
  2713. .name = "memory_migrate",
  2714. .read_u64 = cpuset_read_u64,
  2715. .write_u64 = cpuset_write_u64,
  2716. .private = FILE_MEMORY_MIGRATE,
  2717. },
  2718. {
  2719. .name = "memory_pressure",
  2720. .read_u64 = cpuset_read_u64,
  2721. .private = FILE_MEMORY_PRESSURE,
  2722. },
  2723. {
  2724. .name = "memory_spread_page",
  2725. .read_u64 = cpuset_read_u64,
  2726. .write_u64 = cpuset_write_u64,
  2727. .private = FILE_SPREAD_PAGE,
  2728. },
  2729. {
  2730. .name = "memory_spread_slab",
  2731. .read_u64 = cpuset_read_u64,
  2732. .write_u64 = cpuset_write_u64,
  2733. .private = FILE_SPREAD_SLAB,
  2734. },
  2735. {
  2736. .name = "memory_pressure_enabled",
  2737. .flags = CFTYPE_ONLY_ON_ROOT,
  2738. .read_u64 = cpuset_read_u64,
  2739. .write_u64 = cpuset_write_u64,
  2740. .private = FILE_MEMORY_PRESSURE_ENABLED,
  2741. },
  2742. { } /* terminate */
  2743. };
  2744. /*
  2745. * This is currently a minimal set for the default hierarchy. It can be
  2746. * expanded later on by migrating more features and control files from v1.
  2747. */
  2748. static struct cftype dfl_files[] = {
  2749. {
  2750. .name = "cpus",
  2751. .seq_show = cpuset_common_seq_show,
  2752. .write = cpuset_write_resmask,
  2753. .max_write_len = (100U + 6 * NR_CPUS),
  2754. .private = FILE_CPULIST,
  2755. .flags = CFTYPE_NOT_ON_ROOT,
  2756. },
  2757. {
  2758. .name = "mems",
  2759. .seq_show = cpuset_common_seq_show,
  2760. .write = cpuset_write_resmask,
  2761. .max_write_len = (100U + 6 * MAX_NUMNODES),
  2762. .private = FILE_MEMLIST,
  2763. .flags = CFTYPE_NOT_ON_ROOT,
  2764. },
  2765. {
  2766. .name = "cpus.effective",
  2767. .seq_show = cpuset_common_seq_show,
  2768. .private = FILE_EFFECTIVE_CPULIST,
  2769. },
  2770. {
  2771. .name = "mems.effective",
  2772. .seq_show = cpuset_common_seq_show,
  2773. .private = FILE_EFFECTIVE_MEMLIST,
  2774. },
  2775. {
  2776. .name = "cpus.partition",
  2777. .seq_show = sched_partition_show,
  2778. .write = sched_partition_write,
  2779. .private = FILE_PARTITION_ROOT,
  2780. .flags = CFTYPE_NOT_ON_ROOT,
  2781. .file_offset = offsetof(struct cpuset, partition_file),
  2782. },
  2783. {
  2784. .name = "cpus.subpartitions",
  2785. .seq_show = cpuset_common_seq_show,
  2786. .private = FILE_SUBPARTS_CPULIST,
  2787. .flags = CFTYPE_DEBUG,
  2788. },
  2789. { } /* terminate */
  2790. };
  2791. /*
  2792. * cpuset_css_alloc - allocate a cpuset css
  2793. * cgrp: control group that the new cpuset will be part of
  2794. */
  2795. static struct cgroup_subsys_state *
  2796. cpuset_css_alloc(struct cgroup_subsys_state *parent_css)
  2797. {
  2798. struct cpuset *cs;
  2799. if (!parent_css)
  2800. return &top_cpuset.css;
  2801. cs = kzalloc(sizeof(*cs), GFP_KERNEL);
  2802. if (!cs)
  2803. return ERR_PTR(-ENOMEM);
  2804. if (alloc_cpumasks(cs, NULL)) {
  2805. kfree(cs);
  2806. return ERR_PTR(-ENOMEM);
  2807. }
  2808. __set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
  2809. nodes_clear(cs->mems_allowed);
  2810. nodes_clear(cs->effective_mems);
  2811. fmeter_init(&cs->fmeter);
  2812. cs->relax_domain_level = -1;
  2813. /* Set CS_MEMORY_MIGRATE for default hierarchy */
  2814. if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys))
  2815. __set_bit(CS_MEMORY_MIGRATE, &cs->flags);
  2816. return &cs->css;
  2817. }
  2818. static int cpuset_css_online(struct cgroup_subsys_state *css)
  2819. {
  2820. struct cpuset *cs = css_cs(css);
  2821. struct cpuset *parent = parent_cs(cs);
  2822. struct cpuset *tmp_cs;
  2823. struct cgroup_subsys_state *pos_css;
  2824. if (!parent)
  2825. return 0;
  2826. cpus_read_lock();
  2827. mutex_lock(&cpuset_mutex);
  2828. set_bit(CS_ONLINE, &cs->flags);
  2829. if (is_spread_page(parent))
  2830. set_bit(CS_SPREAD_PAGE, &cs->flags);
  2831. if (is_spread_slab(parent))
  2832. set_bit(CS_SPREAD_SLAB, &cs->flags);
  2833. cpuset_inc();
  2834. spin_lock_irq(&callback_lock);
  2835. if (is_in_v2_mode()) {
  2836. cpumask_copy(cs->effective_cpus, parent->effective_cpus);
  2837. cs->effective_mems = parent->effective_mems;
  2838. cs->use_parent_ecpus = true;
  2839. parent->child_ecpus_count++;
  2840. }
  2841. /*
  2842. * For v2, clear CS_SCHED_LOAD_BALANCE if parent is isolated
  2843. */
  2844. if (cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
  2845. !is_sched_load_balance(parent))
  2846. clear_bit(CS_SCHED_LOAD_BALANCE, &cs->flags);
  2847. spin_unlock_irq(&callback_lock);
  2848. if (!test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags))
  2849. goto out_unlock;
  2850. /*
  2851. * Clone @parent's configuration if CGRP_CPUSET_CLONE_CHILDREN is
  2852. * set. This flag handling is implemented in cgroup core for
  2853. * historical reasons - the flag may be specified during mount.
  2854. *
  2855. * Currently, if any sibling cpusets have exclusive cpus or mem, we
  2856. * refuse to clone the configuration - thereby refusing the task to
  2857. * be entered, and as a result refusing the sys_unshare() or
  2858. * clone() which initiated it. If this becomes a problem for some
  2859. * users who wish to allow that scenario, then this could be
  2860. * changed to grant parent->cpus_allowed-sibling_cpus_exclusive
  2861. * (and likewise for mems) to the new cgroup.
  2862. */
  2863. rcu_read_lock();
  2864. cpuset_for_each_child(tmp_cs, pos_css, parent) {
  2865. if (is_mem_exclusive(tmp_cs) || is_cpu_exclusive(tmp_cs)) {
  2866. rcu_read_unlock();
  2867. goto out_unlock;
  2868. }
  2869. }
  2870. rcu_read_unlock();
  2871. spin_lock_irq(&callback_lock);
  2872. cs->mems_allowed = parent->mems_allowed;
  2873. cs->effective_mems = parent->mems_allowed;
  2874. cpumask_copy(cs->cpus_allowed, parent->cpus_allowed);
  2875. cpumask_copy(cs->cpus_requested, parent->cpus_requested);
  2876. cpumask_copy(cs->effective_cpus, parent->cpus_allowed);
  2877. spin_unlock_irq(&callback_lock);
  2878. out_unlock:
  2879. mutex_unlock(&cpuset_mutex);
  2880. cpus_read_unlock();
  2881. return 0;
  2882. }
  2883. /*
  2884. * If the cpuset being removed has its flag 'sched_load_balance'
  2885. * enabled, then simulate turning sched_load_balance off, which
  2886. * will call rebuild_sched_domains_locked(). That is not needed
  2887. * in the default hierarchy where only changes in partition
  2888. * will cause repartitioning.
  2889. *
  2890. * If the cpuset has the 'sched.partition' flag enabled, simulate
  2891. * turning 'sched.partition" off.
  2892. */
  2893. static void cpuset_css_offline(struct cgroup_subsys_state *css)
  2894. {
  2895. struct cpuset *cs = css_cs(css);
  2896. cpus_read_lock();
  2897. mutex_lock(&cpuset_mutex);
  2898. if (is_partition_valid(cs))
  2899. update_prstate(cs, 0);
  2900. if (!cgroup_subsys_on_dfl(cpuset_cgrp_subsys) &&
  2901. is_sched_load_balance(cs))
  2902. update_flag(CS_SCHED_LOAD_BALANCE, cs, 0);
  2903. if (cs->use_parent_ecpus) {
  2904. struct cpuset *parent = parent_cs(cs);
  2905. cs->use_parent_ecpus = false;
  2906. parent->child_ecpus_count--;
  2907. }
  2908. cpuset_dec();
  2909. clear_bit(CS_ONLINE, &cs->flags);
  2910. mutex_unlock(&cpuset_mutex);
  2911. cpus_read_unlock();
  2912. }
  2913. static void cpuset_css_free(struct cgroup_subsys_state *css)
  2914. {
  2915. struct cpuset *cs = css_cs(css);
  2916. free_cpuset(cs);
  2917. }
  2918. static void cpuset_bind(struct cgroup_subsys_state *root_css)
  2919. {
  2920. mutex_lock(&cpuset_mutex);
  2921. spin_lock_irq(&callback_lock);
  2922. if (is_in_v2_mode()) {
  2923. cpumask_copy(top_cpuset.cpus_allowed, cpu_possible_mask);
  2924. top_cpuset.mems_allowed = node_possible_map;
  2925. } else {
  2926. cpumask_copy(top_cpuset.cpus_allowed,
  2927. top_cpuset.effective_cpus);
  2928. top_cpuset.mems_allowed = top_cpuset.effective_mems;
  2929. }
  2930. spin_unlock_irq(&callback_lock);
  2931. mutex_unlock(&cpuset_mutex);
  2932. }
  2933. /*
  2934. * In case the child is cloned into a cpuset different from its parent,
  2935. * additional checks are done to see if the move is allowed.
  2936. */
  2937. static int cpuset_can_fork(struct task_struct *task, struct css_set *cset)
  2938. {
  2939. struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
  2940. bool same_cs;
  2941. int ret;
  2942. rcu_read_lock();
  2943. same_cs = (cs == task_cs(current));
  2944. rcu_read_unlock();
  2945. if (same_cs)
  2946. return 0;
  2947. lockdep_assert_held(&cgroup_mutex);
  2948. mutex_lock(&cpuset_mutex);
  2949. /* Check to see if task is allowed in the cpuset */
  2950. ret = cpuset_can_attach_check(cs);
  2951. if (ret)
  2952. goto out_unlock;
  2953. ret = task_can_attach(task);
  2954. if (ret)
  2955. goto out_unlock;
  2956. ret = security_task_setscheduler(task);
  2957. if (ret)
  2958. goto out_unlock;
  2959. /*
  2960. * Mark attach is in progress. This makes validate_change() fail
  2961. * changes which zero cpus/mems_allowed.
  2962. */
  2963. cs->attach_in_progress++;
  2964. out_unlock:
  2965. mutex_unlock(&cpuset_mutex);
  2966. return ret;
  2967. }
  2968. static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset)
  2969. {
  2970. struct cpuset *cs = css_cs(cset->subsys[cpuset_cgrp_id]);
  2971. bool same_cs;
  2972. rcu_read_lock();
  2973. same_cs = (cs == task_cs(current));
  2974. rcu_read_unlock();
  2975. if (same_cs)
  2976. return;
  2977. mutex_lock(&cpuset_mutex);
  2978. cs->attach_in_progress--;
  2979. if (!cs->attach_in_progress)
  2980. wake_up(&cpuset_attach_wq);
  2981. mutex_unlock(&cpuset_mutex);
  2982. }
  2983. /*
  2984. * Make sure the new task conform to the current state of its parent,
  2985. * which could have been changed by cpuset just after it inherits the
  2986. * state from the parent and before it sits on the cgroup's task list.
  2987. */
  2988. static void cpuset_fork(struct task_struct *task)
  2989. {
  2990. struct cpuset *cs;
  2991. bool same_cs, inherit_cpus = false;
  2992. rcu_read_lock();
  2993. cs = task_cs(task);
  2994. same_cs = (cs == task_cs(current));
  2995. rcu_read_unlock();
  2996. if (same_cs) {
  2997. if (cs == &top_cpuset)
  2998. return;
  2999. trace_android_rvh_cpuset_fork(task, &inherit_cpus);
  3000. if (!inherit_cpus)
  3001. set_cpus_allowed_ptr(task, current->cpus_ptr);
  3002. task->mems_allowed = current->mems_allowed;
  3003. return;
  3004. }
  3005. /* CLONE_INTO_CGROUP */
  3006. mutex_lock(&cpuset_mutex);
  3007. guarantee_online_mems(cs, &cpuset_attach_nodemask_to);
  3008. cpuset_attach_task(cs, task);
  3009. cs->attach_in_progress--;
  3010. if (!cs->attach_in_progress)
  3011. wake_up(&cpuset_attach_wq);
  3012. mutex_unlock(&cpuset_mutex);
  3013. }
  3014. struct cgroup_subsys cpuset_cgrp_subsys = {
  3015. .css_alloc = cpuset_css_alloc,
  3016. .css_online = cpuset_css_online,
  3017. .css_offline = cpuset_css_offline,
  3018. .css_free = cpuset_css_free,
  3019. .can_attach = cpuset_can_attach,
  3020. .cancel_attach = cpuset_cancel_attach,
  3021. .attach = cpuset_attach,
  3022. .post_attach = cpuset_post_attach,
  3023. .bind = cpuset_bind,
  3024. .can_fork = cpuset_can_fork,
  3025. .cancel_fork = cpuset_cancel_fork,
  3026. .fork = cpuset_fork,
  3027. .legacy_cftypes = legacy_files,
  3028. .dfl_cftypes = dfl_files,
  3029. .early_init = true,
  3030. .threaded = true,
  3031. };
  3032. /**
  3033. * cpuset_init - initialize cpusets at system boot
  3034. *
  3035. * Description: Initialize top_cpuset
  3036. **/
  3037. int __init cpuset_init(void)
  3038. {
  3039. BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_allowed, GFP_KERNEL));
  3040. BUG_ON(!alloc_cpumask_var(&top_cpuset.effective_cpus, GFP_KERNEL));
  3041. BUG_ON(!zalloc_cpumask_var(&top_cpuset.subparts_cpus, GFP_KERNEL));
  3042. BUG_ON(!alloc_cpumask_var(&top_cpuset.cpus_requested, GFP_KERNEL));
  3043. cpumask_setall(top_cpuset.cpus_allowed);
  3044. cpumask_setall(top_cpuset.cpus_requested);
  3045. nodes_setall(top_cpuset.mems_allowed);
  3046. cpumask_setall(top_cpuset.effective_cpus);
  3047. nodes_setall(top_cpuset.effective_mems);
  3048. fmeter_init(&top_cpuset.fmeter);
  3049. set_bit(CS_SCHED_LOAD_BALANCE, &top_cpuset.flags);
  3050. top_cpuset.relax_domain_level = -1;
  3051. BUG_ON(!alloc_cpumask_var(&cpus_attach, GFP_KERNEL));
  3052. return 0;
  3053. }
  3054. /*
  3055. * If CPU and/or memory hotplug handlers, below, unplug any CPUs
  3056. * or memory nodes, we need to walk over the cpuset hierarchy,
  3057. * removing that CPU or node from all cpusets. If this removes the
  3058. * last CPU or node from a cpuset, then move the tasks in the empty
  3059. * cpuset to its next-highest non-empty parent.
  3060. */
  3061. static void remove_tasks_in_empty_cpuset(struct cpuset *cs)
  3062. {
  3063. struct cpuset *parent;
  3064. /*
  3065. * Find its next-highest non-empty parent, (top cpuset
  3066. * has online cpus, so can't be empty).
  3067. */
  3068. parent = parent_cs(cs);
  3069. while (cpumask_empty(parent->cpus_allowed) ||
  3070. nodes_empty(parent->mems_allowed))
  3071. parent = parent_cs(parent);
  3072. if (cgroup_transfer_tasks(parent->css.cgroup, cs->css.cgroup)) {
  3073. pr_err("cpuset: failed to transfer tasks out of empty cpuset ");
  3074. pr_cont_cgroup_name(cs->css.cgroup);
  3075. pr_cont("\n");
  3076. }
  3077. }
  3078. static void
  3079. hotplug_update_tasks_legacy(struct cpuset *cs,
  3080. struct cpumask *new_cpus, nodemask_t *new_mems,
  3081. bool cpus_updated, bool mems_updated)
  3082. {
  3083. bool is_empty;
  3084. spin_lock_irq(&callback_lock);
  3085. cpumask_copy(cs->cpus_allowed, new_cpus);
  3086. cpumask_copy(cs->effective_cpus, new_cpus);
  3087. cs->mems_allowed = *new_mems;
  3088. cs->effective_mems = *new_mems;
  3089. spin_unlock_irq(&callback_lock);
  3090. /*
  3091. * Don't call update_tasks_cpumask() if the cpuset becomes empty,
  3092. * as the tasks will be migrated to an ancestor.
  3093. */
  3094. if (cpus_updated && !cpumask_empty(cs->cpus_allowed))
  3095. update_tasks_cpumask(cs, new_cpus);
  3096. if (mems_updated && !nodes_empty(cs->mems_allowed))
  3097. update_tasks_nodemask(cs);
  3098. is_empty = cpumask_empty(cs->cpus_allowed) ||
  3099. nodes_empty(cs->mems_allowed);
  3100. mutex_unlock(&cpuset_mutex);
  3101. /*
  3102. * Move tasks to the nearest ancestor with execution resources,
  3103. * This is full cgroup operation which will also call back into
  3104. * cpuset. Should be done outside any lock.
  3105. */
  3106. if (is_empty)
  3107. remove_tasks_in_empty_cpuset(cs);
  3108. mutex_lock(&cpuset_mutex);
  3109. }
  3110. static void
  3111. hotplug_update_tasks(struct cpuset *cs,
  3112. struct cpumask *new_cpus, nodemask_t *new_mems,
  3113. bool cpus_updated, bool mems_updated)
  3114. {
  3115. /* A partition root is allowed to have empty effective cpus */
  3116. if (cpumask_empty(new_cpus) && !is_partition_valid(cs))
  3117. cpumask_copy(new_cpus, parent_cs(cs)->effective_cpus);
  3118. if (nodes_empty(*new_mems))
  3119. *new_mems = parent_cs(cs)->effective_mems;
  3120. spin_lock_irq(&callback_lock);
  3121. cpumask_copy(cs->effective_cpus, new_cpus);
  3122. cs->effective_mems = *new_mems;
  3123. spin_unlock_irq(&callback_lock);
  3124. if (cpus_updated)
  3125. update_tasks_cpumask(cs, new_cpus);
  3126. if (mems_updated)
  3127. update_tasks_nodemask(cs);
  3128. }
  3129. static bool force_rebuild;
  3130. void cpuset_force_rebuild(void)
  3131. {
  3132. force_rebuild = true;
  3133. }
  3134. /**
  3135. * cpuset_hotplug_update_tasks - update tasks in a cpuset for hotunplug
  3136. * @cs: cpuset in interest
  3137. * @tmp: the tmpmasks structure pointer
  3138. *
  3139. * Compare @cs's cpu and mem masks against top_cpuset and if some have gone
  3140. * offline, update @cs accordingly. If @cs ends up with no CPU or memory,
  3141. * all its tasks are moved to the nearest ancestor with both resources.
  3142. */
  3143. static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp)
  3144. {
  3145. static cpumask_t new_cpus;
  3146. static nodemask_t new_mems;
  3147. bool cpus_updated;
  3148. bool mems_updated;
  3149. struct cpuset *parent;
  3150. retry:
  3151. wait_event(cpuset_attach_wq, cs->attach_in_progress == 0);
  3152. mutex_lock(&cpuset_mutex);
  3153. /*
  3154. * We have raced with task attaching. We wait until attaching
  3155. * is finished, so we won't attach a task to an empty cpuset.
  3156. */
  3157. if (cs->attach_in_progress) {
  3158. mutex_unlock(&cpuset_mutex);
  3159. goto retry;
  3160. }
  3161. parent = parent_cs(cs);
  3162. compute_effective_cpumask(&new_cpus, cs, parent);
  3163. nodes_and(new_mems, cs->mems_allowed, parent->effective_mems);
  3164. if (cs->nr_subparts_cpus)
  3165. /*
  3166. * Make sure that CPUs allocated to child partitions
  3167. * do not show up in effective_cpus.
  3168. */
  3169. cpumask_andnot(&new_cpus, &new_cpus, cs->subparts_cpus);
  3170. if (!tmp || !cs->partition_root_state)
  3171. goto update_tasks;
  3172. /*
  3173. * In the unlikely event that a partition root has empty
  3174. * effective_cpus with tasks, we will have to invalidate child
  3175. * partitions, if present, by setting nr_subparts_cpus to 0 to
  3176. * reclaim their cpus.
  3177. */
  3178. if (cs->nr_subparts_cpus && is_partition_valid(cs) &&
  3179. cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)) {
  3180. spin_lock_irq(&callback_lock);
  3181. cs->nr_subparts_cpus = 0;
  3182. cpumask_clear(cs->subparts_cpus);
  3183. spin_unlock_irq(&callback_lock);
  3184. compute_effective_cpumask(&new_cpus, cs, parent);
  3185. }
  3186. /*
  3187. * Force the partition to become invalid if either one of
  3188. * the following conditions hold:
  3189. * 1) empty effective cpus but not valid empty partition.
  3190. * 2) parent is invalid or doesn't grant any cpus to child
  3191. * partitions.
  3192. */
  3193. if (is_partition_valid(cs) && (!parent->nr_subparts_cpus ||
  3194. (cpumask_empty(&new_cpus) && partition_is_populated(cs, NULL)))) {
  3195. int old_prs, parent_prs;
  3196. update_parent_subparts_cpumask(cs, partcmd_disable, NULL, tmp);
  3197. if (cs->nr_subparts_cpus) {
  3198. spin_lock_irq(&callback_lock);
  3199. cs->nr_subparts_cpus = 0;
  3200. cpumask_clear(cs->subparts_cpus);
  3201. spin_unlock_irq(&callback_lock);
  3202. compute_effective_cpumask(&new_cpus, cs, parent);
  3203. }
  3204. old_prs = cs->partition_root_state;
  3205. parent_prs = parent->partition_root_state;
  3206. if (is_partition_valid(cs)) {
  3207. spin_lock_irq(&callback_lock);
  3208. make_partition_invalid(cs);
  3209. spin_unlock_irq(&callback_lock);
  3210. if (is_prs_invalid(parent_prs))
  3211. WRITE_ONCE(cs->prs_err, PERR_INVPARENT);
  3212. else if (!parent_prs)
  3213. WRITE_ONCE(cs->prs_err, PERR_NOTPART);
  3214. else
  3215. WRITE_ONCE(cs->prs_err, PERR_HOTPLUG);
  3216. notify_partition_change(cs, old_prs);
  3217. }
  3218. cpuset_force_rebuild();
  3219. }
  3220. /*
  3221. * On the other hand, an invalid partition root may be transitioned
  3222. * back to a regular one.
  3223. */
  3224. else if (is_partition_valid(parent) && is_partition_invalid(cs)) {
  3225. update_parent_subparts_cpumask(cs, partcmd_update, NULL, tmp);
  3226. if (is_partition_valid(cs))
  3227. cpuset_force_rebuild();
  3228. }
  3229. update_tasks:
  3230. cpus_updated = !cpumask_equal(&new_cpus, cs->effective_cpus);
  3231. mems_updated = !nodes_equal(new_mems, cs->effective_mems);
  3232. if (mems_updated)
  3233. check_insane_mems_config(&new_mems);
  3234. if (is_in_v2_mode())
  3235. hotplug_update_tasks(cs, &new_cpus, &new_mems,
  3236. cpus_updated, mems_updated);
  3237. else
  3238. hotplug_update_tasks_legacy(cs, &new_cpus, &new_mems,
  3239. cpus_updated, mems_updated);
  3240. mutex_unlock(&cpuset_mutex);
  3241. }
  3242. /**
  3243. * cpuset_hotplug_workfn - handle CPU/memory hotunplug for a cpuset
  3244. *
  3245. * This function is called after either CPU or memory configuration has
  3246. * changed and updates cpuset accordingly. The top_cpuset is always
  3247. * synchronized to cpu_active_mask and N_MEMORY, which is necessary in
  3248. * order to make cpusets transparent (of no affect) on systems that are
  3249. * actively using CPU hotplug but making no active use of cpusets.
  3250. *
  3251. * Non-root cpusets are only affected by offlining. If any CPUs or memory
  3252. * nodes have been taken down, cpuset_hotplug_update_tasks() is invoked on
  3253. * all descendants.
  3254. *
  3255. * Note that CPU offlining during suspend is ignored. We don't modify
  3256. * cpusets across suspend/resume cycles at all.
  3257. */
  3258. static void cpuset_hotplug_workfn(struct work_struct *work)
  3259. {
  3260. static cpumask_t new_cpus;
  3261. static nodemask_t new_mems;
  3262. bool cpus_updated, mems_updated;
  3263. bool on_dfl = is_in_v2_mode();
  3264. struct tmpmasks tmp, *ptmp = NULL;
  3265. if (on_dfl && !alloc_cpumasks(NULL, &tmp))
  3266. ptmp = &tmp;
  3267. mutex_lock(&cpuset_mutex);
  3268. /* fetch the available cpus/mems and find out which changed how */
  3269. cpumask_copy(&new_cpus, cpu_active_mask);
  3270. new_mems = node_states[N_MEMORY];
  3271. /*
  3272. * If subparts_cpus is populated, it is likely that the check below
  3273. * will produce a false positive on cpus_updated when the cpu list
  3274. * isn't changed. It is extra work, but it is better to be safe.
  3275. */
  3276. cpus_updated = !cpumask_equal(top_cpuset.effective_cpus, &new_cpus);
  3277. mems_updated = !nodes_equal(top_cpuset.effective_mems, new_mems);
  3278. /*
  3279. * In the rare case that hotplug removes all the cpus in subparts_cpus,
  3280. * we assumed that cpus are updated.
  3281. */
  3282. if (!cpus_updated && top_cpuset.nr_subparts_cpus)
  3283. cpus_updated = true;
  3284. /* synchronize cpus_allowed to cpu_active_mask */
  3285. if (cpus_updated) {
  3286. spin_lock_irq(&callback_lock);
  3287. if (!on_dfl)
  3288. cpumask_copy(top_cpuset.cpus_allowed, &new_cpus);
  3289. /*
  3290. * Make sure that CPUs allocated to child partitions
  3291. * do not show up in effective_cpus. If no CPU is left,
  3292. * we clear the subparts_cpus & let the child partitions
  3293. * fight for the CPUs again.
  3294. */
  3295. if (top_cpuset.nr_subparts_cpus) {
  3296. if (cpumask_subset(&new_cpus,
  3297. top_cpuset.subparts_cpus)) {
  3298. top_cpuset.nr_subparts_cpus = 0;
  3299. cpumask_clear(top_cpuset.subparts_cpus);
  3300. } else {
  3301. cpumask_andnot(&new_cpus, &new_cpus,
  3302. top_cpuset.subparts_cpus);
  3303. }
  3304. }
  3305. cpumask_copy(top_cpuset.effective_cpus, &new_cpus);
  3306. spin_unlock_irq(&callback_lock);
  3307. /* we don't mess with cpumasks of tasks in top_cpuset */
  3308. }
  3309. /* synchronize mems_allowed to N_MEMORY */
  3310. if (mems_updated) {
  3311. spin_lock_irq(&callback_lock);
  3312. if (!on_dfl)
  3313. top_cpuset.mems_allowed = new_mems;
  3314. top_cpuset.effective_mems = new_mems;
  3315. spin_unlock_irq(&callback_lock);
  3316. update_tasks_nodemask(&top_cpuset);
  3317. }
  3318. mutex_unlock(&cpuset_mutex);
  3319. /* if cpus or mems changed, we need to propagate to descendants */
  3320. if (cpus_updated || mems_updated) {
  3321. struct cpuset *cs;
  3322. struct cgroup_subsys_state *pos_css;
  3323. rcu_read_lock();
  3324. cpuset_for_each_descendant_pre(cs, pos_css, &top_cpuset) {
  3325. if (cs == &top_cpuset || !css_tryget_online(&cs->css))
  3326. continue;
  3327. rcu_read_unlock();
  3328. cpuset_hotplug_update_tasks(cs, ptmp);
  3329. rcu_read_lock();
  3330. css_put(&cs->css);
  3331. }
  3332. rcu_read_unlock();
  3333. }
  3334. /* rebuild sched domains if cpus_allowed has changed */
  3335. if (cpus_updated || force_rebuild) {
  3336. force_rebuild = false;
  3337. rebuild_sched_domains();
  3338. }
  3339. free_cpumasks(NULL, ptmp);
  3340. }
  3341. void cpuset_update_active_cpus(void)
  3342. {
  3343. /*
  3344. * We're inside cpu hotplug critical region which usually nests
  3345. * inside cgroup synchronization. Bounce actual hotplug processing
  3346. * to a work item to avoid reverse locking order.
  3347. */
  3348. schedule_work(&cpuset_hotplug_work);
  3349. }
  3350. void cpuset_wait_for_hotplug(void)
  3351. {
  3352. flush_work(&cpuset_hotplug_work);
  3353. }
  3354. /*
  3355. * Keep top_cpuset.mems_allowed tracking node_states[N_MEMORY].
  3356. * Call this routine anytime after node_states[N_MEMORY] changes.
  3357. * See cpuset_update_active_cpus() for CPU hotplug handling.
  3358. */
  3359. static int cpuset_track_online_nodes(struct notifier_block *self,
  3360. unsigned long action, void *arg)
  3361. {
  3362. schedule_work(&cpuset_hotplug_work);
  3363. return NOTIFY_OK;
  3364. }
  3365. static struct notifier_block cpuset_track_online_nodes_nb = {
  3366. .notifier_call = cpuset_track_online_nodes,
  3367. .priority = 10, /* ??! */
  3368. };
  3369. /**
  3370. * cpuset_init_smp - initialize cpus_allowed
  3371. *
  3372. * Description: Finish top cpuset after cpu, node maps are initialized
  3373. */
  3374. void __init cpuset_init_smp(void)
  3375. {
  3376. /*
  3377. * cpus_allowd/mems_allowed set to v2 values in the initial
  3378. * cpuset_bind() call will be reset to v1 values in another
  3379. * cpuset_bind() call when v1 cpuset is mounted.
  3380. */
  3381. top_cpuset.old_mems_allowed = top_cpuset.mems_allowed;
  3382. cpumask_copy(top_cpuset.effective_cpus, cpu_active_mask);
  3383. top_cpuset.effective_mems = node_states[N_MEMORY];
  3384. register_hotmemory_notifier(&cpuset_track_online_nodes_nb);
  3385. cpuset_migrate_mm_wq = alloc_ordered_workqueue("cpuset_migrate_mm", 0);
  3386. BUG_ON(!cpuset_migrate_mm_wq);
  3387. }
  3388. /**
  3389. * cpuset_cpus_allowed - return cpus_allowed mask from a tasks cpuset.
  3390. * @tsk: pointer to task_struct from which to obtain cpuset->cpus_allowed.
  3391. * @pmask: pointer to struct cpumask variable to receive cpus_allowed set.
  3392. *
  3393. * Description: Returns the cpumask_var_t cpus_allowed of the cpuset
  3394. * attached to the specified @tsk. Guaranteed to return some non-empty
  3395. * subset of cpu_online_mask, even if this means going outside the
  3396. * tasks cpuset.
  3397. **/
  3398. void cpuset_cpus_allowed(struct task_struct *tsk, struct cpumask *pmask)
  3399. {
  3400. unsigned long flags;
  3401. spin_lock_irqsave(&callback_lock, flags);
  3402. guarantee_online_cpus(tsk, pmask);
  3403. spin_unlock_irqrestore(&callback_lock, flags);
  3404. }
  3405. EXPORT_SYMBOL_GPL(cpuset_cpus_allowed);
  3406. /**
  3407. * cpuset_cpus_allowed_fallback - final fallback before complete catastrophe.
  3408. * @tsk: pointer to task_struct with which the scheduler is struggling
  3409. *
  3410. * Description: In the case that the scheduler cannot find an allowed cpu in
  3411. * tsk->cpus_allowed, we fall back to task_cs(tsk)->cpus_allowed. In legacy
  3412. * mode however, this value is the same as task_cs(tsk)->effective_cpus,
  3413. * which will not contain a sane cpumask during cases such as cpu hotplugging.
  3414. * This is the absolute last resort for the scheduler and it is only used if
  3415. * _every_ other avenue has been traveled.
  3416. *
  3417. * Returns true if the affinity of @tsk was changed, false otherwise.
  3418. **/
  3419. bool cpuset_cpus_allowed_fallback(struct task_struct *tsk)
  3420. {
  3421. const struct cpumask *possible_mask = task_cpu_possible_mask(tsk);
  3422. const struct cpumask *cs_mask;
  3423. bool changed = false;
  3424. rcu_read_lock();
  3425. cs_mask = task_cs(tsk)->cpus_allowed;
  3426. if (is_in_v2_mode() && cpumask_subset(cs_mask, possible_mask)) {
  3427. do_set_cpus_allowed(tsk, cs_mask);
  3428. changed = true;
  3429. }
  3430. rcu_read_unlock();
  3431. /*
  3432. * We own tsk->cpus_allowed, nobody can change it under us.
  3433. *
  3434. * But we used cs && cs->cpus_allowed lockless and thus can
  3435. * race with cgroup_attach_task() or update_cpumask() and get
  3436. * the wrong tsk->cpus_allowed. However, both cases imply the
  3437. * subsequent cpuset_change_cpumask()->set_cpus_allowed_ptr()
  3438. * which takes task_rq_lock().
  3439. *
  3440. * If we are called after it dropped the lock we must see all
  3441. * changes in tsk_cs()->cpus_allowed. Otherwise we can temporary
  3442. * set any mask even if it is not right from task_cs() pov,
  3443. * the pending set_cpus_allowed_ptr() will fix things.
  3444. *
  3445. * select_fallback_rq() will fix things ups and set cpu_possible_mask
  3446. * if required.
  3447. */
  3448. return changed;
  3449. }
  3450. void __init cpuset_init_current_mems_allowed(void)
  3451. {
  3452. nodes_setall(current->mems_allowed);
  3453. }
  3454. /**
  3455. * cpuset_mems_allowed - return mems_allowed mask from a tasks cpuset.
  3456. * @tsk: pointer to task_struct from which to obtain cpuset->mems_allowed.
  3457. *
  3458. * Description: Returns the nodemask_t mems_allowed of the cpuset
  3459. * attached to the specified @tsk. Guaranteed to return some non-empty
  3460. * subset of node_states[N_MEMORY], even if this means going outside the
  3461. * tasks cpuset.
  3462. **/
  3463. nodemask_t cpuset_mems_allowed(struct task_struct *tsk)
  3464. {
  3465. nodemask_t mask;
  3466. unsigned long flags;
  3467. spin_lock_irqsave(&callback_lock, flags);
  3468. rcu_read_lock();
  3469. guarantee_online_mems(task_cs(tsk), &mask);
  3470. rcu_read_unlock();
  3471. spin_unlock_irqrestore(&callback_lock, flags);
  3472. return mask;
  3473. }
  3474. /**
  3475. * cpuset_nodemask_valid_mems_allowed - check nodemask vs. current mems_allowed
  3476. * @nodemask: the nodemask to be checked
  3477. *
  3478. * Are any of the nodes in the nodemask allowed in current->mems_allowed?
  3479. */
  3480. int cpuset_nodemask_valid_mems_allowed(nodemask_t *nodemask)
  3481. {
  3482. return nodes_intersects(*nodemask, current->mems_allowed);
  3483. }
  3484. /*
  3485. * nearest_hardwall_ancestor() - Returns the nearest mem_exclusive or
  3486. * mem_hardwall ancestor to the specified cpuset. Call holding
  3487. * callback_lock. If no ancestor is mem_exclusive or mem_hardwall
  3488. * (an unusual configuration), then returns the root cpuset.
  3489. */
  3490. static struct cpuset *nearest_hardwall_ancestor(struct cpuset *cs)
  3491. {
  3492. while (!(is_mem_exclusive(cs) || is_mem_hardwall(cs)) && parent_cs(cs))
  3493. cs = parent_cs(cs);
  3494. return cs;
  3495. }
  3496. /*
  3497. * __cpuset_node_allowed - Can we allocate on a memory node?
  3498. * @node: is this an allowed node?
  3499. * @gfp_mask: memory allocation flags
  3500. *
  3501. * If we're in interrupt, yes, we can always allocate. If @node is set in
  3502. * current's mems_allowed, yes. If it's not a __GFP_HARDWALL request and this
  3503. * node is set in the nearest hardwalled cpuset ancestor to current's cpuset,
  3504. * yes. If current has access to memory reserves as an oom victim, yes.
  3505. * Otherwise, no.
  3506. *
  3507. * GFP_USER allocations are marked with the __GFP_HARDWALL bit,
  3508. * and do not allow allocations outside the current tasks cpuset
  3509. * unless the task has been OOM killed.
  3510. * GFP_KERNEL allocations are not so marked, so can escape to the
  3511. * nearest enclosing hardwalled ancestor cpuset.
  3512. *
  3513. * Scanning up parent cpusets requires callback_lock. The
  3514. * __alloc_pages() routine only calls here with __GFP_HARDWALL bit
  3515. * _not_ set if it's a GFP_KERNEL allocation, and all nodes in the
  3516. * current tasks mems_allowed came up empty on the first pass over
  3517. * the zonelist. So only GFP_KERNEL allocations, if all nodes in the
  3518. * cpuset are short of memory, might require taking the callback_lock.
  3519. *
  3520. * The first call here from mm/page_alloc:get_page_from_freelist()
  3521. * has __GFP_HARDWALL set in gfp_mask, enforcing hardwall cpusets,
  3522. * so no allocation on a node outside the cpuset is allowed (unless
  3523. * in interrupt, of course).
  3524. *
  3525. * The second pass through get_page_from_freelist() doesn't even call
  3526. * here for GFP_ATOMIC calls. For those calls, the __alloc_pages()
  3527. * variable 'wait' is not set, and the bit ALLOC_CPUSET is not set
  3528. * in alloc_flags. That logic and the checks below have the combined
  3529. * affect that:
  3530. * in_interrupt - any node ok (current task context irrelevant)
  3531. * GFP_ATOMIC - any node ok
  3532. * tsk_is_oom_victim - any node ok
  3533. * GFP_KERNEL - any node in enclosing hardwalled cpuset ok
  3534. * GFP_USER - only nodes in current tasks mems allowed ok.
  3535. */
  3536. bool __cpuset_node_allowed(int node, gfp_t gfp_mask)
  3537. {
  3538. struct cpuset *cs; /* current cpuset ancestors */
  3539. bool allowed; /* is allocation in zone z allowed? */
  3540. unsigned long flags;
  3541. if (in_interrupt())
  3542. return true;
  3543. if (node_isset(node, current->mems_allowed))
  3544. return true;
  3545. /*
  3546. * Allow tasks that have access to memory reserves because they have
  3547. * been OOM killed to get memory anywhere.
  3548. */
  3549. if (unlikely(tsk_is_oom_victim(current)))
  3550. return true;
  3551. if (gfp_mask & __GFP_HARDWALL) /* If hardwall request, stop here */
  3552. return false;
  3553. if (current->flags & PF_EXITING) /* Let dying task have memory */
  3554. return true;
  3555. /* Not hardwall and node outside mems_allowed: scan up cpusets */
  3556. spin_lock_irqsave(&callback_lock, flags);
  3557. rcu_read_lock();
  3558. cs = nearest_hardwall_ancestor(task_cs(current));
  3559. allowed = node_isset(node, cs->mems_allowed);
  3560. rcu_read_unlock();
  3561. spin_unlock_irqrestore(&callback_lock, flags);
  3562. return allowed;
  3563. }
  3564. /**
  3565. * cpuset_mem_spread_node() - On which node to begin search for a file page
  3566. * cpuset_slab_spread_node() - On which node to begin search for a slab page
  3567. *
  3568. * If a task is marked PF_SPREAD_PAGE or PF_SPREAD_SLAB (as for
  3569. * tasks in a cpuset with is_spread_page or is_spread_slab set),
  3570. * and if the memory allocation used cpuset_mem_spread_node()
  3571. * to determine on which node to start looking, as it will for
  3572. * certain page cache or slab cache pages such as used for file
  3573. * system buffers and inode caches, then instead of starting on the
  3574. * local node to look for a free page, rather spread the starting
  3575. * node around the tasks mems_allowed nodes.
  3576. *
  3577. * We don't have to worry about the returned node being offline
  3578. * because "it can't happen", and even if it did, it would be ok.
  3579. *
  3580. * The routines calling guarantee_online_mems() are careful to
  3581. * only set nodes in task->mems_allowed that are online. So it
  3582. * should not be possible for the following code to return an
  3583. * offline node. But if it did, that would be ok, as this routine
  3584. * is not returning the node where the allocation must be, only
  3585. * the node where the search should start. The zonelist passed to
  3586. * __alloc_pages() will include all nodes. If the slab allocator
  3587. * is passed an offline node, it will fall back to the local node.
  3588. * See kmem_cache_alloc_node().
  3589. */
  3590. static int cpuset_spread_node(int *rotor)
  3591. {
  3592. return *rotor = next_node_in(*rotor, current->mems_allowed);
  3593. }
  3594. int cpuset_mem_spread_node(void)
  3595. {
  3596. if (current->cpuset_mem_spread_rotor == NUMA_NO_NODE)
  3597. current->cpuset_mem_spread_rotor =
  3598. node_random(&current->mems_allowed);
  3599. return cpuset_spread_node(&current->cpuset_mem_spread_rotor);
  3600. }
  3601. int cpuset_slab_spread_node(void)
  3602. {
  3603. if (current->cpuset_slab_spread_rotor == NUMA_NO_NODE)
  3604. current->cpuset_slab_spread_rotor =
  3605. node_random(&current->mems_allowed);
  3606. return cpuset_spread_node(&current->cpuset_slab_spread_rotor);
  3607. }
  3608. EXPORT_SYMBOL_GPL(cpuset_mem_spread_node);
  3609. /**
  3610. * cpuset_mems_allowed_intersects - Does @tsk1's mems_allowed intersect @tsk2's?
  3611. * @tsk1: pointer to task_struct of some task.
  3612. * @tsk2: pointer to task_struct of some other task.
  3613. *
  3614. * Description: Return true if @tsk1's mems_allowed intersects the
  3615. * mems_allowed of @tsk2. Used by the OOM killer to determine if
  3616. * one of the task's memory usage might impact the memory available
  3617. * to the other.
  3618. **/
  3619. int cpuset_mems_allowed_intersects(const struct task_struct *tsk1,
  3620. const struct task_struct *tsk2)
  3621. {
  3622. return nodes_intersects(tsk1->mems_allowed, tsk2->mems_allowed);
  3623. }
  3624. /**
  3625. * cpuset_print_current_mems_allowed - prints current's cpuset and mems_allowed
  3626. *
  3627. * Description: Prints current's name, cpuset name, and cached copy of its
  3628. * mems_allowed to the kernel log.
  3629. */
  3630. void cpuset_print_current_mems_allowed(void)
  3631. {
  3632. struct cgroup *cgrp;
  3633. rcu_read_lock();
  3634. cgrp = task_cs(current)->css.cgroup;
  3635. pr_cont(",cpuset=");
  3636. pr_cont_cgroup_name(cgrp);
  3637. pr_cont(",mems_allowed=%*pbl",
  3638. nodemask_pr_args(&current->mems_allowed));
  3639. rcu_read_unlock();
  3640. }
  3641. /*
  3642. * Collection of memory_pressure is suppressed unless
  3643. * this flag is enabled by writing "1" to the special
  3644. * cpuset file 'memory_pressure_enabled' in the root cpuset.
  3645. */
  3646. int cpuset_memory_pressure_enabled __read_mostly;
  3647. /*
  3648. * __cpuset_memory_pressure_bump - keep stats of per-cpuset reclaims.
  3649. *
  3650. * Keep a running average of the rate of synchronous (direct)
  3651. * page reclaim efforts initiated by tasks in each cpuset.
  3652. *
  3653. * This represents the rate at which some task in the cpuset
  3654. * ran low on memory on all nodes it was allowed to use, and
  3655. * had to enter the kernels page reclaim code in an effort to
  3656. * create more free memory by tossing clean pages or swapping
  3657. * or writing dirty pages.
  3658. *
  3659. * Display to user space in the per-cpuset read-only file
  3660. * "memory_pressure". Value displayed is an integer
  3661. * representing the recent rate of entry into the synchronous
  3662. * (direct) page reclaim by any task attached to the cpuset.
  3663. */
  3664. void __cpuset_memory_pressure_bump(void)
  3665. {
  3666. rcu_read_lock();
  3667. fmeter_markevent(&task_cs(current)->fmeter);
  3668. rcu_read_unlock();
  3669. }
  3670. #ifdef CONFIG_PROC_PID_CPUSET
  3671. /*
  3672. * proc_cpuset_show()
  3673. * - Print tasks cpuset path into seq_file.
  3674. * - Used for /proc/<pid>/cpuset.
  3675. * - No need to task_lock(tsk) on this tsk->cpuset reference, as it
  3676. * doesn't really matter if tsk->cpuset changes after we read it,
  3677. * and we take cpuset_mutex, keeping cpuset_attach() from changing it
  3678. * anyway.
  3679. */
  3680. int proc_cpuset_show(struct seq_file *m, struct pid_namespace *ns,
  3681. struct pid *pid, struct task_struct *tsk)
  3682. {
  3683. char *buf;
  3684. struct cgroup_subsys_state *css;
  3685. int retval;
  3686. retval = -ENOMEM;
  3687. buf = kmalloc(PATH_MAX, GFP_KERNEL);
  3688. if (!buf)
  3689. goto out;
  3690. css = task_get_css(tsk, cpuset_cgrp_id);
  3691. retval = cgroup_path_ns(css->cgroup, buf, PATH_MAX,
  3692. current->nsproxy->cgroup_ns);
  3693. css_put(css);
  3694. if (retval >= PATH_MAX)
  3695. retval = -ENAMETOOLONG;
  3696. if (retval < 0)
  3697. goto out_free;
  3698. seq_puts(m, buf);
  3699. seq_putc(m, '\n');
  3700. retval = 0;
  3701. out_free:
  3702. kfree(buf);
  3703. out:
  3704. return retval;
  3705. }
  3706. #endif /* CONFIG_PROC_PID_CPUSET */
  3707. /* Display task mems_allowed in /proc/<pid>/status file. */
  3708. void cpuset_task_status_allowed(struct seq_file *m, struct task_struct *task)
  3709. {
  3710. seq_printf(m, "Mems_allowed:\t%*pb\n",
  3711. nodemask_pr_args(&task->mems_allowed));
  3712. seq_printf(m, "Mems_allowed_list:\t%*pbl\n",
  3713. nodemask_pr_args(&task->mems_allowed));
  3714. }