cpu.c 67 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785
  1. /* CPU control.
  2. * (C) 2001, 2002, 2003, 2004 Rusty Russell
  3. *
  4. * This code is licenced under the GPL.
  5. */
  6. #include <linux/sched/mm.h>
  7. #include <linux/proc_fs.h>
  8. #include <linux/smp.h>
  9. #include <linux/init.h>
  10. #include <linux/notifier.h>
  11. #include <linux/sched/signal.h>
  12. #include <linux/sched/hotplug.h>
  13. #include <linux/sched/isolation.h>
  14. #include <linux/sched/task.h>
  15. #include <linux/sched/smt.h>
  16. #include <linux/unistd.h>
  17. #include <linux/cpu.h>
  18. #include <linux/oom.h>
  19. #include <linux/rcupdate.h>
  20. #include <linux/export.h>
  21. #include <linux/bug.h>
  22. #include <linux/kthread.h>
  23. #include <linux/stop_machine.h>
  24. #include <linux/mutex.h>
  25. #include <linux/gfp.h>
  26. #include <linux/suspend.h>
  27. #include <linux/lockdep.h>
  28. #include <linux/tick.h>
  29. #include <linux/irq.h>
  30. #include <linux/nmi.h>
  31. #include <linux/smpboot.h>
  32. #include <linux/relay.h>
  33. #include <linux/slab.h>
  34. #include <linux/scs.h>
  35. #include <linux/percpu-rwsem.h>
  36. #include <linux/cpuset.h>
  37. #include <linux/random.h>
  38. #include <linux/cc_platform.h>
  39. #include <trace/events/power.h>
  40. #define CREATE_TRACE_POINTS
  41. #include <trace/events/cpuhp.h>
  42. #include "smpboot.h"
  43. /**
  44. * struct cpuhp_cpu_state - Per cpu hotplug state storage
  45. * @state: The current cpu state
  46. * @target: The target state
  47. * @fail: Current CPU hotplug callback state
  48. * @thread: Pointer to the hotplug thread
  49. * @should_run: Thread should execute
  50. * @rollback: Perform a rollback
  51. * @single: Single callback invocation
  52. * @bringup: Single callback bringup or teardown selector
  53. * @cpu: CPU number
  54. * @node: Remote CPU node; for multi-instance, do a
  55. * single entry callback for install/remove
  56. * @last: For multi-instance rollback, remember how far we got
  57. * @cb_state: The state for a single callback (install/uninstall)
  58. * @result: Result of the operation
  59. * @done_up: Signal completion to the issuer of the task for cpu-up
  60. * @done_down: Signal completion to the issuer of the task for cpu-down
  61. */
  62. struct cpuhp_cpu_state {
  63. enum cpuhp_state state;
  64. enum cpuhp_state target;
  65. enum cpuhp_state fail;
  66. #ifdef CONFIG_SMP
  67. struct task_struct *thread;
  68. bool should_run;
  69. bool rollback;
  70. bool single;
  71. bool bringup;
  72. struct hlist_node *node;
  73. struct hlist_node *last;
  74. enum cpuhp_state cb_state;
  75. int result;
  76. struct completion done_up;
  77. struct completion done_down;
  78. #endif
  79. };
  80. static DEFINE_PER_CPU(struct cpuhp_cpu_state, cpuhp_state) = {
  81. .fail = CPUHP_INVALID,
  82. };
  83. #ifdef CONFIG_SMP
  84. cpumask_t cpus_booted_once_mask;
  85. #endif
  86. #if defined(CONFIG_LOCKDEP) && defined(CONFIG_SMP)
  87. static struct lockdep_map cpuhp_state_up_map =
  88. STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
  89. static struct lockdep_map cpuhp_state_down_map =
  90. STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
  91. static inline void cpuhp_lock_acquire(bool bringup)
  92. {
  93. lock_map_acquire(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  94. }
  95. static inline void cpuhp_lock_release(bool bringup)
  96. {
  97. lock_map_release(bringup ? &cpuhp_state_up_map : &cpuhp_state_down_map);
  98. }
  99. #else
  100. static inline void cpuhp_lock_acquire(bool bringup) { }
  101. static inline void cpuhp_lock_release(bool bringup) { }
  102. #endif
  103. /**
  104. * struct cpuhp_step - Hotplug state machine step
  105. * @name: Name of the step
  106. * @startup: Startup function of the step
  107. * @teardown: Teardown function of the step
  108. * @cant_stop: Bringup/teardown can't be stopped at this step
  109. * @multi_instance: State has multiple instances which get added afterwards
  110. */
  111. struct cpuhp_step {
  112. const char *name;
  113. union {
  114. int (*single)(unsigned int cpu);
  115. int (*multi)(unsigned int cpu,
  116. struct hlist_node *node);
  117. } startup;
  118. union {
  119. int (*single)(unsigned int cpu);
  120. int (*multi)(unsigned int cpu,
  121. struct hlist_node *node);
  122. } teardown;
  123. /* private: */
  124. struct hlist_head list;
  125. /* public: */
  126. bool cant_stop;
  127. bool multi_instance;
  128. };
  129. static DEFINE_MUTEX(cpuhp_state_mutex);
  130. static struct cpuhp_step cpuhp_hp_states[];
  131. static struct cpuhp_step *cpuhp_get_step(enum cpuhp_state state)
  132. {
  133. return cpuhp_hp_states + state;
  134. }
  135. static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step)
  136. {
  137. return bringup ? !step->startup.single : !step->teardown.single;
  138. }
  139. /**
  140. * cpuhp_invoke_callback - Invoke the callbacks for a given state
  141. * @cpu: The cpu for which the callback should be invoked
  142. * @state: The state to do callbacks for
  143. * @bringup: True if the bringup callback should be invoked
  144. * @node: For multi-instance, do a single entry callback for install/remove
  145. * @lastp: For multi-instance rollback, remember how far we got
  146. *
  147. * Called from cpu hotplug and from the state register machinery.
  148. *
  149. * Return: %0 on success or a negative errno code
  150. */
  151. static int cpuhp_invoke_callback(unsigned int cpu, enum cpuhp_state state,
  152. bool bringup, struct hlist_node *node,
  153. struct hlist_node **lastp)
  154. {
  155. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  156. struct cpuhp_step *step = cpuhp_get_step(state);
  157. int (*cbm)(unsigned int cpu, struct hlist_node *node);
  158. int (*cb)(unsigned int cpu);
  159. int ret, cnt;
  160. if (st->fail == state) {
  161. st->fail = CPUHP_INVALID;
  162. return -EAGAIN;
  163. }
  164. if (cpuhp_step_empty(bringup, step)) {
  165. WARN_ON_ONCE(1);
  166. return 0;
  167. }
  168. if (!step->multi_instance) {
  169. WARN_ON_ONCE(lastp && *lastp);
  170. cb = bringup ? step->startup.single : step->teardown.single;
  171. trace_cpuhp_enter(cpu, st->target, state, cb);
  172. ret = cb(cpu);
  173. trace_cpuhp_exit(cpu, st->state, state, ret);
  174. return ret;
  175. }
  176. cbm = bringup ? step->startup.multi : step->teardown.multi;
  177. /* Single invocation for instance add/remove */
  178. if (node) {
  179. WARN_ON_ONCE(lastp && *lastp);
  180. trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
  181. ret = cbm(cpu, node);
  182. trace_cpuhp_exit(cpu, st->state, state, ret);
  183. return ret;
  184. }
  185. /* State transition. Invoke on all instances */
  186. cnt = 0;
  187. hlist_for_each(node, &step->list) {
  188. if (lastp && node == *lastp)
  189. break;
  190. trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
  191. ret = cbm(cpu, node);
  192. trace_cpuhp_exit(cpu, st->state, state, ret);
  193. if (ret) {
  194. if (!lastp)
  195. goto err;
  196. *lastp = node;
  197. return ret;
  198. }
  199. cnt++;
  200. }
  201. if (lastp)
  202. *lastp = NULL;
  203. return 0;
  204. err:
  205. /* Rollback the instances if one failed */
  206. cbm = !bringup ? step->startup.multi : step->teardown.multi;
  207. if (!cbm)
  208. return ret;
  209. hlist_for_each(node, &step->list) {
  210. if (!cnt--)
  211. break;
  212. trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node);
  213. ret = cbm(cpu, node);
  214. trace_cpuhp_exit(cpu, st->state, state, ret);
  215. /*
  216. * Rollback must not fail,
  217. */
  218. WARN_ON_ONCE(ret);
  219. }
  220. return ret;
  221. }
  222. #ifdef CONFIG_SMP
  223. static bool cpuhp_is_ap_state(enum cpuhp_state state)
  224. {
  225. /*
  226. * The extra check for CPUHP_TEARDOWN_CPU is only for documentation
  227. * purposes as that state is handled explicitly in cpu_down.
  228. */
  229. return state > CPUHP_BRINGUP_CPU && state != CPUHP_TEARDOWN_CPU;
  230. }
  231. static inline void wait_for_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
  232. {
  233. struct completion *done = bringup ? &st->done_up : &st->done_down;
  234. wait_for_completion(done);
  235. }
  236. static inline void complete_ap_thread(struct cpuhp_cpu_state *st, bool bringup)
  237. {
  238. struct completion *done = bringup ? &st->done_up : &st->done_down;
  239. complete(done);
  240. }
  241. /*
  242. * The former STARTING/DYING states, ran with IRQs disabled and must not fail.
  243. */
  244. static bool cpuhp_is_atomic_state(enum cpuhp_state state)
  245. {
  246. return CPUHP_AP_IDLE_DEAD <= state && state < CPUHP_AP_ONLINE;
  247. }
  248. /* Serializes the updates to cpu_online_mask, cpu_present_mask */
  249. static DEFINE_MUTEX(cpu_add_remove_lock);
  250. bool cpuhp_tasks_frozen;
  251. EXPORT_SYMBOL_GPL(cpuhp_tasks_frozen);
  252. /*
  253. * The following two APIs (cpu_maps_update_begin/done) must be used when
  254. * attempting to serialize the updates to cpu_online_mask & cpu_present_mask.
  255. */
  256. void cpu_maps_update_begin(void)
  257. {
  258. mutex_lock(&cpu_add_remove_lock);
  259. }
  260. void cpu_maps_update_done(void)
  261. {
  262. mutex_unlock(&cpu_add_remove_lock);
  263. }
  264. /*
  265. * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
  266. * Should always be manipulated under cpu_add_remove_lock
  267. */
  268. static int cpu_hotplug_disabled;
  269. #ifdef CONFIG_HOTPLUG_CPU
  270. DEFINE_STATIC_PERCPU_RWSEM(cpu_hotplug_lock);
  271. void cpus_read_lock(void)
  272. {
  273. percpu_down_read(&cpu_hotplug_lock);
  274. }
  275. EXPORT_SYMBOL_GPL(cpus_read_lock);
  276. int cpus_read_trylock(void)
  277. {
  278. return percpu_down_read_trylock(&cpu_hotplug_lock);
  279. }
  280. EXPORT_SYMBOL_GPL(cpus_read_trylock);
  281. void cpus_read_unlock(void)
  282. {
  283. percpu_up_read(&cpu_hotplug_lock);
  284. }
  285. EXPORT_SYMBOL_GPL(cpus_read_unlock);
  286. void cpus_write_lock(void)
  287. {
  288. percpu_down_write(&cpu_hotplug_lock);
  289. }
  290. void cpus_write_unlock(void)
  291. {
  292. percpu_up_write(&cpu_hotplug_lock);
  293. }
  294. void lockdep_assert_cpus_held(void)
  295. {
  296. /*
  297. * We can't have hotplug operations before userspace starts running,
  298. * and some init codepaths will knowingly not take the hotplug lock.
  299. * This is all valid, so mute lockdep until it makes sense to report
  300. * unheld locks.
  301. */
  302. if (system_state < SYSTEM_RUNNING)
  303. return;
  304. percpu_rwsem_assert_held(&cpu_hotplug_lock);
  305. }
  306. #ifdef CONFIG_LOCKDEP
  307. int lockdep_is_cpus_held(void)
  308. {
  309. return percpu_rwsem_is_held(&cpu_hotplug_lock);
  310. }
  311. #endif
  312. static void lockdep_acquire_cpus_lock(void)
  313. {
  314. rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
  315. }
  316. static void lockdep_release_cpus_lock(void)
  317. {
  318. rwsem_release(&cpu_hotplug_lock.dep_map, _THIS_IP_);
  319. }
  320. /*
  321. * Wait for currently running CPU hotplug operations to complete (if any) and
  322. * disable future CPU hotplug (from sysfs). The 'cpu_add_remove_lock' protects
  323. * the 'cpu_hotplug_disabled' flag. The same lock is also acquired by the
  324. * hotplug path before performing hotplug operations. So acquiring that lock
  325. * guarantees mutual exclusion from any currently running hotplug operations.
  326. */
  327. void cpu_hotplug_disable(void)
  328. {
  329. cpu_maps_update_begin();
  330. cpu_hotplug_disabled++;
  331. cpu_maps_update_done();
  332. }
  333. EXPORT_SYMBOL_GPL(cpu_hotplug_disable);
  334. static void __cpu_hotplug_enable(void)
  335. {
  336. if (WARN_ONCE(!cpu_hotplug_disabled, "Unbalanced cpu hotplug enable\n"))
  337. return;
  338. cpu_hotplug_disabled--;
  339. }
  340. void cpu_hotplug_enable(void)
  341. {
  342. cpu_maps_update_begin();
  343. __cpu_hotplug_enable();
  344. cpu_maps_update_done();
  345. }
  346. EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
  347. #else
  348. static void lockdep_acquire_cpus_lock(void)
  349. {
  350. }
  351. static void lockdep_release_cpus_lock(void)
  352. {
  353. }
  354. #endif /* CONFIG_HOTPLUG_CPU */
  355. /*
  356. * Architectures that need SMT-specific errata handling during SMT hotplug
  357. * should override this.
  358. */
  359. void __weak arch_smt_update(void) { }
  360. #ifdef CONFIG_HOTPLUG_SMT
  361. enum cpuhp_smt_control cpu_smt_control __read_mostly = CPU_SMT_ENABLED;
  362. void __init cpu_smt_disable(bool force)
  363. {
  364. if (!cpu_smt_possible())
  365. return;
  366. if (force) {
  367. pr_info("SMT: Force disabled\n");
  368. cpu_smt_control = CPU_SMT_FORCE_DISABLED;
  369. } else {
  370. pr_info("SMT: disabled\n");
  371. cpu_smt_control = CPU_SMT_DISABLED;
  372. }
  373. }
  374. /*
  375. * The decision whether SMT is supported can only be done after the full
  376. * CPU identification. Called from architecture code.
  377. */
  378. void __init cpu_smt_check_topology(void)
  379. {
  380. if (!topology_smt_supported())
  381. cpu_smt_control = CPU_SMT_NOT_SUPPORTED;
  382. }
  383. static int __init smt_cmdline_disable(char *str)
  384. {
  385. cpu_smt_disable(str && !strcmp(str, "force"));
  386. return 0;
  387. }
  388. early_param("nosmt", smt_cmdline_disable);
  389. static inline bool cpu_smt_allowed(unsigned int cpu)
  390. {
  391. if (cpu_smt_control == CPU_SMT_ENABLED)
  392. return true;
  393. if (topology_is_primary_thread(cpu))
  394. return true;
  395. /*
  396. * On x86 it's required to boot all logical CPUs at least once so
  397. * that the init code can get a chance to set CR4.MCE on each
  398. * CPU. Otherwise, a broadcasted MCE observing CR4.MCE=0b on any
  399. * core will shutdown the machine.
  400. */
  401. return !cpumask_test_cpu(cpu, &cpus_booted_once_mask);
  402. }
  403. /* Returns true if SMT is not supported of forcefully (irreversibly) disabled */
  404. bool cpu_smt_possible(void)
  405. {
  406. return cpu_smt_control != CPU_SMT_FORCE_DISABLED &&
  407. cpu_smt_control != CPU_SMT_NOT_SUPPORTED;
  408. }
  409. EXPORT_SYMBOL_GPL(cpu_smt_possible);
  410. #else
  411. static inline bool cpu_smt_allowed(unsigned int cpu) { return true; }
  412. #endif
  413. static inline enum cpuhp_state
  414. cpuhp_set_state(int cpu, struct cpuhp_cpu_state *st, enum cpuhp_state target)
  415. {
  416. enum cpuhp_state prev_state = st->state;
  417. bool bringup = st->state < target;
  418. st->rollback = false;
  419. st->last = NULL;
  420. st->target = target;
  421. st->single = false;
  422. st->bringup = bringup;
  423. if (cpu_dying(cpu) != !bringup)
  424. set_cpu_dying(cpu, !bringup);
  425. return prev_state;
  426. }
  427. static inline void
  428. cpuhp_reset_state(int cpu, struct cpuhp_cpu_state *st,
  429. enum cpuhp_state prev_state)
  430. {
  431. bool bringup = !st->bringup;
  432. st->target = prev_state;
  433. /*
  434. * Already rolling back. No need invert the bringup value or to change
  435. * the current state.
  436. */
  437. if (st->rollback)
  438. return;
  439. st->rollback = true;
  440. /*
  441. * If we have st->last we need to undo partial multi_instance of this
  442. * state first. Otherwise start undo at the previous state.
  443. */
  444. if (!st->last) {
  445. if (st->bringup)
  446. st->state--;
  447. else
  448. st->state++;
  449. }
  450. st->bringup = bringup;
  451. if (cpu_dying(cpu) != !bringup)
  452. set_cpu_dying(cpu, !bringup);
  453. }
  454. /* Regular hotplug invocation of the AP hotplug thread */
  455. static void __cpuhp_kick_ap(struct cpuhp_cpu_state *st)
  456. {
  457. if (!st->single && st->state == st->target)
  458. return;
  459. st->result = 0;
  460. /*
  461. * Make sure the above stores are visible before should_run becomes
  462. * true. Paired with the mb() above in cpuhp_thread_fun()
  463. */
  464. smp_mb();
  465. st->should_run = true;
  466. wake_up_process(st->thread);
  467. wait_for_ap_thread(st, st->bringup);
  468. }
  469. static int cpuhp_kick_ap(int cpu, struct cpuhp_cpu_state *st,
  470. enum cpuhp_state target)
  471. {
  472. enum cpuhp_state prev_state;
  473. int ret;
  474. prev_state = cpuhp_set_state(cpu, st, target);
  475. __cpuhp_kick_ap(st);
  476. if ((ret = st->result)) {
  477. cpuhp_reset_state(cpu, st, prev_state);
  478. __cpuhp_kick_ap(st);
  479. }
  480. return ret;
  481. }
  482. static int bringup_wait_for_ap(unsigned int cpu)
  483. {
  484. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  485. /* Wait for the CPU to reach CPUHP_AP_ONLINE_IDLE */
  486. wait_for_ap_thread(st, true);
  487. if (WARN_ON_ONCE((!cpu_online(cpu))))
  488. return -ECANCELED;
  489. /* Unpark the hotplug thread of the target cpu */
  490. kthread_unpark(st->thread);
  491. /*
  492. * SMT soft disabling on X86 requires to bring the CPU out of the
  493. * BIOS 'wait for SIPI' state in order to set the CR4.MCE bit. The
  494. * CPU marked itself as booted_once in notify_cpu_starting() so the
  495. * cpu_smt_allowed() check will now return false if this is not the
  496. * primary sibling.
  497. */
  498. if (!cpu_smt_allowed(cpu))
  499. return -ECANCELED;
  500. if (st->target <= CPUHP_AP_ONLINE_IDLE)
  501. return 0;
  502. return cpuhp_kick_ap(cpu, st, st->target);
  503. }
  504. static int bringup_cpu(unsigned int cpu)
  505. {
  506. struct task_struct *idle = idle_thread_get(cpu);
  507. int ret;
  508. /*
  509. * Reset stale stack state from the last time this CPU was online.
  510. */
  511. scs_task_reset(idle);
  512. kasan_unpoison_task_stack(idle);
  513. /*
  514. * Some architectures have to walk the irq descriptors to
  515. * setup the vector space for the cpu which comes online.
  516. * Prevent irq alloc/free across the bringup.
  517. */
  518. irq_lock_sparse();
  519. /* Arch-specific enabling code. */
  520. ret = __cpu_up(cpu, idle);
  521. irq_unlock_sparse();
  522. if (ret)
  523. return ret;
  524. return bringup_wait_for_ap(cpu);
  525. }
  526. static int finish_cpu(unsigned int cpu)
  527. {
  528. struct task_struct *idle = idle_thread_get(cpu);
  529. struct mm_struct *mm = idle->active_mm;
  530. /*
  531. * idle_task_exit() will have switched to &init_mm, now
  532. * clean up any remaining active_mm state.
  533. */
  534. if (mm != &init_mm)
  535. idle->active_mm = &init_mm;
  536. mmdrop(mm);
  537. return 0;
  538. }
  539. /*
  540. * Hotplug state machine related functions
  541. */
  542. /*
  543. * Get the next state to run. Empty ones will be skipped. Returns true if a
  544. * state must be run.
  545. *
  546. * st->state will be modified ahead of time, to match state_to_run, as if it
  547. * has already ran.
  548. */
  549. static bool cpuhp_next_state(bool bringup,
  550. enum cpuhp_state *state_to_run,
  551. struct cpuhp_cpu_state *st,
  552. enum cpuhp_state target)
  553. {
  554. do {
  555. if (bringup) {
  556. if (st->state >= target)
  557. return false;
  558. *state_to_run = ++st->state;
  559. } else {
  560. if (st->state <= target)
  561. return false;
  562. *state_to_run = st->state--;
  563. }
  564. if (!cpuhp_step_empty(bringup, cpuhp_get_step(*state_to_run)))
  565. break;
  566. } while (true);
  567. return true;
  568. }
  569. static int __cpuhp_invoke_callback_range(bool bringup,
  570. unsigned int cpu,
  571. struct cpuhp_cpu_state *st,
  572. enum cpuhp_state target,
  573. bool nofail)
  574. {
  575. enum cpuhp_state state;
  576. int ret = 0;
  577. while (cpuhp_next_state(bringup, &state, st, target)) {
  578. int err;
  579. err = cpuhp_invoke_callback(cpu, state, bringup, NULL, NULL);
  580. if (!err)
  581. continue;
  582. if (nofail) {
  583. pr_warn("CPU %u %s state %s (%d) failed (%d)\n",
  584. cpu, bringup ? "UP" : "DOWN",
  585. cpuhp_get_step(st->state)->name,
  586. st->state, err);
  587. ret = -1;
  588. } else {
  589. ret = err;
  590. break;
  591. }
  592. }
  593. return ret;
  594. }
  595. static inline int cpuhp_invoke_callback_range(bool bringup,
  596. unsigned int cpu,
  597. struct cpuhp_cpu_state *st,
  598. enum cpuhp_state target)
  599. {
  600. return __cpuhp_invoke_callback_range(bringup, cpu, st, target, false);
  601. }
  602. static inline void cpuhp_invoke_callback_range_nofail(bool bringup,
  603. unsigned int cpu,
  604. struct cpuhp_cpu_state *st,
  605. enum cpuhp_state target)
  606. {
  607. __cpuhp_invoke_callback_range(bringup, cpu, st, target, true);
  608. }
  609. static inline bool can_rollback_cpu(struct cpuhp_cpu_state *st)
  610. {
  611. if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
  612. return true;
  613. /*
  614. * When CPU hotplug is disabled, then taking the CPU down is not
  615. * possible because takedown_cpu() and the architecture and
  616. * subsystem specific mechanisms are not available. So the CPU
  617. * which would be completely unplugged again needs to stay around
  618. * in the current state.
  619. */
  620. return st->state <= CPUHP_BRINGUP_CPU;
  621. }
  622. static int cpuhp_up_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
  623. enum cpuhp_state target)
  624. {
  625. enum cpuhp_state prev_state = st->state;
  626. int ret = 0;
  627. ret = cpuhp_invoke_callback_range(true, cpu, st, target);
  628. if (ret) {
  629. pr_debug("CPU UP failed (%d) CPU %u state %s (%d)\n",
  630. ret, cpu, cpuhp_get_step(st->state)->name,
  631. st->state);
  632. cpuhp_reset_state(cpu, st, prev_state);
  633. if (can_rollback_cpu(st))
  634. WARN_ON(cpuhp_invoke_callback_range(false, cpu, st,
  635. prev_state));
  636. }
  637. return ret;
  638. }
  639. /*
  640. * The cpu hotplug threads manage the bringup and teardown of the cpus
  641. */
  642. static int cpuhp_should_run(unsigned int cpu)
  643. {
  644. struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
  645. return st->should_run;
  646. }
  647. /*
  648. * Execute teardown/startup callbacks on the plugged cpu. Also used to invoke
  649. * callbacks when a state gets [un]installed at runtime.
  650. *
  651. * Each invocation of this function by the smpboot thread does a single AP
  652. * state callback.
  653. *
  654. * It has 3 modes of operation:
  655. * - single: runs st->cb_state
  656. * - up: runs ++st->state, while st->state < st->target
  657. * - down: runs st->state--, while st->state > st->target
  658. *
  659. * When complete or on error, should_run is cleared and the completion is fired.
  660. */
  661. static void cpuhp_thread_fun(unsigned int cpu)
  662. {
  663. struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
  664. bool bringup = st->bringup;
  665. enum cpuhp_state state;
  666. if (WARN_ON_ONCE(!st->should_run))
  667. return;
  668. /*
  669. * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures
  670. * that if we see ->should_run we also see the rest of the state.
  671. */
  672. smp_mb();
  673. /*
  674. * The BP holds the hotplug lock, but we're now running on the AP,
  675. * ensure that anybody asserting the lock is held, will actually find
  676. * it so.
  677. */
  678. lockdep_acquire_cpus_lock();
  679. cpuhp_lock_acquire(bringup);
  680. if (st->single) {
  681. state = st->cb_state;
  682. st->should_run = false;
  683. } else {
  684. st->should_run = cpuhp_next_state(bringup, &state, st, st->target);
  685. if (!st->should_run)
  686. goto end;
  687. }
  688. WARN_ON_ONCE(!cpuhp_is_ap_state(state));
  689. if (cpuhp_is_atomic_state(state)) {
  690. local_irq_disable();
  691. st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
  692. local_irq_enable();
  693. /*
  694. * STARTING/DYING must not fail!
  695. */
  696. WARN_ON_ONCE(st->result);
  697. } else {
  698. st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last);
  699. }
  700. if (st->result) {
  701. /*
  702. * If we fail on a rollback, we're up a creek without no
  703. * paddle, no way forward, no way back. We loose, thanks for
  704. * playing.
  705. */
  706. WARN_ON_ONCE(st->rollback);
  707. st->should_run = false;
  708. }
  709. end:
  710. cpuhp_lock_release(bringup);
  711. lockdep_release_cpus_lock();
  712. if (!st->should_run)
  713. complete_ap_thread(st, bringup);
  714. }
  715. /* Invoke a single callback on a remote cpu */
  716. static int
  717. cpuhp_invoke_ap_callback(int cpu, enum cpuhp_state state, bool bringup,
  718. struct hlist_node *node)
  719. {
  720. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  721. int ret;
  722. if (!cpu_online(cpu))
  723. return 0;
  724. cpuhp_lock_acquire(false);
  725. cpuhp_lock_release(false);
  726. cpuhp_lock_acquire(true);
  727. cpuhp_lock_release(true);
  728. /*
  729. * If we are up and running, use the hotplug thread. For early calls
  730. * we invoke the thread function directly.
  731. */
  732. if (!st->thread)
  733. return cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
  734. st->rollback = false;
  735. st->last = NULL;
  736. st->node = node;
  737. st->bringup = bringup;
  738. st->cb_state = state;
  739. st->single = true;
  740. __cpuhp_kick_ap(st);
  741. /*
  742. * If we failed and did a partial, do a rollback.
  743. */
  744. if ((ret = st->result) && st->last) {
  745. st->rollback = true;
  746. st->bringup = !bringup;
  747. __cpuhp_kick_ap(st);
  748. }
  749. /*
  750. * Clean up the leftovers so the next hotplug operation wont use stale
  751. * data.
  752. */
  753. st->node = st->last = NULL;
  754. return ret;
  755. }
  756. static int cpuhp_kick_ap_work(unsigned int cpu)
  757. {
  758. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  759. enum cpuhp_state prev_state = st->state;
  760. int ret;
  761. cpuhp_lock_acquire(false);
  762. cpuhp_lock_release(false);
  763. cpuhp_lock_acquire(true);
  764. cpuhp_lock_release(true);
  765. trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work);
  766. ret = cpuhp_kick_ap(cpu, st, st->target);
  767. trace_cpuhp_exit(cpu, st->state, prev_state, ret);
  768. return ret;
  769. }
  770. static struct smp_hotplug_thread cpuhp_threads = {
  771. .store = &cpuhp_state.thread,
  772. .thread_should_run = cpuhp_should_run,
  773. .thread_fn = cpuhp_thread_fun,
  774. .thread_comm = "cpuhp/%u",
  775. .selfparking = true,
  776. };
  777. static __init void cpuhp_init_state(void)
  778. {
  779. struct cpuhp_cpu_state *st;
  780. int cpu;
  781. for_each_possible_cpu(cpu) {
  782. st = per_cpu_ptr(&cpuhp_state, cpu);
  783. init_completion(&st->done_up);
  784. init_completion(&st->done_down);
  785. }
  786. }
  787. void __init cpuhp_threads_init(void)
  788. {
  789. cpuhp_init_state();
  790. BUG_ON(smpboot_register_percpu_thread(&cpuhp_threads));
  791. kthread_unpark(this_cpu_read(cpuhp_state.thread));
  792. }
  793. /*
  794. *
  795. * Serialize hotplug trainwrecks outside of the cpu_hotplug_lock
  796. * protected region.
  797. *
  798. * The operation is still serialized against concurrent CPU hotplug via
  799. * cpu_add_remove_lock, i.e. CPU map protection. But it is _not_
  800. * serialized against other hotplug related activity like adding or
  801. * removing of state callbacks and state instances, which invoke either the
  802. * startup or the teardown callback of the affected state.
  803. *
  804. * This is required for subsystems which are unfixable vs. CPU hotplug and
  805. * evade lock inversion problems by scheduling work which has to be
  806. * completed _before_ cpu_up()/_cpu_down() returns.
  807. *
  808. * Don't even think about adding anything to this for any new code or even
  809. * drivers. It's only purpose is to keep existing lock order trainwrecks
  810. * working.
  811. *
  812. * For cpu_down() there might be valid reasons to finish cleanups which are
  813. * not required to be done under cpu_hotplug_lock, but that's a different
  814. * story and would be not invoked via this.
  815. */
  816. static void cpu_up_down_serialize_trainwrecks(bool tasks_frozen)
  817. {
  818. /*
  819. * cpusets delegate hotplug operations to a worker to "solve" the
  820. * lock order problems. Wait for the worker, but only if tasks are
  821. * _not_ frozen (suspend, hibernate) as that would wait forever.
  822. *
  823. * The wait is required because otherwise the hotplug operation
  824. * returns with inconsistent state, which could even be observed in
  825. * user space when a new CPU is brought up. The CPU plug uevent
  826. * would be delivered and user space reacting on it would fail to
  827. * move tasks to the newly plugged CPU up to the point where the
  828. * work has finished because up to that point the newly plugged CPU
  829. * is not assignable in cpusets/cgroups. On unplug that's not
  830. * necessarily a visible issue, but it is still inconsistent state,
  831. * which is the real problem which needs to be "fixed". This can't
  832. * prevent the transient state between scheduling the work and
  833. * returning from waiting for it.
  834. */
  835. if (!tasks_frozen)
  836. cpuset_wait_for_hotplug();
  837. }
  838. #ifdef CONFIG_HOTPLUG_CPU
  839. #ifndef arch_clear_mm_cpumask_cpu
  840. #define arch_clear_mm_cpumask_cpu(cpu, mm) cpumask_clear_cpu(cpu, mm_cpumask(mm))
  841. #endif
  842. /**
  843. * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
  844. * @cpu: a CPU id
  845. *
  846. * This function walks all processes, finds a valid mm struct for each one and
  847. * then clears a corresponding bit in mm's cpumask. While this all sounds
  848. * trivial, there are various non-obvious corner cases, which this function
  849. * tries to solve in a safe manner.
  850. *
  851. * Also note that the function uses a somewhat relaxed locking scheme, so it may
  852. * be called only for an already offlined CPU.
  853. */
  854. void clear_tasks_mm_cpumask(int cpu)
  855. {
  856. struct task_struct *p;
  857. /*
  858. * This function is called after the cpu is taken down and marked
  859. * offline, so its not like new tasks will ever get this cpu set in
  860. * their mm mask. -- Peter Zijlstra
  861. * Thus, we may use rcu_read_lock() here, instead of grabbing
  862. * full-fledged tasklist_lock.
  863. */
  864. WARN_ON(cpu_online(cpu));
  865. rcu_read_lock();
  866. for_each_process(p) {
  867. struct task_struct *t;
  868. /*
  869. * Main thread might exit, but other threads may still have
  870. * a valid mm. Find one.
  871. */
  872. t = find_lock_task_mm(p);
  873. if (!t)
  874. continue;
  875. arch_clear_mm_cpumask_cpu(cpu, t->mm);
  876. task_unlock(t);
  877. }
  878. rcu_read_unlock();
  879. }
  880. /* Take this CPU down. */
  881. static int take_cpu_down(void *_param)
  882. {
  883. struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
  884. enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE);
  885. int err, cpu = smp_processor_id();
  886. /* Ensure this CPU doesn't handle any more interrupts. */
  887. err = __cpu_disable();
  888. if (err < 0)
  889. return err;
  890. /*
  891. * Must be called from CPUHP_TEARDOWN_CPU, which means, as we are going
  892. * down, that the current state is CPUHP_TEARDOWN_CPU - 1.
  893. */
  894. WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1));
  895. /*
  896. * Invoke the former CPU_DYING callbacks. DYING must not fail!
  897. */
  898. cpuhp_invoke_callback_range_nofail(false, cpu, st, target);
  899. /* Give up timekeeping duties */
  900. tick_handover_do_timer();
  901. /* Remove CPU from timer broadcasting */
  902. tick_offline_cpu(cpu);
  903. /* Park the stopper thread */
  904. stop_machine_park(cpu);
  905. return 0;
  906. }
  907. static int takedown_cpu(unsigned int cpu)
  908. {
  909. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  910. int err;
  911. /* Park the smpboot threads */
  912. kthread_park(st->thread);
  913. /*
  914. * Prevent irq alloc/free while the dying cpu reorganizes the
  915. * interrupt affinities.
  916. */
  917. irq_lock_sparse();
  918. /*
  919. * So now all preempt/rcu users must observe !cpu_active().
  920. */
  921. err = stop_machine_cpuslocked(take_cpu_down, NULL, cpumask_of(cpu));
  922. if (err) {
  923. /* CPU refused to die */
  924. irq_unlock_sparse();
  925. /* Unpark the hotplug thread so we can rollback there */
  926. kthread_unpark(st->thread);
  927. return err;
  928. }
  929. BUG_ON(cpu_online(cpu));
  930. /*
  931. * The teardown callback for CPUHP_AP_SCHED_STARTING will have removed
  932. * all runnable tasks from the CPU, there's only the idle task left now
  933. * that the migration thread is done doing the stop_machine thing.
  934. *
  935. * Wait for the stop thread to go away.
  936. */
  937. wait_for_ap_thread(st, false);
  938. BUG_ON(st->state != CPUHP_AP_IDLE_DEAD);
  939. /* Interrupts are moved away from the dying cpu, reenable alloc/free */
  940. irq_unlock_sparse();
  941. hotplug_cpu__broadcast_tick_pull(cpu);
  942. /* This actually kills the CPU. */
  943. __cpu_die(cpu);
  944. tick_cleanup_dead_cpu(cpu);
  945. rcutree_migrate_callbacks(cpu);
  946. return 0;
  947. }
  948. static void cpuhp_complete_idle_dead(void *arg)
  949. {
  950. struct cpuhp_cpu_state *st = arg;
  951. complete_ap_thread(st, false);
  952. }
  953. void cpuhp_report_idle_dead(void)
  954. {
  955. struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
  956. BUG_ON(st->state != CPUHP_AP_OFFLINE);
  957. rcu_report_dead(smp_processor_id());
  958. st->state = CPUHP_AP_IDLE_DEAD;
  959. /*
  960. * We cannot call complete after rcu_report_dead() so we delegate it
  961. * to an online cpu.
  962. */
  963. smp_call_function_single(cpumask_first(cpu_online_mask),
  964. cpuhp_complete_idle_dead, st, 0);
  965. }
  966. static int cpuhp_down_callbacks(unsigned int cpu, struct cpuhp_cpu_state *st,
  967. enum cpuhp_state target)
  968. {
  969. enum cpuhp_state prev_state = st->state;
  970. int ret = 0;
  971. ret = cpuhp_invoke_callback_range(false, cpu, st, target);
  972. if (ret) {
  973. pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n",
  974. ret, cpu, cpuhp_get_step(st->state)->name,
  975. st->state);
  976. cpuhp_reset_state(cpu, st, prev_state);
  977. if (st->state < prev_state)
  978. WARN_ON(cpuhp_invoke_callback_range(true, cpu, st,
  979. prev_state));
  980. }
  981. return ret;
  982. }
  983. /* Requires cpu_add_remove_lock to be held */
  984. static int __ref _cpu_down(unsigned int cpu, int tasks_frozen,
  985. enum cpuhp_state target)
  986. {
  987. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  988. int prev_state, ret = 0;
  989. if (num_online_cpus() == 1)
  990. return -EBUSY;
  991. if (!cpu_present(cpu))
  992. return -EINVAL;
  993. cpus_write_lock();
  994. cpuhp_tasks_frozen = tasks_frozen;
  995. prev_state = cpuhp_set_state(cpu, st, target);
  996. /*
  997. * If the current CPU state is in the range of the AP hotplug thread,
  998. * then we need to kick the thread.
  999. */
  1000. if (st->state > CPUHP_TEARDOWN_CPU) {
  1001. st->target = max((int)target, CPUHP_TEARDOWN_CPU);
  1002. ret = cpuhp_kick_ap_work(cpu);
  1003. /*
  1004. * The AP side has done the error rollback already. Just
  1005. * return the error code..
  1006. */
  1007. if (ret)
  1008. goto out;
  1009. /*
  1010. * We might have stopped still in the range of the AP hotplug
  1011. * thread. Nothing to do anymore.
  1012. */
  1013. if (st->state > CPUHP_TEARDOWN_CPU)
  1014. goto out;
  1015. st->target = target;
  1016. }
  1017. /*
  1018. * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need
  1019. * to do the further cleanups.
  1020. */
  1021. ret = cpuhp_down_callbacks(cpu, st, target);
  1022. if (ret && st->state < prev_state) {
  1023. if (st->state == CPUHP_TEARDOWN_CPU) {
  1024. cpuhp_reset_state(cpu, st, prev_state);
  1025. __cpuhp_kick_ap(st);
  1026. } else {
  1027. WARN(1, "DEAD callback error for CPU%d", cpu);
  1028. }
  1029. }
  1030. out:
  1031. cpus_write_unlock();
  1032. /*
  1033. * Do post unplug cleanup. This is still protected against
  1034. * concurrent CPU hotplug via cpu_add_remove_lock.
  1035. */
  1036. lockup_detector_cleanup();
  1037. arch_smt_update();
  1038. cpu_up_down_serialize_trainwrecks(tasks_frozen);
  1039. return ret;
  1040. }
  1041. struct cpu_down_work {
  1042. unsigned int cpu;
  1043. enum cpuhp_state target;
  1044. };
  1045. static long __cpu_down_maps_locked(void *arg)
  1046. {
  1047. struct cpu_down_work *work = arg;
  1048. return _cpu_down(work->cpu, 0, work->target);
  1049. }
  1050. static int cpu_down_maps_locked(unsigned int cpu, enum cpuhp_state target)
  1051. {
  1052. struct cpu_down_work work = { .cpu = cpu, .target = target, };
  1053. /*
  1054. * If the platform does not support hotplug, report it explicitly to
  1055. * differentiate it from a transient offlining failure.
  1056. */
  1057. if (cc_platform_has(CC_ATTR_HOTPLUG_DISABLED))
  1058. return -EOPNOTSUPP;
  1059. if (cpu_hotplug_disabled)
  1060. return -EBUSY;
  1061. /*
  1062. * Ensure that the control task does not run on the to be offlined
  1063. * CPU to prevent a deadlock against cfs_b->period_timer.
  1064. * Also keep at least one housekeeping cpu onlined to avoid generating
  1065. * an empty sched_domain span.
  1066. */
  1067. for_each_cpu_and(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_DOMAIN)) {
  1068. if (cpu != work.cpu)
  1069. return work_on_cpu(cpu, __cpu_down_maps_locked, &work);
  1070. }
  1071. return -EBUSY;
  1072. }
  1073. static int cpu_down(unsigned int cpu, enum cpuhp_state target)
  1074. {
  1075. int err;
  1076. cpu_maps_update_begin();
  1077. err = cpu_down_maps_locked(cpu, target);
  1078. cpu_maps_update_done();
  1079. return err;
  1080. }
  1081. /**
  1082. * cpu_device_down - Bring down a cpu device
  1083. * @dev: Pointer to the cpu device to offline
  1084. *
  1085. * This function is meant to be used by device core cpu subsystem only.
  1086. *
  1087. * Other subsystems should use remove_cpu() instead.
  1088. *
  1089. * Return: %0 on success or a negative errno code
  1090. */
  1091. int cpu_device_down(struct device *dev)
  1092. {
  1093. return cpu_down(dev->id, CPUHP_OFFLINE);
  1094. }
  1095. int remove_cpu(unsigned int cpu)
  1096. {
  1097. int ret;
  1098. lock_device_hotplug();
  1099. ret = device_offline(get_cpu_device(cpu));
  1100. unlock_device_hotplug();
  1101. return ret;
  1102. }
  1103. EXPORT_SYMBOL_GPL(remove_cpu);
  1104. void smp_shutdown_nonboot_cpus(unsigned int primary_cpu)
  1105. {
  1106. unsigned int cpu;
  1107. int error;
  1108. cpu_maps_update_begin();
  1109. /*
  1110. * Make certain the cpu I'm about to reboot on is online.
  1111. *
  1112. * This is inline to what migrate_to_reboot_cpu() already do.
  1113. */
  1114. if (!cpu_online(primary_cpu))
  1115. primary_cpu = cpumask_first(cpu_online_mask);
  1116. for_each_online_cpu(cpu) {
  1117. if (cpu == primary_cpu)
  1118. continue;
  1119. error = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
  1120. if (error) {
  1121. pr_err("Failed to offline CPU%d - error=%d",
  1122. cpu, error);
  1123. break;
  1124. }
  1125. }
  1126. /*
  1127. * Ensure all but the reboot CPU are offline.
  1128. */
  1129. BUG_ON(num_online_cpus() > 1);
  1130. /*
  1131. * Make sure the CPUs won't be enabled by someone else after this
  1132. * point. Kexec will reboot to a new kernel shortly resetting
  1133. * everything along the way.
  1134. */
  1135. cpu_hotplug_disabled++;
  1136. cpu_maps_update_done();
  1137. }
  1138. #else
  1139. #define takedown_cpu NULL
  1140. #endif /*CONFIG_HOTPLUG_CPU*/
  1141. /**
  1142. * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
  1143. * @cpu: cpu that just started
  1144. *
  1145. * It must be called by the arch code on the new cpu, before the new cpu
  1146. * enables interrupts and before the "boot" cpu returns from __cpu_up().
  1147. */
  1148. void notify_cpu_starting(unsigned int cpu)
  1149. {
  1150. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1151. enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE);
  1152. rcu_cpu_starting(cpu); /* Enables RCU usage on this CPU. */
  1153. cpumask_set_cpu(cpu, &cpus_booted_once_mask);
  1154. /*
  1155. * STARTING must not fail!
  1156. */
  1157. cpuhp_invoke_callback_range_nofail(true, cpu, st, target);
  1158. }
  1159. /*
  1160. * Called from the idle task. Wake up the controlling task which brings the
  1161. * hotplug thread of the upcoming CPU up and then delegates the rest of the
  1162. * online bringup to the hotplug thread.
  1163. */
  1164. void cpuhp_online_idle(enum cpuhp_state state)
  1165. {
  1166. struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
  1167. /* Happens for the boot cpu */
  1168. if (state != CPUHP_AP_ONLINE_IDLE)
  1169. return;
  1170. /*
  1171. * Unpart the stopper thread before we start the idle loop (and start
  1172. * scheduling); this ensures the stopper task is always available.
  1173. */
  1174. stop_machine_unpark(smp_processor_id());
  1175. st->state = CPUHP_AP_ONLINE_IDLE;
  1176. complete_ap_thread(st, true);
  1177. }
  1178. /* Requires cpu_add_remove_lock to be held */
  1179. static int _cpu_up(unsigned int cpu, int tasks_frozen, enum cpuhp_state target)
  1180. {
  1181. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1182. struct task_struct *idle;
  1183. int ret = 0;
  1184. cpus_write_lock();
  1185. if (!cpu_present(cpu)) {
  1186. ret = -EINVAL;
  1187. goto out;
  1188. }
  1189. /*
  1190. * The caller of cpu_up() might have raced with another
  1191. * caller. Nothing to do.
  1192. */
  1193. if (st->state >= target)
  1194. goto out;
  1195. if (st->state == CPUHP_OFFLINE) {
  1196. /* Let it fail before we try to bring the cpu up */
  1197. idle = idle_thread_get(cpu);
  1198. if (IS_ERR(idle)) {
  1199. ret = PTR_ERR(idle);
  1200. goto out;
  1201. }
  1202. }
  1203. cpuhp_tasks_frozen = tasks_frozen;
  1204. cpuhp_set_state(cpu, st, target);
  1205. /*
  1206. * If the current CPU state is in the range of the AP hotplug thread,
  1207. * then we need to kick the thread once more.
  1208. */
  1209. if (st->state > CPUHP_BRINGUP_CPU) {
  1210. ret = cpuhp_kick_ap_work(cpu);
  1211. /*
  1212. * The AP side has done the error rollback already. Just
  1213. * return the error code..
  1214. */
  1215. if (ret)
  1216. goto out;
  1217. }
  1218. /*
  1219. * Try to reach the target state. We max out on the BP at
  1220. * CPUHP_BRINGUP_CPU. After that the AP hotplug thread is
  1221. * responsible for bringing it up to the target state.
  1222. */
  1223. target = min((int)target, CPUHP_BRINGUP_CPU);
  1224. ret = cpuhp_up_callbacks(cpu, st, target);
  1225. out:
  1226. cpus_write_unlock();
  1227. arch_smt_update();
  1228. cpu_up_down_serialize_trainwrecks(tasks_frozen);
  1229. return ret;
  1230. }
  1231. static int cpu_up(unsigned int cpu, enum cpuhp_state target)
  1232. {
  1233. int err = 0;
  1234. if (!cpu_possible(cpu)) {
  1235. pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n",
  1236. cpu);
  1237. #if defined(CONFIG_IA64)
  1238. pr_err("please check additional_cpus= boot parameter\n");
  1239. #endif
  1240. return -EINVAL;
  1241. }
  1242. err = try_online_node(cpu_to_node(cpu));
  1243. if (err)
  1244. return err;
  1245. cpu_maps_update_begin();
  1246. if (cpu_hotplug_disabled) {
  1247. err = -EBUSY;
  1248. goto out;
  1249. }
  1250. if (!cpu_smt_allowed(cpu)) {
  1251. err = -EPERM;
  1252. goto out;
  1253. }
  1254. err = _cpu_up(cpu, 0, target);
  1255. out:
  1256. cpu_maps_update_done();
  1257. return err;
  1258. }
  1259. /**
  1260. * cpu_device_up - Bring up a cpu device
  1261. * @dev: Pointer to the cpu device to online
  1262. *
  1263. * This function is meant to be used by device core cpu subsystem only.
  1264. *
  1265. * Other subsystems should use add_cpu() instead.
  1266. *
  1267. * Return: %0 on success or a negative errno code
  1268. */
  1269. int cpu_device_up(struct device *dev)
  1270. {
  1271. return cpu_up(dev->id, CPUHP_ONLINE);
  1272. }
  1273. int add_cpu(unsigned int cpu)
  1274. {
  1275. int ret;
  1276. lock_device_hotplug();
  1277. ret = device_online(get_cpu_device(cpu));
  1278. unlock_device_hotplug();
  1279. return ret;
  1280. }
  1281. EXPORT_SYMBOL_GPL(add_cpu);
  1282. /**
  1283. * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
  1284. * @sleep_cpu: The cpu we hibernated on and should be brought up.
  1285. *
  1286. * On some architectures like arm64, we can hibernate on any CPU, but on
  1287. * wake up the CPU we hibernated on might be offline as a side effect of
  1288. * using maxcpus= for example.
  1289. *
  1290. * Return: %0 on success or a negative errno code
  1291. */
  1292. int bringup_hibernate_cpu(unsigned int sleep_cpu)
  1293. {
  1294. int ret;
  1295. if (!cpu_online(sleep_cpu)) {
  1296. pr_info("Hibernated on a CPU that is offline! Bringing CPU up.\n");
  1297. ret = cpu_up(sleep_cpu, CPUHP_ONLINE);
  1298. if (ret) {
  1299. pr_err("Failed to bring hibernate-CPU up!\n");
  1300. return ret;
  1301. }
  1302. }
  1303. return 0;
  1304. }
  1305. void bringup_nonboot_cpus(unsigned int setup_max_cpus)
  1306. {
  1307. unsigned int cpu;
  1308. for_each_present_cpu(cpu) {
  1309. if (num_online_cpus() >= setup_max_cpus)
  1310. break;
  1311. if (!cpu_online(cpu))
  1312. cpu_up(cpu, CPUHP_ONLINE);
  1313. }
  1314. }
  1315. #ifdef CONFIG_PM_SLEEP_SMP
  1316. static cpumask_var_t frozen_cpus;
  1317. int freeze_secondary_cpus(int primary)
  1318. {
  1319. int cpu, error = 0;
  1320. cpu_maps_update_begin();
  1321. if (primary == -1) {
  1322. primary = cpumask_first(cpu_online_mask);
  1323. if (!housekeeping_cpu(primary, HK_TYPE_TIMER))
  1324. primary = housekeeping_any_cpu(HK_TYPE_TIMER);
  1325. } else {
  1326. if (!cpu_online(primary))
  1327. primary = cpumask_first(cpu_online_mask);
  1328. }
  1329. /*
  1330. * We take down all of the non-boot CPUs in one shot to avoid races
  1331. * with the userspace trying to use the CPU hotplug at the same time
  1332. */
  1333. cpumask_clear(frozen_cpus);
  1334. pr_info("Disabling non-boot CPUs ...\n");
  1335. for_each_online_cpu(cpu) {
  1336. if (cpu == primary)
  1337. continue;
  1338. if (pm_wakeup_pending()) {
  1339. pr_info("Wakeup pending. Abort CPU freeze\n");
  1340. error = -EBUSY;
  1341. break;
  1342. }
  1343. trace_suspend_resume(TPS("CPU_OFF"), cpu, true);
  1344. error = _cpu_down(cpu, 1, CPUHP_OFFLINE);
  1345. trace_suspend_resume(TPS("CPU_OFF"), cpu, false);
  1346. if (!error)
  1347. cpumask_set_cpu(cpu, frozen_cpus);
  1348. else {
  1349. pr_err("Error taking CPU%d down: %d\n", cpu, error);
  1350. break;
  1351. }
  1352. }
  1353. if (!error)
  1354. BUG_ON(num_online_cpus() > 1);
  1355. else
  1356. pr_err("Non-boot CPUs are not disabled\n");
  1357. /*
  1358. * Make sure the CPUs won't be enabled by someone else. We need to do
  1359. * this even in case of failure as all freeze_secondary_cpus() users are
  1360. * supposed to do thaw_secondary_cpus() on the failure path.
  1361. */
  1362. cpu_hotplug_disabled++;
  1363. cpu_maps_update_done();
  1364. return error;
  1365. }
  1366. void __weak arch_thaw_secondary_cpus_begin(void)
  1367. {
  1368. }
  1369. void __weak arch_thaw_secondary_cpus_end(void)
  1370. {
  1371. }
  1372. void thaw_secondary_cpus(void)
  1373. {
  1374. int cpu, error;
  1375. /* Allow everyone to use the CPU hotplug again */
  1376. cpu_maps_update_begin();
  1377. __cpu_hotplug_enable();
  1378. if (cpumask_empty(frozen_cpus))
  1379. goto out;
  1380. pr_info("Enabling non-boot CPUs ...\n");
  1381. arch_thaw_secondary_cpus_begin();
  1382. for_each_cpu(cpu, frozen_cpus) {
  1383. trace_suspend_resume(TPS("CPU_ON"), cpu, true);
  1384. error = _cpu_up(cpu, 1, CPUHP_ONLINE);
  1385. trace_suspend_resume(TPS("CPU_ON"), cpu, false);
  1386. if (!error) {
  1387. pr_info("CPU%d is up\n", cpu);
  1388. continue;
  1389. }
  1390. pr_warn("Error taking CPU%d up: %d\n", cpu, error);
  1391. }
  1392. arch_thaw_secondary_cpus_end();
  1393. cpumask_clear(frozen_cpus);
  1394. out:
  1395. cpu_maps_update_done();
  1396. }
  1397. static int __init alloc_frozen_cpus(void)
  1398. {
  1399. if (!alloc_cpumask_var(&frozen_cpus, GFP_KERNEL|__GFP_ZERO))
  1400. return -ENOMEM;
  1401. return 0;
  1402. }
  1403. core_initcall(alloc_frozen_cpus);
  1404. /*
  1405. * When callbacks for CPU hotplug notifications are being executed, we must
  1406. * ensure that the state of the system with respect to the tasks being frozen
  1407. * or not, as reported by the notification, remains unchanged *throughout the
  1408. * duration* of the execution of the callbacks.
  1409. * Hence we need to prevent the freezer from racing with regular CPU hotplug.
  1410. *
  1411. * This synchronization is implemented by mutually excluding regular CPU
  1412. * hotplug and Suspend/Hibernate call paths by hooking onto the Suspend/
  1413. * Hibernate notifications.
  1414. */
  1415. static int
  1416. cpu_hotplug_pm_callback(struct notifier_block *nb,
  1417. unsigned long action, void *ptr)
  1418. {
  1419. switch (action) {
  1420. case PM_SUSPEND_PREPARE:
  1421. case PM_HIBERNATION_PREPARE:
  1422. cpu_hotplug_disable();
  1423. break;
  1424. case PM_POST_SUSPEND:
  1425. case PM_POST_HIBERNATION:
  1426. cpu_hotplug_enable();
  1427. break;
  1428. default:
  1429. return NOTIFY_DONE;
  1430. }
  1431. return NOTIFY_OK;
  1432. }
  1433. static int __init cpu_hotplug_pm_sync_init(void)
  1434. {
  1435. /*
  1436. * cpu_hotplug_pm_callback has higher priority than x86
  1437. * bsp_pm_callback which depends on cpu_hotplug_pm_callback
  1438. * to disable cpu hotplug to avoid cpu hotplug race.
  1439. */
  1440. pm_notifier(cpu_hotplug_pm_callback, 0);
  1441. return 0;
  1442. }
  1443. core_initcall(cpu_hotplug_pm_sync_init);
  1444. #endif /* CONFIG_PM_SLEEP_SMP */
  1445. int __boot_cpu_id;
  1446. #endif /* CONFIG_SMP */
  1447. /* Boot processor state steps */
  1448. static struct cpuhp_step cpuhp_hp_states[] = {
  1449. [CPUHP_OFFLINE] = {
  1450. .name = "offline",
  1451. .startup.single = NULL,
  1452. .teardown.single = NULL,
  1453. },
  1454. #ifdef CONFIG_SMP
  1455. [CPUHP_CREATE_THREADS]= {
  1456. .name = "threads:prepare",
  1457. .startup.single = smpboot_create_threads,
  1458. .teardown.single = NULL,
  1459. .cant_stop = true,
  1460. },
  1461. [CPUHP_PERF_PREPARE] = {
  1462. .name = "perf:prepare",
  1463. .startup.single = perf_event_init_cpu,
  1464. .teardown.single = perf_event_exit_cpu,
  1465. },
  1466. [CPUHP_RANDOM_PREPARE] = {
  1467. .name = "random:prepare",
  1468. .startup.single = random_prepare_cpu,
  1469. .teardown.single = NULL,
  1470. },
  1471. [CPUHP_WORKQUEUE_PREP] = {
  1472. .name = "workqueue:prepare",
  1473. .startup.single = workqueue_prepare_cpu,
  1474. .teardown.single = NULL,
  1475. },
  1476. [CPUHP_HRTIMERS_PREPARE] = {
  1477. .name = "hrtimers:prepare",
  1478. .startup.single = hrtimers_prepare_cpu,
  1479. .teardown.single = hrtimers_dead_cpu,
  1480. },
  1481. [CPUHP_SMPCFD_PREPARE] = {
  1482. .name = "smpcfd:prepare",
  1483. .startup.single = smpcfd_prepare_cpu,
  1484. .teardown.single = smpcfd_dead_cpu,
  1485. },
  1486. [CPUHP_RELAY_PREPARE] = {
  1487. .name = "relay:prepare",
  1488. .startup.single = relay_prepare_cpu,
  1489. .teardown.single = NULL,
  1490. },
  1491. [CPUHP_SLAB_PREPARE] = {
  1492. .name = "slab:prepare",
  1493. .startup.single = slab_prepare_cpu,
  1494. .teardown.single = slab_dead_cpu,
  1495. },
  1496. [CPUHP_RCUTREE_PREP] = {
  1497. .name = "RCU/tree:prepare",
  1498. .startup.single = rcutree_prepare_cpu,
  1499. .teardown.single = rcutree_dead_cpu,
  1500. },
  1501. /*
  1502. * On the tear-down path, timers_dead_cpu() must be invoked
  1503. * before blk_mq_queue_reinit_notify() from notify_dead(),
  1504. * otherwise a RCU stall occurs.
  1505. */
  1506. [CPUHP_TIMERS_PREPARE] = {
  1507. .name = "timers:prepare",
  1508. .startup.single = timers_prepare_cpu,
  1509. .teardown.single = timers_dead_cpu,
  1510. },
  1511. /* Kicks the plugged cpu into life */
  1512. [CPUHP_BRINGUP_CPU] = {
  1513. .name = "cpu:bringup",
  1514. .startup.single = bringup_cpu,
  1515. .teardown.single = finish_cpu,
  1516. .cant_stop = true,
  1517. },
  1518. /* Final state before CPU kills itself */
  1519. [CPUHP_AP_IDLE_DEAD] = {
  1520. .name = "idle:dead",
  1521. },
  1522. /*
  1523. * Last state before CPU enters the idle loop to die. Transient state
  1524. * for synchronization.
  1525. */
  1526. [CPUHP_AP_OFFLINE] = {
  1527. .name = "ap:offline",
  1528. .cant_stop = true,
  1529. },
  1530. /* First state is scheduler control. Interrupts are disabled */
  1531. [CPUHP_AP_SCHED_STARTING] = {
  1532. .name = "sched:starting",
  1533. .startup.single = sched_cpu_starting,
  1534. .teardown.single = sched_cpu_dying,
  1535. },
  1536. [CPUHP_AP_RCUTREE_DYING] = {
  1537. .name = "RCU/tree:dying",
  1538. .startup.single = NULL,
  1539. .teardown.single = rcutree_dying_cpu,
  1540. },
  1541. [CPUHP_AP_SMPCFD_DYING] = {
  1542. .name = "smpcfd:dying",
  1543. .startup.single = NULL,
  1544. .teardown.single = smpcfd_dying_cpu,
  1545. },
  1546. /* Entry state on starting. Interrupts enabled from here on. Transient
  1547. * state for synchronsization */
  1548. [CPUHP_AP_ONLINE] = {
  1549. .name = "ap:online",
  1550. },
  1551. /*
  1552. * Handled on control processor until the plugged processor manages
  1553. * this itself.
  1554. */
  1555. [CPUHP_TEARDOWN_CPU] = {
  1556. .name = "cpu:teardown",
  1557. .startup.single = NULL,
  1558. .teardown.single = takedown_cpu,
  1559. .cant_stop = true,
  1560. },
  1561. [CPUHP_AP_SCHED_WAIT_EMPTY] = {
  1562. .name = "sched:waitempty",
  1563. .startup.single = NULL,
  1564. .teardown.single = sched_cpu_wait_empty,
  1565. },
  1566. /* Handle smpboot threads park/unpark */
  1567. [CPUHP_AP_SMPBOOT_THREADS] = {
  1568. .name = "smpboot/threads:online",
  1569. .startup.single = smpboot_unpark_threads,
  1570. .teardown.single = smpboot_park_threads,
  1571. },
  1572. [CPUHP_AP_IRQ_AFFINITY_ONLINE] = {
  1573. .name = "irq/affinity:online",
  1574. .startup.single = irq_affinity_online_cpu,
  1575. .teardown.single = NULL,
  1576. },
  1577. [CPUHP_AP_PERF_ONLINE] = {
  1578. .name = "perf:online",
  1579. .startup.single = perf_event_init_cpu,
  1580. .teardown.single = perf_event_exit_cpu,
  1581. },
  1582. [CPUHP_AP_WATCHDOG_ONLINE] = {
  1583. .name = "lockup_detector:online",
  1584. .startup.single = lockup_detector_online_cpu,
  1585. .teardown.single = lockup_detector_offline_cpu,
  1586. },
  1587. [CPUHP_AP_WORKQUEUE_ONLINE] = {
  1588. .name = "workqueue:online",
  1589. .startup.single = workqueue_online_cpu,
  1590. .teardown.single = workqueue_offline_cpu,
  1591. },
  1592. [CPUHP_AP_RANDOM_ONLINE] = {
  1593. .name = "random:online",
  1594. .startup.single = random_online_cpu,
  1595. .teardown.single = NULL,
  1596. },
  1597. [CPUHP_AP_RCUTREE_ONLINE] = {
  1598. .name = "RCU/tree:online",
  1599. .startup.single = rcutree_online_cpu,
  1600. .teardown.single = rcutree_offline_cpu,
  1601. },
  1602. #endif
  1603. /*
  1604. * The dynamically registered state space is here
  1605. */
  1606. #ifdef CONFIG_SMP
  1607. /* Last state is scheduler control setting the cpu active */
  1608. [CPUHP_AP_ACTIVE] = {
  1609. .name = "sched:active",
  1610. .startup.single = sched_cpu_activate,
  1611. .teardown.single = sched_cpu_deactivate,
  1612. },
  1613. #endif
  1614. /* CPU is fully up and running. */
  1615. [CPUHP_ONLINE] = {
  1616. .name = "online",
  1617. .startup.single = NULL,
  1618. .teardown.single = NULL,
  1619. },
  1620. };
  1621. /* Sanity check for callbacks */
  1622. static int cpuhp_cb_check(enum cpuhp_state state)
  1623. {
  1624. if (state <= CPUHP_OFFLINE || state >= CPUHP_ONLINE)
  1625. return -EINVAL;
  1626. return 0;
  1627. }
  1628. /*
  1629. * Returns a free for dynamic slot assignment of the Online state. The states
  1630. * are protected by the cpuhp_slot_states mutex and an empty slot is identified
  1631. * by having no name assigned.
  1632. */
  1633. static int cpuhp_reserve_state(enum cpuhp_state state)
  1634. {
  1635. enum cpuhp_state i, end;
  1636. struct cpuhp_step *step;
  1637. switch (state) {
  1638. case CPUHP_AP_ONLINE_DYN:
  1639. step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN;
  1640. end = CPUHP_AP_ONLINE_DYN_END;
  1641. break;
  1642. case CPUHP_BP_PREPARE_DYN:
  1643. step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN;
  1644. end = CPUHP_BP_PREPARE_DYN_END;
  1645. break;
  1646. default:
  1647. return -EINVAL;
  1648. }
  1649. for (i = state; i <= end; i++, step++) {
  1650. if (!step->name)
  1651. return i;
  1652. }
  1653. WARN(1, "No more dynamic states available for CPU hotplug\n");
  1654. return -ENOSPC;
  1655. }
  1656. static int cpuhp_store_callbacks(enum cpuhp_state state, const char *name,
  1657. int (*startup)(unsigned int cpu),
  1658. int (*teardown)(unsigned int cpu),
  1659. bool multi_instance)
  1660. {
  1661. /* (Un)Install the callbacks for further cpu hotplug operations */
  1662. struct cpuhp_step *sp;
  1663. int ret = 0;
  1664. /*
  1665. * If name is NULL, then the state gets removed.
  1666. *
  1667. * CPUHP_AP_ONLINE_DYN and CPUHP_BP_PREPARE_DYN are handed out on
  1668. * the first allocation from these dynamic ranges, so the removal
  1669. * would trigger a new allocation and clear the wrong (already
  1670. * empty) state, leaving the callbacks of the to be cleared state
  1671. * dangling, which causes wreckage on the next hotplug operation.
  1672. */
  1673. if (name && (state == CPUHP_AP_ONLINE_DYN ||
  1674. state == CPUHP_BP_PREPARE_DYN)) {
  1675. ret = cpuhp_reserve_state(state);
  1676. if (ret < 0)
  1677. return ret;
  1678. state = ret;
  1679. }
  1680. sp = cpuhp_get_step(state);
  1681. if (name && sp->name)
  1682. return -EBUSY;
  1683. sp->startup.single = startup;
  1684. sp->teardown.single = teardown;
  1685. sp->name = name;
  1686. sp->multi_instance = multi_instance;
  1687. INIT_HLIST_HEAD(&sp->list);
  1688. return ret;
  1689. }
  1690. static void *cpuhp_get_teardown_cb(enum cpuhp_state state)
  1691. {
  1692. return cpuhp_get_step(state)->teardown.single;
  1693. }
  1694. /*
  1695. * Call the startup/teardown function for a step either on the AP or
  1696. * on the current CPU.
  1697. */
  1698. static int cpuhp_issue_call(int cpu, enum cpuhp_state state, bool bringup,
  1699. struct hlist_node *node)
  1700. {
  1701. struct cpuhp_step *sp = cpuhp_get_step(state);
  1702. int ret;
  1703. /*
  1704. * If there's nothing to do, we done.
  1705. * Relies on the union for multi_instance.
  1706. */
  1707. if (cpuhp_step_empty(bringup, sp))
  1708. return 0;
  1709. /*
  1710. * The non AP bound callbacks can fail on bringup. On teardown
  1711. * e.g. module removal we crash for now.
  1712. */
  1713. #ifdef CONFIG_SMP
  1714. if (cpuhp_is_ap_state(state))
  1715. ret = cpuhp_invoke_ap_callback(cpu, state, bringup, node);
  1716. else
  1717. ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
  1718. #else
  1719. ret = cpuhp_invoke_callback(cpu, state, bringup, node, NULL);
  1720. #endif
  1721. BUG_ON(ret && !bringup);
  1722. return ret;
  1723. }
  1724. /*
  1725. * Called from __cpuhp_setup_state on a recoverable failure.
  1726. *
  1727. * Note: The teardown callbacks for rollback are not allowed to fail!
  1728. */
  1729. static void cpuhp_rollback_install(int failedcpu, enum cpuhp_state state,
  1730. struct hlist_node *node)
  1731. {
  1732. int cpu;
  1733. /* Roll back the already executed steps on the other cpus */
  1734. for_each_present_cpu(cpu) {
  1735. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1736. int cpustate = st->state;
  1737. if (cpu >= failedcpu)
  1738. break;
  1739. /* Did we invoke the startup call on that cpu ? */
  1740. if (cpustate >= state)
  1741. cpuhp_issue_call(cpu, state, false, node);
  1742. }
  1743. }
  1744. int __cpuhp_state_add_instance_cpuslocked(enum cpuhp_state state,
  1745. struct hlist_node *node,
  1746. bool invoke)
  1747. {
  1748. struct cpuhp_step *sp;
  1749. int cpu;
  1750. int ret;
  1751. lockdep_assert_cpus_held();
  1752. sp = cpuhp_get_step(state);
  1753. if (sp->multi_instance == false)
  1754. return -EINVAL;
  1755. mutex_lock(&cpuhp_state_mutex);
  1756. if (!invoke || !sp->startup.multi)
  1757. goto add_node;
  1758. /*
  1759. * Try to call the startup callback for each present cpu
  1760. * depending on the hotplug state of the cpu.
  1761. */
  1762. for_each_present_cpu(cpu) {
  1763. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1764. int cpustate = st->state;
  1765. if (cpustate < state)
  1766. continue;
  1767. ret = cpuhp_issue_call(cpu, state, true, node);
  1768. if (ret) {
  1769. if (sp->teardown.multi)
  1770. cpuhp_rollback_install(cpu, state, node);
  1771. goto unlock;
  1772. }
  1773. }
  1774. add_node:
  1775. ret = 0;
  1776. hlist_add_head(node, &sp->list);
  1777. unlock:
  1778. mutex_unlock(&cpuhp_state_mutex);
  1779. return ret;
  1780. }
  1781. int __cpuhp_state_add_instance(enum cpuhp_state state, struct hlist_node *node,
  1782. bool invoke)
  1783. {
  1784. int ret;
  1785. cpus_read_lock();
  1786. ret = __cpuhp_state_add_instance_cpuslocked(state, node, invoke);
  1787. cpus_read_unlock();
  1788. return ret;
  1789. }
  1790. EXPORT_SYMBOL_GPL(__cpuhp_state_add_instance);
  1791. /**
  1792. * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
  1793. * @state: The state to setup
  1794. * @name: Name of the step
  1795. * @invoke: If true, the startup function is invoked for cpus where
  1796. * cpu state >= @state
  1797. * @startup: startup callback function
  1798. * @teardown: teardown callback function
  1799. * @multi_instance: State is set up for multiple instances which get
  1800. * added afterwards.
  1801. *
  1802. * The caller needs to hold cpus read locked while calling this function.
  1803. * Return:
  1804. * On success:
  1805. * Positive state number if @state is CPUHP_AP_ONLINE_DYN;
  1806. * 0 for all other states
  1807. * On failure: proper (negative) error code
  1808. */
  1809. int __cpuhp_setup_state_cpuslocked(enum cpuhp_state state,
  1810. const char *name, bool invoke,
  1811. int (*startup)(unsigned int cpu),
  1812. int (*teardown)(unsigned int cpu),
  1813. bool multi_instance)
  1814. {
  1815. int cpu, ret = 0;
  1816. bool dynstate;
  1817. lockdep_assert_cpus_held();
  1818. if (cpuhp_cb_check(state) || !name)
  1819. return -EINVAL;
  1820. mutex_lock(&cpuhp_state_mutex);
  1821. ret = cpuhp_store_callbacks(state, name, startup, teardown,
  1822. multi_instance);
  1823. dynstate = state == CPUHP_AP_ONLINE_DYN;
  1824. if (ret > 0 && dynstate) {
  1825. state = ret;
  1826. ret = 0;
  1827. }
  1828. if (ret || !invoke || !startup)
  1829. goto out;
  1830. /*
  1831. * Try to call the startup callback for each present cpu
  1832. * depending on the hotplug state of the cpu.
  1833. */
  1834. for_each_present_cpu(cpu) {
  1835. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1836. int cpustate = st->state;
  1837. if (cpustate < state)
  1838. continue;
  1839. ret = cpuhp_issue_call(cpu, state, true, NULL);
  1840. if (ret) {
  1841. if (teardown)
  1842. cpuhp_rollback_install(cpu, state, NULL);
  1843. cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
  1844. goto out;
  1845. }
  1846. }
  1847. out:
  1848. mutex_unlock(&cpuhp_state_mutex);
  1849. /*
  1850. * If the requested state is CPUHP_AP_ONLINE_DYN, return the
  1851. * dynamically allocated state in case of success.
  1852. */
  1853. if (!ret && dynstate)
  1854. return state;
  1855. return ret;
  1856. }
  1857. EXPORT_SYMBOL(__cpuhp_setup_state_cpuslocked);
  1858. int __cpuhp_setup_state(enum cpuhp_state state,
  1859. const char *name, bool invoke,
  1860. int (*startup)(unsigned int cpu),
  1861. int (*teardown)(unsigned int cpu),
  1862. bool multi_instance)
  1863. {
  1864. int ret;
  1865. cpus_read_lock();
  1866. ret = __cpuhp_setup_state_cpuslocked(state, name, invoke, startup,
  1867. teardown, multi_instance);
  1868. cpus_read_unlock();
  1869. return ret;
  1870. }
  1871. EXPORT_SYMBOL(__cpuhp_setup_state);
  1872. int __cpuhp_state_remove_instance(enum cpuhp_state state,
  1873. struct hlist_node *node, bool invoke)
  1874. {
  1875. struct cpuhp_step *sp = cpuhp_get_step(state);
  1876. int cpu;
  1877. BUG_ON(cpuhp_cb_check(state));
  1878. if (!sp->multi_instance)
  1879. return -EINVAL;
  1880. cpus_read_lock();
  1881. mutex_lock(&cpuhp_state_mutex);
  1882. if (!invoke || !cpuhp_get_teardown_cb(state))
  1883. goto remove;
  1884. /*
  1885. * Call the teardown callback for each present cpu depending
  1886. * on the hotplug state of the cpu. This function is not
  1887. * allowed to fail currently!
  1888. */
  1889. for_each_present_cpu(cpu) {
  1890. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1891. int cpustate = st->state;
  1892. if (cpustate >= state)
  1893. cpuhp_issue_call(cpu, state, false, node);
  1894. }
  1895. remove:
  1896. hlist_del(node);
  1897. mutex_unlock(&cpuhp_state_mutex);
  1898. cpus_read_unlock();
  1899. return 0;
  1900. }
  1901. EXPORT_SYMBOL_GPL(__cpuhp_state_remove_instance);
  1902. /**
  1903. * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
  1904. * @state: The state to remove
  1905. * @invoke: If true, the teardown function is invoked for cpus where
  1906. * cpu state >= @state
  1907. *
  1908. * The caller needs to hold cpus read locked while calling this function.
  1909. * The teardown callback is currently not allowed to fail. Think
  1910. * about module removal!
  1911. */
  1912. void __cpuhp_remove_state_cpuslocked(enum cpuhp_state state, bool invoke)
  1913. {
  1914. struct cpuhp_step *sp = cpuhp_get_step(state);
  1915. int cpu;
  1916. BUG_ON(cpuhp_cb_check(state));
  1917. lockdep_assert_cpus_held();
  1918. mutex_lock(&cpuhp_state_mutex);
  1919. if (sp->multi_instance) {
  1920. WARN(!hlist_empty(&sp->list),
  1921. "Error: Removing state %d which has instances left.\n",
  1922. state);
  1923. goto remove;
  1924. }
  1925. if (!invoke || !cpuhp_get_teardown_cb(state))
  1926. goto remove;
  1927. /*
  1928. * Call the teardown callback for each present cpu depending
  1929. * on the hotplug state of the cpu. This function is not
  1930. * allowed to fail currently!
  1931. */
  1932. for_each_present_cpu(cpu) {
  1933. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
  1934. int cpustate = st->state;
  1935. if (cpustate >= state)
  1936. cpuhp_issue_call(cpu, state, false, NULL);
  1937. }
  1938. remove:
  1939. cpuhp_store_callbacks(state, NULL, NULL, NULL, false);
  1940. mutex_unlock(&cpuhp_state_mutex);
  1941. }
  1942. EXPORT_SYMBOL(__cpuhp_remove_state_cpuslocked);
  1943. void __cpuhp_remove_state(enum cpuhp_state state, bool invoke)
  1944. {
  1945. cpus_read_lock();
  1946. __cpuhp_remove_state_cpuslocked(state, invoke);
  1947. cpus_read_unlock();
  1948. }
  1949. EXPORT_SYMBOL(__cpuhp_remove_state);
  1950. #ifdef CONFIG_HOTPLUG_SMT
  1951. static void cpuhp_offline_cpu_device(unsigned int cpu)
  1952. {
  1953. struct device *dev = get_cpu_device(cpu);
  1954. dev->offline = true;
  1955. /* Tell user space about the state change */
  1956. kobject_uevent(&dev->kobj, KOBJ_OFFLINE);
  1957. }
  1958. static void cpuhp_online_cpu_device(unsigned int cpu)
  1959. {
  1960. struct device *dev = get_cpu_device(cpu);
  1961. dev->offline = false;
  1962. /* Tell user space about the state change */
  1963. kobject_uevent(&dev->kobj, KOBJ_ONLINE);
  1964. }
  1965. int cpuhp_smt_disable(enum cpuhp_smt_control ctrlval)
  1966. {
  1967. int cpu, ret = 0;
  1968. cpu_maps_update_begin();
  1969. for_each_online_cpu(cpu) {
  1970. if (topology_is_primary_thread(cpu))
  1971. continue;
  1972. ret = cpu_down_maps_locked(cpu, CPUHP_OFFLINE);
  1973. if (ret)
  1974. break;
  1975. /*
  1976. * As this needs to hold the cpu maps lock it's impossible
  1977. * to call device_offline() because that ends up calling
  1978. * cpu_down() which takes cpu maps lock. cpu maps lock
  1979. * needs to be held as this might race against in kernel
  1980. * abusers of the hotplug machinery (thermal management).
  1981. *
  1982. * So nothing would update device:offline state. That would
  1983. * leave the sysfs entry stale and prevent onlining after
  1984. * smt control has been changed to 'off' again. This is
  1985. * called under the sysfs hotplug lock, so it is properly
  1986. * serialized against the regular offline usage.
  1987. */
  1988. cpuhp_offline_cpu_device(cpu);
  1989. }
  1990. if (!ret)
  1991. cpu_smt_control = ctrlval;
  1992. cpu_maps_update_done();
  1993. return ret;
  1994. }
  1995. int cpuhp_smt_enable(void)
  1996. {
  1997. int cpu, ret = 0;
  1998. cpu_maps_update_begin();
  1999. cpu_smt_control = CPU_SMT_ENABLED;
  2000. for_each_present_cpu(cpu) {
  2001. /* Skip online CPUs and CPUs on offline nodes */
  2002. if (cpu_online(cpu) || !node_online(cpu_to_node(cpu)))
  2003. continue;
  2004. ret = _cpu_up(cpu, 0, CPUHP_ONLINE);
  2005. if (ret)
  2006. break;
  2007. /* See comment in cpuhp_smt_disable() */
  2008. cpuhp_online_cpu_device(cpu);
  2009. }
  2010. cpu_maps_update_done();
  2011. return ret;
  2012. }
  2013. #endif
  2014. #if defined(CONFIG_SYSFS) && defined(CONFIG_HOTPLUG_CPU)
  2015. static ssize_t state_show(struct device *dev,
  2016. struct device_attribute *attr, char *buf)
  2017. {
  2018. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
  2019. return sprintf(buf, "%d\n", st->state);
  2020. }
  2021. static DEVICE_ATTR_RO(state);
  2022. static ssize_t target_store(struct device *dev, struct device_attribute *attr,
  2023. const char *buf, size_t count)
  2024. {
  2025. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
  2026. struct cpuhp_step *sp;
  2027. int target, ret;
  2028. ret = kstrtoint(buf, 10, &target);
  2029. if (ret)
  2030. return ret;
  2031. #ifdef CONFIG_CPU_HOTPLUG_STATE_CONTROL
  2032. if (target < CPUHP_OFFLINE || target > CPUHP_ONLINE)
  2033. return -EINVAL;
  2034. #else
  2035. if (target != CPUHP_OFFLINE && target != CPUHP_ONLINE)
  2036. return -EINVAL;
  2037. #endif
  2038. ret = lock_device_hotplug_sysfs();
  2039. if (ret)
  2040. return ret;
  2041. mutex_lock(&cpuhp_state_mutex);
  2042. sp = cpuhp_get_step(target);
  2043. ret = !sp->name || sp->cant_stop ? -EINVAL : 0;
  2044. mutex_unlock(&cpuhp_state_mutex);
  2045. if (ret)
  2046. goto out;
  2047. if (st->state < target)
  2048. ret = cpu_up(dev->id, target);
  2049. else if (st->state > target)
  2050. ret = cpu_down(dev->id, target);
  2051. else if (WARN_ON(st->target != target))
  2052. st->target = target;
  2053. out:
  2054. unlock_device_hotplug();
  2055. return ret ? ret : count;
  2056. }
  2057. static ssize_t target_show(struct device *dev,
  2058. struct device_attribute *attr, char *buf)
  2059. {
  2060. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
  2061. return sprintf(buf, "%d\n", st->target);
  2062. }
  2063. static DEVICE_ATTR_RW(target);
  2064. static ssize_t fail_store(struct device *dev, struct device_attribute *attr,
  2065. const char *buf, size_t count)
  2066. {
  2067. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
  2068. struct cpuhp_step *sp;
  2069. int fail, ret;
  2070. ret = kstrtoint(buf, 10, &fail);
  2071. if (ret)
  2072. return ret;
  2073. if (fail == CPUHP_INVALID) {
  2074. st->fail = fail;
  2075. return count;
  2076. }
  2077. if (fail < CPUHP_OFFLINE || fail > CPUHP_ONLINE)
  2078. return -EINVAL;
  2079. /*
  2080. * Cannot fail STARTING/DYING callbacks.
  2081. */
  2082. if (cpuhp_is_atomic_state(fail))
  2083. return -EINVAL;
  2084. /*
  2085. * DEAD callbacks cannot fail...
  2086. * ... neither can CPUHP_BRINGUP_CPU during hotunplug. The latter
  2087. * triggering STARTING callbacks, a failure in this state would
  2088. * hinder rollback.
  2089. */
  2090. if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU)
  2091. return -EINVAL;
  2092. /*
  2093. * Cannot fail anything that doesn't have callbacks.
  2094. */
  2095. mutex_lock(&cpuhp_state_mutex);
  2096. sp = cpuhp_get_step(fail);
  2097. if (!sp->startup.single && !sp->teardown.single)
  2098. ret = -EINVAL;
  2099. mutex_unlock(&cpuhp_state_mutex);
  2100. if (ret)
  2101. return ret;
  2102. st->fail = fail;
  2103. return count;
  2104. }
  2105. static ssize_t fail_show(struct device *dev,
  2106. struct device_attribute *attr, char *buf)
  2107. {
  2108. struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id);
  2109. return sprintf(buf, "%d\n", st->fail);
  2110. }
  2111. static DEVICE_ATTR_RW(fail);
  2112. static struct attribute *cpuhp_cpu_attrs[] = {
  2113. &dev_attr_state.attr,
  2114. &dev_attr_target.attr,
  2115. &dev_attr_fail.attr,
  2116. NULL
  2117. };
  2118. static const struct attribute_group cpuhp_cpu_attr_group = {
  2119. .attrs = cpuhp_cpu_attrs,
  2120. .name = "hotplug",
  2121. NULL
  2122. };
  2123. static ssize_t states_show(struct device *dev,
  2124. struct device_attribute *attr, char *buf)
  2125. {
  2126. ssize_t cur, res = 0;
  2127. int i;
  2128. mutex_lock(&cpuhp_state_mutex);
  2129. for (i = CPUHP_OFFLINE; i <= CPUHP_ONLINE; i++) {
  2130. struct cpuhp_step *sp = cpuhp_get_step(i);
  2131. if (sp->name) {
  2132. cur = sprintf(buf, "%3d: %s\n", i, sp->name);
  2133. buf += cur;
  2134. res += cur;
  2135. }
  2136. }
  2137. mutex_unlock(&cpuhp_state_mutex);
  2138. return res;
  2139. }
  2140. static DEVICE_ATTR_RO(states);
  2141. static struct attribute *cpuhp_cpu_root_attrs[] = {
  2142. &dev_attr_states.attr,
  2143. NULL
  2144. };
  2145. static const struct attribute_group cpuhp_cpu_root_attr_group = {
  2146. .attrs = cpuhp_cpu_root_attrs,
  2147. .name = "hotplug",
  2148. NULL
  2149. };
  2150. #ifdef CONFIG_HOTPLUG_SMT
  2151. static ssize_t
  2152. __store_smt_control(struct device *dev, struct device_attribute *attr,
  2153. const char *buf, size_t count)
  2154. {
  2155. int ctrlval, ret;
  2156. if (sysfs_streq(buf, "on"))
  2157. ctrlval = CPU_SMT_ENABLED;
  2158. else if (sysfs_streq(buf, "off"))
  2159. ctrlval = CPU_SMT_DISABLED;
  2160. else if (sysfs_streq(buf, "forceoff"))
  2161. ctrlval = CPU_SMT_FORCE_DISABLED;
  2162. else
  2163. return -EINVAL;
  2164. if (cpu_smt_control == CPU_SMT_FORCE_DISABLED)
  2165. return -EPERM;
  2166. if (cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
  2167. return -ENODEV;
  2168. ret = lock_device_hotplug_sysfs();
  2169. if (ret)
  2170. return ret;
  2171. if (ctrlval != cpu_smt_control) {
  2172. switch (ctrlval) {
  2173. case CPU_SMT_ENABLED:
  2174. ret = cpuhp_smt_enable();
  2175. break;
  2176. case CPU_SMT_DISABLED:
  2177. case CPU_SMT_FORCE_DISABLED:
  2178. ret = cpuhp_smt_disable(ctrlval);
  2179. break;
  2180. }
  2181. }
  2182. unlock_device_hotplug();
  2183. return ret ? ret : count;
  2184. }
  2185. #else /* !CONFIG_HOTPLUG_SMT */
  2186. static ssize_t
  2187. __store_smt_control(struct device *dev, struct device_attribute *attr,
  2188. const char *buf, size_t count)
  2189. {
  2190. return -ENODEV;
  2191. }
  2192. #endif /* CONFIG_HOTPLUG_SMT */
  2193. static const char *smt_states[] = {
  2194. [CPU_SMT_ENABLED] = "on",
  2195. [CPU_SMT_DISABLED] = "off",
  2196. [CPU_SMT_FORCE_DISABLED] = "forceoff",
  2197. [CPU_SMT_NOT_SUPPORTED] = "notsupported",
  2198. [CPU_SMT_NOT_IMPLEMENTED] = "notimplemented",
  2199. };
  2200. static ssize_t control_show(struct device *dev,
  2201. struct device_attribute *attr, char *buf)
  2202. {
  2203. const char *state = smt_states[cpu_smt_control];
  2204. return snprintf(buf, PAGE_SIZE - 2, "%s\n", state);
  2205. }
  2206. static ssize_t control_store(struct device *dev, struct device_attribute *attr,
  2207. const char *buf, size_t count)
  2208. {
  2209. return __store_smt_control(dev, attr, buf, count);
  2210. }
  2211. static DEVICE_ATTR_RW(control);
  2212. static ssize_t active_show(struct device *dev,
  2213. struct device_attribute *attr, char *buf)
  2214. {
  2215. return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active());
  2216. }
  2217. static DEVICE_ATTR_RO(active);
  2218. static struct attribute *cpuhp_smt_attrs[] = {
  2219. &dev_attr_control.attr,
  2220. &dev_attr_active.attr,
  2221. NULL
  2222. };
  2223. static const struct attribute_group cpuhp_smt_attr_group = {
  2224. .attrs = cpuhp_smt_attrs,
  2225. .name = "smt",
  2226. NULL
  2227. };
  2228. static int __init cpu_smt_sysfs_init(void)
  2229. {
  2230. return sysfs_create_group(&cpu_subsys.dev_root->kobj,
  2231. &cpuhp_smt_attr_group);
  2232. }
  2233. static int __init cpuhp_sysfs_init(void)
  2234. {
  2235. int cpu, ret;
  2236. ret = cpu_smt_sysfs_init();
  2237. if (ret)
  2238. return ret;
  2239. ret = sysfs_create_group(&cpu_subsys.dev_root->kobj,
  2240. &cpuhp_cpu_root_attr_group);
  2241. if (ret)
  2242. return ret;
  2243. for_each_possible_cpu(cpu) {
  2244. struct device *dev = get_cpu_device(cpu);
  2245. if (!dev)
  2246. continue;
  2247. ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group);
  2248. if (ret)
  2249. return ret;
  2250. }
  2251. return 0;
  2252. }
  2253. device_initcall(cpuhp_sysfs_init);
  2254. #endif /* CONFIG_SYSFS && CONFIG_HOTPLUG_CPU */
  2255. /*
  2256. * cpu_bit_bitmap[] is a special, "compressed" data structure that
  2257. * represents all NR_CPUS bits binary values of 1<<nr.
  2258. *
  2259. * It is used by cpumask_of() to get a constant address to a CPU
  2260. * mask value that has a single bit set only.
  2261. */
  2262. /* cpu_bit_bitmap[0] is empty - so we can back into it */
  2263. #define MASK_DECLARE_1(x) [x+1][0] = (1UL << (x))
  2264. #define MASK_DECLARE_2(x) MASK_DECLARE_1(x), MASK_DECLARE_1(x+1)
  2265. #define MASK_DECLARE_4(x) MASK_DECLARE_2(x), MASK_DECLARE_2(x+2)
  2266. #define MASK_DECLARE_8(x) MASK_DECLARE_4(x), MASK_DECLARE_4(x+4)
  2267. const unsigned long cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)] = {
  2268. MASK_DECLARE_8(0), MASK_DECLARE_8(8),
  2269. MASK_DECLARE_8(16), MASK_DECLARE_8(24),
  2270. #if BITS_PER_LONG > 32
  2271. MASK_DECLARE_8(32), MASK_DECLARE_8(40),
  2272. MASK_DECLARE_8(48), MASK_DECLARE_8(56),
  2273. #endif
  2274. };
  2275. EXPORT_SYMBOL_GPL(cpu_bit_bitmap);
  2276. const DECLARE_BITMAP(cpu_all_bits, NR_CPUS) = CPU_BITS_ALL;
  2277. EXPORT_SYMBOL(cpu_all_bits);
  2278. #ifdef CONFIG_INIT_ALL_POSSIBLE
  2279. struct cpumask __cpu_possible_mask __read_mostly
  2280. = {CPU_BITS_ALL};
  2281. #else
  2282. struct cpumask __cpu_possible_mask __read_mostly;
  2283. #endif
  2284. EXPORT_SYMBOL(__cpu_possible_mask);
  2285. struct cpumask __cpu_online_mask __read_mostly;
  2286. EXPORT_SYMBOL(__cpu_online_mask);
  2287. struct cpumask __cpu_present_mask __read_mostly;
  2288. EXPORT_SYMBOL(__cpu_present_mask);
  2289. struct cpumask __cpu_active_mask __read_mostly;
  2290. EXPORT_SYMBOL(__cpu_active_mask);
  2291. struct cpumask __cpu_dying_mask __read_mostly;
  2292. EXPORT_SYMBOL(__cpu_dying_mask);
  2293. atomic_t __num_online_cpus __read_mostly;
  2294. EXPORT_SYMBOL(__num_online_cpus);
  2295. void init_cpu_present(const struct cpumask *src)
  2296. {
  2297. cpumask_copy(&__cpu_present_mask, src);
  2298. }
  2299. void init_cpu_possible(const struct cpumask *src)
  2300. {
  2301. cpumask_copy(&__cpu_possible_mask, src);
  2302. }
  2303. void init_cpu_online(const struct cpumask *src)
  2304. {
  2305. cpumask_copy(&__cpu_online_mask, src);
  2306. }
  2307. void set_cpu_online(unsigned int cpu, bool online)
  2308. {
  2309. /*
  2310. * atomic_inc/dec() is required to handle the horrid abuse of this
  2311. * function by the reboot and kexec code which invoke it from
  2312. * IPI/NMI broadcasts when shutting down CPUs. Invocation from
  2313. * regular CPU hotplug is properly serialized.
  2314. *
  2315. * Note, that the fact that __num_online_cpus is of type atomic_t
  2316. * does not protect readers which are not serialized against
  2317. * concurrent hotplug operations.
  2318. */
  2319. if (online) {
  2320. if (!cpumask_test_and_set_cpu(cpu, &__cpu_online_mask))
  2321. atomic_inc(&__num_online_cpus);
  2322. } else {
  2323. if (cpumask_test_and_clear_cpu(cpu, &__cpu_online_mask))
  2324. atomic_dec(&__num_online_cpus);
  2325. }
  2326. }
  2327. /*
  2328. * Activate the first processor.
  2329. */
  2330. void __init boot_cpu_init(void)
  2331. {
  2332. int cpu = smp_processor_id();
  2333. /* Mark the boot cpu "present", "online" etc for SMP and UP case */
  2334. set_cpu_online(cpu, true);
  2335. set_cpu_active(cpu, true);
  2336. set_cpu_present(cpu, true);
  2337. set_cpu_possible(cpu, true);
  2338. #ifdef CONFIG_SMP
  2339. __boot_cpu_id = cpu;
  2340. #endif
  2341. }
  2342. /*
  2343. * Must be called _AFTER_ setting up the per_cpu areas
  2344. */
  2345. void __init boot_cpu_hotplug_init(void)
  2346. {
  2347. #ifdef CONFIG_SMP
  2348. cpumask_set_cpu(smp_processor_id(), &cpus_booted_once_mask);
  2349. #endif
  2350. this_cpu_write(cpuhp_state.state, CPUHP_ONLINE);
  2351. }
  2352. /*
  2353. * These are used for a global "mitigations=" cmdline option for toggling
  2354. * optional CPU mitigations.
  2355. */
  2356. enum cpu_mitigations {
  2357. CPU_MITIGATIONS_OFF,
  2358. CPU_MITIGATIONS_AUTO,
  2359. CPU_MITIGATIONS_AUTO_NOSMT,
  2360. };
  2361. static enum cpu_mitigations cpu_mitigations __ro_after_init =
  2362. CPU_MITIGATIONS_AUTO;
  2363. static int __init mitigations_parse_cmdline(char *arg)
  2364. {
  2365. if (!strcmp(arg, "off"))
  2366. cpu_mitigations = CPU_MITIGATIONS_OFF;
  2367. else if (!strcmp(arg, "auto"))
  2368. cpu_mitigations = CPU_MITIGATIONS_AUTO;
  2369. else if (!strcmp(arg, "auto,nosmt"))
  2370. cpu_mitigations = CPU_MITIGATIONS_AUTO_NOSMT;
  2371. else
  2372. pr_crit("Unsupported mitigations=%s, system may still be vulnerable\n",
  2373. arg);
  2374. return 0;
  2375. }
  2376. early_param("mitigations", mitigations_parse_cmdline);
  2377. /* mitigations=off */
  2378. bool cpu_mitigations_off(void)
  2379. {
  2380. return cpu_mitigations == CPU_MITIGATIONS_OFF;
  2381. }
  2382. EXPORT_SYMBOL_GPL(cpu_mitigations_off);
  2383. /* mitigations=auto,nosmt */
  2384. bool cpu_mitigations_auto_nosmt(void)
  2385. {
  2386. return cpu_mitigations == CPU_MITIGATIONS_AUTO_NOSMT;
  2387. }
  2388. EXPORT_SYMBOL_GPL(cpu_mitigations_auto_nosmt);