padata.c 27 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * padata.c - generic interface to process data streams in parallel
  4. *
  5. * See Documentation/core-api/padata.rst for more information.
  6. *
  7. * Copyright (C) 2008, 2009 secunet Security Networks AG
  8. * Copyright (C) 2008, 2009 Steffen Klassert <[email protected]>
  9. *
  10. * Copyright (c) 2020 Oracle and/or its affiliates.
  11. * Author: Daniel Jordan <[email protected]>
  12. */
  13. #include <linux/completion.h>
  14. #include <linux/export.h>
  15. #include <linux/cpumask.h>
  16. #include <linux/err.h>
  17. #include <linux/cpu.h>
  18. #include <linux/padata.h>
  19. #include <linux/mutex.h>
  20. #include <linux/sched.h>
  21. #include <linux/slab.h>
  22. #include <linux/sysfs.h>
  23. #include <linux/rcupdate.h>
  24. #define PADATA_WORK_ONSTACK 1 /* Work's memory is on stack */
  25. struct padata_work {
  26. struct work_struct pw_work;
  27. struct list_head pw_list; /* padata_free_works linkage */
  28. void *pw_data;
  29. };
  30. static DEFINE_SPINLOCK(padata_works_lock);
  31. static struct padata_work *padata_works;
  32. static LIST_HEAD(padata_free_works);
  33. struct padata_mt_job_state {
  34. spinlock_t lock;
  35. struct completion completion;
  36. struct padata_mt_job *job;
  37. int nworks;
  38. int nworks_fini;
  39. unsigned long chunk_size;
  40. };
  41. static void padata_free_pd(struct parallel_data *pd);
  42. static void __init padata_mt_helper(struct work_struct *work);
  43. static int padata_index_to_cpu(struct parallel_data *pd, int cpu_index)
  44. {
  45. int cpu, target_cpu;
  46. target_cpu = cpumask_first(pd->cpumask.pcpu);
  47. for (cpu = 0; cpu < cpu_index; cpu++)
  48. target_cpu = cpumask_next(target_cpu, pd->cpumask.pcpu);
  49. return target_cpu;
  50. }
  51. static int padata_cpu_hash(struct parallel_data *pd, unsigned int seq_nr)
  52. {
  53. /*
  54. * Hash the sequence numbers to the cpus by taking
  55. * seq_nr mod. number of cpus in use.
  56. */
  57. int cpu_index = seq_nr % cpumask_weight(pd->cpumask.pcpu);
  58. return padata_index_to_cpu(pd, cpu_index);
  59. }
  60. static struct padata_work *padata_work_alloc(void)
  61. {
  62. struct padata_work *pw;
  63. lockdep_assert_held(&padata_works_lock);
  64. if (list_empty(&padata_free_works))
  65. return NULL; /* No more work items allowed to be queued. */
  66. pw = list_first_entry(&padata_free_works, struct padata_work, pw_list);
  67. list_del(&pw->pw_list);
  68. return pw;
  69. }
  70. static void padata_work_init(struct padata_work *pw, work_func_t work_fn,
  71. void *data, int flags)
  72. {
  73. if (flags & PADATA_WORK_ONSTACK)
  74. INIT_WORK_ONSTACK(&pw->pw_work, work_fn);
  75. else
  76. INIT_WORK(&pw->pw_work, work_fn);
  77. pw->pw_data = data;
  78. }
  79. static int __init padata_work_alloc_mt(int nworks, void *data,
  80. struct list_head *head)
  81. {
  82. int i;
  83. spin_lock(&padata_works_lock);
  84. /* Start at 1 because the current task participates in the job. */
  85. for (i = 1; i < nworks; ++i) {
  86. struct padata_work *pw = padata_work_alloc();
  87. if (!pw)
  88. break;
  89. padata_work_init(pw, padata_mt_helper, data, 0);
  90. list_add(&pw->pw_list, head);
  91. }
  92. spin_unlock(&padata_works_lock);
  93. return i;
  94. }
  95. static void padata_work_free(struct padata_work *pw)
  96. {
  97. lockdep_assert_held(&padata_works_lock);
  98. list_add(&pw->pw_list, &padata_free_works);
  99. }
  100. static void __init padata_works_free(struct list_head *works)
  101. {
  102. struct padata_work *cur, *next;
  103. if (list_empty(works))
  104. return;
  105. spin_lock(&padata_works_lock);
  106. list_for_each_entry_safe(cur, next, works, pw_list) {
  107. list_del(&cur->pw_list);
  108. padata_work_free(cur);
  109. }
  110. spin_unlock(&padata_works_lock);
  111. }
  112. static void padata_parallel_worker(struct work_struct *parallel_work)
  113. {
  114. struct padata_work *pw = container_of(parallel_work, struct padata_work,
  115. pw_work);
  116. struct padata_priv *padata = pw->pw_data;
  117. local_bh_disable();
  118. padata->parallel(padata);
  119. spin_lock(&padata_works_lock);
  120. padata_work_free(pw);
  121. spin_unlock(&padata_works_lock);
  122. local_bh_enable();
  123. }
  124. /**
  125. * padata_do_parallel - padata parallelization function
  126. *
  127. * @ps: padatashell
  128. * @padata: object to be parallelized
  129. * @cb_cpu: pointer to the CPU that the serialization callback function should
  130. * run on. If it's not in the serial cpumask of @pinst
  131. * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if
  132. * none found, returns -EINVAL.
  133. *
  134. * The parallelization callback function will run with BHs off.
  135. * Note: Every object which is parallelized by padata_do_parallel
  136. * must be seen by padata_do_serial.
  137. *
  138. * Return: 0 on success or else negative error code.
  139. */
  140. int padata_do_parallel(struct padata_shell *ps,
  141. struct padata_priv *padata, int *cb_cpu)
  142. {
  143. struct padata_instance *pinst = ps->pinst;
  144. int i, cpu, cpu_index, err;
  145. struct parallel_data *pd;
  146. struct padata_work *pw;
  147. rcu_read_lock_bh();
  148. pd = rcu_dereference_bh(ps->pd);
  149. err = -EINVAL;
  150. if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID)
  151. goto out;
  152. if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) {
  153. if (cpumask_empty(pd->cpumask.cbcpu))
  154. goto out;
  155. /* Select an alternate fallback CPU and notify the caller. */
  156. cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu);
  157. cpu = cpumask_first(pd->cpumask.cbcpu);
  158. for (i = 0; i < cpu_index; i++)
  159. cpu = cpumask_next(cpu, pd->cpumask.cbcpu);
  160. *cb_cpu = cpu;
  161. }
  162. err = -EBUSY;
  163. if ((pinst->flags & PADATA_RESET))
  164. goto out;
  165. refcount_inc(&pd->refcnt);
  166. padata->pd = pd;
  167. padata->cb_cpu = *cb_cpu;
  168. spin_lock(&padata_works_lock);
  169. padata->seq_nr = ++pd->seq_nr;
  170. pw = padata_work_alloc();
  171. spin_unlock(&padata_works_lock);
  172. if (!pw) {
  173. /* Maximum works limit exceeded, run in the current task. */
  174. padata->parallel(padata);
  175. }
  176. rcu_read_unlock_bh();
  177. if (pw) {
  178. padata_work_init(pw, padata_parallel_worker, padata, 0);
  179. queue_work(pinst->parallel_wq, &pw->pw_work);
  180. }
  181. return 0;
  182. out:
  183. rcu_read_unlock_bh();
  184. return err;
  185. }
  186. EXPORT_SYMBOL(padata_do_parallel);
  187. /*
  188. * padata_find_next - Find the next object that needs serialization.
  189. *
  190. * Return:
  191. * * A pointer to the control struct of the next object that needs
  192. * serialization, if present in one of the percpu reorder queues.
  193. * * NULL, if the next object that needs serialization will
  194. * be parallel processed by another cpu and is not yet present in
  195. * the cpu's reorder queue.
  196. */
  197. static struct padata_priv *padata_find_next(struct parallel_data *pd,
  198. bool remove_object)
  199. {
  200. struct padata_priv *padata;
  201. struct padata_list *reorder;
  202. int cpu = pd->cpu;
  203. reorder = per_cpu_ptr(pd->reorder_list, cpu);
  204. spin_lock(&reorder->lock);
  205. if (list_empty(&reorder->list)) {
  206. spin_unlock(&reorder->lock);
  207. return NULL;
  208. }
  209. padata = list_entry(reorder->list.next, struct padata_priv, list);
  210. /*
  211. * Checks the rare case where two or more parallel jobs have hashed to
  212. * the same CPU and one of the later ones finishes first.
  213. */
  214. if (padata->seq_nr != pd->processed) {
  215. spin_unlock(&reorder->lock);
  216. return NULL;
  217. }
  218. if (remove_object) {
  219. list_del_init(&padata->list);
  220. ++pd->processed;
  221. pd->cpu = cpumask_next_wrap(cpu, pd->cpumask.pcpu, -1, false);
  222. }
  223. spin_unlock(&reorder->lock);
  224. return padata;
  225. }
  226. static void padata_reorder(struct parallel_data *pd)
  227. {
  228. struct padata_instance *pinst = pd->ps->pinst;
  229. int cb_cpu;
  230. struct padata_priv *padata;
  231. struct padata_serial_queue *squeue;
  232. struct padata_list *reorder;
  233. /*
  234. * We need to ensure that only one cpu can work on dequeueing of
  235. * the reorder queue the time. Calculating in which percpu reorder
  236. * queue the next object will arrive takes some time. A spinlock
  237. * would be highly contended. Also it is not clear in which order
  238. * the objects arrive to the reorder queues. So a cpu could wait to
  239. * get the lock just to notice that there is nothing to do at the
  240. * moment. Therefore we use a trylock and let the holder of the lock
  241. * care for all the objects enqueued during the holdtime of the lock.
  242. */
  243. if (!spin_trylock_bh(&pd->lock))
  244. return;
  245. while (1) {
  246. padata = padata_find_next(pd, true);
  247. /*
  248. * If the next object that needs serialization is parallel
  249. * processed by another cpu and is still on it's way to the
  250. * cpu's reorder queue, nothing to do for now.
  251. */
  252. if (!padata)
  253. break;
  254. cb_cpu = padata->cb_cpu;
  255. squeue = per_cpu_ptr(pd->squeue, cb_cpu);
  256. spin_lock(&squeue->serial.lock);
  257. list_add_tail(&padata->list, &squeue->serial.list);
  258. spin_unlock(&squeue->serial.lock);
  259. queue_work_on(cb_cpu, pinst->serial_wq, &squeue->work);
  260. }
  261. spin_unlock_bh(&pd->lock);
  262. /*
  263. * The next object that needs serialization might have arrived to
  264. * the reorder queues in the meantime.
  265. *
  266. * Ensure reorder queue is read after pd->lock is dropped so we see
  267. * new objects from another task in padata_do_serial. Pairs with
  268. * smp_mb in padata_do_serial.
  269. */
  270. smp_mb();
  271. reorder = per_cpu_ptr(pd->reorder_list, pd->cpu);
  272. if (!list_empty(&reorder->list) && padata_find_next(pd, false))
  273. queue_work(pinst->serial_wq, &pd->reorder_work);
  274. }
  275. static void invoke_padata_reorder(struct work_struct *work)
  276. {
  277. struct parallel_data *pd;
  278. local_bh_disable();
  279. pd = container_of(work, struct parallel_data, reorder_work);
  280. padata_reorder(pd);
  281. local_bh_enable();
  282. }
  283. static void padata_serial_worker(struct work_struct *serial_work)
  284. {
  285. struct padata_serial_queue *squeue;
  286. struct parallel_data *pd;
  287. LIST_HEAD(local_list);
  288. int cnt;
  289. local_bh_disable();
  290. squeue = container_of(serial_work, struct padata_serial_queue, work);
  291. pd = squeue->pd;
  292. spin_lock(&squeue->serial.lock);
  293. list_replace_init(&squeue->serial.list, &local_list);
  294. spin_unlock(&squeue->serial.lock);
  295. cnt = 0;
  296. while (!list_empty(&local_list)) {
  297. struct padata_priv *padata;
  298. padata = list_entry(local_list.next,
  299. struct padata_priv, list);
  300. list_del_init(&padata->list);
  301. padata->serial(padata);
  302. cnt++;
  303. }
  304. local_bh_enable();
  305. if (refcount_sub_and_test(cnt, &pd->refcnt))
  306. padata_free_pd(pd);
  307. }
  308. /**
  309. * padata_do_serial - padata serialization function
  310. *
  311. * @padata: object to be serialized.
  312. *
  313. * padata_do_serial must be called for every parallelized object.
  314. * The serialization callback function will run with BHs off.
  315. */
  316. void padata_do_serial(struct padata_priv *padata)
  317. {
  318. struct parallel_data *pd = padata->pd;
  319. int hashed_cpu = padata_cpu_hash(pd, padata->seq_nr);
  320. struct padata_list *reorder = per_cpu_ptr(pd->reorder_list, hashed_cpu);
  321. struct padata_priv *cur;
  322. struct list_head *pos;
  323. spin_lock(&reorder->lock);
  324. /* Sort in ascending order of sequence number. */
  325. list_for_each_prev(pos, &reorder->list) {
  326. cur = list_entry(pos, struct padata_priv, list);
  327. if (cur->seq_nr < padata->seq_nr)
  328. break;
  329. }
  330. list_add(&padata->list, pos);
  331. spin_unlock(&reorder->lock);
  332. /*
  333. * Ensure the addition to the reorder list is ordered correctly
  334. * with the trylock of pd->lock in padata_reorder. Pairs with smp_mb
  335. * in padata_reorder.
  336. */
  337. smp_mb();
  338. padata_reorder(pd);
  339. }
  340. EXPORT_SYMBOL(padata_do_serial);
  341. static int padata_setup_cpumasks(struct padata_instance *pinst)
  342. {
  343. struct workqueue_attrs *attrs;
  344. int err;
  345. attrs = alloc_workqueue_attrs();
  346. if (!attrs)
  347. return -ENOMEM;
  348. /* Restrict parallel_wq workers to pd->cpumask.pcpu. */
  349. cpumask_copy(attrs->cpumask, pinst->cpumask.pcpu);
  350. err = apply_workqueue_attrs(pinst->parallel_wq, attrs);
  351. free_workqueue_attrs(attrs);
  352. return err;
  353. }
  354. static void __init padata_mt_helper(struct work_struct *w)
  355. {
  356. struct padata_work *pw = container_of(w, struct padata_work, pw_work);
  357. struct padata_mt_job_state *ps = pw->pw_data;
  358. struct padata_mt_job *job = ps->job;
  359. bool done;
  360. spin_lock(&ps->lock);
  361. while (job->size > 0) {
  362. unsigned long start, size, end;
  363. start = job->start;
  364. /* So end is chunk size aligned if enough work remains. */
  365. size = roundup(start + 1, ps->chunk_size) - start;
  366. size = min(size, job->size);
  367. end = start + size;
  368. job->start = end;
  369. job->size -= size;
  370. spin_unlock(&ps->lock);
  371. job->thread_fn(start, end, job->fn_arg);
  372. spin_lock(&ps->lock);
  373. }
  374. ++ps->nworks_fini;
  375. done = (ps->nworks_fini == ps->nworks);
  376. spin_unlock(&ps->lock);
  377. if (done)
  378. complete(&ps->completion);
  379. }
  380. /**
  381. * padata_do_multithreaded - run a multithreaded job
  382. * @job: Description of the job.
  383. *
  384. * See the definition of struct padata_mt_job for more details.
  385. */
  386. void __init padata_do_multithreaded(struct padata_mt_job *job)
  387. {
  388. /* In case threads finish at different times. */
  389. static const unsigned long load_balance_factor = 4;
  390. struct padata_work my_work, *pw;
  391. struct padata_mt_job_state ps;
  392. LIST_HEAD(works);
  393. int nworks;
  394. if (job->size == 0)
  395. return;
  396. /* Ensure at least one thread when size < min_chunk. */
  397. nworks = max(job->size / job->min_chunk, 1ul);
  398. nworks = min(nworks, job->max_threads);
  399. if (nworks == 1) {
  400. /* Single thread, no coordination needed, cut to the chase. */
  401. job->thread_fn(job->start, job->start + job->size, job->fn_arg);
  402. return;
  403. }
  404. spin_lock_init(&ps.lock);
  405. init_completion(&ps.completion);
  406. ps.job = job;
  407. ps.nworks = padata_work_alloc_mt(nworks, &ps, &works);
  408. ps.nworks_fini = 0;
  409. /*
  410. * Chunk size is the amount of work a helper does per call to the
  411. * thread function. Load balance large jobs between threads by
  412. * increasing the number of chunks, guarantee at least the minimum
  413. * chunk size from the caller, and honor the caller's alignment.
  414. */
  415. ps.chunk_size = job->size / (ps.nworks * load_balance_factor);
  416. ps.chunk_size = max(ps.chunk_size, job->min_chunk);
  417. ps.chunk_size = roundup(ps.chunk_size, job->align);
  418. list_for_each_entry(pw, &works, pw_list)
  419. queue_work(system_unbound_wq, &pw->pw_work);
  420. /* Use the current thread, which saves starting a workqueue worker. */
  421. padata_work_init(&my_work, padata_mt_helper, &ps, PADATA_WORK_ONSTACK);
  422. padata_mt_helper(&my_work.pw_work);
  423. /* Wait for all the helpers to finish. */
  424. wait_for_completion(&ps.completion);
  425. destroy_work_on_stack(&my_work.pw_work);
  426. padata_works_free(&works);
  427. }
  428. static void __padata_list_init(struct padata_list *pd_list)
  429. {
  430. INIT_LIST_HEAD(&pd_list->list);
  431. spin_lock_init(&pd_list->lock);
  432. }
  433. /* Initialize all percpu queues used by serial workers */
  434. static void padata_init_squeues(struct parallel_data *pd)
  435. {
  436. int cpu;
  437. struct padata_serial_queue *squeue;
  438. for_each_cpu(cpu, pd->cpumask.cbcpu) {
  439. squeue = per_cpu_ptr(pd->squeue, cpu);
  440. squeue->pd = pd;
  441. __padata_list_init(&squeue->serial);
  442. INIT_WORK(&squeue->work, padata_serial_worker);
  443. }
  444. }
  445. /* Initialize per-CPU reorder lists */
  446. static void padata_init_reorder_list(struct parallel_data *pd)
  447. {
  448. int cpu;
  449. struct padata_list *list;
  450. for_each_cpu(cpu, pd->cpumask.pcpu) {
  451. list = per_cpu_ptr(pd->reorder_list, cpu);
  452. __padata_list_init(list);
  453. }
  454. }
  455. /* Allocate and initialize the internal cpumask dependend resources. */
  456. static struct parallel_data *padata_alloc_pd(struct padata_shell *ps)
  457. {
  458. struct padata_instance *pinst = ps->pinst;
  459. struct parallel_data *pd;
  460. pd = kzalloc(sizeof(struct parallel_data), GFP_KERNEL);
  461. if (!pd)
  462. goto err;
  463. pd->reorder_list = alloc_percpu(struct padata_list);
  464. if (!pd->reorder_list)
  465. goto err_free_pd;
  466. pd->squeue = alloc_percpu(struct padata_serial_queue);
  467. if (!pd->squeue)
  468. goto err_free_reorder_list;
  469. pd->ps = ps;
  470. if (!alloc_cpumask_var(&pd->cpumask.pcpu, GFP_KERNEL))
  471. goto err_free_squeue;
  472. if (!alloc_cpumask_var(&pd->cpumask.cbcpu, GFP_KERNEL))
  473. goto err_free_pcpu;
  474. cpumask_and(pd->cpumask.pcpu, pinst->cpumask.pcpu, cpu_online_mask);
  475. cpumask_and(pd->cpumask.cbcpu, pinst->cpumask.cbcpu, cpu_online_mask);
  476. padata_init_reorder_list(pd);
  477. padata_init_squeues(pd);
  478. pd->seq_nr = -1;
  479. refcount_set(&pd->refcnt, 1);
  480. spin_lock_init(&pd->lock);
  481. pd->cpu = cpumask_first(pd->cpumask.pcpu);
  482. INIT_WORK(&pd->reorder_work, invoke_padata_reorder);
  483. return pd;
  484. err_free_pcpu:
  485. free_cpumask_var(pd->cpumask.pcpu);
  486. err_free_squeue:
  487. free_percpu(pd->squeue);
  488. err_free_reorder_list:
  489. free_percpu(pd->reorder_list);
  490. err_free_pd:
  491. kfree(pd);
  492. err:
  493. return NULL;
  494. }
  495. static void padata_free_pd(struct parallel_data *pd)
  496. {
  497. free_cpumask_var(pd->cpumask.pcpu);
  498. free_cpumask_var(pd->cpumask.cbcpu);
  499. free_percpu(pd->reorder_list);
  500. free_percpu(pd->squeue);
  501. kfree(pd);
  502. }
  503. static void __padata_start(struct padata_instance *pinst)
  504. {
  505. pinst->flags |= PADATA_INIT;
  506. }
  507. static void __padata_stop(struct padata_instance *pinst)
  508. {
  509. if (!(pinst->flags & PADATA_INIT))
  510. return;
  511. pinst->flags &= ~PADATA_INIT;
  512. synchronize_rcu();
  513. }
  514. /* Replace the internal control structure with a new one. */
  515. static int padata_replace_one(struct padata_shell *ps)
  516. {
  517. struct parallel_data *pd_new;
  518. pd_new = padata_alloc_pd(ps);
  519. if (!pd_new)
  520. return -ENOMEM;
  521. ps->opd = rcu_dereference_protected(ps->pd, 1);
  522. rcu_assign_pointer(ps->pd, pd_new);
  523. return 0;
  524. }
  525. static int padata_replace(struct padata_instance *pinst)
  526. {
  527. struct padata_shell *ps;
  528. int err = 0;
  529. pinst->flags |= PADATA_RESET;
  530. list_for_each_entry(ps, &pinst->pslist, list) {
  531. err = padata_replace_one(ps);
  532. if (err)
  533. break;
  534. }
  535. synchronize_rcu();
  536. list_for_each_entry_continue_reverse(ps, &pinst->pslist, list)
  537. if (refcount_dec_and_test(&ps->opd->refcnt))
  538. padata_free_pd(ps->opd);
  539. pinst->flags &= ~PADATA_RESET;
  540. return err;
  541. }
  542. /* If cpumask contains no active cpu, we mark the instance as invalid. */
  543. static bool padata_validate_cpumask(struct padata_instance *pinst,
  544. const struct cpumask *cpumask)
  545. {
  546. if (!cpumask_intersects(cpumask, cpu_online_mask)) {
  547. pinst->flags |= PADATA_INVALID;
  548. return false;
  549. }
  550. pinst->flags &= ~PADATA_INVALID;
  551. return true;
  552. }
  553. static int __padata_set_cpumasks(struct padata_instance *pinst,
  554. cpumask_var_t pcpumask,
  555. cpumask_var_t cbcpumask)
  556. {
  557. int valid;
  558. int err;
  559. valid = padata_validate_cpumask(pinst, pcpumask);
  560. if (!valid) {
  561. __padata_stop(pinst);
  562. goto out_replace;
  563. }
  564. valid = padata_validate_cpumask(pinst, cbcpumask);
  565. if (!valid)
  566. __padata_stop(pinst);
  567. out_replace:
  568. cpumask_copy(pinst->cpumask.pcpu, pcpumask);
  569. cpumask_copy(pinst->cpumask.cbcpu, cbcpumask);
  570. err = padata_setup_cpumasks(pinst) ?: padata_replace(pinst);
  571. if (valid)
  572. __padata_start(pinst);
  573. return err;
  574. }
  575. /**
  576. * padata_set_cpumask - Sets specified by @cpumask_type cpumask to the value
  577. * equivalent to @cpumask.
  578. * @pinst: padata instance
  579. * @cpumask_type: PADATA_CPU_SERIAL or PADATA_CPU_PARALLEL corresponding
  580. * to parallel and serial cpumasks respectively.
  581. * @cpumask: the cpumask to use
  582. *
  583. * Return: 0 on success or negative error code
  584. */
  585. int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type,
  586. cpumask_var_t cpumask)
  587. {
  588. struct cpumask *serial_mask, *parallel_mask;
  589. int err = -EINVAL;
  590. cpus_read_lock();
  591. mutex_lock(&pinst->lock);
  592. switch (cpumask_type) {
  593. case PADATA_CPU_PARALLEL:
  594. serial_mask = pinst->cpumask.cbcpu;
  595. parallel_mask = cpumask;
  596. break;
  597. case PADATA_CPU_SERIAL:
  598. parallel_mask = pinst->cpumask.pcpu;
  599. serial_mask = cpumask;
  600. break;
  601. default:
  602. goto out;
  603. }
  604. err = __padata_set_cpumasks(pinst, parallel_mask, serial_mask);
  605. out:
  606. mutex_unlock(&pinst->lock);
  607. cpus_read_unlock();
  608. return err;
  609. }
  610. EXPORT_SYMBOL(padata_set_cpumask);
  611. #ifdef CONFIG_HOTPLUG_CPU
  612. static int __padata_add_cpu(struct padata_instance *pinst, int cpu)
  613. {
  614. int err = 0;
  615. if (cpumask_test_cpu(cpu, cpu_online_mask)) {
  616. err = padata_replace(pinst);
  617. if (padata_validate_cpumask(pinst, pinst->cpumask.pcpu) &&
  618. padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
  619. __padata_start(pinst);
  620. }
  621. return err;
  622. }
  623. static int __padata_remove_cpu(struct padata_instance *pinst, int cpu)
  624. {
  625. int err = 0;
  626. if (!cpumask_test_cpu(cpu, cpu_online_mask)) {
  627. if (!padata_validate_cpumask(pinst, pinst->cpumask.pcpu) ||
  628. !padata_validate_cpumask(pinst, pinst->cpumask.cbcpu))
  629. __padata_stop(pinst);
  630. err = padata_replace(pinst);
  631. }
  632. return err;
  633. }
  634. static inline int pinst_has_cpu(struct padata_instance *pinst, int cpu)
  635. {
  636. return cpumask_test_cpu(cpu, pinst->cpumask.pcpu) ||
  637. cpumask_test_cpu(cpu, pinst->cpumask.cbcpu);
  638. }
  639. static int padata_cpu_online(unsigned int cpu, struct hlist_node *node)
  640. {
  641. struct padata_instance *pinst;
  642. int ret;
  643. pinst = hlist_entry_safe(node, struct padata_instance, cpu_online_node);
  644. if (!pinst_has_cpu(pinst, cpu))
  645. return 0;
  646. mutex_lock(&pinst->lock);
  647. ret = __padata_add_cpu(pinst, cpu);
  648. mutex_unlock(&pinst->lock);
  649. return ret;
  650. }
  651. static int padata_cpu_dead(unsigned int cpu, struct hlist_node *node)
  652. {
  653. struct padata_instance *pinst;
  654. int ret;
  655. pinst = hlist_entry_safe(node, struct padata_instance, cpu_dead_node);
  656. if (!pinst_has_cpu(pinst, cpu))
  657. return 0;
  658. mutex_lock(&pinst->lock);
  659. ret = __padata_remove_cpu(pinst, cpu);
  660. mutex_unlock(&pinst->lock);
  661. return ret;
  662. }
  663. static enum cpuhp_state hp_online;
  664. #endif
  665. static void __padata_free(struct padata_instance *pinst)
  666. {
  667. #ifdef CONFIG_HOTPLUG_CPU
  668. cpuhp_state_remove_instance_nocalls(CPUHP_PADATA_DEAD,
  669. &pinst->cpu_dead_node);
  670. cpuhp_state_remove_instance_nocalls(hp_online, &pinst->cpu_online_node);
  671. #endif
  672. WARN_ON(!list_empty(&pinst->pslist));
  673. free_cpumask_var(pinst->cpumask.pcpu);
  674. free_cpumask_var(pinst->cpumask.cbcpu);
  675. destroy_workqueue(pinst->serial_wq);
  676. destroy_workqueue(pinst->parallel_wq);
  677. kfree(pinst);
  678. }
  679. #define kobj2pinst(_kobj) \
  680. container_of(_kobj, struct padata_instance, kobj)
  681. #define attr2pentry(_attr) \
  682. container_of(_attr, struct padata_sysfs_entry, attr)
  683. static void padata_sysfs_release(struct kobject *kobj)
  684. {
  685. struct padata_instance *pinst = kobj2pinst(kobj);
  686. __padata_free(pinst);
  687. }
  688. struct padata_sysfs_entry {
  689. struct attribute attr;
  690. ssize_t (*show)(struct padata_instance *, struct attribute *, char *);
  691. ssize_t (*store)(struct padata_instance *, struct attribute *,
  692. const char *, size_t);
  693. };
  694. static ssize_t show_cpumask(struct padata_instance *pinst,
  695. struct attribute *attr, char *buf)
  696. {
  697. struct cpumask *cpumask;
  698. ssize_t len;
  699. mutex_lock(&pinst->lock);
  700. if (!strcmp(attr->name, "serial_cpumask"))
  701. cpumask = pinst->cpumask.cbcpu;
  702. else
  703. cpumask = pinst->cpumask.pcpu;
  704. len = snprintf(buf, PAGE_SIZE, "%*pb\n",
  705. nr_cpu_ids, cpumask_bits(cpumask));
  706. mutex_unlock(&pinst->lock);
  707. return len < PAGE_SIZE ? len : -EINVAL;
  708. }
  709. static ssize_t store_cpumask(struct padata_instance *pinst,
  710. struct attribute *attr,
  711. const char *buf, size_t count)
  712. {
  713. cpumask_var_t new_cpumask;
  714. ssize_t ret;
  715. int mask_type;
  716. if (!alloc_cpumask_var(&new_cpumask, GFP_KERNEL))
  717. return -ENOMEM;
  718. ret = bitmap_parse(buf, count, cpumask_bits(new_cpumask),
  719. nr_cpumask_bits);
  720. if (ret < 0)
  721. goto out;
  722. mask_type = !strcmp(attr->name, "serial_cpumask") ?
  723. PADATA_CPU_SERIAL : PADATA_CPU_PARALLEL;
  724. ret = padata_set_cpumask(pinst, mask_type, new_cpumask);
  725. if (!ret)
  726. ret = count;
  727. out:
  728. free_cpumask_var(new_cpumask);
  729. return ret;
  730. }
  731. #define PADATA_ATTR_RW(_name, _show_name, _store_name) \
  732. static struct padata_sysfs_entry _name##_attr = \
  733. __ATTR(_name, 0644, _show_name, _store_name)
  734. #define PADATA_ATTR_RO(_name, _show_name) \
  735. static struct padata_sysfs_entry _name##_attr = \
  736. __ATTR(_name, 0400, _show_name, NULL)
  737. PADATA_ATTR_RW(serial_cpumask, show_cpumask, store_cpumask);
  738. PADATA_ATTR_RW(parallel_cpumask, show_cpumask, store_cpumask);
  739. /*
  740. * Padata sysfs provides the following objects:
  741. * serial_cpumask [RW] - cpumask for serial workers
  742. * parallel_cpumask [RW] - cpumask for parallel workers
  743. */
  744. static struct attribute *padata_default_attrs[] = {
  745. &serial_cpumask_attr.attr,
  746. &parallel_cpumask_attr.attr,
  747. NULL,
  748. };
  749. ATTRIBUTE_GROUPS(padata_default);
  750. static ssize_t padata_sysfs_show(struct kobject *kobj,
  751. struct attribute *attr, char *buf)
  752. {
  753. struct padata_instance *pinst;
  754. struct padata_sysfs_entry *pentry;
  755. ssize_t ret = -EIO;
  756. pinst = kobj2pinst(kobj);
  757. pentry = attr2pentry(attr);
  758. if (pentry->show)
  759. ret = pentry->show(pinst, attr, buf);
  760. return ret;
  761. }
  762. static ssize_t padata_sysfs_store(struct kobject *kobj, struct attribute *attr,
  763. const char *buf, size_t count)
  764. {
  765. struct padata_instance *pinst;
  766. struct padata_sysfs_entry *pentry;
  767. ssize_t ret = -EIO;
  768. pinst = kobj2pinst(kobj);
  769. pentry = attr2pentry(attr);
  770. if (pentry->show)
  771. ret = pentry->store(pinst, attr, buf, count);
  772. return ret;
  773. }
  774. static const struct sysfs_ops padata_sysfs_ops = {
  775. .show = padata_sysfs_show,
  776. .store = padata_sysfs_store,
  777. };
  778. static struct kobj_type padata_attr_type = {
  779. .sysfs_ops = &padata_sysfs_ops,
  780. .default_groups = padata_default_groups,
  781. .release = padata_sysfs_release,
  782. };
  783. /**
  784. * padata_alloc - allocate and initialize a padata instance
  785. * @name: used to identify the instance
  786. *
  787. * Return: new instance on success, NULL on error
  788. */
  789. struct padata_instance *padata_alloc(const char *name)
  790. {
  791. struct padata_instance *pinst;
  792. pinst = kzalloc(sizeof(struct padata_instance), GFP_KERNEL);
  793. if (!pinst)
  794. goto err;
  795. pinst->parallel_wq = alloc_workqueue("%s_parallel", WQ_UNBOUND, 0,
  796. name);
  797. if (!pinst->parallel_wq)
  798. goto err_free_inst;
  799. cpus_read_lock();
  800. pinst->serial_wq = alloc_workqueue("%s_serial", WQ_MEM_RECLAIM |
  801. WQ_CPU_INTENSIVE, 1, name);
  802. if (!pinst->serial_wq)
  803. goto err_put_cpus;
  804. if (!alloc_cpumask_var(&pinst->cpumask.pcpu, GFP_KERNEL))
  805. goto err_free_serial_wq;
  806. if (!alloc_cpumask_var(&pinst->cpumask.cbcpu, GFP_KERNEL)) {
  807. free_cpumask_var(pinst->cpumask.pcpu);
  808. goto err_free_serial_wq;
  809. }
  810. INIT_LIST_HEAD(&pinst->pslist);
  811. cpumask_copy(pinst->cpumask.pcpu, cpu_possible_mask);
  812. cpumask_copy(pinst->cpumask.cbcpu, cpu_possible_mask);
  813. if (padata_setup_cpumasks(pinst))
  814. goto err_free_masks;
  815. __padata_start(pinst);
  816. kobject_init(&pinst->kobj, &padata_attr_type);
  817. mutex_init(&pinst->lock);
  818. #ifdef CONFIG_HOTPLUG_CPU
  819. cpuhp_state_add_instance_nocalls_cpuslocked(hp_online,
  820. &pinst->cpu_online_node);
  821. cpuhp_state_add_instance_nocalls_cpuslocked(CPUHP_PADATA_DEAD,
  822. &pinst->cpu_dead_node);
  823. #endif
  824. cpus_read_unlock();
  825. return pinst;
  826. err_free_masks:
  827. free_cpumask_var(pinst->cpumask.pcpu);
  828. free_cpumask_var(pinst->cpumask.cbcpu);
  829. err_free_serial_wq:
  830. destroy_workqueue(pinst->serial_wq);
  831. err_put_cpus:
  832. cpus_read_unlock();
  833. destroy_workqueue(pinst->parallel_wq);
  834. err_free_inst:
  835. kfree(pinst);
  836. err:
  837. return NULL;
  838. }
  839. EXPORT_SYMBOL(padata_alloc);
  840. /**
  841. * padata_free - free a padata instance
  842. *
  843. * @pinst: padata instance to free
  844. */
  845. void padata_free(struct padata_instance *pinst)
  846. {
  847. kobject_put(&pinst->kobj);
  848. }
  849. EXPORT_SYMBOL(padata_free);
  850. /**
  851. * padata_alloc_shell - Allocate and initialize padata shell.
  852. *
  853. * @pinst: Parent padata_instance object.
  854. *
  855. * Return: new shell on success, NULL on error
  856. */
  857. struct padata_shell *padata_alloc_shell(struct padata_instance *pinst)
  858. {
  859. struct parallel_data *pd;
  860. struct padata_shell *ps;
  861. ps = kzalloc(sizeof(*ps), GFP_KERNEL);
  862. if (!ps)
  863. goto out;
  864. ps->pinst = pinst;
  865. cpus_read_lock();
  866. pd = padata_alloc_pd(ps);
  867. cpus_read_unlock();
  868. if (!pd)
  869. goto out_free_ps;
  870. mutex_lock(&pinst->lock);
  871. RCU_INIT_POINTER(ps->pd, pd);
  872. list_add(&ps->list, &pinst->pslist);
  873. mutex_unlock(&pinst->lock);
  874. return ps;
  875. out_free_ps:
  876. kfree(ps);
  877. out:
  878. return NULL;
  879. }
  880. EXPORT_SYMBOL(padata_alloc_shell);
  881. /**
  882. * padata_free_shell - free a padata shell
  883. *
  884. * @ps: padata shell to free
  885. */
  886. void padata_free_shell(struct padata_shell *ps)
  887. {
  888. struct parallel_data *pd;
  889. if (!ps)
  890. return;
  891. mutex_lock(&ps->pinst->lock);
  892. list_del(&ps->list);
  893. pd = rcu_dereference_protected(ps->pd, 1);
  894. if (refcount_dec_and_test(&pd->refcnt))
  895. padata_free_pd(pd);
  896. mutex_unlock(&ps->pinst->lock);
  897. kfree(ps);
  898. }
  899. EXPORT_SYMBOL(padata_free_shell);
  900. void __init padata_init(void)
  901. {
  902. unsigned int i, possible_cpus;
  903. #ifdef CONFIG_HOTPLUG_CPU
  904. int ret;
  905. ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "padata:online",
  906. padata_cpu_online, NULL);
  907. if (ret < 0)
  908. goto err;
  909. hp_online = ret;
  910. ret = cpuhp_setup_state_multi(CPUHP_PADATA_DEAD, "padata:dead",
  911. NULL, padata_cpu_dead);
  912. if (ret < 0)
  913. goto remove_online_state;
  914. #endif
  915. possible_cpus = num_possible_cpus();
  916. padata_works = kmalloc_array(possible_cpus, sizeof(struct padata_work),
  917. GFP_KERNEL);
  918. if (!padata_works)
  919. goto remove_dead_state;
  920. for (i = 0; i < possible_cpus; ++i)
  921. list_add(&padata_works[i].pw_list, &padata_free_works);
  922. return;
  923. remove_dead_state:
  924. #ifdef CONFIG_HOTPLUG_CPU
  925. cpuhp_remove_multi_state(CPUHP_PADATA_DEAD);
  926. remove_online_state:
  927. cpuhp_remove_multi_state(hp_online);
  928. err:
  929. #endif
  930. pr_warn("padata: initialization failed\n");
  931. }