isolation.c 6.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Housekeeping management. Manage the targets for routine code that can run on
  4. * any CPU: unbound workqueues, timers, kthreads and any offloadable work.
  5. *
  6. * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
  7. * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
  8. *
  9. */
  10. enum hk_flags {
  11. HK_FLAG_TIMER = BIT(HK_TYPE_TIMER),
  12. HK_FLAG_RCU = BIT(HK_TYPE_RCU),
  13. HK_FLAG_MISC = BIT(HK_TYPE_MISC),
  14. HK_FLAG_SCHED = BIT(HK_TYPE_SCHED),
  15. HK_FLAG_TICK = BIT(HK_TYPE_TICK),
  16. HK_FLAG_DOMAIN = BIT(HK_TYPE_DOMAIN),
  17. HK_FLAG_WQ = BIT(HK_TYPE_WQ),
  18. HK_FLAG_MANAGED_IRQ = BIT(HK_TYPE_MANAGED_IRQ),
  19. HK_FLAG_KTHREAD = BIT(HK_TYPE_KTHREAD),
  20. };
  21. DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
  22. EXPORT_SYMBOL_GPL(housekeeping_overridden);
  23. struct housekeeping {
  24. cpumask_var_t cpumasks[HK_TYPE_MAX];
  25. unsigned long flags;
  26. };
  27. static struct housekeeping housekeeping;
  28. bool housekeeping_enabled(enum hk_type type)
  29. {
  30. return !!(housekeeping.flags & BIT(type));
  31. }
  32. EXPORT_SYMBOL_GPL(housekeeping_enabled);
  33. int housekeeping_any_cpu(enum hk_type type)
  34. {
  35. int cpu;
  36. if (static_branch_unlikely(&housekeeping_overridden)) {
  37. if (housekeeping.flags & BIT(type)) {
  38. cpu = sched_numa_find_closest(housekeeping.cpumasks[type], smp_processor_id());
  39. if (cpu < nr_cpu_ids)
  40. return cpu;
  41. return cpumask_any_and(housekeeping.cpumasks[type], cpu_online_mask);
  42. }
  43. }
  44. return smp_processor_id();
  45. }
  46. EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
  47. const struct cpumask *housekeeping_cpumask(enum hk_type type)
  48. {
  49. if (static_branch_unlikely(&housekeeping_overridden))
  50. if (housekeeping.flags & BIT(type))
  51. return housekeeping.cpumasks[type];
  52. return cpu_possible_mask;
  53. }
  54. EXPORT_SYMBOL_GPL(housekeeping_cpumask);
  55. void housekeeping_affine(struct task_struct *t, enum hk_type type)
  56. {
  57. if (static_branch_unlikely(&housekeeping_overridden))
  58. if (housekeeping.flags & BIT(type))
  59. set_cpus_allowed_ptr(t, housekeeping.cpumasks[type]);
  60. }
  61. EXPORT_SYMBOL_GPL(housekeeping_affine);
  62. bool housekeeping_test_cpu(int cpu, enum hk_type type)
  63. {
  64. if (static_branch_unlikely(&housekeeping_overridden))
  65. if (housekeeping.flags & BIT(type))
  66. return cpumask_test_cpu(cpu, housekeeping.cpumasks[type]);
  67. return true;
  68. }
  69. EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
  70. void __init housekeeping_init(void)
  71. {
  72. enum hk_type type;
  73. if (!housekeeping.flags)
  74. return;
  75. static_branch_enable(&housekeeping_overridden);
  76. if (housekeeping.flags & HK_FLAG_TICK)
  77. sched_tick_offload_init();
  78. for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
  79. /* We need at least one CPU to handle housekeeping work */
  80. WARN_ON_ONCE(cpumask_empty(housekeeping.cpumasks[type]));
  81. }
  82. }
  83. static void __init housekeeping_setup_type(enum hk_type type,
  84. cpumask_var_t housekeeping_staging)
  85. {
  86. alloc_bootmem_cpumask_var(&housekeeping.cpumasks[type]);
  87. cpumask_copy(housekeeping.cpumasks[type],
  88. housekeeping_staging);
  89. }
  90. static int __init housekeeping_setup(char *str, unsigned long flags)
  91. {
  92. cpumask_var_t non_housekeeping_mask, housekeeping_staging;
  93. int err = 0;
  94. if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK)) {
  95. if (!IS_ENABLED(CONFIG_NO_HZ_FULL)) {
  96. pr_warn("Housekeeping: nohz unsupported."
  97. " Build with CONFIG_NO_HZ_FULL\n");
  98. return 0;
  99. }
  100. }
  101. alloc_bootmem_cpumask_var(&non_housekeeping_mask);
  102. if (cpulist_parse(str, non_housekeeping_mask) < 0) {
  103. pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
  104. goto free_non_housekeeping_mask;
  105. }
  106. alloc_bootmem_cpumask_var(&housekeeping_staging);
  107. cpumask_andnot(housekeeping_staging,
  108. cpu_possible_mask, non_housekeeping_mask);
  109. if (!cpumask_intersects(cpu_present_mask, housekeeping_staging)) {
  110. __cpumask_set_cpu(smp_processor_id(), housekeeping_staging);
  111. __cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
  112. if (!housekeeping.flags) {
  113. pr_warn("Housekeeping: must include one present CPU, "
  114. "using boot CPU:%d\n", smp_processor_id());
  115. }
  116. }
  117. if (!housekeeping.flags) {
  118. /* First setup call ("nohz_full=" or "isolcpus=") */
  119. enum hk_type type;
  120. for_each_set_bit(type, &flags, HK_TYPE_MAX)
  121. housekeeping_setup_type(type, housekeeping_staging);
  122. } else {
  123. /* Second setup call ("nohz_full=" after "isolcpus=" or the reverse) */
  124. enum hk_type type;
  125. unsigned long iter_flags = flags & housekeeping.flags;
  126. for_each_set_bit(type, &iter_flags, HK_TYPE_MAX) {
  127. if (!cpumask_equal(housekeeping_staging,
  128. housekeeping.cpumasks[type])) {
  129. pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
  130. goto free_housekeeping_staging;
  131. }
  132. }
  133. iter_flags = flags & ~housekeeping.flags;
  134. for_each_set_bit(type, &iter_flags, HK_TYPE_MAX)
  135. housekeeping_setup_type(type, housekeeping_staging);
  136. }
  137. if ((flags & HK_FLAG_TICK) && !(housekeeping.flags & HK_FLAG_TICK))
  138. tick_nohz_full_setup(non_housekeeping_mask);
  139. housekeeping.flags |= flags;
  140. err = 1;
  141. free_housekeeping_staging:
  142. free_bootmem_cpumask_var(housekeeping_staging);
  143. free_non_housekeeping_mask:
  144. free_bootmem_cpumask_var(non_housekeeping_mask);
  145. return err;
  146. }
  147. static int __init housekeeping_nohz_full_setup(char *str)
  148. {
  149. unsigned long flags;
  150. flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
  151. HK_FLAG_MISC | HK_FLAG_KTHREAD;
  152. return housekeeping_setup(str, flags);
  153. }
  154. __setup("nohz_full=", housekeeping_nohz_full_setup);
  155. static int __init housekeeping_isolcpus_setup(char *str)
  156. {
  157. unsigned long flags = 0;
  158. bool illegal = false;
  159. char *par;
  160. int len;
  161. while (isalpha(*str)) {
  162. if (!strncmp(str, "nohz,", 5)) {
  163. str += 5;
  164. flags |= HK_FLAG_TICK;
  165. continue;
  166. }
  167. if (!strncmp(str, "domain,", 7)) {
  168. str += 7;
  169. flags |= HK_FLAG_DOMAIN;
  170. continue;
  171. }
  172. if (!strncmp(str, "managed_irq,", 12)) {
  173. str += 12;
  174. flags |= HK_FLAG_MANAGED_IRQ;
  175. continue;
  176. }
  177. /*
  178. * Skip unknown sub-parameter and validate that it is not
  179. * containing an invalid character.
  180. */
  181. for (par = str, len = 0; *str && *str != ','; str++, len++) {
  182. if (!isalpha(*str) && *str != '_')
  183. illegal = true;
  184. }
  185. if (illegal) {
  186. pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
  187. return 0;
  188. }
  189. pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
  190. str++;
  191. }
  192. /* Default behaviour for isolcpus without flags */
  193. if (!flags)
  194. flags |= HK_FLAG_DOMAIN;
  195. return housekeeping_setup(str, flags);
  196. }
  197. __setup("isolcpus=", housekeeping_isolcpus_setup);