topology.c 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245
  1. /*
  2. * arch/arm/kernel/topology.c
  3. *
  4. * Copyright (C) 2011 Linaro Limited.
  5. * Written by: Vincent Guittot
  6. *
  7. * based on arch/sh/kernel/topology.c
  8. *
  9. * This file is subject to the terms and conditions of the GNU General Public
  10. * License. See the file "COPYING" in the main directory of this archive
  11. * for more details.
  12. */
  13. #include <linux/arch_topology.h>
  14. #include <linux/cpu.h>
  15. #include <linux/cpufreq.h>
  16. #include <linux/cpumask.h>
  17. #include <linux/export.h>
  18. #include <linux/init.h>
  19. #include <linux/percpu.h>
  20. #include <linux/node.h>
  21. #include <linux/nodemask.h>
  22. #include <linux/of.h>
  23. #include <linux/sched.h>
  24. #include <linux/sched/topology.h>
  25. #include <linux/slab.h>
  26. #include <linux/string.h>
  27. #include <asm/cpu.h>
  28. #include <asm/cputype.h>
  29. #include <asm/topology.h>
  30. /*
  31. * cpu capacity scale management
  32. */
  33. /*
  34. * cpu capacity table
  35. * This per cpu data structure describes the relative capacity of each core.
  36. * On a heteregenous system, cores don't have the same computation capacity
  37. * and we reflect that difference in the cpu_capacity field so the scheduler
  38. * can take this difference into account during load balance. A per cpu
  39. * structure is preferred because each CPU updates its own cpu_capacity field
  40. * during the load balance except for idle cores. One idle core is selected
  41. * to run the rebalance_domains for all idle cores and the cpu_capacity can be
  42. * updated during this sequence.
  43. */
  44. #ifdef CONFIG_OF
  45. struct cpu_efficiency {
  46. const char *compatible;
  47. unsigned long efficiency;
  48. };
  49. /*
  50. * Table of relative efficiency of each processors
  51. * The efficiency value must fit in 20bit and the final
  52. * cpu_scale value must be in the range
  53. * 0 < cpu_scale < 3*SCHED_CAPACITY_SCALE/2
  54. * in order to return at most 1 when DIV_ROUND_CLOSEST
  55. * is used to compute the capacity of a CPU.
  56. * Processors that are not defined in the table,
  57. * use the default SCHED_CAPACITY_SCALE value for cpu_scale.
  58. */
  59. static const struct cpu_efficiency table_efficiency[] = {
  60. {"arm,cortex-a15", 3891},
  61. {"arm,cortex-a7", 2048},
  62. {NULL, },
  63. };
  64. static unsigned long *__cpu_capacity;
  65. #define cpu_capacity(cpu) __cpu_capacity[cpu]
  66. static unsigned long middle_capacity = 1;
  67. static bool cap_from_dt = true;
  68. /*
  69. * Iterate all CPUs' descriptor in DT and compute the efficiency
  70. * (as per table_efficiency). Also calculate a middle efficiency
  71. * as close as possible to (max{eff_i} - min{eff_i}) / 2
  72. * This is later used to scale the cpu_capacity field such that an
  73. * 'average' CPU is of middle capacity. Also see the comments near
  74. * table_efficiency[] and update_cpu_capacity().
  75. */
  76. static void __init parse_dt_topology(void)
  77. {
  78. const struct cpu_efficiency *cpu_eff;
  79. struct device_node *cn = NULL;
  80. unsigned long min_capacity = ULONG_MAX;
  81. unsigned long max_capacity = 0;
  82. unsigned long capacity = 0;
  83. int cpu = 0;
  84. __cpu_capacity = kcalloc(nr_cpu_ids, sizeof(*__cpu_capacity),
  85. GFP_NOWAIT);
  86. for_each_possible_cpu(cpu) {
  87. const __be32 *rate;
  88. int len;
  89. /* too early to use cpu->of_node */
  90. cn = of_get_cpu_node(cpu, NULL);
  91. if (!cn) {
  92. pr_err("missing device node for CPU %d\n", cpu);
  93. continue;
  94. }
  95. if (topology_parse_cpu_capacity(cn, cpu)) {
  96. of_node_put(cn);
  97. continue;
  98. }
  99. cap_from_dt = false;
  100. for (cpu_eff = table_efficiency; cpu_eff->compatible; cpu_eff++)
  101. if (of_device_is_compatible(cn, cpu_eff->compatible))
  102. break;
  103. if (cpu_eff->compatible == NULL)
  104. continue;
  105. rate = of_get_property(cn, "clock-frequency", &len);
  106. if (!rate || len != 4) {
  107. pr_err("%pOF missing clock-frequency property\n", cn);
  108. continue;
  109. }
  110. capacity = ((be32_to_cpup(rate)) >> 20) * cpu_eff->efficiency;
  111. /* Save min capacity of the system */
  112. if (capacity < min_capacity)
  113. min_capacity = capacity;
  114. /* Save max capacity of the system */
  115. if (capacity > max_capacity)
  116. max_capacity = capacity;
  117. cpu_capacity(cpu) = capacity;
  118. }
  119. /* If min and max capacities are equals, we bypass the update of the
  120. * cpu_scale because all CPUs have the same capacity. Otherwise, we
  121. * compute a middle_capacity factor that will ensure that the capacity
  122. * of an 'average' CPU of the system will be as close as possible to
  123. * SCHED_CAPACITY_SCALE, which is the default value, but with the
  124. * constraint explained near table_efficiency[].
  125. */
  126. if (4*max_capacity < (3*(max_capacity + min_capacity)))
  127. middle_capacity = (min_capacity + max_capacity)
  128. >> (SCHED_CAPACITY_SHIFT+1);
  129. else
  130. middle_capacity = ((max_capacity / 3)
  131. >> (SCHED_CAPACITY_SHIFT-1)) + 1;
  132. if (cap_from_dt)
  133. topology_normalize_cpu_scale();
  134. }
  135. /*
  136. * Look for a customed capacity of a CPU in the cpu_capacity table during the
  137. * boot. The update of all CPUs is in O(n^2) for heteregeneous system but the
  138. * function returns directly for SMP system.
  139. */
  140. static void update_cpu_capacity(unsigned int cpu)
  141. {
  142. if (!cpu_capacity(cpu) || cap_from_dt)
  143. return;
  144. topology_set_cpu_scale(cpu, cpu_capacity(cpu) / middle_capacity);
  145. pr_info("CPU%u: update cpu_capacity %lu\n",
  146. cpu, topology_get_cpu_scale(cpu));
  147. }
  148. #else
  149. static inline void parse_dt_topology(void) {}
  150. static inline void update_cpu_capacity(unsigned int cpuid) {}
  151. #endif
  152. /*
  153. * store_cpu_topology is called at boot when only one cpu is running
  154. * and with the mutex cpu_hotplug.lock locked, when several cpus have booted,
  155. * which prevents simultaneous write access to cpu_topology array
  156. */
  157. void store_cpu_topology(unsigned int cpuid)
  158. {
  159. struct cpu_topology *cpuid_topo = &cpu_topology[cpuid];
  160. unsigned int mpidr;
  161. if (cpuid_topo->package_id != -1)
  162. goto topology_populated;
  163. mpidr = read_cpuid_mpidr();
  164. /* create cpu topology mapping */
  165. if ((mpidr & MPIDR_SMP_BITMASK) == MPIDR_SMP_VALUE) {
  166. /*
  167. * This is a multiprocessor system
  168. * multiprocessor format & multiprocessor mode field are set
  169. */
  170. if (mpidr & MPIDR_MT_BITMASK) {
  171. /* core performance interdependency */
  172. cpuid_topo->thread_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  173. cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  174. cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 2);
  175. } else {
  176. /* largely independent cores */
  177. cpuid_topo->thread_id = -1;
  178. cpuid_topo->core_id = MPIDR_AFFINITY_LEVEL(mpidr, 0);
  179. cpuid_topo->package_id = MPIDR_AFFINITY_LEVEL(mpidr, 1);
  180. }
  181. } else {
  182. /*
  183. * This is an uniprocessor system
  184. * we are in multiprocessor format but uniprocessor system
  185. * or in the old uniprocessor format
  186. */
  187. cpuid_topo->thread_id = -1;
  188. cpuid_topo->core_id = 0;
  189. cpuid_topo->package_id = -1;
  190. }
  191. update_cpu_capacity(cpuid);
  192. pr_info("CPU%u: thread %d, cpu %d, socket %d, mpidr %x\n",
  193. cpuid, cpu_topology[cpuid].thread_id,
  194. cpu_topology[cpuid].core_id,
  195. cpu_topology[cpuid].package_id, mpidr);
  196. topology_populated:
  197. update_siblings_masks(cpuid);
  198. }
  199. /*
  200. * init_cpu_topology is called at boot when only one cpu is running
  201. * which prevent simultaneous write access to cpu_topology array
  202. */
  203. void __init init_cpu_topology(void)
  204. {
  205. reset_cpu_topology();
  206. smp_wmb();
  207. parse_dt_topology();
  208. }