smp.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634
  1. /*
  2. * Xtensa SMP support functions.
  3. *
  4. * This file is subject to the terms and conditions of the GNU General Public
  5. * License. See the file "COPYING" in the main directory of this archive
  6. * for more details.
  7. *
  8. * Copyright (C) 2008 - 2013 Tensilica Inc.
  9. *
  10. * Chris Zankel <[email protected]>
  11. * Joe Taylor <[email protected]>
  12. * Pete Delaney <[email protected]
  13. */
  14. #include <linux/cpu.h>
  15. #include <linux/cpumask.h>
  16. #include <linux/delay.h>
  17. #include <linux/init.h>
  18. #include <linux/interrupt.h>
  19. #include <linux/irqdomain.h>
  20. #include <linux/irq.h>
  21. #include <linux/kdebug.h>
  22. #include <linux/module.h>
  23. #include <linux/sched/mm.h>
  24. #include <linux/sched/hotplug.h>
  25. #include <linux/sched/task_stack.h>
  26. #include <linux/reboot.h>
  27. #include <linux/seq_file.h>
  28. #include <linux/smp.h>
  29. #include <linux/thread_info.h>
  30. #include <asm/cacheflush.h>
  31. #include <asm/coprocessor.h>
  32. #include <asm/kdebug.h>
  33. #include <asm/mmu_context.h>
  34. #include <asm/mxregs.h>
  35. #include <asm/platform.h>
  36. #include <asm/tlbflush.h>
  37. #include <asm/traps.h>
  38. #ifdef CONFIG_SMP
  39. # if XCHAL_HAVE_S32C1I == 0
  40. # error "The S32C1I option is required for SMP."
  41. # endif
  42. #endif
  43. static void system_invalidate_dcache_range(unsigned long start,
  44. unsigned long size);
  45. static void system_flush_invalidate_dcache_range(unsigned long start,
  46. unsigned long size);
  47. /* IPI (Inter Process Interrupt) */
  48. #define IPI_IRQ 0
  49. static irqreturn_t ipi_interrupt(int irq, void *dev_id);
  50. void ipi_init(void)
  51. {
  52. unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
  53. if (request_irq(irq, ipi_interrupt, IRQF_PERCPU, "ipi", NULL))
  54. pr_err("Failed to request irq %u (ipi)\n", irq);
  55. }
  56. static inline unsigned int get_core_count(void)
  57. {
  58. /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
  59. unsigned int syscfgid = get_er(SYSCFGID);
  60. return ((syscfgid >> 18) & 0xf) + 1;
  61. }
  62. static inline int get_core_id(void)
  63. {
  64. /* Bits 0...18 of SYSCFGID contain the core id */
  65. unsigned int core_id = get_er(SYSCFGID);
  66. return core_id & 0x3fff;
  67. }
  68. void __init smp_prepare_cpus(unsigned int max_cpus)
  69. {
  70. unsigned i;
  71. for_each_possible_cpu(i)
  72. set_cpu_present(i, true);
  73. }
  74. void __init smp_init_cpus(void)
  75. {
  76. unsigned i;
  77. unsigned int ncpus = get_core_count();
  78. unsigned int core_id = get_core_id();
  79. pr_info("%s: Core Count = %d\n", __func__, ncpus);
  80. pr_info("%s: Core Id = %d\n", __func__, core_id);
  81. if (ncpus > NR_CPUS) {
  82. ncpus = NR_CPUS;
  83. pr_info("%s: limiting core count by %d\n", __func__, ncpus);
  84. }
  85. for (i = 0; i < ncpus; ++i)
  86. set_cpu_possible(i, true);
  87. }
  88. void __init smp_prepare_boot_cpu(void)
  89. {
  90. unsigned int cpu = smp_processor_id();
  91. BUG_ON(cpu != 0);
  92. cpu_asid_cache(cpu) = ASID_USER_FIRST;
  93. }
  94. void __init smp_cpus_done(unsigned int max_cpus)
  95. {
  96. }
  97. static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
  98. static DECLARE_COMPLETION(cpu_running);
  99. void secondary_start_kernel(void)
  100. {
  101. struct mm_struct *mm = &init_mm;
  102. unsigned int cpu = smp_processor_id();
  103. init_mmu();
  104. #ifdef CONFIG_DEBUG_MISC
  105. if (boot_secondary_processors == 0) {
  106. pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
  107. __func__, boot_secondary_processors, cpu);
  108. for (;;)
  109. __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
  110. }
  111. pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
  112. __func__, boot_secondary_processors, cpu);
  113. #endif
  114. /* Init EXCSAVE1 */
  115. secondary_trap_init();
  116. /* All kernel threads share the same mm context. */
  117. mmget(mm);
  118. mmgrab(mm);
  119. current->active_mm = mm;
  120. cpumask_set_cpu(cpu, mm_cpumask(mm));
  121. enter_lazy_tlb(mm, current);
  122. trace_hardirqs_off();
  123. calibrate_delay();
  124. notify_cpu_starting(cpu);
  125. secondary_init_irq();
  126. local_timer_setup(cpu);
  127. set_cpu_online(cpu, true);
  128. local_irq_enable();
  129. complete(&cpu_running);
  130. cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
  131. }
  132. static void mx_cpu_start(void *p)
  133. {
  134. unsigned cpu = (unsigned)p;
  135. unsigned long run_stall_mask = get_er(MPSCORE);
  136. set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
  137. pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
  138. __func__, cpu, run_stall_mask, get_er(MPSCORE));
  139. }
  140. static void mx_cpu_stop(void *p)
  141. {
  142. unsigned cpu = (unsigned)p;
  143. unsigned long run_stall_mask = get_er(MPSCORE);
  144. set_er(run_stall_mask | (1u << cpu), MPSCORE);
  145. pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
  146. __func__, cpu, run_stall_mask, get_er(MPSCORE));
  147. }
  148. #ifdef CONFIG_HOTPLUG_CPU
  149. unsigned long cpu_start_id __cacheline_aligned;
  150. #endif
  151. unsigned long cpu_start_ccount;
  152. static int boot_secondary(unsigned int cpu, struct task_struct *ts)
  153. {
  154. unsigned long timeout = jiffies + msecs_to_jiffies(1000);
  155. unsigned long ccount;
  156. int i;
  157. #ifdef CONFIG_HOTPLUG_CPU
  158. WRITE_ONCE(cpu_start_id, cpu);
  159. /* Pairs with the third memw in the cpu_restart */
  160. mb();
  161. system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
  162. sizeof(cpu_start_id));
  163. #endif
  164. smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
  165. for (i = 0; i < 2; ++i) {
  166. do
  167. ccount = get_ccount();
  168. while (!ccount);
  169. WRITE_ONCE(cpu_start_ccount, ccount);
  170. do {
  171. /*
  172. * Pairs with the first two memws in the
  173. * .Lboot_secondary.
  174. */
  175. mb();
  176. ccount = READ_ONCE(cpu_start_ccount);
  177. } while (ccount && time_before(jiffies, timeout));
  178. if (ccount) {
  179. smp_call_function_single(0, mx_cpu_stop,
  180. (void *)cpu, 1);
  181. WRITE_ONCE(cpu_start_ccount, 0);
  182. return -EIO;
  183. }
  184. }
  185. return 0;
  186. }
  187. int __cpu_up(unsigned int cpu, struct task_struct *idle)
  188. {
  189. int ret = 0;
  190. if (cpu_asid_cache(cpu) == 0)
  191. cpu_asid_cache(cpu) = ASID_USER_FIRST;
  192. start_info.stack = (unsigned long)task_pt_regs(idle);
  193. wmb();
  194. pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
  195. __func__, cpu, idle, start_info.stack);
  196. init_completion(&cpu_running);
  197. ret = boot_secondary(cpu, idle);
  198. if (ret == 0) {
  199. wait_for_completion_timeout(&cpu_running,
  200. msecs_to_jiffies(1000));
  201. if (!cpu_online(cpu))
  202. ret = -EIO;
  203. }
  204. if (ret)
  205. pr_err("CPU %u failed to boot\n", cpu);
  206. return ret;
  207. }
  208. #ifdef CONFIG_HOTPLUG_CPU
  209. /*
  210. * __cpu_disable runs on the processor to be shutdown.
  211. */
  212. int __cpu_disable(void)
  213. {
  214. unsigned int cpu = smp_processor_id();
  215. /*
  216. * Take this CPU offline. Once we clear this, we can't return,
  217. * and we must not schedule until we're ready to give up the cpu.
  218. */
  219. set_cpu_online(cpu, false);
  220. #if XTENSA_HAVE_COPROCESSORS
  221. /*
  222. * Flush coprocessor contexts that are active on the current CPU.
  223. */
  224. local_coprocessors_flush_release_all();
  225. #endif
  226. /*
  227. * OK - migrate IRQs away from this CPU
  228. */
  229. migrate_irqs();
  230. /*
  231. * Flush user cache and TLB mappings, and then remove this CPU
  232. * from the vm mask set of all processes.
  233. */
  234. local_flush_cache_all();
  235. local_flush_tlb_all();
  236. invalidate_page_directory();
  237. clear_tasks_mm_cpumask(cpu);
  238. return 0;
  239. }
  240. static void platform_cpu_kill(unsigned int cpu)
  241. {
  242. smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
  243. }
  244. /*
  245. * called on the thread which is asking for a CPU to be shutdown -
  246. * waits until shutdown has completed, or it is timed out.
  247. */
  248. void __cpu_die(unsigned int cpu)
  249. {
  250. unsigned long timeout = jiffies + msecs_to_jiffies(1000);
  251. while (time_before(jiffies, timeout)) {
  252. system_invalidate_dcache_range((unsigned long)&cpu_start_id,
  253. sizeof(cpu_start_id));
  254. /* Pairs with the second memw in the cpu_restart */
  255. mb();
  256. if (READ_ONCE(cpu_start_id) == -cpu) {
  257. platform_cpu_kill(cpu);
  258. return;
  259. }
  260. }
  261. pr_err("CPU%u: unable to kill\n", cpu);
  262. }
  263. void arch_cpu_idle_dead(void)
  264. {
  265. cpu_die();
  266. }
  267. /*
  268. * Called from the idle thread for the CPU which has been shutdown.
  269. *
  270. * Note that we disable IRQs here, but do not re-enable them
  271. * before returning to the caller. This is also the behaviour
  272. * of the other hotplug-cpu capable cores, so presumably coming
  273. * out of idle fixes this.
  274. */
  275. void __ref cpu_die(void)
  276. {
  277. idle_task_exit();
  278. local_irq_disable();
  279. __asm__ __volatile__(
  280. " movi a2, cpu_restart\n"
  281. " jx a2\n");
  282. }
  283. #endif /* CONFIG_HOTPLUG_CPU */
  284. enum ipi_msg_type {
  285. IPI_RESCHEDULE = 0,
  286. IPI_CALL_FUNC,
  287. IPI_CPU_STOP,
  288. IPI_MAX
  289. };
  290. static const struct {
  291. const char *short_text;
  292. const char *long_text;
  293. } ipi_text[] = {
  294. { .short_text = "RES", .long_text = "Rescheduling interrupts" },
  295. { .short_text = "CAL", .long_text = "Function call interrupts" },
  296. { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
  297. };
  298. struct ipi_data {
  299. unsigned long ipi_count[IPI_MAX];
  300. };
  301. static DEFINE_PER_CPU(struct ipi_data, ipi_data);
  302. static void send_ipi_message(const struct cpumask *callmask,
  303. enum ipi_msg_type msg_id)
  304. {
  305. int index;
  306. unsigned long mask = 0;
  307. for_each_cpu(index, callmask)
  308. mask |= 1 << index;
  309. set_er(mask, MIPISET(msg_id));
  310. }
  311. void arch_send_call_function_ipi_mask(const struct cpumask *mask)
  312. {
  313. send_ipi_message(mask, IPI_CALL_FUNC);
  314. }
  315. void arch_send_call_function_single_ipi(int cpu)
  316. {
  317. send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
  318. }
  319. void smp_send_reschedule(int cpu)
  320. {
  321. send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
  322. }
  323. void smp_send_stop(void)
  324. {
  325. struct cpumask targets;
  326. cpumask_copy(&targets, cpu_online_mask);
  327. cpumask_clear_cpu(smp_processor_id(), &targets);
  328. send_ipi_message(&targets, IPI_CPU_STOP);
  329. }
  330. static void ipi_cpu_stop(unsigned int cpu)
  331. {
  332. set_cpu_online(cpu, false);
  333. machine_halt();
  334. }
  335. irqreturn_t ipi_interrupt(int irq, void *dev_id)
  336. {
  337. unsigned int cpu = smp_processor_id();
  338. struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
  339. for (;;) {
  340. unsigned int msg;
  341. msg = get_er(MIPICAUSE(cpu));
  342. set_er(msg, MIPICAUSE(cpu));
  343. if (!msg)
  344. break;
  345. if (msg & (1 << IPI_CALL_FUNC)) {
  346. ++ipi->ipi_count[IPI_CALL_FUNC];
  347. generic_smp_call_function_interrupt();
  348. }
  349. if (msg & (1 << IPI_RESCHEDULE)) {
  350. ++ipi->ipi_count[IPI_RESCHEDULE];
  351. scheduler_ipi();
  352. }
  353. if (msg & (1 << IPI_CPU_STOP)) {
  354. ++ipi->ipi_count[IPI_CPU_STOP];
  355. ipi_cpu_stop(cpu);
  356. }
  357. }
  358. return IRQ_HANDLED;
  359. }
  360. void show_ipi_list(struct seq_file *p, int prec)
  361. {
  362. unsigned int cpu;
  363. unsigned i;
  364. for (i = 0; i < IPI_MAX; ++i) {
  365. seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
  366. for_each_online_cpu(cpu)
  367. seq_printf(p, " %10lu",
  368. per_cpu(ipi_data, cpu).ipi_count[i]);
  369. seq_printf(p, " %s\n", ipi_text[i].long_text);
  370. }
  371. }
  372. int setup_profiling_timer(unsigned int multiplier)
  373. {
  374. pr_debug("setup_profiling_timer %d\n", multiplier);
  375. return 0;
  376. }
  377. /* TLB flush functions */
  378. struct flush_data {
  379. struct vm_area_struct *vma;
  380. unsigned long addr1;
  381. unsigned long addr2;
  382. };
  383. static void ipi_flush_tlb_all(void *arg)
  384. {
  385. local_flush_tlb_all();
  386. }
  387. void flush_tlb_all(void)
  388. {
  389. on_each_cpu(ipi_flush_tlb_all, NULL, 1);
  390. }
  391. static void ipi_flush_tlb_mm(void *arg)
  392. {
  393. local_flush_tlb_mm(arg);
  394. }
  395. void flush_tlb_mm(struct mm_struct *mm)
  396. {
  397. on_each_cpu(ipi_flush_tlb_mm, mm, 1);
  398. }
  399. static void ipi_flush_tlb_page(void *arg)
  400. {
  401. struct flush_data *fd = arg;
  402. local_flush_tlb_page(fd->vma, fd->addr1);
  403. }
  404. void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
  405. {
  406. struct flush_data fd = {
  407. .vma = vma,
  408. .addr1 = addr,
  409. };
  410. on_each_cpu(ipi_flush_tlb_page, &fd, 1);
  411. }
  412. static void ipi_flush_tlb_range(void *arg)
  413. {
  414. struct flush_data *fd = arg;
  415. local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
  416. }
  417. void flush_tlb_range(struct vm_area_struct *vma,
  418. unsigned long start, unsigned long end)
  419. {
  420. struct flush_data fd = {
  421. .vma = vma,
  422. .addr1 = start,
  423. .addr2 = end,
  424. };
  425. on_each_cpu(ipi_flush_tlb_range, &fd, 1);
  426. }
  427. static void ipi_flush_tlb_kernel_range(void *arg)
  428. {
  429. struct flush_data *fd = arg;
  430. local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
  431. }
  432. void flush_tlb_kernel_range(unsigned long start, unsigned long end)
  433. {
  434. struct flush_data fd = {
  435. .addr1 = start,
  436. .addr2 = end,
  437. };
  438. on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
  439. }
  440. /* Cache flush functions */
  441. static void ipi_flush_cache_all(void *arg)
  442. {
  443. local_flush_cache_all();
  444. }
  445. void flush_cache_all(void)
  446. {
  447. on_each_cpu(ipi_flush_cache_all, NULL, 1);
  448. }
  449. static void ipi_flush_cache_page(void *arg)
  450. {
  451. struct flush_data *fd = arg;
  452. local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
  453. }
  454. void flush_cache_page(struct vm_area_struct *vma,
  455. unsigned long address, unsigned long pfn)
  456. {
  457. struct flush_data fd = {
  458. .vma = vma,
  459. .addr1 = address,
  460. .addr2 = pfn,
  461. };
  462. on_each_cpu(ipi_flush_cache_page, &fd, 1);
  463. }
  464. static void ipi_flush_cache_range(void *arg)
  465. {
  466. struct flush_data *fd = arg;
  467. local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
  468. }
  469. void flush_cache_range(struct vm_area_struct *vma,
  470. unsigned long start, unsigned long end)
  471. {
  472. struct flush_data fd = {
  473. .vma = vma,
  474. .addr1 = start,
  475. .addr2 = end,
  476. };
  477. on_each_cpu(ipi_flush_cache_range, &fd, 1);
  478. }
  479. static void ipi_flush_icache_range(void *arg)
  480. {
  481. struct flush_data *fd = arg;
  482. local_flush_icache_range(fd->addr1, fd->addr2);
  483. }
  484. void flush_icache_range(unsigned long start, unsigned long end)
  485. {
  486. struct flush_data fd = {
  487. .addr1 = start,
  488. .addr2 = end,
  489. };
  490. on_each_cpu(ipi_flush_icache_range, &fd, 1);
  491. }
  492. EXPORT_SYMBOL(flush_icache_range);
  493. /* ------------------------------------------------------------------------- */
  494. static void ipi_invalidate_dcache_range(void *arg)
  495. {
  496. struct flush_data *fd = arg;
  497. __invalidate_dcache_range(fd->addr1, fd->addr2);
  498. }
  499. static void system_invalidate_dcache_range(unsigned long start,
  500. unsigned long size)
  501. {
  502. struct flush_data fd = {
  503. .addr1 = start,
  504. .addr2 = size,
  505. };
  506. on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
  507. }
  508. static void ipi_flush_invalidate_dcache_range(void *arg)
  509. {
  510. struct flush_data *fd = arg;
  511. __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
  512. }
  513. static void system_flush_invalidate_dcache_range(unsigned long start,
  514. unsigned long size)
  515. {
  516. struct flush_data fd = {
  517. .addr1 = start,
  518. .addr2 = size,
  519. };
  520. on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
  521. }