smp.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. ** SMP Support
  4. **
  5. ** Copyright (C) 1999 Walt Drummond <[email protected]>
  6. ** Copyright (C) 1999 David Mosberger-Tang <[email protected]>
  7. ** Copyright (C) 2001,2004 Grant Grundler <[email protected]>
  8. **
  9. ** Lots of stuff stolen from arch/alpha/kernel/smp.c
  10. ** ...and then parisc stole from arch/ia64/kernel/smp.c. Thanks David! :^)
  11. **
  12. ** Thanks to John Curry and Ullas Ponnadi. I learned a lot from their work.
  13. ** -grant (1/12/2001)
  14. **
  15. */
  16. #include <linux/types.h>
  17. #include <linux/spinlock.h>
  18. #include <linux/kernel.h>
  19. #include <linux/module.h>
  20. #include <linux/sched/mm.h>
  21. #include <linux/init.h>
  22. #include <linux/interrupt.h>
  23. #include <linux/smp.h>
  24. #include <linux/kernel_stat.h>
  25. #include <linux/mm.h>
  26. #include <linux/err.h>
  27. #include <linux/delay.h>
  28. #include <linux/bitops.h>
  29. #include <linux/ftrace.h>
  30. #include <linux/cpu.h>
  31. #include <linux/kgdb.h>
  32. #include <linux/sched/hotplug.h>
  33. #include <linux/atomic.h>
  34. #include <asm/current.h>
  35. #include <asm/delay.h>
  36. #include <asm/tlbflush.h>
  37. #include <asm/io.h>
  38. #include <asm/irq.h> /* for CPU_IRQ_REGION and friends */
  39. #include <asm/mmu_context.h>
  40. #include <asm/page.h>
  41. #include <asm/processor.h>
  42. #include <asm/ptrace.h>
  43. #include <asm/unistd.h>
  44. #include <asm/cacheflush.h>
  45. #undef DEBUG_SMP
  46. #ifdef DEBUG_SMP
  47. static int smp_debug_lvl = 0;
  48. #define smp_debug(lvl, printargs...) \
  49. if (lvl >= smp_debug_lvl) \
  50. printk(printargs);
  51. #else
  52. #define smp_debug(lvl, ...) do { } while(0)
  53. #endif /* DEBUG_SMP */
  54. volatile struct task_struct *smp_init_current_idle_task;
  55. /* track which CPU is booting */
  56. static volatile int cpu_now_booting;
  57. static DEFINE_PER_CPU(spinlock_t, ipi_lock);
  58. enum ipi_message_type {
  59. IPI_NOP=0,
  60. IPI_RESCHEDULE=1,
  61. IPI_CALL_FUNC,
  62. IPI_CPU_START,
  63. IPI_CPU_STOP,
  64. IPI_CPU_TEST,
  65. #ifdef CONFIG_KGDB
  66. IPI_ENTER_KGDB,
  67. #endif
  68. };
  69. /********** SMP inter processor interrupt and communication routines */
  70. #undef PER_CPU_IRQ_REGION
  71. #ifdef PER_CPU_IRQ_REGION
  72. /* XXX REVISIT Ignore for now.
  73. ** *May* need this "hook" to register IPI handler
  74. ** once we have perCPU ExtIntr switch tables.
  75. */
  76. static void
  77. ipi_init(int cpuid)
  78. {
  79. #error verify IRQ_OFFSET(IPI_IRQ) is ipi_interrupt() in new IRQ region
  80. if(cpu_online(cpuid) )
  81. {
  82. switch_to_idle_task(current);
  83. }
  84. return;
  85. }
  86. #endif
  87. /*
  88. ** Yoink this CPU from the runnable list...
  89. **
  90. */
  91. static void
  92. halt_processor(void)
  93. {
  94. /* REVISIT : redirect I/O Interrupts to another CPU? */
  95. /* REVISIT : does PM *know* this CPU isn't available? */
  96. set_cpu_online(smp_processor_id(), false);
  97. local_irq_disable();
  98. __pdc_cpu_rendezvous();
  99. for (;;)
  100. ;
  101. }
  102. irqreturn_t __irq_entry
  103. ipi_interrupt(int irq, void *dev_id)
  104. {
  105. int this_cpu = smp_processor_id();
  106. struct cpuinfo_parisc *p = &per_cpu(cpu_data, this_cpu);
  107. unsigned long ops;
  108. unsigned long flags;
  109. for (;;) {
  110. spinlock_t *lock = &per_cpu(ipi_lock, this_cpu);
  111. spin_lock_irqsave(lock, flags);
  112. ops = p->pending_ipi;
  113. p->pending_ipi = 0;
  114. spin_unlock_irqrestore(lock, flags);
  115. mb(); /* Order bit clearing and data access. */
  116. if (!ops)
  117. break;
  118. while (ops) {
  119. unsigned long which = ffz(~ops);
  120. ops &= ~(1 << which);
  121. switch (which) {
  122. case IPI_NOP:
  123. smp_debug(100, KERN_DEBUG "CPU%d IPI_NOP\n", this_cpu);
  124. break;
  125. case IPI_RESCHEDULE:
  126. smp_debug(100, KERN_DEBUG "CPU%d IPI_RESCHEDULE\n", this_cpu);
  127. inc_irq_stat(irq_resched_count);
  128. scheduler_ipi();
  129. break;
  130. case IPI_CALL_FUNC:
  131. smp_debug(100, KERN_DEBUG "CPU%d IPI_CALL_FUNC\n", this_cpu);
  132. inc_irq_stat(irq_call_count);
  133. generic_smp_call_function_interrupt();
  134. break;
  135. case IPI_CPU_START:
  136. smp_debug(100, KERN_DEBUG "CPU%d IPI_CPU_START\n", this_cpu);
  137. break;
  138. case IPI_CPU_STOP:
  139. smp_debug(100, KERN_DEBUG "CPU%d IPI_CPU_STOP\n", this_cpu);
  140. halt_processor();
  141. break;
  142. case IPI_CPU_TEST:
  143. smp_debug(100, KERN_DEBUG "CPU%d is alive!\n", this_cpu);
  144. break;
  145. #ifdef CONFIG_KGDB
  146. case IPI_ENTER_KGDB:
  147. smp_debug(100, KERN_DEBUG "CPU%d ENTER_KGDB\n", this_cpu);
  148. kgdb_nmicallback(raw_smp_processor_id(), get_irq_regs());
  149. break;
  150. #endif
  151. default:
  152. printk(KERN_CRIT "Unknown IPI num on CPU%d: %lu\n",
  153. this_cpu, which);
  154. return IRQ_NONE;
  155. } /* Switch */
  156. /* before doing more, let in any pending interrupts */
  157. if (ops) {
  158. local_irq_enable();
  159. local_irq_disable();
  160. }
  161. } /* while (ops) */
  162. }
  163. return IRQ_HANDLED;
  164. }
  165. static inline void
  166. ipi_send(int cpu, enum ipi_message_type op)
  167. {
  168. struct cpuinfo_parisc *p = &per_cpu(cpu_data, cpu);
  169. spinlock_t *lock = &per_cpu(ipi_lock, cpu);
  170. unsigned long flags;
  171. spin_lock_irqsave(lock, flags);
  172. p->pending_ipi |= 1 << op;
  173. gsc_writel(IPI_IRQ - CPU_IRQ_BASE, p->hpa);
  174. spin_unlock_irqrestore(lock, flags);
  175. }
  176. static void
  177. send_IPI_mask(const struct cpumask *mask, enum ipi_message_type op)
  178. {
  179. int cpu;
  180. for_each_cpu(cpu, mask)
  181. ipi_send(cpu, op);
  182. }
  183. static inline void
  184. send_IPI_single(int dest_cpu, enum ipi_message_type op)
  185. {
  186. BUG_ON(dest_cpu == NO_PROC_ID);
  187. ipi_send(dest_cpu, op);
  188. }
  189. static inline void
  190. send_IPI_allbutself(enum ipi_message_type op)
  191. {
  192. int i;
  193. preempt_disable();
  194. for_each_online_cpu(i) {
  195. if (i != smp_processor_id())
  196. send_IPI_single(i, op);
  197. }
  198. preempt_enable();
  199. }
  200. #ifdef CONFIG_KGDB
  201. void kgdb_roundup_cpus(void)
  202. {
  203. send_IPI_allbutself(IPI_ENTER_KGDB);
  204. }
  205. #endif
  206. inline void
  207. smp_send_stop(void) { send_IPI_allbutself(IPI_CPU_STOP); }
  208. void
  209. smp_send_reschedule(int cpu) { send_IPI_single(cpu, IPI_RESCHEDULE); }
  210. void
  211. smp_send_all_nop(void)
  212. {
  213. send_IPI_allbutself(IPI_NOP);
  214. }
  215. void arch_send_call_function_ipi_mask(const struct cpumask *mask)
  216. {
  217. send_IPI_mask(mask, IPI_CALL_FUNC);
  218. }
  219. void arch_send_call_function_single_ipi(int cpu)
  220. {
  221. send_IPI_single(cpu, IPI_CALL_FUNC);
  222. }
  223. /*
  224. * Called by secondaries to update state and initialize CPU registers.
  225. */
  226. static void
  227. smp_cpu_init(int cpunum)
  228. {
  229. extern void init_IRQ(void); /* arch/parisc/kernel/irq.c */
  230. extern void start_cpu_itimer(void); /* arch/parisc/kernel/time.c */
  231. /* Set modes and Enable floating point coprocessor */
  232. init_per_cpu(cpunum);
  233. disable_sr_hashing();
  234. mb();
  235. /* Well, support 2.4 linux scheme as well. */
  236. if (cpu_online(cpunum)) {
  237. extern void machine_halt(void); /* arch/parisc.../process.c */
  238. printk(KERN_CRIT "CPU#%d already initialized!\n", cpunum);
  239. machine_halt();
  240. }
  241. notify_cpu_starting(cpunum);
  242. set_cpu_online(cpunum, true);
  243. /* Initialise the idle task for this CPU */
  244. mmgrab(&init_mm);
  245. current->active_mm = &init_mm;
  246. BUG_ON(current->mm);
  247. enter_lazy_tlb(&init_mm, current);
  248. init_IRQ(); /* make sure no IRQs are enabled or pending */
  249. start_cpu_itimer();
  250. }
  251. /*
  252. * Slaves start using C here. Indirectly called from smp_slave_stext.
  253. * Do what start_kernel() and main() do for boot strap processor (aka monarch)
  254. */
  255. void smp_callin(unsigned long pdce_proc)
  256. {
  257. int slave_id = cpu_now_booting;
  258. #ifdef CONFIG_64BIT
  259. WARN_ON(((unsigned long)(PAGE0->mem_pdc_hi) << 32
  260. | PAGE0->mem_pdc) != pdce_proc);
  261. #endif
  262. smp_cpu_init(slave_id);
  263. flush_cache_all_local(); /* start with known state */
  264. flush_tlb_all_local(NULL);
  265. local_irq_enable(); /* Interrupts have been off until now */
  266. cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
  267. /* NOTREACHED */
  268. panic("smp_callin() AAAAaaaaahhhh....\n");
  269. }
  270. /*
  271. * Bring one cpu online.
  272. */
  273. static int smp_boot_one_cpu(int cpuid, struct task_struct *idle)
  274. {
  275. const struct cpuinfo_parisc *p = &per_cpu(cpu_data, cpuid);
  276. long timeout;
  277. #ifdef CONFIG_HOTPLUG_CPU
  278. int i;
  279. /* reset irq statistics for this CPU */
  280. memset(&per_cpu(irq_stat, cpuid), 0, sizeof(irq_cpustat_t));
  281. for (i = 0; i < NR_IRQS; i++) {
  282. struct irq_desc *desc = irq_to_desc(i);
  283. if (desc && desc->kstat_irqs)
  284. *per_cpu_ptr(desc->kstat_irqs, cpuid) = 0;
  285. }
  286. #endif
  287. /* wait until last booting CPU has started. */
  288. while (cpu_now_booting)
  289. ;
  290. /* Let _start know what logical CPU we're booting
  291. ** (offset into init_tasks[],cpu_data[])
  292. */
  293. cpu_now_booting = cpuid;
  294. /*
  295. ** boot strap code needs to know the task address since
  296. ** it also contains the process stack.
  297. */
  298. smp_init_current_idle_task = idle ;
  299. mb();
  300. printk(KERN_INFO "Releasing cpu %d now, hpa=%lx\n", cpuid, p->hpa);
  301. /*
  302. ** This gets PDC to release the CPU from a very tight loop.
  303. **
  304. ** From the PA-RISC 2.0 Firmware Architecture Reference Specification:
  305. ** "The MEM_RENDEZ vector specifies the location of OS_RENDEZ which
  306. ** is executed after receiving the rendezvous signal (an interrupt to
  307. ** EIR{0}). MEM_RENDEZ is valid only when it is nonzero and the
  308. ** contents of memory are valid."
  309. */
  310. gsc_writel(TIMER_IRQ - CPU_IRQ_BASE, p->hpa);
  311. mb();
  312. /*
  313. * OK, wait a bit for that CPU to finish staggering about.
  314. * Slave will set a bit when it reaches smp_cpu_init().
  315. * Once the "monarch CPU" sees the bit change, it can move on.
  316. */
  317. for (timeout = 0; timeout < 10000; timeout++) {
  318. if(cpu_online(cpuid)) {
  319. /* Which implies Slave has started up */
  320. cpu_now_booting = 0;
  321. goto alive ;
  322. }
  323. udelay(100);
  324. barrier();
  325. }
  326. printk(KERN_CRIT "SMP: CPU:%d is stuck.\n", cpuid);
  327. return -1;
  328. alive:
  329. /* Remember the Slave data */
  330. smp_debug(100, KERN_DEBUG "SMP: CPU:%d came alive after %ld _us\n",
  331. cpuid, timeout * 100);
  332. return 0;
  333. }
  334. void __init smp_prepare_boot_cpu(void)
  335. {
  336. int bootstrap_processor = per_cpu(cpu_data, 0).cpuid;
  337. /* Setup BSP mappings */
  338. printk(KERN_INFO "SMP: bootstrap CPU ID is %d\n", bootstrap_processor);
  339. set_cpu_online(bootstrap_processor, true);
  340. set_cpu_present(bootstrap_processor, true);
  341. }
  342. /*
  343. ** inventory.c:do_inventory() hasn't yet been run and thus we
  344. ** don't 'discover' the additional CPUs until later.
  345. */
  346. void __init smp_prepare_cpus(unsigned int max_cpus)
  347. {
  348. int cpu;
  349. for_each_possible_cpu(cpu)
  350. spin_lock_init(&per_cpu(ipi_lock, cpu));
  351. init_cpu_present(cpumask_of(0));
  352. }
  353. void __init smp_cpus_done(unsigned int cpu_max)
  354. {
  355. }
  356. int __cpu_up(unsigned int cpu, struct task_struct *tidle)
  357. {
  358. if (cpu_online(cpu))
  359. return 0;
  360. if (num_online_cpus() < nr_cpu_ids &&
  361. num_online_cpus() < setup_max_cpus &&
  362. smp_boot_one_cpu(cpu, tidle))
  363. return -EIO;
  364. return cpu_online(cpu) ? 0 : -EIO;
  365. }
  366. /*
  367. * __cpu_disable runs on the processor to be shutdown.
  368. */
  369. int __cpu_disable(void)
  370. {
  371. #ifdef CONFIG_HOTPLUG_CPU
  372. unsigned int cpu = smp_processor_id();
  373. remove_cpu_topology(cpu);
  374. /*
  375. * Take this CPU offline. Once we clear this, we can't return,
  376. * and we must not schedule until we're ready to give up the cpu.
  377. */
  378. set_cpu_online(cpu, false);
  379. /* Find a new timesync master */
  380. if (cpu == time_keeper_id) {
  381. time_keeper_id = cpumask_first(cpu_online_mask);
  382. pr_info("CPU %d is now promoted to time-keeper master\n", time_keeper_id);
  383. }
  384. disable_percpu_irq(IPI_IRQ);
  385. irq_migrate_all_off_this_cpu();
  386. flush_cache_all_local();
  387. flush_tlb_all_local(NULL);
  388. /* disable all irqs, including timer irq */
  389. local_irq_disable();
  390. /* wait for next timer irq ... */
  391. mdelay(1000/HZ+100);
  392. /* ... and then clear all pending external irqs */
  393. set_eiem(0);
  394. mtctl(~0UL, CR_EIRR);
  395. mfctl(CR_EIRR);
  396. mtctl(0, CR_EIRR);
  397. #endif
  398. return 0;
  399. }
  400. /*
  401. * called on the thread which is asking for a CPU to be shutdown -
  402. * waits until shutdown has completed, or it is timed out.
  403. */
  404. void __cpu_die(unsigned int cpu)
  405. {
  406. pdc_cpu_rendezvous_lock();
  407. if (!cpu_wait_death(cpu, 5)) {
  408. pr_crit("CPU%u: cpu didn't die\n", cpu);
  409. return;
  410. }
  411. pr_info("CPU%u: is shutting down\n", cpu);
  412. /* set task's state to interruptible sleep */
  413. set_current_state(TASK_INTERRUPTIBLE);
  414. schedule_timeout((IS_ENABLED(CONFIG_64BIT) ? 8:2) * HZ);
  415. pdc_cpu_rendezvous_unlock();
  416. }