pm-cps.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2014 Imagination Technologies
  4. * Author: Paul Burton <[email protected]>
  5. */
  6. #include <linux/cpuhotplug.h>
  7. #include <linux/init.h>
  8. #include <linux/percpu.h>
  9. #include <linux/slab.h>
  10. #include <linux/suspend.h>
  11. #include <asm/asm-offsets.h>
  12. #include <asm/cacheflush.h>
  13. #include <asm/cacheops.h>
  14. #include <asm/idle.h>
  15. #include <asm/mips-cps.h>
  16. #include <asm/mipsmtregs.h>
  17. #include <asm/pm.h>
  18. #include <asm/pm-cps.h>
  19. #include <asm/smp-cps.h>
  20. #include <asm/uasm.h>
  21. /*
  22. * cps_nc_entry_fn - type of a generated non-coherent state entry function
  23. * @online: the count of online coupled VPEs
  24. * @nc_ready_count: pointer to a non-coherent mapping of the core ready_count
  25. *
  26. * The code entering & exiting non-coherent states is generated at runtime
  27. * using uasm, in order to ensure that the compiler cannot insert a stray
  28. * memory access at an unfortunate time and to allow the generation of optimal
  29. * core-specific code particularly for cache routines. If coupled_coherence
  30. * is non-zero and this is the entry function for the CPS_PM_NC_WAIT state,
  31. * returns the number of VPEs that were in the wait state at the point this
  32. * VPE left it. Returns garbage if coupled_coherence is zero or this is not
  33. * the entry function for CPS_PM_NC_WAIT.
  34. */
  35. typedef unsigned (*cps_nc_entry_fn)(unsigned online, u32 *nc_ready_count);
  36. /*
  37. * The entry point of the generated non-coherent idle state entry/exit
  38. * functions. Actually per-core rather than per-CPU.
  39. */
  40. static DEFINE_PER_CPU_READ_MOSTLY(cps_nc_entry_fn[CPS_PM_STATE_COUNT],
  41. nc_asm_enter);
  42. /* Bitmap indicating which states are supported by the system */
  43. static DECLARE_BITMAP(state_support, CPS_PM_STATE_COUNT);
  44. /*
  45. * Indicates the number of coupled VPEs ready to operate in a non-coherent
  46. * state. Actually per-core rather than per-CPU.
  47. */
  48. static DEFINE_PER_CPU_ALIGNED(u32*, ready_count);
  49. /* Indicates online CPUs coupled with the current CPU */
  50. static DEFINE_PER_CPU_ALIGNED(cpumask_t, online_coupled);
  51. /*
  52. * Used to synchronize entry to deep idle states. Actually per-core rather
  53. * than per-CPU.
  54. */
  55. static DEFINE_PER_CPU_ALIGNED(atomic_t, pm_barrier);
  56. /* Saved CPU state across the CPS_PM_POWER_GATED state */
  57. DEFINE_PER_CPU_ALIGNED(struct mips_static_suspend_state, cps_cpu_state);
  58. /* A somewhat arbitrary number of labels & relocs for uasm */
  59. static struct uasm_label labels[32];
  60. static struct uasm_reloc relocs[32];
  61. enum mips_reg {
  62. zero, at, v0, v1, a0, a1, a2, a3,
  63. t0, t1, t2, t3, t4, t5, t6, t7,
  64. s0, s1, s2, s3, s4, s5, s6, s7,
  65. t8, t9, k0, k1, gp, sp, fp, ra,
  66. };
  67. bool cps_pm_support_state(enum cps_pm_state state)
  68. {
  69. return test_bit(state, state_support);
  70. }
  71. static void coupled_barrier(atomic_t *a, unsigned online)
  72. {
  73. /*
  74. * This function is effectively the same as
  75. * cpuidle_coupled_parallel_barrier, which can't be used here since
  76. * there's no cpuidle device.
  77. */
  78. if (!coupled_coherence)
  79. return;
  80. smp_mb__before_atomic();
  81. atomic_inc(a);
  82. while (atomic_read(a) < online)
  83. cpu_relax();
  84. if (atomic_inc_return(a) == online * 2) {
  85. atomic_set(a, 0);
  86. return;
  87. }
  88. while (atomic_read(a) > online)
  89. cpu_relax();
  90. }
  91. int cps_pm_enter_state(enum cps_pm_state state)
  92. {
  93. unsigned cpu = smp_processor_id();
  94. unsigned core = cpu_core(&current_cpu_data);
  95. unsigned online, left;
  96. cpumask_t *coupled_mask = this_cpu_ptr(&online_coupled);
  97. u32 *core_ready_count, *nc_core_ready_count;
  98. void *nc_addr;
  99. cps_nc_entry_fn entry;
  100. struct core_boot_config *core_cfg;
  101. struct vpe_boot_config *vpe_cfg;
  102. /* Check that there is an entry function for this state */
  103. entry = per_cpu(nc_asm_enter, core)[state];
  104. if (!entry)
  105. return -EINVAL;
  106. /* Calculate which coupled CPUs (VPEs) are online */
  107. #if defined(CONFIG_MIPS_MT) || defined(CONFIG_CPU_MIPSR6)
  108. if (cpu_online(cpu)) {
  109. cpumask_and(coupled_mask, cpu_online_mask,
  110. &cpu_sibling_map[cpu]);
  111. online = cpumask_weight(coupled_mask);
  112. cpumask_clear_cpu(cpu, coupled_mask);
  113. } else
  114. #endif
  115. {
  116. cpumask_clear(coupled_mask);
  117. online = 1;
  118. }
  119. /* Setup the VPE to run mips_cps_pm_restore when started again */
  120. if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) {
  121. /* Power gating relies upon CPS SMP */
  122. if (!mips_cps_smp_in_use())
  123. return -EINVAL;
  124. core_cfg = &mips_cps_core_bootcfg[core];
  125. vpe_cfg = &core_cfg->vpe_config[cpu_vpe_id(&current_cpu_data)];
  126. vpe_cfg->pc = (unsigned long)mips_cps_pm_restore;
  127. vpe_cfg->gp = (unsigned long)current_thread_info();
  128. vpe_cfg->sp = 0;
  129. }
  130. /* Indicate that this CPU might not be coherent */
  131. cpumask_clear_cpu(cpu, &cpu_coherent_mask);
  132. smp_mb__after_atomic();
  133. /* Create a non-coherent mapping of the core ready_count */
  134. core_ready_count = per_cpu(ready_count, core);
  135. nc_addr = kmap_noncoherent(virt_to_page(core_ready_count),
  136. (unsigned long)core_ready_count);
  137. nc_addr += ((unsigned long)core_ready_count & ~PAGE_MASK);
  138. nc_core_ready_count = nc_addr;
  139. /* Ensure ready_count is zero-initialised before the assembly runs */
  140. WRITE_ONCE(*nc_core_ready_count, 0);
  141. coupled_barrier(&per_cpu(pm_barrier, core), online);
  142. /* Run the generated entry code */
  143. left = entry(online, nc_core_ready_count);
  144. /* Remove the non-coherent mapping of ready_count */
  145. kunmap_noncoherent();
  146. /* Indicate that this CPU is definitely coherent */
  147. cpumask_set_cpu(cpu, &cpu_coherent_mask);
  148. /*
  149. * If this VPE is the first to leave the non-coherent wait state then
  150. * it needs to wake up any coupled VPEs still running their wait
  151. * instruction so that they return to cpuidle, which can then complete
  152. * coordination between the coupled VPEs & provide the governor with
  153. * a chance to reflect on the length of time the VPEs were in the
  154. * idle state.
  155. */
  156. if (coupled_coherence && (state == CPS_PM_NC_WAIT) && (left == online))
  157. arch_send_call_function_ipi_mask(coupled_mask);
  158. return 0;
  159. }
  160. static void cps_gen_cache_routine(u32 **pp, struct uasm_label **pl,
  161. struct uasm_reloc **pr,
  162. const struct cache_desc *cache,
  163. unsigned op, int lbl)
  164. {
  165. unsigned cache_size = cache->ways << cache->waybit;
  166. unsigned i;
  167. const unsigned unroll_lines = 32;
  168. /* If the cache isn't present this function has it easy */
  169. if (cache->flags & MIPS_CACHE_NOT_PRESENT)
  170. return;
  171. /* Load base address */
  172. UASM_i_LA(pp, t0, (long)CKSEG0);
  173. /* Calculate end address */
  174. if (cache_size < 0x8000)
  175. uasm_i_addiu(pp, t1, t0, cache_size);
  176. else
  177. UASM_i_LA(pp, t1, (long)(CKSEG0 + cache_size));
  178. /* Start of cache op loop */
  179. uasm_build_label(pl, *pp, lbl);
  180. /* Generate the cache ops */
  181. for (i = 0; i < unroll_lines; i++) {
  182. if (cpu_has_mips_r6) {
  183. uasm_i_cache(pp, op, 0, t0);
  184. uasm_i_addiu(pp, t0, t0, cache->linesz);
  185. } else {
  186. uasm_i_cache(pp, op, i * cache->linesz, t0);
  187. }
  188. }
  189. if (!cpu_has_mips_r6)
  190. /* Update the base address */
  191. uasm_i_addiu(pp, t0, t0, unroll_lines * cache->linesz);
  192. /* Loop if we haven't reached the end address yet */
  193. uasm_il_bne(pp, pr, t0, t1, lbl);
  194. uasm_i_nop(pp);
  195. }
  196. static int cps_gen_flush_fsb(u32 **pp, struct uasm_label **pl,
  197. struct uasm_reloc **pr,
  198. const struct cpuinfo_mips *cpu_info,
  199. int lbl)
  200. {
  201. unsigned i, fsb_size = 8;
  202. unsigned num_loads = (fsb_size * 3) / 2;
  203. unsigned line_stride = 2;
  204. unsigned line_size = cpu_info->dcache.linesz;
  205. unsigned perf_counter, perf_event;
  206. unsigned revision = cpu_info->processor_id & PRID_REV_MASK;
  207. /*
  208. * Determine whether this CPU requires an FSB flush, and if so which
  209. * performance counter/event reflect stalls due to a full FSB.
  210. */
  211. switch (__get_cpu_type(cpu_info->cputype)) {
  212. case CPU_INTERAPTIV:
  213. perf_counter = 1;
  214. perf_event = 51;
  215. break;
  216. case CPU_PROAPTIV:
  217. /* Newer proAptiv cores don't require this workaround */
  218. if (revision >= PRID_REV_ENCODE_332(1, 1, 0))
  219. return 0;
  220. /* On older ones it's unavailable */
  221. return -1;
  222. default:
  223. /* Assume that the CPU does not need this workaround */
  224. return 0;
  225. }
  226. /*
  227. * Ensure that the fill/store buffer (FSB) is not holding the results
  228. * of a prefetch, since if it is then the CPC sequencer may become
  229. * stuck in the D3 (ClrBus) state whilst entering a low power state.
  230. */
  231. /* Preserve perf counter setup */
  232. uasm_i_mfc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */
  233. uasm_i_mfc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */
  234. /* Setup perf counter to count FSB full pipeline stalls */
  235. uasm_i_addiu(pp, t0, zero, (perf_event << 5) | 0xf);
  236. uasm_i_mtc0(pp, t0, 25, (perf_counter * 2) + 0); /* PerfCtlN */
  237. uasm_i_ehb(pp);
  238. uasm_i_mtc0(pp, zero, 25, (perf_counter * 2) + 1); /* PerfCntN */
  239. uasm_i_ehb(pp);
  240. /* Base address for loads */
  241. UASM_i_LA(pp, t0, (long)CKSEG0);
  242. /* Start of clear loop */
  243. uasm_build_label(pl, *pp, lbl);
  244. /* Perform some loads to fill the FSB */
  245. for (i = 0; i < num_loads; i++)
  246. uasm_i_lw(pp, zero, i * line_size * line_stride, t0);
  247. /*
  248. * Invalidate the new D-cache entries so that the cache will need
  249. * refilling (via the FSB) if the loop is executed again.
  250. */
  251. for (i = 0; i < num_loads; i++) {
  252. uasm_i_cache(pp, Hit_Invalidate_D,
  253. i * line_size * line_stride, t0);
  254. uasm_i_cache(pp, Hit_Writeback_Inv_SD,
  255. i * line_size * line_stride, t0);
  256. }
  257. /* Barrier ensuring previous cache invalidates are complete */
  258. uasm_i_sync(pp, __SYNC_full);
  259. uasm_i_ehb(pp);
  260. /* Check whether the pipeline stalled due to the FSB being full */
  261. uasm_i_mfc0(pp, t1, 25, (perf_counter * 2) + 1); /* PerfCntN */
  262. /* Loop if it didn't */
  263. uasm_il_beqz(pp, pr, t1, lbl);
  264. uasm_i_nop(pp);
  265. /* Restore perf counter 1. The count may well now be wrong... */
  266. uasm_i_mtc0(pp, t2, 25, (perf_counter * 2) + 0); /* PerfCtlN */
  267. uasm_i_ehb(pp);
  268. uasm_i_mtc0(pp, t3, 25, (perf_counter * 2) + 1); /* PerfCntN */
  269. uasm_i_ehb(pp);
  270. return 0;
  271. }
  272. static void cps_gen_set_top_bit(u32 **pp, struct uasm_label **pl,
  273. struct uasm_reloc **pr,
  274. unsigned r_addr, int lbl)
  275. {
  276. uasm_i_lui(pp, t0, uasm_rel_hi(0x80000000));
  277. uasm_build_label(pl, *pp, lbl);
  278. uasm_i_ll(pp, t1, 0, r_addr);
  279. uasm_i_or(pp, t1, t1, t0);
  280. uasm_i_sc(pp, t1, 0, r_addr);
  281. uasm_il_beqz(pp, pr, t1, lbl);
  282. uasm_i_nop(pp);
  283. }
  284. static void *cps_gen_entry_code(unsigned cpu, enum cps_pm_state state)
  285. {
  286. struct uasm_label *l = labels;
  287. struct uasm_reloc *r = relocs;
  288. u32 *buf, *p;
  289. const unsigned r_online = a0;
  290. const unsigned r_nc_count = a1;
  291. const unsigned r_pcohctl = t7;
  292. const unsigned max_instrs = 256;
  293. unsigned cpc_cmd;
  294. int err;
  295. enum {
  296. lbl_incready = 1,
  297. lbl_poll_cont,
  298. lbl_secondary_hang,
  299. lbl_disable_coherence,
  300. lbl_flush_fsb,
  301. lbl_invicache,
  302. lbl_flushdcache,
  303. lbl_hang,
  304. lbl_set_cont,
  305. lbl_secondary_cont,
  306. lbl_decready,
  307. };
  308. /* Allocate a buffer to hold the generated code */
  309. p = buf = kcalloc(max_instrs, sizeof(u32), GFP_KERNEL);
  310. if (!buf)
  311. return NULL;
  312. /* Clear labels & relocs ready for (re)use */
  313. memset(labels, 0, sizeof(labels));
  314. memset(relocs, 0, sizeof(relocs));
  315. if (IS_ENABLED(CONFIG_CPU_PM) && state == CPS_PM_POWER_GATED) {
  316. /* Power gating relies upon CPS SMP */
  317. if (!mips_cps_smp_in_use())
  318. goto out_err;
  319. /*
  320. * Save CPU state. Note the non-standard calling convention
  321. * with the return address placed in v0 to avoid clobbering
  322. * the ra register before it is saved.
  323. */
  324. UASM_i_LA(&p, t0, (long)mips_cps_pm_save);
  325. uasm_i_jalr(&p, v0, t0);
  326. uasm_i_nop(&p);
  327. }
  328. /*
  329. * Load addresses of required CM & CPC registers. This is done early
  330. * because they're needed in both the enable & disable coherence steps
  331. * but in the coupled case the enable step will only run on one VPE.
  332. */
  333. UASM_i_LA(&p, r_pcohctl, (long)addr_gcr_cl_coherence());
  334. if (coupled_coherence) {
  335. /* Increment ready_count */
  336. uasm_i_sync(&p, __SYNC_mb);
  337. uasm_build_label(&l, p, lbl_incready);
  338. uasm_i_ll(&p, t1, 0, r_nc_count);
  339. uasm_i_addiu(&p, t2, t1, 1);
  340. uasm_i_sc(&p, t2, 0, r_nc_count);
  341. uasm_il_beqz(&p, &r, t2, lbl_incready);
  342. uasm_i_addiu(&p, t1, t1, 1);
  343. /* Barrier ensuring all CPUs see the updated r_nc_count value */
  344. uasm_i_sync(&p, __SYNC_mb);
  345. /*
  346. * If this is the last VPE to become ready for non-coherence
  347. * then it should branch below.
  348. */
  349. uasm_il_beq(&p, &r, t1, r_online, lbl_disable_coherence);
  350. uasm_i_nop(&p);
  351. if (state < CPS_PM_POWER_GATED) {
  352. /*
  353. * Otherwise this is not the last VPE to become ready
  354. * for non-coherence. It needs to wait until coherence
  355. * has been disabled before proceeding, which it will do
  356. * by polling for the top bit of ready_count being set.
  357. */
  358. uasm_i_addiu(&p, t1, zero, -1);
  359. uasm_build_label(&l, p, lbl_poll_cont);
  360. uasm_i_lw(&p, t0, 0, r_nc_count);
  361. uasm_il_bltz(&p, &r, t0, lbl_secondary_cont);
  362. uasm_i_ehb(&p);
  363. if (cpu_has_mipsmt)
  364. uasm_i_yield(&p, zero, t1);
  365. uasm_il_b(&p, &r, lbl_poll_cont);
  366. uasm_i_nop(&p);
  367. } else {
  368. /*
  369. * The core will lose power & this VPE will not continue
  370. * so it can simply halt here.
  371. */
  372. if (cpu_has_mipsmt) {
  373. /* Halt the VPE via C0 tchalt register */
  374. uasm_i_addiu(&p, t0, zero, TCHALT_H);
  375. uasm_i_mtc0(&p, t0, 2, 4);
  376. } else if (cpu_has_vp) {
  377. /* Halt the VP via the CPC VP_STOP register */
  378. unsigned int vpe_id;
  379. vpe_id = cpu_vpe_id(&cpu_data[cpu]);
  380. uasm_i_addiu(&p, t0, zero, 1 << vpe_id);
  381. UASM_i_LA(&p, t1, (long)addr_cpc_cl_vp_stop());
  382. uasm_i_sw(&p, t0, 0, t1);
  383. } else {
  384. BUG();
  385. }
  386. uasm_build_label(&l, p, lbl_secondary_hang);
  387. uasm_il_b(&p, &r, lbl_secondary_hang);
  388. uasm_i_nop(&p);
  389. }
  390. }
  391. /*
  392. * This is the point of no return - this VPE will now proceed to
  393. * disable coherence. At this point we *must* be sure that no other
  394. * VPE within the core will interfere with the L1 dcache.
  395. */
  396. uasm_build_label(&l, p, lbl_disable_coherence);
  397. /* Invalidate the L1 icache */
  398. cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].icache,
  399. Index_Invalidate_I, lbl_invicache);
  400. /* Writeback & invalidate the L1 dcache */
  401. cps_gen_cache_routine(&p, &l, &r, &cpu_data[cpu].dcache,
  402. Index_Writeback_Inv_D, lbl_flushdcache);
  403. /* Barrier ensuring previous cache invalidates are complete */
  404. uasm_i_sync(&p, __SYNC_full);
  405. uasm_i_ehb(&p);
  406. if (mips_cm_revision() < CM_REV_CM3) {
  407. /*
  408. * Disable all but self interventions. The load from COHCTL is
  409. * defined by the interAptiv & proAptiv SUMs as ensuring that the
  410. * operation resulting from the preceding store is complete.
  411. */
  412. uasm_i_addiu(&p, t0, zero, 1 << cpu_core(&cpu_data[cpu]));
  413. uasm_i_sw(&p, t0, 0, r_pcohctl);
  414. uasm_i_lw(&p, t0, 0, r_pcohctl);
  415. /* Barrier to ensure write to coherence control is complete */
  416. uasm_i_sync(&p, __SYNC_full);
  417. uasm_i_ehb(&p);
  418. }
  419. /* Disable coherence */
  420. uasm_i_sw(&p, zero, 0, r_pcohctl);
  421. uasm_i_lw(&p, t0, 0, r_pcohctl);
  422. if (state >= CPS_PM_CLOCK_GATED) {
  423. err = cps_gen_flush_fsb(&p, &l, &r, &cpu_data[cpu],
  424. lbl_flush_fsb);
  425. if (err)
  426. goto out_err;
  427. /* Determine the CPC command to issue */
  428. switch (state) {
  429. case CPS_PM_CLOCK_GATED:
  430. cpc_cmd = CPC_Cx_CMD_CLOCKOFF;
  431. break;
  432. case CPS_PM_POWER_GATED:
  433. cpc_cmd = CPC_Cx_CMD_PWRDOWN;
  434. break;
  435. default:
  436. BUG();
  437. goto out_err;
  438. }
  439. /* Issue the CPC command */
  440. UASM_i_LA(&p, t0, (long)addr_cpc_cl_cmd());
  441. uasm_i_addiu(&p, t1, zero, cpc_cmd);
  442. uasm_i_sw(&p, t1, 0, t0);
  443. if (state == CPS_PM_POWER_GATED) {
  444. /* If anything goes wrong just hang */
  445. uasm_build_label(&l, p, lbl_hang);
  446. uasm_il_b(&p, &r, lbl_hang);
  447. uasm_i_nop(&p);
  448. /*
  449. * There's no point generating more code, the core is
  450. * powered down & if powered back up will run from the
  451. * reset vector not from here.
  452. */
  453. goto gen_done;
  454. }
  455. /* Barrier to ensure write to CPC command is complete */
  456. uasm_i_sync(&p, __SYNC_full);
  457. uasm_i_ehb(&p);
  458. }
  459. if (state == CPS_PM_NC_WAIT) {
  460. /*
  461. * At this point it is safe for all VPEs to proceed with
  462. * execution. This VPE will set the top bit of ready_count
  463. * to indicate to the other VPEs that they may continue.
  464. */
  465. if (coupled_coherence)
  466. cps_gen_set_top_bit(&p, &l, &r, r_nc_count,
  467. lbl_set_cont);
  468. /*
  469. * VPEs which did not disable coherence will continue
  470. * executing, after coherence has been disabled, from this
  471. * point.
  472. */
  473. uasm_build_label(&l, p, lbl_secondary_cont);
  474. /* Now perform our wait */
  475. uasm_i_wait(&p, 0);
  476. }
  477. /*
  478. * Re-enable coherence. Note that for CPS_PM_NC_WAIT all coupled VPEs
  479. * will run this. The first will actually re-enable coherence & the
  480. * rest will just be performing a rather unusual nop.
  481. */
  482. uasm_i_addiu(&p, t0, zero, mips_cm_revision() < CM_REV_CM3
  483. ? CM_GCR_Cx_COHERENCE_COHDOMAINEN
  484. : CM3_GCR_Cx_COHERENCE_COHEN);
  485. uasm_i_sw(&p, t0, 0, r_pcohctl);
  486. uasm_i_lw(&p, t0, 0, r_pcohctl);
  487. /* Barrier to ensure write to coherence control is complete */
  488. uasm_i_sync(&p, __SYNC_full);
  489. uasm_i_ehb(&p);
  490. if (coupled_coherence && (state == CPS_PM_NC_WAIT)) {
  491. /* Decrement ready_count */
  492. uasm_build_label(&l, p, lbl_decready);
  493. uasm_i_sync(&p, __SYNC_mb);
  494. uasm_i_ll(&p, t1, 0, r_nc_count);
  495. uasm_i_addiu(&p, t2, t1, -1);
  496. uasm_i_sc(&p, t2, 0, r_nc_count);
  497. uasm_il_beqz(&p, &r, t2, lbl_decready);
  498. uasm_i_andi(&p, v0, t1, (1 << fls(smp_num_siblings)) - 1);
  499. /* Barrier ensuring all CPUs see the updated r_nc_count value */
  500. uasm_i_sync(&p, __SYNC_mb);
  501. }
  502. if (coupled_coherence && (state == CPS_PM_CLOCK_GATED)) {
  503. /*
  504. * At this point it is safe for all VPEs to proceed with
  505. * execution. This VPE will set the top bit of ready_count
  506. * to indicate to the other VPEs that they may continue.
  507. */
  508. cps_gen_set_top_bit(&p, &l, &r, r_nc_count, lbl_set_cont);
  509. /*
  510. * This core will be reliant upon another core sending a
  511. * power-up command to the CPC in order to resume operation.
  512. * Thus an arbitrary VPE can't trigger the core leaving the
  513. * idle state and the one that disables coherence might as well
  514. * be the one to re-enable it. The rest will continue from here
  515. * after that has been done.
  516. */
  517. uasm_build_label(&l, p, lbl_secondary_cont);
  518. /* Barrier ensuring all CPUs see the updated r_nc_count value */
  519. uasm_i_sync(&p, __SYNC_mb);
  520. }
  521. /* The core is coherent, time to return to C code */
  522. uasm_i_jr(&p, ra);
  523. uasm_i_nop(&p);
  524. gen_done:
  525. /* Ensure the code didn't exceed the resources allocated for it */
  526. BUG_ON((p - buf) > max_instrs);
  527. BUG_ON((l - labels) > ARRAY_SIZE(labels));
  528. BUG_ON((r - relocs) > ARRAY_SIZE(relocs));
  529. /* Patch branch offsets */
  530. uasm_resolve_relocs(relocs, labels);
  531. /* Flush the icache */
  532. local_flush_icache_range((unsigned long)buf, (unsigned long)p);
  533. return buf;
  534. out_err:
  535. kfree(buf);
  536. return NULL;
  537. }
  538. static int cps_pm_online_cpu(unsigned int cpu)
  539. {
  540. enum cps_pm_state state;
  541. unsigned core = cpu_core(&cpu_data[cpu]);
  542. void *entry_fn, *core_rc;
  543. for (state = CPS_PM_NC_WAIT; state < CPS_PM_STATE_COUNT; state++) {
  544. if (per_cpu(nc_asm_enter, core)[state])
  545. continue;
  546. if (!test_bit(state, state_support))
  547. continue;
  548. entry_fn = cps_gen_entry_code(cpu, state);
  549. if (!entry_fn) {
  550. pr_err("Failed to generate core %u state %u entry\n",
  551. core, state);
  552. clear_bit(state, state_support);
  553. }
  554. per_cpu(nc_asm_enter, core)[state] = entry_fn;
  555. }
  556. if (!per_cpu(ready_count, core)) {
  557. core_rc = kmalloc(sizeof(u32), GFP_KERNEL);
  558. if (!core_rc) {
  559. pr_err("Failed allocate core %u ready_count\n", core);
  560. return -ENOMEM;
  561. }
  562. per_cpu(ready_count, core) = core_rc;
  563. }
  564. return 0;
  565. }
  566. static int cps_pm_power_notifier(struct notifier_block *this,
  567. unsigned long event, void *ptr)
  568. {
  569. unsigned int stat;
  570. switch (event) {
  571. case PM_SUSPEND_PREPARE:
  572. stat = read_cpc_cl_stat_conf();
  573. /*
  574. * If we're attempting to suspend the system and power down all
  575. * of the cores, the JTAG detect bit indicates that the CPC will
  576. * instead put the cores into clock-off state. In this state
  577. * a connected debugger can cause the CPU to attempt
  578. * interactions with the powered down system. At best this will
  579. * fail. At worst, it can hang the NoC, requiring a hard reset.
  580. * To avoid this, just block system suspend if a JTAG probe
  581. * is detected.
  582. */
  583. if (stat & CPC_Cx_STAT_CONF_EJTAG_PROBE) {
  584. pr_warn("JTAG probe is connected - abort suspend\n");
  585. return NOTIFY_BAD;
  586. }
  587. return NOTIFY_DONE;
  588. default:
  589. return NOTIFY_DONE;
  590. }
  591. }
  592. static int __init cps_pm_init(void)
  593. {
  594. /* A CM is required for all non-coherent states */
  595. if (!mips_cm_present()) {
  596. pr_warn("pm-cps: no CM, non-coherent states unavailable\n");
  597. return 0;
  598. }
  599. /*
  600. * If interrupts were enabled whilst running a wait instruction on a
  601. * non-coherent core then the VPE may end up processing interrupts
  602. * whilst non-coherent. That would be bad.
  603. */
  604. if (cpu_wait == r4k_wait_irqoff)
  605. set_bit(CPS_PM_NC_WAIT, state_support);
  606. else
  607. pr_warn("pm-cps: non-coherent wait unavailable\n");
  608. /* Detect whether a CPC is present */
  609. if (mips_cpc_present()) {
  610. /* Detect whether clock gating is implemented */
  611. if (read_cpc_cl_stat_conf() & CPC_Cx_STAT_CONF_CLKGAT_IMPL)
  612. set_bit(CPS_PM_CLOCK_GATED, state_support);
  613. else
  614. pr_warn("pm-cps: CPC does not support clock gating\n");
  615. /* Power gating is available with CPS SMP & any CPC */
  616. if (mips_cps_smp_in_use())
  617. set_bit(CPS_PM_POWER_GATED, state_support);
  618. else
  619. pr_warn("pm-cps: CPS SMP not in use, power gating unavailable\n");
  620. } else {
  621. pr_warn("pm-cps: no CPC, clock & power gating unavailable\n");
  622. }
  623. pm_notifier(cps_pm_power_notifier, 0);
  624. return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mips/cps_pm:online",
  625. cps_pm_online_cpu, NULL);
  626. }
  627. arch_initcall(cps_pm_init);