vfpmodule.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/arch/arm/vfp/vfpmodule.c
  4. *
  5. * Copyright (C) 2004 ARM Limited.
  6. * Written by Deep Blue Solutions Limited.
  7. */
  8. #include <linux/types.h>
  9. #include <linux/cpu.h>
  10. #include <linux/cpu_pm.h>
  11. #include <linux/hardirq.h>
  12. #include <linux/kernel.h>
  13. #include <linux/notifier.h>
  14. #include <linux/signal.h>
  15. #include <linux/sched/signal.h>
  16. #include <linux/smp.h>
  17. #include <linux/init.h>
  18. #include <linux/uaccess.h>
  19. #include <linux/user.h>
  20. #include <linux/export.h>
  21. #include <asm/cp15.h>
  22. #include <asm/cputype.h>
  23. #include <asm/system_info.h>
  24. #include <asm/thread_notify.h>
  25. #include <asm/traps.h>
  26. #include <asm/vfp.h>
  27. #include "vfpinstr.h"
  28. #include "vfp.h"
  29. /*
  30. * Our undef handlers (in entry.S)
  31. */
  32. asmlinkage void vfp_support_entry(void);
  33. asmlinkage void vfp_null_entry(void);
  34. asmlinkage void (*vfp_vector)(void) = vfp_null_entry;
  35. /*
  36. * Dual-use variable.
  37. * Used in startup: set to non-zero if VFP checks fail
  38. * After startup, holds VFP architecture
  39. */
  40. static unsigned int __initdata VFP_arch;
  41. /*
  42. * The pointer to the vfpstate structure of the thread which currently
  43. * owns the context held in the VFP hardware, or NULL if the hardware
  44. * context is invalid.
  45. *
  46. * For UP, this is sufficient to tell which thread owns the VFP context.
  47. * However, for SMP, we also need to check the CPU number stored in the
  48. * saved state too to catch migrations.
  49. */
  50. union vfp_state *vfp_current_hw_state[NR_CPUS];
  51. /*
  52. * Is 'thread's most up to date state stored in this CPUs hardware?
  53. * Must be called from non-preemptible context.
  54. */
  55. static bool vfp_state_in_hw(unsigned int cpu, struct thread_info *thread)
  56. {
  57. #ifdef CONFIG_SMP
  58. if (thread->vfpstate.hard.cpu != cpu)
  59. return false;
  60. #endif
  61. return vfp_current_hw_state[cpu] == &thread->vfpstate;
  62. }
  63. /*
  64. * Force a reload of the VFP context from the thread structure. We do
  65. * this by ensuring that access to the VFP hardware is disabled, and
  66. * clear vfp_current_hw_state. Must be called from non-preemptible context.
  67. */
  68. static void vfp_force_reload(unsigned int cpu, struct thread_info *thread)
  69. {
  70. if (vfp_state_in_hw(cpu, thread)) {
  71. fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
  72. vfp_current_hw_state[cpu] = NULL;
  73. }
  74. #ifdef CONFIG_SMP
  75. thread->vfpstate.hard.cpu = NR_CPUS;
  76. #endif
  77. }
  78. /*
  79. * Per-thread VFP initialization.
  80. */
  81. static void vfp_thread_flush(struct thread_info *thread)
  82. {
  83. union vfp_state *vfp = &thread->vfpstate;
  84. unsigned int cpu;
  85. /*
  86. * Disable VFP to ensure we initialize it first. We must ensure
  87. * that the modification of vfp_current_hw_state[] and hardware
  88. * disable are done for the same CPU and without preemption.
  89. *
  90. * Do this first to ensure that preemption won't overwrite our
  91. * state saving should access to the VFP be enabled at this point.
  92. */
  93. cpu = get_cpu();
  94. if (vfp_current_hw_state[cpu] == vfp)
  95. vfp_current_hw_state[cpu] = NULL;
  96. fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
  97. put_cpu();
  98. memset(vfp, 0, sizeof(union vfp_state));
  99. vfp->hard.fpexc = FPEXC_EN;
  100. vfp->hard.fpscr = FPSCR_ROUND_NEAREST;
  101. #ifdef CONFIG_SMP
  102. vfp->hard.cpu = NR_CPUS;
  103. #endif
  104. }
  105. static void vfp_thread_exit(struct thread_info *thread)
  106. {
  107. /* release case: Per-thread VFP cleanup. */
  108. union vfp_state *vfp = &thread->vfpstate;
  109. unsigned int cpu = get_cpu();
  110. if (vfp_current_hw_state[cpu] == vfp)
  111. vfp_current_hw_state[cpu] = NULL;
  112. put_cpu();
  113. }
  114. static void vfp_thread_copy(struct thread_info *thread)
  115. {
  116. struct thread_info *parent = current_thread_info();
  117. vfp_sync_hwstate(parent);
  118. thread->vfpstate = parent->vfpstate;
  119. #ifdef CONFIG_SMP
  120. thread->vfpstate.hard.cpu = NR_CPUS;
  121. #endif
  122. }
  123. /*
  124. * When this function is called with the following 'cmd's, the following
  125. * is true while this function is being run:
  126. * THREAD_NOFTIFY_SWTICH:
  127. * - the previously running thread will not be scheduled onto another CPU.
  128. * - the next thread to be run (v) will not be running on another CPU.
  129. * - thread->cpu is the local CPU number
  130. * - not preemptible as we're called in the middle of a thread switch
  131. * THREAD_NOTIFY_FLUSH:
  132. * - the thread (v) will be running on the local CPU, so
  133. * v === current_thread_info()
  134. * - thread->cpu is the local CPU number at the time it is accessed,
  135. * but may change at any time.
  136. * - we could be preempted if tree preempt rcu is enabled, so
  137. * it is unsafe to use thread->cpu.
  138. * THREAD_NOTIFY_EXIT
  139. * - we could be preempted if tree preempt rcu is enabled, so
  140. * it is unsafe to use thread->cpu.
  141. */
  142. static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v)
  143. {
  144. struct thread_info *thread = v;
  145. u32 fpexc;
  146. #ifdef CONFIG_SMP
  147. unsigned int cpu;
  148. #endif
  149. switch (cmd) {
  150. case THREAD_NOTIFY_SWITCH:
  151. fpexc = fmrx(FPEXC);
  152. #ifdef CONFIG_SMP
  153. cpu = thread->cpu;
  154. /*
  155. * On SMP, if VFP is enabled, save the old state in
  156. * case the thread migrates to a different CPU. The
  157. * restoring is done lazily.
  158. */
  159. if ((fpexc & FPEXC_EN) && vfp_current_hw_state[cpu])
  160. vfp_save_state(vfp_current_hw_state[cpu], fpexc);
  161. #endif
  162. /*
  163. * Always disable VFP so we can lazily save/restore the
  164. * old state.
  165. */
  166. fmxr(FPEXC, fpexc & ~FPEXC_EN);
  167. break;
  168. case THREAD_NOTIFY_FLUSH:
  169. vfp_thread_flush(thread);
  170. break;
  171. case THREAD_NOTIFY_EXIT:
  172. vfp_thread_exit(thread);
  173. break;
  174. case THREAD_NOTIFY_COPY:
  175. vfp_thread_copy(thread);
  176. break;
  177. }
  178. return NOTIFY_DONE;
  179. }
  180. static struct notifier_block vfp_notifier_block = {
  181. .notifier_call = vfp_notifier,
  182. };
  183. /*
  184. * Raise a SIGFPE for the current process.
  185. * sicode describes the signal being raised.
  186. */
  187. static void vfp_raise_sigfpe(unsigned int sicode, struct pt_regs *regs)
  188. {
  189. /*
  190. * This is the same as NWFPE, because it's not clear what
  191. * this is used for
  192. */
  193. current->thread.error_code = 0;
  194. current->thread.trap_no = 6;
  195. send_sig_fault(SIGFPE, sicode,
  196. (void __user *)(instruction_pointer(regs) - 4),
  197. current);
  198. }
  199. static void vfp_panic(char *reason, u32 inst)
  200. {
  201. int i;
  202. pr_err("VFP: Error: %s\n", reason);
  203. pr_err("VFP: EXC 0x%08x SCR 0x%08x INST 0x%08x\n",
  204. fmrx(FPEXC), fmrx(FPSCR), inst);
  205. for (i = 0; i < 32; i += 2)
  206. pr_err("VFP: s%2u: 0x%08x s%2u: 0x%08x\n",
  207. i, vfp_get_float(i), i+1, vfp_get_float(i+1));
  208. }
  209. /*
  210. * Process bitmask of exception conditions.
  211. */
  212. static void vfp_raise_exceptions(u32 exceptions, u32 inst, u32 fpscr, struct pt_regs *regs)
  213. {
  214. int si_code = 0;
  215. pr_debug("VFP: raising exceptions %08x\n", exceptions);
  216. if (exceptions == VFP_EXCEPTION_ERROR) {
  217. vfp_panic("unhandled bounce", inst);
  218. vfp_raise_sigfpe(FPE_FLTINV, regs);
  219. return;
  220. }
  221. /*
  222. * If any of the status flags are set, update the FPSCR.
  223. * Comparison instructions always return at least one of
  224. * these flags set.
  225. */
  226. if (exceptions & (FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V))
  227. fpscr &= ~(FPSCR_N|FPSCR_Z|FPSCR_C|FPSCR_V);
  228. fpscr |= exceptions;
  229. fmxr(FPSCR, fpscr);
  230. #define RAISE(stat,en,sig) \
  231. if (exceptions & stat && fpscr & en) \
  232. si_code = sig;
  233. /*
  234. * These are arranged in priority order, least to highest.
  235. */
  236. RAISE(FPSCR_DZC, FPSCR_DZE, FPE_FLTDIV);
  237. RAISE(FPSCR_IXC, FPSCR_IXE, FPE_FLTRES);
  238. RAISE(FPSCR_UFC, FPSCR_UFE, FPE_FLTUND);
  239. RAISE(FPSCR_OFC, FPSCR_OFE, FPE_FLTOVF);
  240. RAISE(FPSCR_IOC, FPSCR_IOE, FPE_FLTINV);
  241. if (si_code)
  242. vfp_raise_sigfpe(si_code, regs);
  243. }
  244. /*
  245. * Emulate a VFP instruction.
  246. */
  247. static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs)
  248. {
  249. u32 exceptions = VFP_EXCEPTION_ERROR;
  250. pr_debug("VFP: emulate: INST=0x%08x SCR=0x%08x\n", inst, fpscr);
  251. if (INST_CPRTDO(inst)) {
  252. if (!INST_CPRT(inst)) {
  253. /*
  254. * CPDO
  255. */
  256. if (vfp_single(inst)) {
  257. exceptions = vfp_single_cpdo(inst, fpscr);
  258. } else {
  259. exceptions = vfp_double_cpdo(inst, fpscr);
  260. }
  261. } else {
  262. /*
  263. * A CPRT instruction can not appear in FPINST2, nor
  264. * can it cause an exception. Therefore, we do not
  265. * have to emulate it.
  266. */
  267. }
  268. } else {
  269. /*
  270. * A CPDT instruction can not appear in FPINST2, nor can
  271. * it cause an exception. Therefore, we do not have to
  272. * emulate it.
  273. */
  274. }
  275. return exceptions & ~VFP_NAN_FLAG;
  276. }
  277. /*
  278. * Package up a bounce condition.
  279. */
  280. void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs)
  281. {
  282. u32 fpscr, orig_fpscr, fpsid, exceptions;
  283. pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc);
  284. /*
  285. * At this point, FPEXC can have the following configuration:
  286. *
  287. * EX DEX IXE
  288. * 0 1 x - synchronous exception
  289. * 1 x 0 - asynchronous exception
  290. * 1 x 1 - sychronous on VFP subarch 1 and asynchronous on later
  291. * 0 0 1 - synchronous on VFP9 (non-standard subarch 1
  292. * implementation), undefined otherwise
  293. *
  294. * Clear various bits and enable access to the VFP so we can
  295. * handle the bounce.
  296. */
  297. fmxr(FPEXC, fpexc & ~(FPEXC_EX|FPEXC_DEX|FPEXC_FP2V|FPEXC_VV|FPEXC_TRAP_MASK));
  298. fpsid = fmrx(FPSID);
  299. orig_fpscr = fpscr = fmrx(FPSCR);
  300. /*
  301. * Check for the special VFP subarch 1 and FPSCR.IXE bit case
  302. */
  303. if ((fpsid & FPSID_ARCH_MASK) == (1 << FPSID_ARCH_BIT)
  304. && (fpscr & FPSCR_IXE)) {
  305. /*
  306. * Synchronous exception, emulate the trigger instruction
  307. */
  308. goto emulate;
  309. }
  310. if (fpexc & FPEXC_EX) {
  311. #ifndef CONFIG_CPU_FEROCEON
  312. /*
  313. * Asynchronous exception. The instruction is read from FPINST
  314. * and the interrupted instruction has to be restarted.
  315. */
  316. trigger = fmrx(FPINST);
  317. regs->ARM_pc -= 4;
  318. #endif
  319. } else if (!(fpexc & FPEXC_DEX)) {
  320. /*
  321. * Illegal combination of bits. It can be caused by an
  322. * unallocated VFP instruction but with FPSCR.IXE set and not
  323. * on VFP subarch 1.
  324. */
  325. vfp_raise_exceptions(VFP_EXCEPTION_ERROR, trigger, fpscr, regs);
  326. goto exit;
  327. }
  328. /*
  329. * Modify fpscr to indicate the number of iterations remaining.
  330. * If FPEXC.EX is 0, FPEXC.DEX is 1 and the FPEXC.VV bit indicates
  331. * whether FPEXC.VECITR or FPSCR.LEN is used.
  332. */
  333. if (fpexc & (FPEXC_EX | FPEXC_VV)) {
  334. u32 len;
  335. len = fpexc + (1 << FPEXC_LENGTH_BIT);
  336. fpscr &= ~FPSCR_LENGTH_MASK;
  337. fpscr |= (len & FPEXC_LENGTH_MASK) << (FPSCR_LENGTH_BIT - FPEXC_LENGTH_BIT);
  338. }
  339. /*
  340. * Handle the first FP instruction. We used to take note of the
  341. * FPEXC bounce reason, but this appears to be unreliable.
  342. * Emulate the bounced instruction instead.
  343. */
  344. exceptions = vfp_emulate_instruction(trigger, fpscr, regs);
  345. if (exceptions)
  346. vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
  347. /*
  348. * If there isn't a second FP instruction, exit now. Note that
  349. * the FPEXC.FP2V bit is valid only if FPEXC.EX is 1.
  350. */
  351. if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V))
  352. goto exit;
  353. /*
  354. * The barrier() here prevents fpinst2 being read
  355. * before the condition above.
  356. */
  357. barrier();
  358. trigger = fmrx(FPINST2);
  359. emulate:
  360. exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs);
  361. if (exceptions)
  362. vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs);
  363. exit:
  364. preempt_enable();
  365. }
  366. static void vfp_enable(void *unused)
  367. {
  368. u32 access;
  369. BUG_ON(preemptible());
  370. access = get_copro_access();
  371. /*
  372. * Enable full access to VFP (cp10 and cp11)
  373. */
  374. set_copro_access(access | CPACC_FULL(10) | CPACC_FULL(11));
  375. }
  376. /* Called by platforms on which we want to disable VFP because it may not be
  377. * present on all CPUs within a SMP complex. Needs to be called prior to
  378. * vfp_init().
  379. */
  380. void __init vfp_disable(void)
  381. {
  382. if (VFP_arch) {
  383. pr_debug("%s: should be called prior to vfp_init\n", __func__);
  384. return;
  385. }
  386. VFP_arch = 1;
  387. }
  388. #ifdef CONFIG_CPU_PM
  389. static int vfp_pm_suspend(void)
  390. {
  391. struct thread_info *ti = current_thread_info();
  392. u32 fpexc = fmrx(FPEXC);
  393. /* if vfp is on, then save state for resumption */
  394. if (fpexc & FPEXC_EN) {
  395. pr_debug("%s: saving vfp state\n", __func__);
  396. vfp_save_state(&ti->vfpstate, fpexc);
  397. /* disable, just in case */
  398. fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
  399. } else if (vfp_current_hw_state[ti->cpu]) {
  400. #ifndef CONFIG_SMP
  401. fmxr(FPEXC, fpexc | FPEXC_EN);
  402. vfp_save_state(vfp_current_hw_state[ti->cpu], fpexc);
  403. fmxr(FPEXC, fpexc);
  404. #endif
  405. }
  406. /* clear any information we had about last context state */
  407. vfp_current_hw_state[ti->cpu] = NULL;
  408. return 0;
  409. }
  410. static void vfp_pm_resume(void)
  411. {
  412. /* ensure we have access to the vfp */
  413. vfp_enable(NULL);
  414. /* and disable it to ensure the next usage restores the state */
  415. fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
  416. }
  417. static int vfp_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd,
  418. void *v)
  419. {
  420. switch (cmd) {
  421. case CPU_PM_ENTER:
  422. vfp_pm_suspend();
  423. break;
  424. case CPU_PM_ENTER_FAILED:
  425. case CPU_PM_EXIT:
  426. vfp_pm_resume();
  427. break;
  428. }
  429. return NOTIFY_OK;
  430. }
  431. static struct notifier_block vfp_cpu_pm_notifier_block = {
  432. .notifier_call = vfp_cpu_pm_notifier,
  433. };
  434. static void vfp_pm_init(void)
  435. {
  436. cpu_pm_register_notifier(&vfp_cpu_pm_notifier_block);
  437. }
  438. #else
  439. static inline void vfp_pm_init(void) { }
  440. #endif /* CONFIG_CPU_PM */
  441. /*
  442. * Ensure that the VFP state stored in 'thread->vfpstate' is up to date
  443. * with the hardware state.
  444. */
  445. void vfp_sync_hwstate(struct thread_info *thread)
  446. {
  447. unsigned int cpu = get_cpu();
  448. if (vfp_state_in_hw(cpu, thread)) {
  449. u32 fpexc = fmrx(FPEXC);
  450. /*
  451. * Save the last VFP state on this CPU.
  452. */
  453. fmxr(FPEXC, fpexc | FPEXC_EN);
  454. vfp_save_state(&thread->vfpstate, fpexc | FPEXC_EN);
  455. fmxr(FPEXC, fpexc);
  456. }
  457. put_cpu();
  458. }
  459. /* Ensure that the thread reloads the hardware VFP state on the next use. */
  460. void vfp_flush_hwstate(struct thread_info *thread)
  461. {
  462. unsigned int cpu = get_cpu();
  463. vfp_force_reload(cpu, thread);
  464. put_cpu();
  465. }
  466. /*
  467. * Save the current VFP state into the provided structures and prepare
  468. * for entry into a new function (signal handler).
  469. */
  470. int vfp_preserve_user_clear_hwstate(struct user_vfp *ufp,
  471. struct user_vfp_exc *ufp_exc)
  472. {
  473. struct thread_info *thread = current_thread_info();
  474. struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
  475. /* Ensure that the saved hwstate is up-to-date. */
  476. vfp_sync_hwstate(thread);
  477. /*
  478. * Copy the floating point registers. There can be unused
  479. * registers see asm/hwcap.h for details.
  480. */
  481. memcpy(&ufp->fpregs, &hwstate->fpregs, sizeof(hwstate->fpregs));
  482. /*
  483. * Copy the status and control register.
  484. */
  485. ufp->fpscr = hwstate->fpscr;
  486. /*
  487. * Copy the exception registers.
  488. */
  489. ufp_exc->fpexc = hwstate->fpexc;
  490. ufp_exc->fpinst = hwstate->fpinst;
  491. ufp_exc->fpinst2 = hwstate->fpinst2;
  492. /* Ensure that VFP is disabled. */
  493. vfp_flush_hwstate(thread);
  494. /*
  495. * As per the PCS, clear the length and stride bits for function
  496. * entry.
  497. */
  498. hwstate->fpscr &= ~(FPSCR_LENGTH_MASK | FPSCR_STRIDE_MASK);
  499. return 0;
  500. }
  501. /* Sanitise and restore the current VFP state from the provided structures. */
  502. int vfp_restore_user_hwstate(struct user_vfp *ufp, struct user_vfp_exc *ufp_exc)
  503. {
  504. struct thread_info *thread = current_thread_info();
  505. struct vfp_hard_struct *hwstate = &thread->vfpstate.hard;
  506. unsigned long fpexc;
  507. /* Disable VFP to avoid corrupting the new thread state. */
  508. vfp_flush_hwstate(thread);
  509. /*
  510. * Copy the floating point registers. There can be unused
  511. * registers see asm/hwcap.h for details.
  512. */
  513. memcpy(&hwstate->fpregs, &ufp->fpregs, sizeof(hwstate->fpregs));
  514. /*
  515. * Copy the status and control register.
  516. */
  517. hwstate->fpscr = ufp->fpscr;
  518. /*
  519. * Sanitise and restore the exception registers.
  520. */
  521. fpexc = ufp_exc->fpexc;
  522. /* Ensure the VFP is enabled. */
  523. fpexc |= FPEXC_EN;
  524. /* Ensure FPINST2 is invalid and the exception flag is cleared. */
  525. fpexc &= ~(FPEXC_EX | FPEXC_FP2V);
  526. hwstate->fpexc = fpexc;
  527. hwstate->fpinst = ufp_exc->fpinst;
  528. hwstate->fpinst2 = ufp_exc->fpinst2;
  529. return 0;
  530. }
  531. /*
  532. * VFP hardware can lose all context when a CPU goes offline.
  533. * As we will be running in SMP mode with CPU hotplug, we will save the
  534. * hardware state at every thread switch. We clear our held state when
  535. * a CPU has been killed, indicating that the VFP hardware doesn't contain
  536. * a threads VFP state. When a CPU starts up, we re-enable access to the
  537. * VFP hardware. The callbacks below are called on the CPU which
  538. * is being offlined/onlined.
  539. */
  540. static int vfp_dying_cpu(unsigned int cpu)
  541. {
  542. vfp_current_hw_state[cpu] = NULL;
  543. return 0;
  544. }
  545. static int vfp_starting_cpu(unsigned int unused)
  546. {
  547. vfp_enable(NULL);
  548. return 0;
  549. }
  550. #ifdef CONFIG_KERNEL_MODE_NEON
  551. static int vfp_kmode_exception(struct pt_regs *regs, unsigned int instr)
  552. {
  553. /*
  554. * If we reach this point, a floating point exception has been raised
  555. * while running in kernel mode. If the NEON/VFP unit was enabled at the
  556. * time, it means a VFP instruction has been issued that requires
  557. * software assistance to complete, something which is not currently
  558. * supported in kernel mode.
  559. * If the NEON/VFP unit was disabled, and the location pointed to below
  560. * is properly preceded by a call to kernel_neon_begin(), something has
  561. * caused the task to be scheduled out and back in again. In this case,
  562. * rebuilding and running with CONFIG_DEBUG_ATOMIC_SLEEP enabled should
  563. * be helpful in localizing the problem.
  564. */
  565. if (fmrx(FPEXC) & FPEXC_EN)
  566. pr_crit("BUG: unsupported FP instruction in kernel mode\n");
  567. else
  568. pr_crit("BUG: FP instruction issued in kernel mode with FP unit disabled\n");
  569. pr_crit("FPEXC == 0x%08x\n", fmrx(FPEXC));
  570. return 1;
  571. }
  572. static struct undef_hook vfp_kmode_exception_hook[] = {{
  573. .instr_mask = 0xfe000000,
  574. .instr_val = 0xf2000000,
  575. .cpsr_mask = MODE_MASK | PSR_T_BIT,
  576. .cpsr_val = SVC_MODE,
  577. .fn = vfp_kmode_exception,
  578. }, {
  579. .instr_mask = 0xff100000,
  580. .instr_val = 0xf4000000,
  581. .cpsr_mask = MODE_MASK | PSR_T_BIT,
  582. .cpsr_val = SVC_MODE,
  583. .fn = vfp_kmode_exception,
  584. }, {
  585. .instr_mask = 0xef000000,
  586. .instr_val = 0xef000000,
  587. .cpsr_mask = MODE_MASK | PSR_T_BIT,
  588. .cpsr_val = SVC_MODE | PSR_T_BIT,
  589. .fn = vfp_kmode_exception,
  590. }, {
  591. .instr_mask = 0xff100000,
  592. .instr_val = 0xf9000000,
  593. .cpsr_mask = MODE_MASK | PSR_T_BIT,
  594. .cpsr_val = SVC_MODE | PSR_T_BIT,
  595. .fn = vfp_kmode_exception,
  596. }, {
  597. .instr_mask = 0x0c000e00,
  598. .instr_val = 0x0c000a00,
  599. .cpsr_mask = MODE_MASK,
  600. .cpsr_val = SVC_MODE,
  601. .fn = vfp_kmode_exception,
  602. }};
  603. static int __init vfp_kmode_exception_hook_init(void)
  604. {
  605. int i;
  606. for (i = 0; i < ARRAY_SIZE(vfp_kmode_exception_hook); i++)
  607. register_undef_hook(&vfp_kmode_exception_hook[i]);
  608. return 0;
  609. }
  610. subsys_initcall(vfp_kmode_exception_hook_init);
  611. /*
  612. * Kernel-side NEON support functions
  613. */
  614. void kernel_neon_begin(void)
  615. {
  616. struct thread_info *thread = current_thread_info();
  617. unsigned int cpu;
  618. u32 fpexc;
  619. /*
  620. * Kernel mode NEON is only allowed outside of interrupt context
  621. * with preemption disabled. This will make sure that the kernel
  622. * mode NEON register contents never need to be preserved.
  623. */
  624. BUG_ON(in_interrupt());
  625. cpu = get_cpu();
  626. fpexc = fmrx(FPEXC) | FPEXC_EN;
  627. fmxr(FPEXC, fpexc);
  628. /*
  629. * Save the userland NEON/VFP state. Under UP,
  630. * the owner could be a task other than 'current'
  631. */
  632. if (vfp_state_in_hw(cpu, thread))
  633. vfp_save_state(&thread->vfpstate, fpexc);
  634. #ifndef CONFIG_SMP
  635. else if (vfp_current_hw_state[cpu] != NULL)
  636. vfp_save_state(vfp_current_hw_state[cpu], fpexc);
  637. #endif
  638. vfp_current_hw_state[cpu] = NULL;
  639. }
  640. EXPORT_SYMBOL(kernel_neon_begin);
  641. void kernel_neon_end(void)
  642. {
  643. /* Disable the NEON/VFP unit. */
  644. fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN);
  645. put_cpu();
  646. }
  647. EXPORT_SYMBOL(kernel_neon_end);
  648. #endif /* CONFIG_KERNEL_MODE_NEON */
  649. static int __init vfp_detect(struct pt_regs *regs, unsigned int instr)
  650. {
  651. VFP_arch = UINT_MAX; /* mark as not present */
  652. regs->ARM_pc += 4;
  653. return 0;
  654. }
  655. static struct undef_hook vfp_detect_hook __initdata = {
  656. .instr_mask = 0x0c000e00,
  657. .instr_val = 0x0c000a00,
  658. .cpsr_mask = MODE_MASK,
  659. .cpsr_val = SVC_MODE,
  660. .fn = vfp_detect,
  661. };
  662. /*
  663. * VFP support code initialisation.
  664. */
  665. static int __init vfp_init(void)
  666. {
  667. unsigned int vfpsid;
  668. unsigned int cpu_arch = cpu_architecture();
  669. /*
  670. * Enable the access to the VFP on all online CPUs so the
  671. * following test on FPSID will succeed.
  672. */
  673. if (cpu_arch >= CPU_ARCH_ARMv6)
  674. on_each_cpu(vfp_enable, NULL, 1);
  675. /*
  676. * First check that there is a VFP that we can use.
  677. * The handler is already setup to just log calls, so
  678. * we just need to read the VFPSID register.
  679. */
  680. register_undef_hook(&vfp_detect_hook);
  681. barrier();
  682. vfpsid = fmrx(FPSID);
  683. barrier();
  684. unregister_undef_hook(&vfp_detect_hook);
  685. vfp_vector = vfp_null_entry;
  686. pr_info("VFP support v0.3: ");
  687. if (VFP_arch) {
  688. pr_cont("not present\n");
  689. return 0;
  690. /* Extract the architecture on CPUID scheme */
  691. } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) {
  692. VFP_arch = vfpsid & FPSID_CPUID_ARCH_MASK;
  693. VFP_arch >>= FPSID_ARCH_BIT;
  694. /*
  695. * Check for the presence of the Advanced SIMD
  696. * load/store instructions, integer and single
  697. * precision floating point operations. Only check
  698. * for NEON if the hardware has the MVFR registers.
  699. */
  700. if (IS_ENABLED(CONFIG_NEON) &&
  701. (fmrx(MVFR1) & 0x000fff00) == 0x00011100)
  702. elf_hwcap |= HWCAP_NEON;
  703. if (IS_ENABLED(CONFIG_VFPv3)) {
  704. u32 mvfr0 = fmrx(MVFR0);
  705. if (((mvfr0 & MVFR0_DP_MASK) >> MVFR0_DP_BIT) == 0x2 ||
  706. ((mvfr0 & MVFR0_SP_MASK) >> MVFR0_SP_BIT) == 0x2) {
  707. elf_hwcap |= HWCAP_VFPv3;
  708. /*
  709. * Check for VFPv3 D16 and VFPv4 D16. CPUs in
  710. * this configuration only have 16 x 64bit
  711. * registers.
  712. */
  713. if ((mvfr0 & MVFR0_A_SIMD_MASK) == 1)
  714. /* also v4-D16 */
  715. elf_hwcap |= HWCAP_VFPv3D16;
  716. else
  717. elf_hwcap |= HWCAP_VFPD32;
  718. }
  719. if ((fmrx(MVFR1) & 0xf0000000) == 0x10000000)
  720. elf_hwcap |= HWCAP_VFPv4;
  721. }
  722. /* Extract the architecture version on pre-cpuid scheme */
  723. } else {
  724. if (vfpsid & FPSID_NODOUBLE) {
  725. pr_cont("no double precision support\n");
  726. return 0;
  727. }
  728. VFP_arch = (vfpsid & FPSID_ARCH_MASK) >> FPSID_ARCH_BIT;
  729. }
  730. cpuhp_setup_state_nocalls(CPUHP_AP_ARM_VFP_STARTING,
  731. "arm/vfp:starting", vfp_starting_cpu,
  732. vfp_dying_cpu);
  733. vfp_vector = vfp_support_entry;
  734. thread_register_notifier(&vfp_notifier_block);
  735. vfp_pm_init();
  736. /*
  737. * We detected VFP, and the support code is
  738. * in place; report VFP support to userspace.
  739. */
  740. elf_hwcap |= HWCAP_VFP;
  741. pr_cont("implementor %02x architecture %d part %02x variant %x rev %x\n",
  742. (vfpsid & FPSID_IMPLEMENTER_MASK) >> FPSID_IMPLEMENTER_BIT,
  743. VFP_arch,
  744. (vfpsid & FPSID_PART_MASK) >> FPSID_PART_BIT,
  745. (vfpsid & FPSID_VARIANT_MASK) >> FPSID_VARIANT_BIT,
  746. (vfpsid & FPSID_REV_MASK) >> FPSID_REV_BIT);
  747. return 0;
  748. }
  749. core_initcall(vfp_init);