entry.S 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * S390 low-level entry points.
  4. *
  5. * Copyright IBM Corp. 1999, 2012
  6. * Author(s): Martin Schwidefsky ([email protected]),
  7. * Hartmut Penner ([email protected]),
  8. * Denis Joseph Barrow ([email protected],[email protected]),
  9. */
  10. #include <linux/init.h>
  11. #include <linux/linkage.h>
  12. #include <asm/asm-extable.h>
  13. #include <asm/alternative-asm.h>
  14. #include <asm/processor.h>
  15. #include <asm/cache.h>
  16. #include <asm/dwarf.h>
  17. #include <asm/errno.h>
  18. #include <asm/ptrace.h>
  19. #include <asm/thread_info.h>
  20. #include <asm/asm-offsets.h>
  21. #include <asm/unistd.h>
  22. #include <asm/page.h>
  23. #include <asm/sigp.h>
  24. #include <asm/irq.h>
  25. #include <asm/vx-insn.h>
  26. #include <asm/setup.h>
  27. #include <asm/nmi.h>
  28. #include <asm/export.h>
  29. #include <asm/nospec-insn.h>
  30. STACK_SHIFT = PAGE_SHIFT + THREAD_SIZE_ORDER
  31. STACK_SIZE = 1 << STACK_SHIFT
  32. STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE
  33. _LPP_OFFSET = __LC_LPP
  34. .macro STBEAR address
  35. ALTERNATIVE "nop", ".insn s,0xb2010000,\address", 193
  36. .endm
  37. .macro LBEAR address
  38. ALTERNATIVE "nop", ".insn s,0xb2000000,\address", 193
  39. .endm
  40. .macro LPSWEY address,lpswe
  41. ALTERNATIVE "b \lpswe; nopr", ".insn siy,0xeb0000000071,\address,0", 193
  42. .endm
  43. .macro MBEAR reg
  44. ALTERNATIVE "brcl 0,0", __stringify(mvc __PT_LAST_BREAK(8,\reg),__LC_LAST_BREAK), 193
  45. .endm
  46. .macro CHECK_STACK savearea
  47. #ifdef CONFIG_CHECK_STACK
  48. tml %r15,STACK_SIZE - CONFIG_STACK_GUARD
  49. lghi %r14,\savearea
  50. jz stack_overflow
  51. #endif
  52. .endm
  53. .macro CHECK_VMAP_STACK savearea,oklabel
  54. #ifdef CONFIG_VMAP_STACK
  55. lgr %r14,%r15
  56. nill %r14,0x10000 - STACK_SIZE
  57. oill %r14,STACK_INIT
  58. clg %r14,__LC_KERNEL_STACK
  59. je \oklabel
  60. clg %r14,__LC_ASYNC_STACK
  61. je \oklabel
  62. clg %r14,__LC_MCCK_STACK
  63. je \oklabel
  64. clg %r14,__LC_NODAT_STACK
  65. je \oklabel
  66. clg %r14,__LC_RESTART_STACK
  67. je \oklabel
  68. lghi %r14,\savearea
  69. j stack_overflow
  70. #else
  71. j \oklabel
  72. #endif
  73. .endm
  74. /*
  75. * The TSTMSK macro generates a test-under-mask instruction by
  76. * calculating the memory offset for the specified mask value.
  77. * Mask value can be any constant. The macro shifts the mask
  78. * value to calculate the memory offset for the test-under-mask
  79. * instruction.
  80. */
  81. .macro TSTMSK addr, mask, size=8, bytepos=0
  82. .if (\bytepos < \size) && (\mask >> 8)
  83. .if (\mask & 0xff)
  84. .error "Mask exceeds byte boundary"
  85. .endif
  86. TSTMSK \addr, "(\mask >> 8)", \size, "(\bytepos + 1)"
  87. .exitm
  88. .endif
  89. .ifeq \mask
  90. .error "Mask must not be zero"
  91. .endif
  92. off = \size - \bytepos - 1
  93. tm off+\addr, \mask
  94. .endm
  95. .macro BPOFF
  96. ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,12,0", 82
  97. .endm
  98. .macro BPON
  99. ALTERNATIVE "nop", ".insn rrf,0xb2e80000,0,0,13,0", 82
  100. .endm
  101. .macro BPENTER tif_ptr,tif_mask
  102. ALTERNATIVE "TSTMSK \tif_ptr,\tif_mask; jz .+8; .insn rrf,0xb2e80000,0,0,13,0", \
  103. "j .+12; nop; nop", 82
  104. .endm
  105. .macro BPEXIT tif_ptr,tif_mask
  106. TSTMSK \tif_ptr,\tif_mask
  107. ALTERNATIVE "jz .+8; .insn rrf,0xb2e80000,0,0,12,0", \
  108. "jnz .+8; .insn rrf,0xb2e80000,0,0,13,0", 82
  109. .endm
  110. /*
  111. * The CHKSTG macro jumps to the provided label in case the
  112. * machine check interruption code reports one of unrecoverable
  113. * storage errors:
  114. * - Storage error uncorrected
  115. * - Storage key error uncorrected
  116. * - Storage degradation with Failing-storage-address validity
  117. */
  118. .macro CHKSTG errlabel
  119. TSTMSK __LC_MCCK_CODE,(MCCK_CODE_STG_ERROR|MCCK_CODE_STG_KEY_ERROR)
  120. jnz \errlabel
  121. TSTMSK __LC_MCCK_CODE,MCCK_CODE_STG_DEGRAD
  122. jz .Loklabel\@
  123. TSTMSK __LC_MCCK_CODE,MCCK_CODE_STG_FAIL_ADDR
  124. jnz \errlabel
  125. .Loklabel\@:
  126. .endm
  127. #if IS_ENABLED(CONFIG_KVM)
  128. /*
  129. * The OUTSIDE macro jumps to the provided label in case the value
  130. * in the provided register is outside of the provided range. The
  131. * macro is useful for checking whether a PSW stored in a register
  132. * pair points inside or outside of a block of instructions.
  133. * @reg: register to check
  134. * @start: start of the range
  135. * @end: end of the range
  136. * @outside_label: jump here if @reg is outside of [@start..@end)
  137. */
  138. .macro OUTSIDE reg,start,end,outside_label
  139. lgr %r14,\reg
  140. larl %r13,\start
  141. slgr %r14,%r13
  142. #ifdef CONFIG_AS_IS_LLVM
  143. clgfrl %r14,.Lrange_size\@
  144. #else
  145. clgfi %r14,\end - \start
  146. #endif
  147. jhe \outside_label
  148. #ifdef CONFIG_AS_IS_LLVM
  149. .section .rodata, "a"
  150. .align 4
  151. .Lrange_size\@:
  152. .long \end - \start
  153. .previous
  154. #endif
  155. .endm
  156. .macro SIEEXIT
  157. lg %r9,__SF_SIE_CONTROL(%r15) # get control block pointer
  158. ni __SIE_PROG0C+3(%r9),0xfe # no longer in SIE
  159. lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
  160. larl %r9,sie_exit # skip forward to sie_exit
  161. .endm
  162. #endif
  163. GEN_BR_THUNK %r14
  164. .section .kprobes.text, "ax"
  165. .Ldummy:
  166. /*
  167. * This nop exists only in order to avoid that __bpon starts at
  168. * the beginning of the kprobes text section. In that case we would
  169. * have several symbols at the same address. E.g. objdump would take
  170. * an arbitrary symbol name when disassembling this code.
  171. * With the added nop in between the __bpon symbol is unique
  172. * again.
  173. */
  174. nop 0
  175. ENTRY(__bpon)
  176. .globl __bpon
  177. BPON
  178. BR_EX %r14
  179. ENDPROC(__bpon)
  180. /*
  181. * Scheduler resume function, called by switch_to
  182. * gpr2 = (task_struct *) prev
  183. * gpr3 = (task_struct *) next
  184. * Returns:
  185. * gpr2 = prev
  186. */
  187. ENTRY(__switch_to)
  188. stmg %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task
  189. lghi %r4,__TASK_stack
  190. lghi %r1,__TASK_thread
  191. llill %r5,STACK_INIT
  192. stg %r15,__THREAD_ksp(%r1,%r2) # store kernel stack of prev
  193. lg %r15,0(%r4,%r3) # start of kernel stack of next
  194. agr %r15,%r5 # end of kernel stack of next
  195. stg %r3,__LC_CURRENT # store task struct of next
  196. stg %r15,__LC_KERNEL_STACK # store end of kernel stack
  197. lg %r15,__THREAD_ksp(%r1,%r3) # load kernel stack of next
  198. aghi %r3,__TASK_pid
  199. mvc __LC_CURRENT_PID(4,%r0),0(%r3) # store pid of next
  200. lmg %r6,%r15,__SF_GPRS(%r15) # load gprs of next task
  201. ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
  202. BR_EX %r14
  203. ENDPROC(__switch_to)
  204. #if IS_ENABLED(CONFIG_KVM)
  205. /*
  206. * sie64a calling convention:
  207. * %r2 pointer to sie control block
  208. * %r3 guest register save area
  209. */
  210. ENTRY(sie64a)
  211. stmg %r6,%r14,__SF_GPRS(%r15) # save kernel registers
  212. lg %r12,__LC_CURRENT
  213. stg %r2,__SF_SIE_CONTROL(%r15) # save control block pointer
  214. stg %r3,__SF_SIE_SAVEAREA(%r15) # save guest register save area
  215. xc __SF_SIE_REASON(8,%r15),__SF_SIE_REASON(%r15) # reason code = 0
  216. mvc __SF_SIE_FLAGS(8,%r15),__TI_flags(%r12) # copy thread flags
  217. lmg %r0,%r13,0(%r3) # load guest gprs 0-13
  218. lg %r14,__LC_GMAP # get gmap pointer
  219. ltgr %r14,%r14
  220. jz .Lsie_gmap
  221. lctlg %c1,%c1,__GMAP_ASCE(%r14) # load primary asce
  222. .Lsie_gmap:
  223. lg %r14,__SF_SIE_CONTROL(%r15) # get control block pointer
  224. oi __SIE_PROG0C+3(%r14),1 # we are going into SIE now
  225. tm __SIE_PROG20+3(%r14),3 # last exit...
  226. jnz .Lsie_skip
  227. TSTMSK __LC_CPU_FLAGS,_CIF_FPU
  228. jo .Lsie_skip # exit if fp/vx regs changed
  229. BPEXIT __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
  230. .Lsie_entry:
  231. sie 0(%r14)
  232. # Let the next instruction be NOP to avoid triggering a machine check
  233. # and handling it in a guest as result of the instruction execution.
  234. nopr 7
  235. .Lsie_leave:
  236. BPOFF
  237. BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
  238. .Lsie_skip:
  239. ni __SIE_PROG0C+3(%r14),0xfe # no longer in SIE
  240. lctlg %c1,%c1,__LC_KERNEL_ASCE # load primary asce
  241. .Lsie_done:
  242. # some program checks are suppressing. C code (e.g. do_protection_exception)
  243. # will rewind the PSW by the ILC, which is often 4 bytes in case of SIE. There
  244. # are some corner cases (e.g. runtime instrumentation) where ILC is unpredictable.
  245. # Other instructions between sie64a and .Lsie_done should not cause program
  246. # interrupts. So lets use 3 nops as a landing pad for all possible rewinds.
  247. .Lrewind_pad6:
  248. nopr 7
  249. .Lrewind_pad4:
  250. nopr 7
  251. .Lrewind_pad2:
  252. nopr 7
  253. .globl sie_exit
  254. sie_exit:
  255. lg %r14,__SF_SIE_SAVEAREA(%r15) # load guest register save area
  256. stmg %r0,%r13,0(%r14) # save guest gprs 0-13
  257. xgr %r0,%r0 # clear guest registers to
  258. xgr %r1,%r1 # prevent speculative use
  259. xgr %r3,%r3
  260. xgr %r4,%r4
  261. xgr %r5,%r5
  262. lmg %r6,%r14,__SF_GPRS(%r15) # restore kernel registers
  263. lg %r2,__SF_SIE_REASON(%r15) # return exit reason code
  264. BR_EX %r14
  265. .Lsie_fault:
  266. lghi %r14,-EFAULT
  267. stg %r14,__SF_SIE_REASON(%r15) # set exit reason code
  268. j sie_exit
  269. EX_TABLE(.Lrewind_pad6,.Lsie_fault)
  270. EX_TABLE(.Lrewind_pad4,.Lsie_fault)
  271. EX_TABLE(.Lrewind_pad2,.Lsie_fault)
  272. EX_TABLE(sie_exit,.Lsie_fault)
  273. ENDPROC(sie64a)
  274. EXPORT_SYMBOL(sie64a)
  275. EXPORT_SYMBOL(sie_exit)
  276. #endif
  277. /*
  278. * SVC interrupt handler routine. System calls are synchronous events and
  279. * are entered with interrupts disabled.
  280. */
  281. ENTRY(system_call)
  282. stpt __LC_SYS_ENTER_TIMER
  283. stmg %r8,%r15,__LC_SAVE_AREA_SYNC
  284. BPOFF
  285. lghi %r14,0
  286. .Lsysc_per:
  287. STBEAR __LC_LAST_BREAK
  288. lctlg %c1,%c1,__LC_KERNEL_ASCE
  289. lg %r12,__LC_CURRENT
  290. lg %r15,__LC_KERNEL_STACK
  291. xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
  292. stmg %r0,%r7,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
  293. BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
  294. # clear user controlled register to prevent speculative use
  295. xgr %r0,%r0
  296. xgr %r1,%r1
  297. xgr %r4,%r4
  298. xgr %r5,%r5
  299. xgr %r6,%r6
  300. xgr %r7,%r7
  301. xgr %r8,%r8
  302. xgr %r9,%r9
  303. xgr %r10,%r10
  304. xgr %r11,%r11
  305. la %r2,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs
  306. mvc __PT_R8(64,%r2),__LC_SAVE_AREA_SYNC
  307. MBEAR %r2
  308. lgr %r3,%r14
  309. brasl %r14,__do_syscall
  310. lctlg %c1,%c1,__LC_USER_ASCE
  311. mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
  312. BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
  313. LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
  314. lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
  315. stpt __LC_EXIT_TIMER
  316. LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
  317. ENDPROC(system_call)
  318. #
  319. # a new process exits the kernel with ret_from_fork
  320. #
  321. ENTRY(ret_from_fork)
  322. lgr %r3,%r11
  323. brasl %r14,__ret_from_fork
  324. lctlg %c1,%c1,__LC_USER_ASCE
  325. mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
  326. BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
  327. LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
  328. lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
  329. stpt __LC_EXIT_TIMER
  330. LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
  331. ENDPROC(ret_from_fork)
  332. /*
  333. * Program check handler routine
  334. */
  335. ENTRY(pgm_check_handler)
  336. stpt __LC_SYS_ENTER_TIMER
  337. BPOFF
  338. stmg %r8,%r15,__LC_SAVE_AREA_SYNC
  339. lg %r12,__LC_CURRENT
  340. lghi %r10,0
  341. lmg %r8,%r9,__LC_PGM_OLD_PSW
  342. tmhh %r8,0x0001 # coming from user space?
  343. jno .Lpgm_skip_asce
  344. lctlg %c1,%c1,__LC_KERNEL_ASCE
  345. j 3f # -> fault in user space
  346. .Lpgm_skip_asce:
  347. #if IS_ENABLED(CONFIG_KVM)
  348. # cleanup critical section for program checks in sie64a
  349. OUTSIDE %r9,.Lsie_gmap,.Lsie_done,1f
  350. SIEEXIT
  351. lghi %r10,_PIF_GUEST_FAULT
  352. #endif
  353. 1: tmhh %r8,0x4000 # PER bit set in old PSW ?
  354. jnz 2f # -> enabled, can't be a double fault
  355. tm __LC_PGM_ILC+3,0x80 # check for per exception
  356. jnz .Lpgm_svcper # -> single stepped svc
  357. 2: CHECK_STACK __LC_SAVE_AREA_SYNC
  358. aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
  359. # CHECK_VMAP_STACK branches to stack_overflow or 4f
  360. CHECK_VMAP_STACK __LC_SAVE_AREA_SYNC,4f
  361. 3: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
  362. lg %r15,__LC_KERNEL_STACK
  363. 4: la %r11,STACK_FRAME_OVERHEAD(%r15)
  364. stg %r10,__PT_FLAGS(%r11)
  365. xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
  366. stmg %r0,%r7,__PT_R0(%r11)
  367. mvc __PT_R8(64,%r11),__LC_SAVE_AREA_SYNC
  368. mvc __PT_LAST_BREAK(8,%r11),__LC_PGM_LAST_BREAK
  369. stmg %r8,%r9,__PT_PSW(%r11)
  370. # clear user controlled registers to prevent speculative use
  371. xgr %r0,%r0
  372. xgr %r1,%r1
  373. xgr %r3,%r3
  374. xgr %r4,%r4
  375. xgr %r5,%r5
  376. xgr %r6,%r6
  377. xgr %r7,%r7
  378. lgr %r2,%r11
  379. brasl %r14,__do_pgm_check
  380. tmhh %r8,0x0001 # returning to user space?
  381. jno .Lpgm_exit_kernel
  382. lctlg %c1,%c1,__LC_USER_ASCE
  383. BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
  384. stpt __LC_EXIT_TIMER
  385. .Lpgm_exit_kernel:
  386. mvc __LC_RETURN_PSW(16),STACK_FRAME_OVERHEAD+__PT_PSW(%r15)
  387. LBEAR STACK_FRAME_OVERHEAD+__PT_LAST_BREAK(%r15)
  388. lmg %r0,%r15,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
  389. LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
  390. #
  391. # single stepped system call
  392. #
  393. .Lpgm_svcper:
  394. mvc __LC_RETURN_PSW(8),__LC_SVC_NEW_PSW
  395. larl %r14,.Lsysc_per
  396. stg %r14,__LC_RETURN_PSW+8
  397. lghi %r14,1
  398. LBEAR __LC_PGM_LAST_BREAK
  399. LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE # branch to .Lsysc_per
  400. ENDPROC(pgm_check_handler)
  401. /*
  402. * Interrupt handler macro used for external and IO interrupts.
  403. */
  404. .macro INT_HANDLER name,lc_old_psw,handler
  405. ENTRY(\name)
  406. stckf __LC_INT_CLOCK
  407. stpt __LC_SYS_ENTER_TIMER
  408. STBEAR __LC_LAST_BREAK
  409. BPOFF
  410. stmg %r8,%r15,__LC_SAVE_AREA_ASYNC
  411. lg %r12,__LC_CURRENT
  412. lmg %r8,%r9,\lc_old_psw
  413. tmhh %r8,0x0001 # interrupting from user ?
  414. jnz 1f
  415. #if IS_ENABLED(CONFIG_KVM)
  416. OUTSIDE %r9,.Lsie_gmap,.Lsie_done,0f
  417. BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
  418. SIEEXIT
  419. #endif
  420. 0: CHECK_STACK __LC_SAVE_AREA_ASYNC
  421. aghi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE)
  422. j 2f
  423. 1: BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
  424. lctlg %c1,%c1,__LC_KERNEL_ASCE
  425. lg %r15,__LC_KERNEL_STACK
  426. 2: xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
  427. la %r11,STACK_FRAME_OVERHEAD(%r15)
  428. stmg %r0,%r7,__PT_R0(%r11)
  429. # clear user controlled registers to prevent speculative use
  430. xgr %r0,%r0
  431. xgr %r1,%r1
  432. xgr %r3,%r3
  433. xgr %r4,%r4
  434. xgr %r5,%r5
  435. xgr %r6,%r6
  436. xgr %r7,%r7
  437. xgr %r10,%r10
  438. xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
  439. mvc __PT_R8(64,%r11),__LC_SAVE_AREA_ASYNC
  440. MBEAR %r11
  441. stmg %r8,%r9,__PT_PSW(%r11)
  442. lgr %r2,%r11 # pass pointer to pt_regs
  443. brasl %r14,\handler
  444. mvc __LC_RETURN_PSW(16),__PT_PSW(%r11)
  445. tmhh %r8,0x0001 # returning to user ?
  446. jno 2f
  447. lctlg %c1,%c1,__LC_USER_ASCE
  448. BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
  449. stpt __LC_EXIT_TIMER
  450. 2: LBEAR __PT_LAST_BREAK(%r11)
  451. lmg %r0,%r15,__PT_R0(%r11)
  452. LPSWEY __LC_RETURN_PSW,__LC_RETURN_LPSWE
  453. ENDPROC(\name)
  454. .endm
  455. INT_HANDLER ext_int_handler,__LC_EXT_OLD_PSW,do_ext_irq
  456. INT_HANDLER io_int_handler,__LC_IO_OLD_PSW,do_io_irq
  457. /*
  458. * Load idle PSW.
  459. */
  460. ENTRY(psw_idle)
  461. stg %r14,(__SF_GPRS+8*8)(%r15)
  462. stg %r3,__SF_EMPTY(%r15)
  463. larl %r1,psw_idle_exit
  464. stg %r1,__SF_EMPTY+8(%r15)
  465. larl %r1,smp_cpu_mtid
  466. llgf %r1,0(%r1)
  467. ltgr %r1,%r1
  468. jz .Lpsw_idle_stcctm
  469. .insn rsy,0xeb0000000017,%r1,5,__MT_CYCLES_ENTER(%r2)
  470. .Lpsw_idle_stcctm:
  471. oi __LC_CPU_FLAGS+7,_CIF_ENABLED_WAIT
  472. BPON
  473. stckf __CLOCK_IDLE_ENTER(%r2)
  474. stpt __TIMER_IDLE_ENTER(%r2)
  475. lpswe __SF_EMPTY(%r15)
  476. .globl psw_idle_exit
  477. psw_idle_exit:
  478. BR_EX %r14
  479. ENDPROC(psw_idle)
  480. /*
  481. * Machine check handler routines
  482. */
  483. ENTRY(mcck_int_handler)
  484. stckf __LC_MCCK_CLOCK
  485. BPOFF
  486. la %r1,4095 # validate r1
  487. spt __LC_CPU_TIMER_SAVE_AREA-4095(%r1) # validate cpu timer
  488. LBEAR __LC_LAST_BREAK_SAVE_AREA-4095(%r1) # validate bear
  489. lmg %r0,%r15,__LC_GPREGS_SAVE_AREA-4095(%r1)# validate gprs
  490. lg %r12,__LC_CURRENT
  491. lmg %r8,%r9,__LC_MCK_OLD_PSW
  492. TSTMSK __LC_MCCK_CODE,MCCK_CODE_SYSTEM_DAMAGE
  493. jo .Lmcck_panic # yes -> rest of mcck code invalid
  494. TSTMSK __LC_MCCK_CODE,MCCK_CODE_CR_VALID
  495. jno .Lmcck_panic # control registers invalid -> panic
  496. la %r14,4095
  497. lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r14) # validate ctl regs
  498. ptlb
  499. lghi %r14,__LC_CPU_TIMER_SAVE_AREA
  500. mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
  501. TSTMSK __LC_MCCK_CODE,MCCK_CODE_CPU_TIMER_VALID
  502. jo 3f
  503. la %r14,__LC_SYS_ENTER_TIMER
  504. clc 0(8,%r14),__LC_EXIT_TIMER
  505. jl 1f
  506. la %r14,__LC_EXIT_TIMER
  507. 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER
  508. jl 2f
  509. la %r14,__LC_LAST_UPDATE_TIMER
  510. 2: spt 0(%r14)
  511. mvc __LC_MCCK_ENTER_TIMER(8),0(%r14)
  512. 3: TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_MWP_VALID
  513. jno .Lmcck_panic
  514. tmhh %r8,0x0001 # interrupting from user ?
  515. jnz 6f
  516. TSTMSK __LC_MCCK_CODE,MCCK_CODE_PSW_IA_VALID
  517. jno .Lmcck_panic
  518. #if IS_ENABLED(CONFIG_KVM)
  519. OUTSIDE %r9,.Lsie_gmap,.Lsie_done,6f
  520. OUTSIDE %r9,.Lsie_entry,.Lsie_leave,4f
  521. oi __LC_CPU_FLAGS+7, _CIF_MCCK_GUEST
  522. j 5f
  523. 4: CHKSTG .Lmcck_panic
  524. 5: larl %r14,.Lstosm_tmp
  525. stosm 0(%r14),0x04 # turn dat on, keep irqs off
  526. BPENTER __SF_SIE_FLAGS(%r15),(_TIF_ISOLATE_BP|_TIF_ISOLATE_BP_GUEST)
  527. SIEEXIT
  528. j .Lmcck_stack
  529. #endif
  530. 6: CHKSTG .Lmcck_panic
  531. larl %r14,.Lstosm_tmp
  532. stosm 0(%r14),0x04 # turn dat on, keep irqs off
  533. tmhh %r8,0x0001 # interrupting from user ?
  534. jz .Lmcck_stack
  535. BPENTER __TI_flags(%r12),_TIF_ISOLATE_BP
  536. .Lmcck_stack:
  537. lg %r15,__LC_MCCK_STACK
  538. la %r11,STACK_FRAME_OVERHEAD(%r15)
  539. stctg %c1,%c1,__PT_CR1(%r11)
  540. lctlg %c1,%c1,__LC_KERNEL_ASCE
  541. xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
  542. lghi %r14,__LC_GPREGS_SAVE_AREA+64
  543. stmg %r0,%r7,__PT_R0(%r11)
  544. # clear user controlled registers to prevent speculative use
  545. xgr %r0,%r0
  546. xgr %r1,%r1
  547. xgr %r3,%r3
  548. xgr %r4,%r4
  549. xgr %r5,%r5
  550. xgr %r6,%r6
  551. xgr %r7,%r7
  552. xgr %r10,%r10
  553. mvc __PT_R8(64,%r11),0(%r14)
  554. stmg %r8,%r9,__PT_PSW(%r11)
  555. xc __PT_FLAGS(8,%r11),__PT_FLAGS(%r11)
  556. xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
  557. lgr %r2,%r11 # pass pointer to pt_regs
  558. brasl %r14,s390_do_machine_check
  559. cghi %r2,0
  560. je .Lmcck_return
  561. lg %r1,__LC_KERNEL_STACK # switch to kernel stack
  562. mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11)
  563. xc __SF_BACKCHAIN(8,%r1),__SF_BACKCHAIN(%r1)
  564. la %r11,STACK_FRAME_OVERHEAD(%r1)
  565. lgr %r2,%r11
  566. lgr %r15,%r1
  567. brasl %r14,s390_handle_mcck
  568. .Lmcck_return:
  569. lctlg %c1,%c1,__PT_CR1(%r11)
  570. lmg %r0,%r10,__PT_R0(%r11)
  571. mvc __LC_RETURN_MCCK_PSW(16),__PT_PSW(%r11) # move return PSW
  572. tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ?
  573. jno 0f
  574. BPEXIT __TI_flags(%r12),_TIF_ISOLATE_BP
  575. stpt __LC_EXIT_TIMER
  576. 0: ALTERNATIVE "nop", __stringify(lghi %r12,__LC_LAST_BREAK_SAVE_AREA),193
  577. LBEAR 0(%r12)
  578. lmg %r11,%r15,__PT_R11(%r11)
  579. LPSWEY __LC_RETURN_MCCK_PSW,__LC_RETURN_MCCK_LPSWE
  580. .Lmcck_panic:
  581. /*
  582. * Iterate over all possible CPU addresses in the range 0..0xffff
  583. * and stop each CPU using signal processor. Use compare and swap
  584. * to allow just one CPU-stopper and prevent concurrent CPUs from
  585. * stopping each other while leaving the others running.
  586. */
  587. lhi %r5,0
  588. lhi %r6,1
  589. larl %r7,.Lstop_lock
  590. cs %r5,%r6,0(%r7) # single CPU-stopper only
  591. jnz 4f
  592. larl %r7,.Lthis_cpu
  593. stap 0(%r7) # this CPU address
  594. lh %r4,0(%r7)
  595. nilh %r4,0
  596. lhi %r0,1
  597. sll %r0,16 # CPU counter
  598. lhi %r3,0 # next CPU address
  599. 0: cr %r3,%r4
  600. je 2f
  601. 1: sigp %r1,%r3,SIGP_STOP # stop next CPU
  602. brc SIGP_CC_BUSY,1b
  603. 2: ahi %r3,1
  604. brct %r0,0b
  605. 3: sigp %r1,%r4,SIGP_STOP # stop this CPU
  606. brc SIGP_CC_BUSY,3b
  607. 4: j 4b
  608. ENDPROC(mcck_int_handler)
  609. ENTRY(restart_int_handler)
  610. ALTERNATIVE "nop", "lpp _LPP_OFFSET", 40
  611. stg %r15,__LC_SAVE_AREA_RESTART
  612. TSTMSK __LC_RESTART_FLAGS,RESTART_FLAG_CTLREGS,4
  613. jz 0f
  614. la %r15,4095
  615. lctlg %c0,%c15,__LC_CREGS_SAVE_AREA-4095(%r15)
  616. 0: larl %r15,.Lstosm_tmp
  617. stosm 0(%r15),0x04 # turn dat on, keep irqs off
  618. lg %r15,__LC_RESTART_STACK
  619. xc STACK_FRAME_OVERHEAD(__PT_SIZE,%r15),STACK_FRAME_OVERHEAD(%r15)
  620. stmg %r0,%r14,STACK_FRAME_OVERHEAD+__PT_R0(%r15)
  621. mvc STACK_FRAME_OVERHEAD+__PT_R15(8,%r15),__LC_SAVE_AREA_RESTART
  622. mvc STACK_FRAME_OVERHEAD+__PT_PSW(16,%r15),__LC_RST_OLD_PSW
  623. xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15)
  624. lg %r1,__LC_RESTART_FN # load fn, parm & source cpu
  625. lg %r2,__LC_RESTART_DATA
  626. lgf %r3,__LC_RESTART_SOURCE
  627. ltgr %r3,%r3 # test source cpu address
  628. jm 1f # negative -> skip source stop
  629. 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu
  630. brc 10,0b # wait for status stored
  631. 1: basr %r14,%r1 # call function
  632. stap __SF_EMPTY(%r15) # store cpu address
  633. llgh %r3,__SF_EMPTY(%r15)
  634. 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu
  635. brc 2,2b
  636. 3: j 3b
  637. ENDPROC(restart_int_handler)
  638. .section .kprobes.text, "ax"
  639. #if defined(CONFIG_CHECK_STACK) || defined(CONFIG_VMAP_STACK)
  640. /*
  641. * The synchronous or the asynchronous stack overflowed. We are dead.
  642. * No need to properly save the registers, we are going to panic anyway.
  643. * Setup a pt_regs so that show_trace can provide a good call trace.
  644. */
  645. ENTRY(stack_overflow)
  646. lg %r15,__LC_NODAT_STACK # change to panic stack
  647. la %r11,STACK_FRAME_OVERHEAD(%r15)
  648. stmg %r0,%r7,__PT_R0(%r11)
  649. stmg %r8,%r9,__PT_PSW(%r11)
  650. mvc __PT_R8(64,%r11),0(%r14)
  651. stg %r10,__PT_ORIG_GPR2(%r11) # store last break to orig_gpr2
  652. xc __SF_BACKCHAIN(8,%r15),__SF_BACKCHAIN(%r15)
  653. lgr %r2,%r11 # pass pointer to pt_regs
  654. jg kernel_stack_overflow
  655. ENDPROC(stack_overflow)
  656. #endif
  657. .section .data, "aw"
  658. .align 4
  659. .Lstop_lock: .long 0
  660. .Lthis_cpu: .short 0
  661. .Lstosm_tmp: .byte 0
  662. .section .rodata, "a"
  663. #define SYSCALL(esame,emu) .quad __s390x_ ## esame
  664. .globl sys_call_table
  665. sys_call_table:
  666. #include "asm/syscall_table.h"
  667. #undef SYSCALL
  668. #ifdef CONFIG_COMPAT
  669. #define SYSCALL(esame,emu) .quad __s390_ ## emu
  670. .globl sys_call_table_emu
  671. sys_call_table_emu:
  672. #include "asm/syscall_table.h"
  673. #undef SYSCALL
  674. #endif