uprobes.c 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * User-space Probes (UProbes) for x86
  4. *
  5. * Copyright (C) IBM Corporation, 2008-2011
  6. * Authors:
  7. * Srikar Dronamraju
  8. * Jim Keniston
  9. */
  10. #include <linux/kernel.h>
  11. #include <linux/sched.h>
  12. #include <linux/ptrace.h>
  13. #include <linux/uprobes.h>
  14. #include <linux/uaccess.h>
  15. #include <linux/kdebug.h>
  16. #include <asm/processor.h>
  17. #include <asm/insn.h>
  18. #include <asm/mmu_context.h>
  19. /* Post-execution fixups. */
  20. /* Adjust IP back to vicinity of actual insn */
  21. #define UPROBE_FIX_IP 0x01
  22. /* Adjust the return address of a call insn */
  23. #define UPROBE_FIX_CALL 0x02
  24. /* Instruction will modify TF, don't change it */
  25. #define UPROBE_FIX_SETF 0x04
  26. #define UPROBE_FIX_RIP_SI 0x08
  27. #define UPROBE_FIX_RIP_DI 0x10
  28. #define UPROBE_FIX_RIP_BX 0x20
  29. #define UPROBE_FIX_RIP_MASK \
  30. (UPROBE_FIX_RIP_SI | UPROBE_FIX_RIP_DI | UPROBE_FIX_RIP_BX)
  31. #define UPROBE_TRAP_NR UINT_MAX
  32. /* Adaptations for mhiramat x86 decoder v14. */
  33. #define OPCODE1(insn) ((insn)->opcode.bytes[0])
  34. #define OPCODE2(insn) ((insn)->opcode.bytes[1])
  35. #define OPCODE3(insn) ((insn)->opcode.bytes[2])
  36. #define MODRM_REG(insn) X86_MODRM_REG((insn)->modrm.value)
  37. #define W(row, b0, b1, b2, b3, b4, b5, b6, b7, b8, b9, ba, bb, bc, bd, be, bf)\
  38. (((b0##UL << 0x0)|(b1##UL << 0x1)|(b2##UL << 0x2)|(b3##UL << 0x3) | \
  39. (b4##UL << 0x4)|(b5##UL << 0x5)|(b6##UL << 0x6)|(b7##UL << 0x7) | \
  40. (b8##UL << 0x8)|(b9##UL << 0x9)|(ba##UL << 0xa)|(bb##UL << 0xb) | \
  41. (bc##UL << 0xc)|(bd##UL << 0xd)|(be##UL << 0xe)|(bf##UL << 0xf)) \
  42. << (row % 32))
  43. /*
  44. * Good-instruction tables for 32-bit apps. This is non-const and volatile
  45. * to keep gcc from statically optimizing it out, as variable_test_bit makes
  46. * some versions of gcc to think only *(unsigned long*) is used.
  47. *
  48. * Opcodes we'll probably never support:
  49. * 6c-6f - ins,outs. SEGVs if used in userspace
  50. * e4-e7 - in,out imm. SEGVs if used in userspace
  51. * ec-ef - in,out acc. SEGVs if used in userspace
  52. * cc - int3. SIGTRAP if used in userspace
  53. * ce - into. Not used in userspace - no kernel support to make it useful. SEGVs
  54. * (why we support bound (62) then? it's similar, and similarly unused...)
  55. * f1 - int1. SIGTRAP if used in userspace
  56. * f4 - hlt. SEGVs if used in userspace
  57. * fa - cli. SEGVs if used in userspace
  58. * fb - sti. SEGVs if used in userspace
  59. *
  60. * Opcodes which need some work to be supported:
  61. * 07,17,1f - pop es/ss/ds
  62. * Normally not used in userspace, but would execute if used.
  63. * Can cause GP or stack exception if tries to load wrong segment descriptor.
  64. * We hesitate to run them under single step since kernel's handling
  65. * of userspace single-stepping (TF flag) is fragile.
  66. * We can easily refuse to support push es/cs/ss/ds (06/0e/16/1e)
  67. * on the same grounds that they are never used.
  68. * cd - int N.
  69. * Used by userspace for "int 80" syscall entry. (Other "int N"
  70. * cause GP -> SEGV since their IDT gates don't allow calls from CPL 3).
  71. * Not supported since kernel's handling of userspace single-stepping
  72. * (TF flag) is fragile.
  73. * cf - iret. Normally not used in userspace. Doesn't SEGV unless arguments are bad
  74. */
  75. #if defined(CONFIG_X86_32) || defined(CONFIG_IA32_EMULATION)
  76. static volatile u32 good_insns_32[256 / 32] = {
  77. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  78. /* ---------------------------------------------- */
  79. W(0x00, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) | /* 00 */
  80. W(0x10, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 10 */
  81. W(0x20, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
  82. W(0x30, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 30 */
  83. W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
  84. W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
  85. W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
  86. W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
  87. W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
  88. W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
  89. W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
  90. W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
  91. W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
  92. W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
  93. W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0) | /* e0 */
  94. W(0xf0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1) /* f0 */
  95. /* ---------------------------------------------- */
  96. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  97. };
  98. #else
  99. #define good_insns_32 NULL
  100. #endif
  101. /* Good-instruction tables for 64-bit apps.
  102. *
  103. * Genuinely invalid opcodes:
  104. * 06,07 - formerly push/pop es
  105. * 0e - formerly push cs
  106. * 16,17 - formerly push/pop ss
  107. * 1e,1f - formerly push/pop ds
  108. * 27,2f,37,3f - formerly daa/das/aaa/aas
  109. * 60,61 - formerly pusha/popa
  110. * 62 - formerly bound. EVEX prefix for AVX512 (not yet supported)
  111. * 82 - formerly redundant encoding of Group1
  112. * 9a - formerly call seg:ofs
  113. * ce - formerly into
  114. * d4,d5 - formerly aam/aad
  115. * d6 - formerly undocumented salc
  116. * ea - formerly jmp seg:ofs
  117. *
  118. * Opcodes we'll probably never support:
  119. * 6c-6f - ins,outs. SEGVs if used in userspace
  120. * e4-e7 - in,out imm. SEGVs if used in userspace
  121. * ec-ef - in,out acc. SEGVs if used in userspace
  122. * cc - int3. SIGTRAP if used in userspace
  123. * f1 - int1. SIGTRAP if used in userspace
  124. * f4 - hlt. SEGVs if used in userspace
  125. * fa - cli. SEGVs if used in userspace
  126. * fb - sti. SEGVs if used in userspace
  127. *
  128. * Opcodes which need some work to be supported:
  129. * cd - int N.
  130. * Used by userspace for "int 80" syscall entry. (Other "int N"
  131. * cause GP -> SEGV since their IDT gates don't allow calls from CPL 3).
  132. * Not supported since kernel's handling of userspace single-stepping
  133. * (TF flag) is fragile.
  134. * cf - iret. Normally not used in userspace. Doesn't SEGV unless arguments are bad
  135. */
  136. #if defined(CONFIG_X86_64)
  137. static volatile u32 good_insns_64[256 / 32] = {
  138. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  139. /* ---------------------------------------------- */
  140. W(0x00, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1) | /* 00 */
  141. W(0x10, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0) , /* 10 */
  142. W(0x20, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) | /* 20 */
  143. W(0x30, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0) , /* 30 */
  144. W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
  145. W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
  146. W(0x60, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* 60 */
  147. W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 70 */
  148. W(0x80, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
  149. W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1) , /* 90 */
  150. W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* a0 */
  151. W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
  152. W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0) | /* c0 */
  153. W(0xd0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
  154. W(0xe0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0) | /* e0 */
  155. W(0xf0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1) /* f0 */
  156. /* ---------------------------------------------- */
  157. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  158. };
  159. #else
  160. #define good_insns_64 NULL
  161. #endif
  162. /* Using this for both 64-bit and 32-bit apps.
  163. * Opcodes we don't support:
  164. * 0f 00 - SLDT/STR/LLDT/LTR/VERR/VERW/-/- group. System insns
  165. * 0f 01 - SGDT/SIDT/LGDT/LIDT/SMSW/-/LMSW/INVLPG group.
  166. * Also encodes tons of other system insns if mod=11.
  167. * Some are in fact non-system: xend, xtest, rdtscp, maybe more
  168. * 0f 05 - syscall
  169. * 0f 06 - clts (CPL0 insn)
  170. * 0f 07 - sysret
  171. * 0f 08 - invd (CPL0 insn)
  172. * 0f 09 - wbinvd (CPL0 insn)
  173. * 0f 0b - ud2
  174. * 0f 30 - wrmsr (CPL0 insn) (then why rdmsr is allowed, it's also CPL0 insn?)
  175. * 0f 34 - sysenter
  176. * 0f 35 - sysexit
  177. * 0f 37 - getsec
  178. * 0f 78 - vmread (Intel VMX. CPL0 insn)
  179. * 0f 79 - vmwrite (Intel VMX. CPL0 insn)
  180. * Note: with prefixes, these two opcodes are
  181. * extrq/insertq/AVX512 convert vector ops.
  182. * 0f ae - group15: [f]xsave,[f]xrstor,[v]{ld,st}mxcsr,clflush[opt],
  183. * {rd,wr}{fs,gs}base,{s,l,m}fence.
  184. * Why? They are all user-executable.
  185. */
  186. static volatile u32 good_2byte_insns[256 / 32] = {
  187. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  188. /* ---------------------------------------------- */
  189. W(0x00, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1) | /* 00 */
  190. W(0x10, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 10 */
  191. W(0x20, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 20 */
  192. W(0x30, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1) , /* 30 */
  193. W(0x40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 40 */
  194. W(0x50, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 50 */
  195. W(0x60, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 60 */
  196. W(0x70, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1) , /* 70 */
  197. W(0x80, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* 80 */
  198. W(0x90, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* 90 */
  199. W(0xa0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1) | /* a0 */
  200. W(0xb0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* b0 */
  201. W(0xc0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* c0 */
  202. W(0xd0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) , /* d0 */
  203. W(0xe0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) | /* e0 */
  204. W(0xf0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1) /* f0 */
  205. /* ---------------------------------------------- */
  206. /* 0 1 2 3 4 5 6 7 8 9 a b c d e f */
  207. };
  208. #undef W
  209. /*
  210. * opcodes we may need to refine support for:
  211. *
  212. * 0f - 2-byte instructions: For many of these instructions, the validity
  213. * depends on the prefix and/or the reg field. On such instructions, we
  214. * just consider the opcode combination valid if it corresponds to any
  215. * valid instruction.
  216. *
  217. * 8f - Group 1 - only reg = 0 is OK
  218. * c6-c7 - Group 11 - only reg = 0 is OK
  219. * d9-df - fpu insns with some illegal encodings
  220. * f2, f3 - repnz, repz prefixes. These are also the first byte for
  221. * certain floating-point instructions, such as addsd.
  222. *
  223. * fe - Group 4 - only reg = 0 or 1 is OK
  224. * ff - Group 5 - only reg = 0-6 is OK
  225. *
  226. * others -- Do we need to support these?
  227. *
  228. * 0f - (floating-point?) prefetch instructions
  229. * 07, 17, 1f - pop es, pop ss, pop ds
  230. * 26, 2e, 36, 3e - es:, cs:, ss:, ds: segment prefixes --
  231. * but 64 and 65 (fs: and gs:) seem to be used, so we support them
  232. * 67 - addr16 prefix
  233. * ce - into
  234. * f0 - lock prefix
  235. */
  236. /*
  237. * TODO:
  238. * - Where necessary, examine the modrm byte and allow only valid instructions
  239. * in the different Groups and fpu instructions.
  240. */
  241. static bool is_prefix_bad(struct insn *insn)
  242. {
  243. insn_byte_t p;
  244. int i;
  245. for_each_insn_prefix(insn, i, p) {
  246. insn_attr_t attr;
  247. attr = inat_get_opcode_attribute(p);
  248. switch (attr) {
  249. case INAT_MAKE_PREFIX(INAT_PFX_ES):
  250. case INAT_MAKE_PREFIX(INAT_PFX_CS):
  251. case INAT_MAKE_PREFIX(INAT_PFX_DS):
  252. case INAT_MAKE_PREFIX(INAT_PFX_SS):
  253. case INAT_MAKE_PREFIX(INAT_PFX_LOCK):
  254. return true;
  255. }
  256. }
  257. return false;
  258. }
  259. static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64)
  260. {
  261. enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32;
  262. u32 volatile *good_insns;
  263. int ret;
  264. ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m);
  265. if (ret < 0)
  266. return -ENOEXEC;
  267. if (is_prefix_bad(insn))
  268. return -ENOTSUPP;
  269. /* We should not singlestep on the exception masking instructions */
  270. if (insn_masking_exception(insn))
  271. return -ENOTSUPP;
  272. if (x86_64)
  273. good_insns = good_insns_64;
  274. else
  275. good_insns = good_insns_32;
  276. if (test_bit(OPCODE1(insn), (unsigned long *)good_insns))
  277. return 0;
  278. if (insn->opcode.nbytes == 2) {
  279. if (test_bit(OPCODE2(insn), (unsigned long *)good_2byte_insns))
  280. return 0;
  281. }
  282. return -ENOTSUPP;
  283. }
  284. #ifdef CONFIG_X86_64
  285. /*
  286. * If arch_uprobe->insn doesn't use rip-relative addressing, return
  287. * immediately. Otherwise, rewrite the instruction so that it accesses
  288. * its memory operand indirectly through a scratch register. Set
  289. * defparam->fixups accordingly. (The contents of the scratch register
  290. * will be saved before we single-step the modified instruction,
  291. * and restored afterward).
  292. *
  293. * We do this because a rip-relative instruction can access only a
  294. * relatively small area (+/- 2 GB from the instruction), and the XOL
  295. * area typically lies beyond that area. At least for instructions
  296. * that store to memory, we can't execute the original instruction
  297. * and "fix things up" later, because the misdirected store could be
  298. * disastrous.
  299. *
  300. * Some useful facts about rip-relative instructions:
  301. *
  302. * - There's always a modrm byte with bit layout "00 reg 101".
  303. * - There's never a SIB byte.
  304. * - The displacement is always 4 bytes.
  305. * - REX.B=1 bit in REX prefix, which normally extends r/m field,
  306. * has no effect on rip-relative mode. It doesn't make modrm byte
  307. * with r/m=101 refer to register 1101 = R13.
  308. */
  309. static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
  310. {
  311. u8 *cursor;
  312. u8 reg;
  313. u8 reg2;
  314. if (!insn_rip_relative(insn))
  315. return;
  316. /*
  317. * insn_rip_relative() would have decoded rex_prefix, vex_prefix, modrm.
  318. * Clear REX.b bit (extension of MODRM.rm field):
  319. * we want to encode low numbered reg, not r8+.
  320. */
  321. if (insn->rex_prefix.nbytes) {
  322. cursor = auprobe->insn + insn_offset_rex_prefix(insn);
  323. /* REX byte has 0100wrxb layout, clearing REX.b bit */
  324. *cursor &= 0xfe;
  325. }
  326. /*
  327. * Similar treatment for VEX3/EVEX prefix.
  328. * TODO: add XOP treatment when insn decoder supports them
  329. */
  330. if (insn->vex_prefix.nbytes >= 3) {
  331. /*
  332. * vex2: c5 rvvvvLpp (has no b bit)
  333. * vex3/xop: c4/8f rxbmmmmm wvvvvLpp
  334. * evex: 62 rxbR00mm wvvvv1pp zllBVaaa
  335. * Setting VEX3.b (setting because it has inverted meaning).
  336. * Setting EVEX.x since (in non-SIB encoding) EVEX.x
  337. * is the 4th bit of MODRM.rm, and needs the same treatment.
  338. * For VEX3-encoded insns, VEX3.x value has no effect in
  339. * non-SIB encoding, the change is superfluous but harmless.
  340. */
  341. cursor = auprobe->insn + insn_offset_vex_prefix(insn) + 1;
  342. *cursor |= 0x60;
  343. }
  344. /*
  345. * Convert from rip-relative addressing to register-relative addressing
  346. * via a scratch register.
  347. *
  348. * This is tricky since there are insns with modrm byte
  349. * which also use registers not encoded in modrm byte:
  350. * [i]div/[i]mul: implicitly use dx:ax
  351. * shift ops: implicitly use cx
  352. * cmpxchg: implicitly uses ax
  353. * cmpxchg8/16b: implicitly uses dx:ax and bx:cx
  354. * Encoding: 0f c7/1 modrm
  355. * The code below thinks that reg=1 (cx), chooses si as scratch.
  356. * mulx: implicitly uses dx: mulx r/m,r1,r2 does r1:r2 = dx * r/m.
  357. * First appeared in Haswell (BMI2 insn). It is vex-encoded.
  358. * Example where none of bx,cx,dx can be used as scratch reg:
  359. * c4 e2 63 f6 0d disp32 mulx disp32(%rip),%ebx,%ecx
  360. * [v]pcmpistri: implicitly uses cx, xmm0
  361. * [v]pcmpistrm: implicitly uses xmm0
  362. * [v]pcmpestri: implicitly uses ax, dx, cx, xmm0
  363. * [v]pcmpestrm: implicitly uses ax, dx, xmm0
  364. * Evil SSE4.2 string comparison ops from hell.
  365. * maskmovq/[v]maskmovdqu: implicitly uses (ds:rdi) as destination.
  366. * Encoding: 0f f7 modrm, 66 0f f7 modrm, vex-encoded: c5 f9 f7 modrm.
  367. * Store op1, byte-masked by op2 msb's in each byte, to (ds:rdi).
  368. * AMD says it has no 3-operand form (vex.vvvv must be 1111)
  369. * and that it can have only register operands, not mem
  370. * (its modrm byte must have mode=11).
  371. * If these restrictions will ever be lifted,
  372. * we'll need code to prevent selection of di as scratch reg!
  373. *
  374. * Summary: I don't know any insns with modrm byte which
  375. * use SI register implicitly. DI register is used only
  376. * by one insn (maskmovq) and BX register is used
  377. * only by one too (cmpxchg8b).
  378. * BP is stack-segment based (may be a problem?).
  379. * AX, DX, CX are off-limits (many implicit users).
  380. * SP is unusable (it's stack pointer - think about "pop mem";
  381. * also, rsp+disp32 needs sib encoding -> insn length change).
  382. */
  383. reg = MODRM_REG(insn); /* Fetch modrm.reg */
  384. reg2 = 0xff; /* Fetch vex.vvvv */
  385. if (insn->vex_prefix.nbytes)
  386. reg2 = insn->vex_prefix.bytes[2];
  387. /*
  388. * TODO: add XOP vvvv reading.
  389. *
  390. * vex.vvvv field is in bits 6-3, bits are inverted.
  391. * But in 32-bit mode, high-order bit may be ignored.
  392. * Therefore, let's consider only 3 low-order bits.
  393. */
  394. reg2 = ((reg2 >> 3) & 0x7) ^ 0x7;
  395. /*
  396. * Register numbering is ax,cx,dx,bx, sp,bp,si,di, r8..r15.
  397. *
  398. * Choose scratch reg. Order is important: must not select bx
  399. * if we can use si (cmpxchg8b case!)
  400. */
  401. if (reg != 6 && reg2 != 6) {
  402. reg2 = 6;
  403. auprobe->defparam.fixups |= UPROBE_FIX_RIP_SI;
  404. } else if (reg != 7 && reg2 != 7) {
  405. reg2 = 7;
  406. auprobe->defparam.fixups |= UPROBE_FIX_RIP_DI;
  407. /* TODO (paranoia): force maskmovq to not use di */
  408. } else {
  409. reg2 = 3;
  410. auprobe->defparam.fixups |= UPROBE_FIX_RIP_BX;
  411. }
  412. /*
  413. * Point cursor at the modrm byte. The next 4 bytes are the
  414. * displacement. Beyond the displacement, for some instructions,
  415. * is the immediate operand.
  416. */
  417. cursor = auprobe->insn + insn_offset_modrm(insn);
  418. /*
  419. * Change modrm from "00 reg 101" to "10 reg reg2". Example:
  420. * 89 05 disp32 mov %eax,disp32(%rip) becomes
  421. * 89 86 disp32 mov %eax,disp32(%rsi)
  422. */
  423. *cursor = 0x80 | (reg << 3) | reg2;
  424. }
  425. static inline unsigned long *
  426. scratch_reg(struct arch_uprobe *auprobe, struct pt_regs *regs)
  427. {
  428. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_SI)
  429. return &regs->si;
  430. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_DI)
  431. return &regs->di;
  432. return &regs->bx;
  433. }
  434. /*
  435. * If we're emulating a rip-relative instruction, save the contents
  436. * of the scratch register and store the target address in that register.
  437. */
  438. static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  439. {
  440. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
  441. struct uprobe_task *utask = current->utask;
  442. unsigned long *sr = scratch_reg(auprobe, regs);
  443. utask->autask.saved_scratch_register = *sr;
  444. *sr = utask->vaddr + auprobe->defparam.ilen;
  445. }
  446. }
  447. static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  448. {
  449. if (auprobe->defparam.fixups & UPROBE_FIX_RIP_MASK) {
  450. struct uprobe_task *utask = current->utask;
  451. unsigned long *sr = scratch_reg(auprobe, regs);
  452. *sr = utask->autask.saved_scratch_register;
  453. }
  454. }
  455. #else /* 32-bit: */
  456. /*
  457. * No RIP-relative addressing on 32-bit
  458. */
  459. static void riprel_analyze(struct arch_uprobe *auprobe, struct insn *insn)
  460. {
  461. }
  462. static void riprel_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  463. {
  464. }
  465. static void riprel_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  466. {
  467. }
  468. #endif /* CONFIG_X86_64 */
  469. struct uprobe_xol_ops {
  470. bool (*emulate)(struct arch_uprobe *, struct pt_regs *);
  471. int (*pre_xol)(struct arch_uprobe *, struct pt_regs *);
  472. int (*post_xol)(struct arch_uprobe *, struct pt_regs *);
  473. void (*abort)(struct arch_uprobe *, struct pt_regs *);
  474. };
  475. static inline int sizeof_long(struct pt_regs *regs)
  476. {
  477. /*
  478. * Check registers for mode as in_xxx_syscall() does not apply here.
  479. */
  480. return user_64bit_mode(regs) ? 8 : 4;
  481. }
  482. static int default_pre_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  483. {
  484. riprel_pre_xol(auprobe, regs);
  485. return 0;
  486. }
  487. static int emulate_push_stack(struct pt_regs *regs, unsigned long val)
  488. {
  489. unsigned long new_sp = regs->sp - sizeof_long(regs);
  490. if (copy_to_user((void __user *)new_sp, &val, sizeof_long(regs)))
  491. return -EFAULT;
  492. regs->sp = new_sp;
  493. return 0;
  494. }
  495. /*
  496. * We have to fix things up as follows:
  497. *
  498. * Typically, the new ip is relative to the copied instruction. We need
  499. * to make it relative to the original instruction (FIX_IP). Exceptions
  500. * are return instructions and absolute or indirect jump or call instructions.
  501. *
  502. * If the single-stepped instruction was a call, the return address that
  503. * is atop the stack is the address following the copied instruction. We
  504. * need to make it the address following the original instruction (FIX_CALL).
  505. *
  506. * If the original instruction was a rip-relative instruction such as
  507. * "movl %edx,0xnnnn(%rip)", we have instead executed an equivalent
  508. * instruction using a scratch register -- e.g., "movl %edx,0xnnnn(%rsi)".
  509. * We need to restore the contents of the scratch register
  510. * (FIX_RIP_reg).
  511. */
  512. static int default_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  513. {
  514. struct uprobe_task *utask = current->utask;
  515. riprel_post_xol(auprobe, regs);
  516. if (auprobe->defparam.fixups & UPROBE_FIX_IP) {
  517. long correction = utask->vaddr - utask->xol_vaddr;
  518. regs->ip += correction;
  519. } else if (auprobe->defparam.fixups & UPROBE_FIX_CALL) {
  520. regs->sp += sizeof_long(regs); /* Pop incorrect return address */
  521. if (emulate_push_stack(regs, utask->vaddr + auprobe->defparam.ilen))
  522. return -ERESTART;
  523. }
  524. /* popf; tell the caller to not touch TF */
  525. if (auprobe->defparam.fixups & UPROBE_FIX_SETF)
  526. utask->autask.saved_tf = true;
  527. return 0;
  528. }
  529. static void default_abort_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  530. {
  531. riprel_post_xol(auprobe, regs);
  532. }
  533. static const struct uprobe_xol_ops default_xol_ops = {
  534. .pre_xol = default_pre_xol_op,
  535. .post_xol = default_post_xol_op,
  536. .abort = default_abort_op,
  537. };
  538. static bool branch_is_call(struct arch_uprobe *auprobe)
  539. {
  540. return auprobe->branch.opc1 == 0xe8;
  541. }
  542. #define CASE_COND \
  543. COND(70, 71, XF(OF)) \
  544. COND(72, 73, XF(CF)) \
  545. COND(74, 75, XF(ZF)) \
  546. COND(78, 79, XF(SF)) \
  547. COND(7a, 7b, XF(PF)) \
  548. COND(76, 77, XF(CF) || XF(ZF)) \
  549. COND(7c, 7d, XF(SF) != XF(OF)) \
  550. COND(7e, 7f, XF(ZF) || XF(SF) != XF(OF))
  551. #define COND(op_y, op_n, expr) \
  552. case 0x ## op_y: DO((expr) != 0) \
  553. case 0x ## op_n: DO((expr) == 0)
  554. #define XF(xf) (!!(flags & X86_EFLAGS_ ## xf))
  555. static bool is_cond_jmp_opcode(u8 opcode)
  556. {
  557. switch (opcode) {
  558. #define DO(expr) \
  559. return true;
  560. CASE_COND
  561. #undef DO
  562. default:
  563. return false;
  564. }
  565. }
  566. static bool check_jmp_cond(struct arch_uprobe *auprobe, struct pt_regs *regs)
  567. {
  568. unsigned long flags = regs->flags;
  569. switch (auprobe->branch.opc1) {
  570. #define DO(expr) \
  571. return expr;
  572. CASE_COND
  573. #undef DO
  574. default: /* not a conditional jmp */
  575. return true;
  576. }
  577. }
  578. #undef XF
  579. #undef COND
  580. #undef CASE_COND
  581. static bool branch_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  582. {
  583. unsigned long new_ip = regs->ip += auprobe->branch.ilen;
  584. unsigned long offs = (long)auprobe->branch.offs;
  585. if (branch_is_call(auprobe)) {
  586. /*
  587. * If it fails we execute this (mangled, see the comment in
  588. * branch_clear_offset) insn out-of-line. In the likely case
  589. * this should trigger the trap, and the probed application
  590. * should die or restart the same insn after it handles the
  591. * signal, arch_uprobe_post_xol() won't be even called.
  592. *
  593. * But there is corner case, see the comment in ->post_xol().
  594. */
  595. if (emulate_push_stack(regs, new_ip))
  596. return false;
  597. } else if (!check_jmp_cond(auprobe, regs)) {
  598. offs = 0;
  599. }
  600. regs->ip = new_ip + offs;
  601. return true;
  602. }
  603. static bool push_emulate_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  604. {
  605. unsigned long *src_ptr = (void *)regs + auprobe->push.reg_offset;
  606. if (emulate_push_stack(regs, *src_ptr))
  607. return false;
  608. regs->ip += auprobe->push.ilen;
  609. return true;
  610. }
  611. static int branch_post_xol_op(struct arch_uprobe *auprobe, struct pt_regs *regs)
  612. {
  613. BUG_ON(!branch_is_call(auprobe));
  614. /*
  615. * We can only get here if branch_emulate_op() failed to push the ret
  616. * address _and_ another thread expanded our stack before the (mangled)
  617. * "call" insn was executed out-of-line. Just restore ->sp and restart.
  618. * We could also restore ->ip and try to call branch_emulate_op() again.
  619. */
  620. regs->sp += sizeof_long(regs);
  621. return -ERESTART;
  622. }
  623. static void branch_clear_offset(struct arch_uprobe *auprobe, struct insn *insn)
  624. {
  625. /*
  626. * Turn this insn into "call 1f; 1:", this is what we will execute
  627. * out-of-line if ->emulate() fails. We only need this to generate
  628. * a trap, so that the probed task receives the correct signal with
  629. * the properly filled siginfo.
  630. *
  631. * But see the comment in ->post_xol(), in the unlikely case it can
  632. * succeed. So we need to ensure that the new ->ip can not fall into
  633. * the non-canonical area and trigger #GP.
  634. *
  635. * We could turn it into (say) "pushf", but then we would need to
  636. * divorce ->insn[] and ->ixol[]. We need to preserve the 1st byte
  637. * of ->insn[] for set_orig_insn().
  638. */
  639. memset(auprobe->insn + insn_offset_immediate(insn),
  640. 0, insn->immediate.nbytes);
  641. }
  642. static const struct uprobe_xol_ops branch_xol_ops = {
  643. .emulate = branch_emulate_op,
  644. .post_xol = branch_post_xol_op,
  645. };
  646. static const struct uprobe_xol_ops push_xol_ops = {
  647. .emulate = push_emulate_op,
  648. };
  649. /* Returns -ENOSYS if branch_xol_ops doesn't handle this insn */
  650. static int branch_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
  651. {
  652. u8 opc1 = OPCODE1(insn);
  653. insn_byte_t p;
  654. int i;
  655. switch (opc1) {
  656. case 0xeb: /* jmp 8 */
  657. case 0xe9: /* jmp 32 */
  658. break;
  659. case 0x90: /* prefix* + nop; same as jmp with .offs = 0 */
  660. goto setup;
  661. case 0xe8: /* call relative */
  662. branch_clear_offset(auprobe, insn);
  663. break;
  664. case 0x0f:
  665. if (insn->opcode.nbytes != 2)
  666. return -ENOSYS;
  667. /*
  668. * If it is a "near" conditional jmp, OPCODE2() - 0x10 matches
  669. * OPCODE1() of the "short" jmp which checks the same condition.
  670. */
  671. opc1 = OPCODE2(insn) - 0x10;
  672. fallthrough;
  673. default:
  674. if (!is_cond_jmp_opcode(opc1))
  675. return -ENOSYS;
  676. }
  677. /*
  678. * 16-bit overrides such as CALLW (66 e8 nn nn) are not supported.
  679. * Intel and AMD behavior differ in 64-bit mode: Intel ignores 66 prefix.
  680. * No one uses these insns, reject any branch insns with such prefix.
  681. */
  682. for_each_insn_prefix(insn, i, p) {
  683. if (p == 0x66)
  684. return -ENOTSUPP;
  685. }
  686. setup:
  687. auprobe->branch.opc1 = opc1;
  688. auprobe->branch.ilen = insn->length;
  689. auprobe->branch.offs = insn->immediate.value;
  690. auprobe->ops = &branch_xol_ops;
  691. return 0;
  692. }
  693. /* Returns -ENOSYS if push_xol_ops doesn't handle this insn */
  694. static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn)
  695. {
  696. u8 opc1 = OPCODE1(insn), reg_offset = 0;
  697. if (opc1 < 0x50 || opc1 > 0x57)
  698. return -ENOSYS;
  699. if (insn->length > 2)
  700. return -ENOSYS;
  701. if (insn->length == 2) {
  702. /* only support rex_prefix 0x41 (x64 only) */
  703. #ifdef CONFIG_X86_64
  704. if (insn->rex_prefix.nbytes != 1 ||
  705. insn->rex_prefix.bytes[0] != 0x41)
  706. return -ENOSYS;
  707. switch (opc1) {
  708. case 0x50:
  709. reg_offset = offsetof(struct pt_regs, r8);
  710. break;
  711. case 0x51:
  712. reg_offset = offsetof(struct pt_regs, r9);
  713. break;
  714. case 0x52:
  715. reg_offset = offsetof(struct pt_regs, r10);
  716. break;
  717. case 0x53:
  718. reg_offset = offsetof(struct pt_regs, r11);
  719. break;
  720. case 0x54:
  721. reg_offset = offsetof(struct pt_regs, r12);
  722. break;
  723. case 0x55:
  724. reg_offset = offsetof(struct pt_regs, r13);
  725. break;
  726. case 0x56:
  727. reg_offset = offsetof(struct pt_regs, r14);
  728. break;
  729. case 0x57:
  730. reg_offset = offsetof(struct pt_regs, r15);
  731. break;
  732. }
  733. #else
  734. return -ENOSYS;
  735. #endif
  736. } else {
  737. switch (opc1) {
  738. case 0x50:
  739. reg_offset = offsetof(struct pt_regs, ax);
  740. break;
  741. case 0x51:
  742. reg_offset = offsetof(struct pt_regs, cx);
  743. break;
  744. case 0x52:
  745. reg_offset = offsetof(struct pt_regs, dx);
  746. break;
  747. case 0x53:
  748. reg_offset = offsetof(struct pt_regs, bx);
  749. break;
  750. case 0x54:
  751. reg_offset = offsetof(struct pt_regs, sp);
  752. break;
  753. case 0x55:
  754. reg_offset = offsetof(struct pt_regs, bp);
  755. break;
  756. case 0x56:
  757. reg_offset = offsetof(struct pt_regs, si);
  758. break;
  759. case 0x57:
  760. reg_offset = offsetof(struct pt_regs, di);
  761. break;
  762. }
  763. }
  764. auprobe->push.reg_offset = reg_offset;
  765. auprobe->push.ilen = insn->length;
  766. auprobe->ops = &push_xol_ops;
  767. return 0;
  768. }
  769. /**
  770. * arch_uprobe_analyze_insn - instruction analysis including validity and fixups.
  771. * @auprobe: the probepoint information.
  772. * @mm: the probed address space.
  773. * @addr: virtual address at which to install the probepoint
  774. * Return 0 on success or a -ve number on error.
  775. */
  776. int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr)
  777. {
  778. struct insn insn;
  779. u8 fix_ip_or_call = UPROBE_FIX_IP;
  780. int ret;
  781. ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm));
  782. if (ret)
  783. return ret;
  784. ret = branch_setup_xol_ops(auprobe, &insn);
  785. if (ret != -ENOSYS)
  786. return ret;
  787. ret = push_setup_xol_ops(auprobe, &insn);
  788. if (ret != -ENOSYS)
  789. return ret;
  790. /*
  791. * Figure out which fixups default_post_xol_op() will need to perform,
  792. * and annotate defparam->fixups accordingly.
  793. */
  794. switch (OPCODE1(&insn)) {
  795. case 0x9d: /* popf */
  796. auprobe->defparam.fixups |= UPROBE_FIX_SETF;
  797. break;
  798. case 0xc3: /* ret or lret -- ip is correct */
  799. case 0xcb:
  800. case 0xc2:
  801. case 0xca:
  802. case 0xea: /* jmp absolute -- ip is correct */
  803. fix_ip_or_call = 0;
  804. break;
  805. case 0x9a: /* call absolute - Fix return addr, not ip */
  806. fix_ip_or_call = UPROBE_FIX_CALL;
  807. break;
  808. case 0xff:
  809. switch (MODRM_REG(&insn)) {
  810. case 2: case 3: /* call or lcall, indirect */
  811. fix_ip_or_call = UPROBE_FIX_CALL;
  812. break;
  813. case 4: case 5: /* jmp or ljmp, indirect */
  814. fix_ip_or_call = 0;
  815. break;
  816. }
  817. fallthrough;
  818. default:
  819. riprel_analyze(auprobe, &insn);
  820. }
  821. auprobe->defparam.ilen = insn.length;
  822. auprobe->defparam.fixups |= fix_ip_or_call;
  823. auprobe->ops = &default_xol_ops;
  824. return 0;
  825. }
  826. /*
  827. * arch_uprobe_pre_xol - prepare to execute out of line.
  828. * @auprobe: the probepoint information.
  829. * @regs: reflects the saved user state of current task.
  830. */
  831. int arch_uprobe_pre_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  832. {
  833. struct uprobe_task *utask = current->utask;
  834. if (auprobe->ops->pre_xol) {
  835. int err = auprobe->ops->pre_xol(auprobe, regs);
  836. if (err)
  837. return err;
  838. }
  839. regs->ip = utask->xol_vaddr;
  840. utask->autask.saved_trap_nr = current->thread.trap_nr;
  841. current->thread.trap_nr = UPROBE_TRAP_NR;
  842. utask->autask.saved_tf = !!(regs->flags & X86_EFLAGS_TF);
  843. regs->flags |= X86_EFLAGS_TF;
  844. if (test_tsk_thread_flag(current, TIF_BLOCKSTEP))
  845. set_task_blockstep(current, false);
  846. return 0;
  847. }
  848. /*
  849. * If xol insn itself traps and generates a signal(Say,
  850. * SIGILL/SIGSEGV/etc), then detect the case where a singlestepped
  851. * instruction jumps back to its own address. It is assumed that anything
  852. * like do_page_fault/do_trap/etc sets thread.trap_nr != -1.
  853. *
  854. * arch_uprobe_pre_xol/arch_uprobe_post_xol save/restore thread.trap_nr,
  855. * arch_uprobe_xol_was_trapped() simply checks that ->trap_nr is not equal to
  856. * UPROBE_TRAP_NR == -1 set by arch_uprobe_pre_xol().
  857. */
  858. bool arch_uprobe_xol_was_trapped(struct task_struct *t)
  859. {
  860. if (t->thread.trap_nr != UPROBE_TRAP_NR)
  861. return true;
  862. return false;
  863. }
  864. /*
  865. * Called after single-stepping. To avoid the SMP problems that can
  866. * occur when we temporarily put back the original opcode to
  867. * single-step, we single-stepped a copy of the instruction.
  868. *
  869. * This function prepares to resume execution after the single-step.
  870. */
  871. int arch_uprobe_post_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  872. {
  873. struct uprobe_task *utask = current->utask;
  874. bool send_sigtrap = utask->autask.saved_tf;
  875. int err = 0;
  876. WARN_ON_ONCE(current->thread.trap_nr != UPROBE_TRAP_NR);
  877. current->thread.trap_nr = utask->autask.saved_trap_nr;
  878. if (auprobe->ops->post_xol) {
  879. err = auprobe->ops->post_xol(auprobe, regs);
  880. if (err) {
  881. /*
  882. * Restore ->ip for restart or post mortem analysis.
  883. * ->post_xol() must not return -ERESTART unless this
  884. * is really possible.
  885. */
  886. regs->ip = utask->vaddr;
  887. if (err == -ERESTART)
  888. err = 0;
  889. send_sigtrap = false;
  890. }
  891. }
  892. /*
  893. * arch_uprobe_pre_xol() doesn't save the state of TIF_BLOCKSTEP
  894. * so we can get an extra SIGTRAP if we do not clear TF. We need
  895. * to examine the opcode to make it right.
  896. */
  897. if (send_sigtrap)
  898. send_sig(SIGTRAP, current, 0);
  899. if (!utask->autask.saved_tf)
  900. regs->flags &= ~X86_EFLAGS_TF;
  901. return err;
  902. }
  903. /* callback routine for handling exceptions. */
  904. int arch_uprobe_exception_notify(struct notifier_block *self, unsigned long val, void *data)
  905. {
  906. struct die_args *args = data;
  907. struct pt_regs *regs = args->regs;
  908. int ret = NOTIFY_DONE;
  909. /* We are only interested in userspace traps */
  910. if (regs && !user_mode(regs))
  911. return NOTIFY_DONE;
  912. switch (val) {
  913. case DIE_INT3:
  914. if (uprobe_pre_sstep_notifier(regs))
  915. ret = NOTIFY_STOP;
  916. break;
  917. case DIE_DEBUG:
  918. if (uprobe_post_sstep_notifier(regs))
  919. ret = NOTIFY_STOP;
  920. break;
  921. default:
  922. break;
  923. }
  924. return ret;
  925. }
  926. /*
  927. * This function gets called when XOL instruction either gets trapped or
  928. * the thread has a fatal signal. Reset the instruction pointer to its
  929. * probed address for the potential restart or for post mortem analysis.
  930. */
  931. void arch_uprobe_abort_xol(struct arch_uprobe *auprobe, struct pt_regs *regs)
  932. {
  933. struct uprobe_task *utask = current->utask;
  934. if (auprobe->ops->abort)
  935. auprobe->ops->abort(auprobe, regs);
  936. current->thread.trap_nr = utask->autask.saved_trap_nr;
  937. regs->ip = utask->vaddr;
  938. /* clear TF if it was set by us in arch_uprobe_pre_xol() */
  939. if (!utask->autask.saved_tf)
  940. regs->flags &= ~X86_EFLAGS_TF;
  941. }
  942. static bool __skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
  943. {
  944. if (auprobe->ops->emulate)
  945. return auprobe->ops->emulate(auprobe, regs);
  946. return false;
  947. }
  948. bool arch_uprobe_skip_sstep(struct arch_uprobe *auprobe, struct pt_regs *regs)
  949. {
  950. bool ret = __skip_sstep(auprobe, regs);
  951. if (ret && (regs->flags & X86_EFLAGS_TF))
  952. send_sig(SIGTRAP, current, 0);
  953. return ret;
  954. }
  955. unsigned long
  956. arch_uretprobe_hijack_return_addr(unsigned long trampoline_vaddr, struct pt_regs *regs)
  957. {
  958. int rasize = sizeof_long(regs), nleft;
  959. unsigned long orig_ret_vaddr = 0; /* clear high bits for 32-bit apps */
  960. if (copy_from_user(&orig_ret_vaddr, (void __user *)regs->sp, rasize))
  961. return -1;
  962. /* check whether address has been already hijacked */
  963. if (orig_ret_vaddr == trampoline_vaddr)
  964. return orig_ret_vaddr;
  965. nleft = copy_to_user((void __user *)regs->sp, &trampoline_vaddr, rasize);
  966. if (likely(!nleft))
  967. return orig_ret_vaddr;
  968. if (nleft != rasize) {
  969. pr_err("return address clobbered: pid=%d, %%sp=%#lx, %%ip=%#lx\n",
  970. current->pid, regs->sp, regs->ip);
  971. force_sig(SIGSEGV);
  972. }
  973. return -1;
  974. }
  975. bool arch_uretprobe_is_alive(struct return_instance *ret, enum rp_check ctx,
  976. struct pt_regs *regs)
  977. {
  978. if (ctx == RP_CHECK_CALL) /* sp was just decremented by "call" insn */
  979. return regs->sp < ret->stack;
  980. else
  981. return regs->sp <= ret->stack;
  982. }