ftrace.c 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310
  1. /*
  2. * Dynamic function tracing support.
  3. *
  4. * Copyright (C) 2008 Abhishek Sagar <[email protected]>
  5. * Copyright (C) 2010 Rabin Vincent <[email protected]>
  6. *
  7. * For licencing details, see COPYING.
  8. *
  9. * Defines low-level handling of mcount calls when the kernel
  10. * is compiled with the -pg flag. When using dynamic ftrace, the
  11. * mcount call-sites get patched with NOP till they are enabled.
  12. * All code mutation routines here are called under stop_machine().
  13. */
  14. #include <linux/ftrace.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/module.h>
  17. #include <linux/stop_machine.h>
  18. #include <asm/cacheflush.h>
  19. #include <asm/opcodes.h>
  20. #include <asm/ftrace.h>
  21. #include <asm/insn.h>
  22. #include <asm/set_memory.h>
  23. #include <asm/stacktrace.h>
  24. #include <asm/patch.h>
  25. /*
  26. * The compiler emitted profiling hook consists of
  27. *
  28. * PUSH {LR}
  29. * BL __gnu_mcount_nc
  30. *
  31. * To turn this combined sequence into a NOP, we need to restore the value of
  32. * SP before the PUSH. Let's use an ADD rather than a POP into LR, as LR is not
  33. * modified anyway, and reloading LR from memory is highly likely to be less
  34. * efficient.
  35. */
  36. #ifdef CONFIG_THUMB2_KERNEL
  37. #define NOP 0xf10d0d04 /* add.w sp, sp, #4 */
  38. #else
  39. #define NOP 0xe28dd004 /* add sp, sp, #4 */
  40. #endif
  41. #ifdef CONFIG_DYNAMIC_FTRACE
  42. static int __ftrace_modify_code(void *data)
  43. {
  44. int *command = data;
  45. ftrace_modify_all_code(*command);
  46. return 0;
  47. }
  48. void arch_ftrace_update_code(int command)
  49. {
  50. stop_machine(__ftrace_modify_code, &command, NULL);
  51. }
  52. static unsigned long ftrace_nop_replace(struct dyn_ftrace *rec)
  53. {
  54. return NOP;
  55. }
  56. void ftrace_caller_from_init(void);
  57. void ftrace_regs_caller_from_init(void);
  58. static unsigned long __ref adjust_address(struct dyn_ftrace *rec,
  59. unsigned long addr)
  60. {
  61. if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE) ||
  62. system_state >= SYSTEM_FREEING_INITMEM ||
  63. likely(!is_kernel_inittext(rec->ip)))
  64. return addr;
  65. if (!IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS) ||
  66. addr == (unsigned long)&ftrace_caller)
  67. return (unsigned long)&ftrace_caller_from_init;
  68. return (unsigned long)&ftrace_regs_caller_from_init;
  69. }
  70. void ftrace_arch_code_modify_prepare(void)
  71. {
  72. }
  73. void ftrace_arch_code_modify_post_process(void)
  74. {
  75. /* Make sure any TLB misses during machine stop are cleared. */
  76. flush_tlb_all();
  77. }
  78. static unsigned long ftrace_call_replace(unsigned long pc, unsigned long addr,
  79. bool warn)
  80. {
  81. return arm_gen_branch_link(pc, addr, warn);
  82. }
  83. static int ftrace_modify_code(unsigned long pc, unsigned long old,
  84. unsigned long new, bool validate)
  85. {
  86. unsigned long replaced;
  87. if (IS_ENABLED(CONFIG_THUMB2_KERNEL))
  88. old = __opcode_to_mem_thumb32(old);
  89. else
  90. old = __opcode_to_mem_arm(old);
  91. if (validate) {
  92. if (copy_from_kernel_nofault(&replaced, (void *)pc,
  93. MCOUNT_INSN_SIZE))
  94. return -EFAULT;
  95. if (replaced != old)
  96. return -EINVAL;
  97. }
  98. __patch_text((void *)pc, new);
  99. return 0;
  100. }
  101. int ftrace_update_ftrace_func(ftrace_func_t func)
  102. {
  103. unsigned long pc;
  104. unsigned long new;
  105. int ret;
  106. pc = (unsigned long)&ftrace_call;
  107. new = ftrace_call_replace(pc, (unsigned long)func, true);
  108. ret = ftrace_modify_code(pc, 0, new, false);
  109. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  110. if (!ret) {
  111. pc = (unsigned long)&ftrace_regs_call;
  112. new = ftrace_call_replace(pc, (unsigned long)func, true);
  113. ret = ftrace_modify_code(pc, 0, new, false);
  114. }
  115. #endif
  116. return ret;
  117. }
  118. int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
  119. {
  120. unsigned long new, old;
  121. unsigned long ip = rec->ip;
  122. unsigned long aaddr = adjust_address(rec, addr);
  123. struct module *mod = NULL;
  124. #ifdef CONFIG_ARM_MODULE_PLTS
  125. mod = rec->arch.mod;
  126. #endif
  127. old = ftrace_nop_replace(rec);
  128. new = ftrace_call_replace(ip, aaddr, !mod);
  129. #ifdef CONFIG_ARM_MODULE_PLTS
  130. if (!new && mod) {
  131. aaddr = get_module_plt(mod, ip, aaddr);
  132. new = ftrace_call_replace(ip, aaddr, true);
  133. }
  134. #endif
  135. return ftrace_modify_code(rec->ip, old, new, true);
  136. }
  137. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  138. int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
  139. unsigned long addr)
  140. {
  141. unsigned long new, old;
  142. unsigned long ip = rec->ip;
  143. old = ftrace_call_replace(ip, adjust_address(rec, old_addr), true);
  144. new = ftrace_call_replace(ip, adjust_address(rec, addr), true);
  145. return ftrace_modify_code(rec->ip, old, new, true);
  146. }
  147. #endif
  148. int ftrace_make_nop(struct module *mod,
  149. struct dyn_ftrace *rec, unsigned long addr)
  150. {
  151. unsigned long aaddr = adjust_address(rec, addr);
  152. unsigned long ip = rec->ip;
  153. unsigned long old;
  154. unsigned long new;
  155. int ret;
  156. #ifdef CONFIG_ARM_MODULE_PLTS
  157. /* mod is only supplied during module loading */
  158. if (!mod)
  159. mod = rec->arch.mod;
  160. else
  161. rec->arch.mod = mod;
  162. #endif
  163. old = ftrace_call_replace(ip, aaddr,
  164. !IS_ENABLED(CONFIG_ARM_MODULE_PLTS) || !mod);
  165. #ifdef CONFIG_ARM_MODULE_PLTS
  166. if (!old && mod) {
  167. aaddr = get_module_plt(mod, ip, aaddr);
  168. old = ftrace_call_replace(ip, aaddr, true);
  169. }
  170. #endif
  171. new = ftrace_nop_replace(rec);
  172. /*
  173. * Locations in .init.text may call __gnu_mcount_mc via a linker
  174. * emitted veneer if they are too far away from its implementation, and
  175. * so validation may fail spuriously in such cases. Let's work around
  176. * this by omitting those from validation.
  177. */
  178. ret = ftrace_modify_code(ip, old, new, !is_kernel_inittext(ip));
  179. return ret;
  180. }
  181. #endif /* CONFIG_DYNAMIC_FTRACE */
  182. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  183. asmlinkage
  184. void prepare_ftrace_return(unsigned long *parent, unsigned long self_addr,
  185. unsigned long frame_pointer,
  186. unsigned long stack_pointer)
  187. {
  188. unsigned long return_hooker = (unsigned long) &return_to_handler;
  189. unsigned long old;
  190. if (unlikely(atomic_read(&current->tracing_graph_pause)))
  191. return;
  192. if (IS_ENABLED(CONFIG_UNWINDER_FRAME_POINTER)) {
  193. /* FP points one word below parent's top of stack */
  194. frame_pointer += 4;
  195. } else {
  196. struct stackframe frame = {
  197. .fp = frame_pointer,
  198. .sp = stack_pointer,
  199. .lr = self_addr,
  200. .pc = self_addr,
  201. };
  202. if (unwind_frame(&frame) < 0)
  203. return;
  204. if (frame.lr != self_addr)
  205. parent = frame.lr_addr;
  206. frame_pointer = frame.sp;
  207. }
  208. old = *parent;
  209. *parent = return_hooker;
  210. if (function_graph_enter(old, self_addr, frame_pointer, NULL))
  211. *parent = old;
  212. }
  213. #ifdef CONFIG_DYNAMIC_FTRACE
  214. extern unsigned long ftrace_graph_call;
  215. extern unsigned long ftrace_graph_call_old;
  216. extern void ftrace_graph_caller_old(void);
  217. extern unsigned long ftrace_graph_regs_call;
  218. extern void ftrace_graph_regs_caller(void);
  219. static int __ftrace_modify_caller(unsigned long *callsite,
  220. void (*func) (void), bool enable)
  221. {
  222. unsigned long caller_fn = (unsigned long) func;
  223. unsigned long pc = (unsigned long) callsite;
  224. unsigned long branch = arm_gen_branch(pc, caller_fn);
  225. unsigned long nop = arm_gen_nop();
  226. unsigned long old = enable ? nop : branch;
  227. unsigned long new = enable ? branch : nop;
  228. return ftrace_modify_code(pc, old, new, true);
  229. }
  230. static int ftrace_modify_graph_caller(bool enable)
  231. {
  232. int ret;
  233. ret = __ftrace_modify_caller(&ftrace_graph_call,
  234. ftrace_graph_caller,
  235. enable);
  236. #ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
  237. if (!ret)
  238. ret = __ftrace_modify_caller(&ftrace_graph_regs_call,
  239. ftrace_graph_regs_caller,
  240. enable);
  241. #endif
  242. return ret;
  243. }
  244. int ftrace_enable_ftrace_graph_caller(void)
  245. {
  246. return ftrace_modify_graph_caller(true);
  247. }
  248. int ftrace_disable_ftrace_graph_caller(void)
  249. {
  250. return ftrace_modify_graph_caller(false);
  251. }
  252. #endif /* CONFIG_DYNAMIC_FTRACE */
  253. #endif /* CONFIG_FUNCTION_GRAPH_TRACER */