rethook.c 9.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346
  1. // SPDX-License-Identifier: GPL-2.0
  2. #define pr_fmt(fmt) "rethook: " fmt
  3. #include <linux/bug.h>
  4. #include <linux/kallsyms.h>
  5. #include <linux/kprobes.h>
  6. #include <linux/preempt.h>
  7. #include <linux/rethook.h>
  8. #include <linux/slab.h>
  9. #include <linux/sort.h>
  10. /* Return hook list (shadow stack by list) */
  11. /*
  12. * This function is called from delayed_put_task_struct() when a task is
  13. * dead and cleaned up to recycle any kretprobe instances associated with
  14. * this task. These left over instances represent probed functions that
  15. * have been called but will never return.
  16. */
  17. void rethook_flush_task(struct task_struct *tk)
  18. {
  19. struct rethook_node *rhn;
  20. struct llist_node *node;
  21. node = __llist_del_all(&tk->rethooks);
  22. while (node) {
  23. rhn = container_of(node, struct rethook_node, llist);
  24. node = node->next;
  25. preempt_disable();
  26. rethook_recycle(rhn);
  27. preempt_enable();
  28. }
  29. }
  30. static void rethook_free_rcu(struct rcu_head *head)
  31. {
  32. struct rethook *rh = container_of(head, struct rethook, rcu);
  33. struct rethook_node *rhn;
  34. struct freelist_node *node;
  35. int count = 1;
  36. node = rh->pool.head;
  37. while (node) {
  38. rhn = container_of(node, struct rethook_node, freelist);
  39. node = node->next;
  40. kfree(rhn);
  41. count++;
  42. }
  43. /* The rh->ref is the number of pooled node + 1 */
  44. if (refcount_sub_and_test(count, &rh->ref))
  45. kfree(rh);
  46. }
  47. /**
  48. * rethook_stop() - Stop using a rethook.
  49. * @rh: the struct rethook to stop.
  50. *
  51. * Stop using a rethook to prepare for freeing it. If you want to wait for
  52. * all running rethook handler before calling rethook_free(), you need to
  53. * call this first and wait RCU, and call rethook_free().
  54. */
  55. void rethook_stop(struct rethook *rh)
  56. {
  57. rcu_assign_pointer(rh->handler, NULL);
  58. }
  59. /**
  60. * rethook_free() - Free struct rethook.
  61. * @rh: the struct rethook to be freed.
  62. *
  63. * Free the rethook. Before calling this function, user must ensure the
  64. * @rh::data is cleaned if needed (or, the handler can access it after
  65. * calling this function.) This function will set the @rh to be freed
  66. * after all rethook_node are freed (not soon). And the caller must
  67. * not touch @rh after calling this.
  68. */
  69. void rethook_free(struct rethook *rh)
  70. {
  71. rethook_stop(rh);
  72. call_rcu(&rh->rcu, rethook_free_rcu);
  73. }
  74. static inline rethook_handler_t rethook_get_handler(struct rethook *rh)
  75. {
  76. return (rethook_handler_t)rcu_dereference_check(rh->handler,
  77. rcu_read_lock_any_held());
  78. }
  79. /**
  80. * rethook_alloc() - Allocate struct rethook.
  81. * @data: a data to pass the @handler when hooking the return.
  82. * @handler: the return hook callback function.
  83. *
  84. * Allocate and initialize a new rethook with @data and @handler.
  85. * Return NULL if memory allocation fails or @handler is NULL.
  86. * Note that @handler == NULL means this rethook is going to be freed.
  87. */
  88. struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
  89. {
  90. struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
  91. if (!rh || !handler) {
  92. kfree(rh);
  93. return NULL;
  94. }
  95. rh->data = data;
  96. rcu_assign_pointer(rh->handler, handler);
  97. rh->pool.head = NULL;
  98. refcount_set(&rh->ref, 1);
  99. return rh;
  100. }
  101. /**
  102. * rethook_add_node() - Add a new node to the rethook.
  103. * @rh: the struct rethook.
  104. * @node: the struct rethook_node to be added.
  105. *
  106. * Add @node to @rh. User must allocate @node (as a part of user's
  107. * data structure.) The @node fields are initialized in this function.
  108. */
  109. void rethook_add_node(struct rethook *rh, struct rethook_node *node)
  110. {
  111. node->rethook = rh;
  112. freelist_add(&node->freelist, &rh->pool);
  113. refcount_inc(&rh->ref);
  114. }
  115. static void free_rethook_node_rcu(struct rcu_head *head)
  116. {
  117. struct rethook_node *node = container_of(head, struct rethook_node, rcu);
  118. if (refcount_dec_and_test(&node->rethook->ref))
  119. kfree(node->rethook);
  120. kfree(node);
  121. }
  122. /**
  123. * rethook_recycle() - return the node to rethook.
  124. * @node: The struct rethook_node to be returned.
  125. *
  126. * Return back the @node to @node::rethook. If the @node::rethook is already
  127. * marked as freed, this will free the @node.
  128. */
  129. void rethook_recycle(struct rethook_node *node)
  130. {
  131. rethook_handler_t handler;
  132. handler = rethook_get_handler(node->rethook);
  133. if (likely(handler))
  134. freelist_add(&node->freelist, &node->rethook->pool);
  135. else
  136. call_rcu(&node->rcu, free_rethook_node_rcu);
  137. }
  138. NOKPROBE_SYMBOL(rethook_recycle);
  139. /**
  140. * rethook_try_get() - get an unused rethook node.
  141. * @rh: The struct rethook which pools the nodes.
  142. *
  143. * Get an unused rethook node from @rh. If the node pool is empty, this
  144. * will return NULL. Caller must disable preemption.
  145. */
  146. struct rethook_node *rethook_try_get(struct rethook *rh)
  147. {
  148. rethook_handler_t handler = rethook_get_handler(rh);
  149. struct freelist_node *fn;
  150. /* Check whether @rh is going to be freed. */
  151. if (unlikely(!handler))
  152. return NULL;
  153. /*
  154. * This expects the caller will set up a rethook on a function entry.
  155. * When the function returns, the rethook will eventually be reclaimed
  156. * or released in the rethook_recycle() with call_rcu().
  157. * This means the caller must be run in the RCU-availabe context.
  158. */
  159. if (unlikely(!rcu_is_watching()))
  160. return NULL;
  161. fn = freelist_try_get(&rh->pool);
  162. if (!fn)
  163. return NULL;
  164. return container_of(fn, struct rethook_node, freelist);
  165. }
  166. NOKPROBE_SYMBOL(rethook_try_get);
  167. /**
  168. * rethook_hook() - Hook the current function return.
  169. * @node: The struct rethook node to hook the function return.
  170. * @regs: The struct pt_regs for the function entry.
  171. * @mcount: True if this is called from mcount(ftrace) context.
  172. *
  173. * Hook the current running function return. This must be called when the
  174. * function entry (or at least @regs must be the registers of the function
  175. * entry.) @mcount is used for identifying the context. If this is called
  176. * from ftrace (mcount) callback, @mcount must be set true. If this is called
  177. * from the real function entry (e.g. kprobes) @mcount must be set false.
  178. * This is because the way to hook the function return depends on the context.
  179. */
  180. void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
  181. {
  182. arch_rethook_prepare(node, regs, mcount);
  183. __llist_add(&node->llist, &current->rethooks);
  184. }
  185. NOKPROBE_SYMBOL(rethook_hook);
  186. /* This assumes the 'tsk' is the current task or is not running. */
  187. static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
  188. struct llist_node **cur)
  189. {
  190. struct rethook_node *rh = NULL;
  191. struct llist_node *node = *cur;
  192. if (!node)
  193. node = tsk->rethooks.first;
  194. else
  195. node = node->next;
  196. while (node) {
  197. rh = container_of(node, struct rethook_node, llist);
  198. if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
  199. *cur = node;
  200. return rh->ret_addr;
  201. }
  202. node = node->next;
  203. }
  204. return 0;
  205. }
  206. NOKPROBE_SYMBOL(__rethook_find_ret_addr);
  207. /**
  208. * rethook_find_ret_addr -- Find correct return address modified by rethook
  209. * @tsk: Target task
  210. * @frame: A frame pointer
  211. * @cur: a storage of the loop cursor llist_node pointer for next call
  212. *
  213. * Find the correct return address modified by a rethook on @tsk in unsigned
  214. * long type.
  215. * The @tsk must be 'current' or a task which is not running. @frame is a hint
  216. * to get the currect return address - which is compared with the
  217. * rethook::frame field. The @cur is a loop cursor for searching the
  218. * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
  219. * first call, but '@cur' itself must NOT NULL.
  220. *
  221. * Returns found address value or zero if not found.
  222. */
  223. unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
  224. struct llist_node **cur)
  225. {
  226. struct rethook_node *rhn = NULL;
  227. unsigned long ret;
  228. if (WARN_ON_ONCE(!cur))
  229. return 0;
  230. if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
  231. return 0;
  232. do {
  233. ret = __rethook_find_ret_addr(tsk, cur);
  234. if (!ret)
  235. break;
  236. rhn = container_of(*cur, struct rethook_node, llist);
  237. } while (rhn->frame != frame);
  238. return ret;
  239. }
  240. NOKPROBE_SYMBOL(rethook_find_ret_addr);
  241. void __weak arch_rethook_fixup_return(struct pt_regs *regs,
  242. unsigned long correct_ret_addr)
  243. {
  244. /*
  245. * Do nothing by default. If the architecture which uses a
  246. * frame pointer to record real return address on the stack,
  247. * it should fill this function to fixup the return address
  248. * so that stacktrace works from the rethook handler.
  249. */
  250. }
  251. /* This function will be called from each arch-defined trampoline. */
  252. unsigned long rethook_trampoline_handler(struct pt_regs *regs,
  253. unsigned long frame)
  254. {
  255. struct llist_node *first, *node = NULL;
  256. unsigned long correct_ret_addr;
  257. rethook_handler_t handler;
  258. struct rethook_node *rhn;
  259. correct_ret_addr = __rethook_find_ret_addr(current, &node);
  260. if (!correct_ret_addr) {
  261. pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
  262. BUG_ON(1);
  263. }
  264. instruction_pointer_set(regs, correct_ret_addr);
  265. /*
  266. * These loops must be protected from rethook_free_rcu() because those
  267. * are accessing 'rhn->rethook'.
  268. */
  269. preempt_disable_notrace();
  270. /*
  271. * Run the handler on the shadow stack. Do not unlink the list here because
  272. * stackdump inside the handlers needs to decode it.
  273. */
  274. first = current->rethooks.first;
  275. while (first) {
  276. rhn = container_of(first, struct rethook_node, llist);
  277. if (WARN_ON_ONCE(rhn->frame != frame))
  278. break;
  279. handler = rethook_get_handler(rhn->rethook);
  280. if (handler)
  281. handler(rhn, rhn->rethook->data, regs);
  282. if (first == node)
  283. break;
  284. first = first->next;
  285. }
  286. /* Fixup registers for returning to correct address. */
  287. arch_rethook_fixup_return(regs, correct_ret_addr);
  288. /* Unlink used shadow stack */
  289. first = current->rethooks.first;
  290. current->rethooks.first = node->next;
  291. node->next = NULL;
  292. while (first) {
  293. rhn = container_of(first, struct rethook_node, llist);
  294. first = first->next;
  295. rethook_recycle(rhn);
  296. }
  297. preempt_enable_notrace();
  298. return correct_ret_addr;
  299. }
  300. NOKPROBE_SYMBOL(rethook_trampoline_handler);