123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346 |
- // SPDX-License-Identifier: GPL-2.0
- #define pr_fmt(fmt) "rethook: " fmt
- #include <linux/bug.h>
- #include <linux/kallsyms.h>
- #include <linux/kprobes.h>
- #include <linux/preempt.h>
- #include <linux/rethook.h>
- #include <linux/slab.h>
- #include <linux/sort.h>
- /* Return hook list (shadow stack by list) */
- /*
- * This function is called from delayed_put_task_struct() when a task is
- * dead and cleaned up to recycle any kretprobe instances associated with
- * this task. These left over instances represent probed functions that
- * have been called but will never return.
- */
- void rethook_flush_task(struct task_struct *tk)
- {
- struct rethook_node *rhn;
- struct llist_node *node;
- node = __llist_del_all(&tk->rethooks);
- while (node) {
- rhn = container_of(node, struct rethook_node, llist);
- node = node->next;
- preempt_disable();
- rethook_recycle(rhn);
- preempt_enable();
- }
- }
- static void rethook_free_rcu(struct rcu_head *head)
- {
- struct rethook *rh = container_of(head, struct rethook, rcu);
- struct rethook_node *rhn;
- struct freelist_node *node;
- int count = 1;
- node = rh->pool.head;
- while (node) {
- rhn = container_of(node, struct rethook_node, freelist);
- node = node->next;
- kfree(rhn);
- count++;
- }
- /* The rh->ref is the number of pooled node + 1 */
- if (refcount_sub_and_test(count, &rh->ref))
- kfree(rh);
- }
- /**
- * rethook_stop() - Stop using a rethook.
- * @rh: the struct rethook to stop.
- *
- * Stop using a rethook to prepare for freeing it. If you want to wait for
- * all running rethook handler before calling rethook_free(), you need to
- * call this first and wait RCU, and call rethook_free().
- */
- void rethook_stop(struct rethook *rh)
- {
- rcu_assign_pointer(rh->handler, NULL);
- }
- /**
- * rethook_free() - Free struct rethook.
- * @rh: the struct rethook to be freed.
- *
- * Free the rethook. Before calling this function, user must ensure the
- * @rh::data is cleaned if needed (or, the handler can access it after
- * calling this function.) This function will set the @rh to be freed
- * after all rethook_node are freed (not soon). And the caller must
- * not touch @rh after calling this.
- */
- void rethook_free(struct rethook *rh)
- {
- rethook_stop(rh);
- call_rcu(&rh->rcu, rethook_free_rcu);
- }
- static inline rethook_handler_t rethook_get_handler(struct rethook *rh)
- {
- return (rethook_handler_t)rcu_dereference_check(rh->handler,
- rcu_read_lock_any_held());
- }
- /**
- * rethook_alloc() - Allocate struct rethook.
- * @data: a data to pass the @handler when hooking the return.
- * @handler: the return hook callback function.
- *
- * Allocate and initialize a new rethook with @data and @handler.
- * Return NULL if memory allocation fails or @handler is NULL.
- * Note that @handler == NULL means this rethook is going to be freed.
- */
- struct rethook *rethook_alloc(void *data, rethook_handler_t handler)
- {
- struct rethook *rh = kzalloc(sizeof(struct rethook), GFP_KERNEL);
- if (!rh || !handler) {
- kfree(rh);
- return NULL;
- }
- rh->data = data;
- rcu_assign_pointer(rh->handler, handler);
- rh->pool.head = NULL;
- refcount_set(&rh->ref, 1);
- return rh;
- }
- /**
- * rethook_add_node() - Add a new node to the rethook.
- * @rh: the struct rethook.
- * @node: the struct rethook_node to be added.
- *
- * Add @node to @rh. User must allocate @node (as a part of user's
- * data structure.) The @node fields are initialized in this function.
- */
- void rethook_add_node(struct rethook *rh, struct rethook_node *node)
- {
- node->rethook = rh;
- freelist_add(&node->freelist, &rh->pool);
- refcount_inc(&rh->ref);
- }
- static void free_rethook_node_rcu(struct rcu_head *head)
- {
- struct rethook_node *node = container_of(head, struct rethook_node, rcu);
- if (refcount_dec_and_test(&node->rethook->ref))
- kfree(node->rethook);
- kfree(node);
- }
- /**
- * rethook_recycle() - return the node to rethook.
- * @node: The struct rethook_node to be returned.
- *
- * Return back the @node to @node::rethook. If the @node::rethook is already
- * marked as freed, this will free the @node.
- */
- void rethook_recycle(struct rethook_node *node)
- {
- rethook_handler_t handler;
- handler = rethook_get_handler(node->rethook);
- if (likely(handler))
- freelist_add(&node->freelist, &node->rethook->pool);
- else
- call_rcu(&node->rcu, free_rethook_node_rcu);
- }
- NOKPROBE_SYMBOL(rethook_recycle);
- /**
- * rethook_try_get() - get an unused rethook node.
- * @rh: The struct rethook which pools the nodes.
- *
- * Get an unused rethook node from @rh. If the node pool is empty, this
- * will return NULL. Caller must disable preemption.
- */
- struct rethook_node *rethook_try_get(struct rethook *rh)
- {
- rethook_handler_t handler = rethook_get_handler(rh);
- struct freelist_node *fn;
- /* Check whether @rh is going to be freed. */
- if (unlikely(!handler))
- return NULL;
- /*
- * This expects the caller will set up a rethook on a function entry.
- * When the function returns, the rethook will eventually be reclaimed
- * or released in the rethook_recycle() with call_rcu().
- * This means the caller must be run in the RCU-availabe context.
- */
- if (unlikely(!rcu_is_watching()))
- return NULL;
- fn = freelist_try_get(&rh->pool);
- if (!fn)
- return NULL;
- return container_of(fn, struct rethook_node, freelist);
- }
- NOKPROBE_SYMBOL(rethook_try_get);
- /**
- * rethook_hook() - Hook the current function return.
- * @node: The struct rethook node to hook the function return.
- * @regs: The struct pt_regs for the function entry.
- * @mcount: True if this is called from mcount(ftrace) context.
- *
- * Hook the current running function return. This must be called when the
- * function entry (or at least @regs must be the registers of the function
- * entry.) @mcount is used for identifying the context. If this is called
- * from ftrace (mcount) callback, @mcount must be set true. If this is called
- * from the real function entry (e.g. kprobes) @mcount must be set false.
- * This is because the way to hook the function return depends on the context.
- */
- void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount)
- {
- arch_rethook_prepare(node, regs, mcount);
- __llist_add(&node->llist, ¤t->rethooks);
- }
- NOKPROBE_SYMBOL(rethook_hook);
- /* This assumes the 'tsk' is the current task or is not running. */
- static unsigned long __rethook_find_ret_addr(struct task_struct *tsk,
- struct llist_node **cur)
- {
- struct rethook_node *rh = NULL;
- struct llist_node *node = *cur;
- if (!node)
- node = tsk->rethooks.first;
- else
- node = node->next;
- while (node) {
- rh = container_of(node, struct rethook_node, llist);
- if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) {
- *cur = node;
- return rh->ret_addr;
- }
- node = node->next;
- }
- return 0;
- }
- NOKPROBE_SYMBOL(__rethook_find_ret_addr);
- /**
- * rethook_find_ret_addr -- Find correct return address modified by rethook
- * @tsk: Target task
- * @frame: A frame pointer
- * @cur: a storage of the loop cursor llist_node pointer for next call
- *
- * Find the correct return address modified by a rethook on @tsk in unsigned
- * long type.
- * The @tsk must be 'current' or a task which is not running. @frame is a hint
- * to get the currect return address - which is compared with the
- * rethook::frame field. The @cur is a loop cursor for searching the
- * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
- * first call, but '@cur' itself must NOT NULL.
- *
- * Returns found address value or zero if not found.
- */
- unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame,
- struct llist_node **cur)
- {
- struct rethook_node *rhn = NULL;
- unsigned long ret;
- if (WARN_ON_ONCE(!cur))
- return 0;
- if (WARN_ON_ONCE(tsk != current && task_is_running(tsk)))
- return 0;
- do {
- ret = __rethook_find_ret_addr(tsk, cur);
- if (!ret)
- break;
- rhn = container_of(*cur, struct rethook_node, llist);
- } while (rhn->frame != frame);
- return ret;
- }
- NOKPROBE_SYMBOL(rethook_find_ret_addr);
- void __weak arch_rethook_fixup_return(struct pt_regs *regs,
- unsigned long correct_ret_addr)
- {
- /*
- * Do nothing by default. If the architecture which uses a
- * frame pointer to record real return address on the stack,
- * it should fill this function to fixup the return address
- * so that stacktrace works from the rethook handler.
- */
- }
- /* This function will be called from each arch-defined trampoline. */
- unsigned long rethook_trampoline_handler(struct pt_regs *regs,
- unsigned long frame)
- {
- struct llist_node *first, *node = NULL;
- unsigned long correct_ret_addr;
- rethook_handler_t handler;
- struct rethook_node *rhn;
- correct_ret_addr = __rethook_find_ret_addr(current, &node);
- if (!correct_ret_addr) {
- pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n");
- BUG_ON(1);
- }
- instruction_pointer_set(regs, correct_ret_addr);
- /*
- * These loops must be protected from rethook_free_rcu() because those
- * are accessing 'rhn->rethook'.
- */
- preempt_disable_notrace();
- /*
- * Run the handler on the shadow stack. Do not unlink the list here because
- * stackdump inside the handlers needs to decode it.
- */
- first = current->rethooks.first;
- while (first) {
- rhn = container_of(first, struct rethook_node, llist);
- if (WARN_ON_ONCE(rhn->frame != frame))
- break;
- handler = rethook_get_handler(rhn->rethook);
- if (handler)
- handler(rhn, rhn->rethook->data, regs);
- if (first == node)
- break;
- first = first->next;
- }
- /* Fixup registers for returning to correct address. */
- arch_rethook_fixup_return(regs, correct_ret_addr);
- /* Unlink used shadow stack */
- first = current->rethooks.first;
- current->rethooks.first = node->next;
- node->next = NULL;
- while (first) {
- rhn = container_of(first, struct rethook_node, llist);
- first = first->next;
- rethook_recycle(rhn);
- }
- preempt_enable_notrace();
- return correct_ret_addr;
- }
- NOKPROBE_SYMBOL(rethook_trampoline_handler);
|