|
|
|
@@ -86,6 +86,18 @@ static DEFINE_SPINLOCK(kcov_remote_lock);
|
|
|
|
|
static DEFINE_HASHTABLE(kcov_remote_map, 4);
|
|
|
|
|
static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas);
|
|
|
|
|
|
|
|
|
|
struct kcov_percpu_data {
|
|
|
|
|
void *irq_area;
|
|
|
|
|
|
|
|
|
|
unsigned int saved_mode;
|
|
|
|
|
unsigned int saved_size;
|
|
|
|
|
void *saved_area;
|
|
|
|
|
struct kcov *saved_kcov;
|
|
|
|
|
int saved_sequence;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data);
|
|
|
|
|
|
|
|
|
|
/* Must be called with kcov_remote_lock locked. */
|
|
|
|
|
static struct kcov_remote *kcov_remote_find(u64 handle)
|
|
|
|
|
{
|
|
|
|
@@ -98,6 +110,7 @@ static struct kcov_remote *kcov_remote_find(u64 handle)
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Must be called with kcov_remote_lock locked. */
|
|
|
|
|
static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle)
|
|
|
|
|
{
|
|
|
|
|
struct kcov_remote *remote;
|
|
|
|
@@ -119,16 +132,13 @@ static struct kcov_remote_area *kcov_remote_area_get(unsigned int size)
|
|
|
|
|
struct kcov_remote_area *area;
|
|
|
|
|
struct list_head *pos;
|
|
|
|
|
|
|
|
|
|
kcov_debug("size = %u\n", size);
|
|
|
|
|
list_for_each(pos, &kcov_remote_areas) {
|
|
|
|
|
area = list_entry(pos, struct kcov_remote_area, list);
|
|
|
|
|
if (area->size == size) {
|
|
|
|
|
list_del(&area->list);
|
|
|
|
|
kcov_debug("rv = %px\n", area);
|
|
|
|
|
return area;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
kcov_debug("rv = NULL\n");
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -136,7 +146,6 @@ static struct kcov_remote_area *kcov_remote_area_get(unsigned int size)
|
|
|
|
|
static void kcov_remote_area_put(struct kcov_remote_area *area,
|
|
|
|
|
unsigned int size)
|
|
|
|
|
{
|
|
|
|
|
kcov_debug("area = %px, size = %u\n", area, size);
|
|
|
|
|
INIT_LIST_HEAD(&area->list);
|
|
|
|
|
area->size = size;
|
|
|
|
|
list_add(&area->list, &kcov_remote_areas);
|
|
|
|
@@ -148,9 +157,10 @@ static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_stru
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We are interested in code coverage as a function of a syscall inputs,
|
|
|
|
|
* so we ignore code executed in interrupts.
|
|
|
|
|
* so we ignore code executed in interrupts, unless we are in a remote
|
|
|
|
|
* coverage collection section in a softirq.
|
|
|
|
|
*/
|
|
|
|
|
if (!in_task())
|
|
|
|
|
if (!in_task() && !(in_serving_softirq() && t->kcov_softirq))
|
|
|
|
|
return false;
|
|
|
|
|
mode = READ_ONCE(t->kcov_mode);
|
|
|
|
|
/*
|
|
|
|
@@ -312,23 +322,26 @@ void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
|
|
|
|
|
EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
|
|
|
|
|
#endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
|
|
|
|
|
|
|
|
|
|
static void kcov_start(struct task_struct *t, unsigned int size,
|
|
|
|
|
void *area, enum kcov_mode mode, int sequence)
|
|
|
|
|
static void kcov_start(struct task_struct *t, struct kcov *kcov,
|
|
|
|
|
unsigned int size, void *area, enum kcov_mode mode,
|
|
|
|
|
int sequence)
|
|
|
|
|
{
|
|
|
|
|
kcov_debug("t = %px, size = %u, area = %px\n", t, size, area);
|
|
|
|
|
t->kcov = kcov;
|
|
|
|
|
/* Cache in task struct for performance. */
|
|
|
|
|
t->kcov_size = size;
|
|
|
|
|
t->kcov_area = area;
|
|
|
|
|
t->kcov_sequence = sequence;
|
|
|
|
|
/* See comment in check_kcov_mode(). */
|
|
|
|
|
barrier();
|
|
|
|
|
WRITE_ONCE(t->kcov_mode, mode);
|
|
|
|
|
t->kcov_sequence = sequence;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void kcov_stop(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
|
|
|
|
|
barrier();
|
|
|
|
|
t->kcov = NULL;
|
|
|
|
|
t->kcov_size = 0;
|
|
|
|
|
t->kcov_area = NULL;
|
|
|
|
|
}
|
|
|
|
@@ -336,7 +349,6 @@ static void kcov_stop(struct task_struct *t)
|
|
|
|
|
static void kcov_task_reset(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
kcov_stop(t);
|
|
|
|
|
t->kcov = NULL;
|
|
|
|
|
t->kcov_sequence = 0;
|
|
|
|
|
t->kcov_handle = 0;
|
|
|
|
|
}
|
|
|
|
@@ -361,18 +373,18 @@ static void kcov_remote_reset(struct kcov *kcov)
|
|
|
|
|
int bkt;
|
|
|
|
|
struct kcov_remote *remote;
|
|
|
|
|
struct hlist_node *tmp;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
spin_lock(&kcov_remote_lock);
|
|
|
|
|
spin_lock_irqsave(&kcov_remote_lock, flags);
|
|
|
|
|
hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) {
|
|
|
|
|
if (remote->kcov != kcov)
|
|
|
|
|
continue;
|
|
|
|
|
kcov_debug("removing handle %llx\n", remote->handle);
|
|
|
|
|
hash_del(&remote->hnode);
|
|
|
|
|
kfree(remote);
|
|
|
|
|
}
|
|
|
|
|
/* Do reset before unlock to prevent races with kcov_remote_start(). */
|
|
|
|
|
kcov_reset(kcov);
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock, flags);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void kcov_disable(struct task_struct *t, struct kcov *kcov)
|
|
|
|
@@ -401,12 +413,13 @@ static void kcov_put(struct kcov *kcov)
|
|
|
|
|
void kcov_task_exit(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
struct kcov *kcov;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
kcov = t->kcov;
|
|
|
|
|
if (kcov == NULL)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
spin_lock(&kcov->lock);
|
|
|
|
|
spin_lock_irqsave(&kcov->lock, flags);
|
|
|
|
|
kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
|
|
|
|
|
/*
|
|
|
|
|
* For KCOV_ENABLE devices we want to make sure that t->kcov->t == t,
|
|
|
|
@@ -430,12 +443,12 @@ void kcov_task_exit(struct task_struct *t)
|
|
|
|
|
* By combining all three checks into one we get:
|
|
|
|
|
*/
|
|
|
|
|
if (WARN_ON(kcov->t != t)) {
|
|
|
|
|
spin_unlock(&kcov->lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov->lock, flags);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
/* Just to not leave dangling references behind. */
|
|
|
|
|
kcov_disable(t, kcov);
|
|
|
|
|
spin_unlock(&kcov->lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov->lock, flags);
|
|
|
|
|
kcov_put(kcov);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -446,12 +459,13 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
|
|
|
|
|
struct kcov *kcov = vma->vm_file->private_data;
|
|
|
|
|
unsigned long size, off;
|
|
|
|
|
struct page *page;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
area = vmalloc_user(vma->vm_end - vma->vm_start);
|
|
|
|
|
if (!area)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
|
|
spin_lock(&kcov->lock);
|
|
|
|
|
spin_lock_irqsave(&kcov->lock, flags);
|
|
|
|
|
size = kcov->size * sizeof(unsigned long);
|
|
|
|
|
if (kcov->mode != KCOV_MODE_INIT || vma->vm_pgoff != 0 ||
|
|
|
|
|
vma->vm_end - vma->vm_start != size) {
|
|
|
|
@@ -461,7 +475,7 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
|
|
|
|
|
if (!kcov->area) {
|
|
|
|
|
kcov->area = area;
|
|
|
|
|
vma->vm_flags |= VM_DONTEXPAND;
|
|
|
|
|
spin_unlock(&kcov->lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov->lock, flags);
|
|
|
|
|
for (off = 0; off < size; off += PAGE_SIZE) {
|
|
|
|
|
page = vmalloc_to_page(kcov->area + off);
|
|
|
|
|
if (vm_insert_page(vma, vma->vm_start + off, page))
|
|
|
|
@@ -470,7 +484,7 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
exit:
|
|
|
|
|
spin_unlock(&kcov->lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov->lock, flags);
|
|
|
|
|
vfree(area);
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
@@ -550,10 +564,10 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
|
|
|
|
|
int mode, i;
|
|
|
|
|
struct kcov_remote_arg *remote_arg;
|
|
|
|
|
struct kcov_remote *remote;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
switch (cmd) {
|
|
|
|
|
case KCOV_INIT_TRACE:
|
|
|
|
|
kcov_debug("KCOV_INIT_TRACE\n");
|
|
|
|
|
/*
|
|
|
|
|
* Enable kcov in trace mode and setup buffer size.
|
|
|
|
|
* Must happen before anything else.
|
|
|
|
@@ -572,7 +586,6 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
|
|
|
|
|
kcov->mode = KCOV_MODE_INIT;
|
|
|
|
|
return 0;
|
|
|
|
|
case KCOV_ENABLE:
|
|
|
|
|
kcov_debug("KCOV_ENABLE\n");
|
|
|
|
|
/*
|
|
|
|
|
* Enable coverage for the current task.
|
|
|
|
|
* At this point user must have been enabled trace mode,
|
|
|
|
@@ -590,15 +603,13 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
|
|
|
|
|
return mode;
|
|
|
|
|
kcov_fault_in_area(kcov);
|
|
|
|
|
kcov->mode = mode;
|
|
|
|
|
kcov_start(t, kcov->size, kcov->area, kcov->mode,
|
|
|
|
|
kcov_start(t, kcov, kcov->size, kcov->area, kcov->mode,
|
|
|
|
|
kcov->sequence);
|
|
|
|
|
t->kcov = kcov;
|
|
|
|
|
kcov->t = t;
|
|
|
|
|
/* Put either in kcov_task_exit() or in KCOV_DISABLE. */
|
|
|
|
|
kcov_get(kcov);
|
|
|
|
|
return 0;
|
|
|
|
|
case KCOV_DISABLE:
|
|
|
|
|
kcov_debug("KCOV_DISABLE\n");
|
|
|
|
|
/* Disable coverage for the current task. */
|
|
|
|
|
unused = arg;
|
|
|
|
|
if (unused != 0 || current->kcov != kcov)
|
|
|
|
@@ -610,7 +621,6 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
|
|
|
|
|
kcov_put(kcov);
|
|
|
|
|
return 0;
|
|
|
|
|
case KCOV_REMOTE_ENABLE:
|
|
|
|
|
kcov_debug("KCOV_REMOTE_ENABLE\n");
|
|
|
|
|
if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
t = current;
|
|
|
|
@@ -627,41 +637,42 @@ static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
|
|
|
|
|
kcov->t = t;
|
|
|
|
|
kcov->remote = true;
|
|
|
|
|
kcov->remote_size = remote_arg->area_size;
|
|
|
|
|
spin_lock(&kcov_remote_lock);
|
|
|
|
|
spin_lock_irqsave(&kcov_remote_lock, flags);
|
|
|
|
|
for (i = 0; i < remote_arg->num_handles; i++) {
|
|
|
|
|
kcov_debug("handle %llx\n", remote_arg->handles[i]);
|
|
|
|
|
if (!kcov_check_handle(remote_arg->handles[i],
|
|
|
|
|
false, true, false)) {
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock,
|
|
|
|
|
flags);
|
|
|
|
|
kcov_disable(t, kcov);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
remote = kcov_remote_add(kcov, remote_arg->handles[i]);
|
|
|
|
|
if (IS_ERR(remote)) {
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock,
|
|
|
|
|
flags);
|
|
|
|
|
kcov_disable(t, kcov);
|
|
|
|
|
return PTR_ERR(remote);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (remote_arg->common_handle) {
|
|
|
|
|
kcov_debug("common handle %llx\n",
|
|
|
|
|
remote_arg->common_handle);
|
|
|
|
|
if (!kcov_check_handle(remote_arg->common_handle,
|
|
|
|
|
true, false, false)) {
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock,
|
|
|
|
|
flags);
|
|
|
|
|
kcov_disable(t, kcov);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
remote = kcov_remote_add(kcov,
|
|
|
|
|
remote_arg->common_handle);
|
|
|
|
|
if (IS_ERR(remote)) {
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock,
|
|
|
|
|
flags);
|
|
|
|
|
kcov_disable(t, kcov);
|
|
|
|
|
return PTR_ERR(remote);
|
|
|
|
|
}
|
|
|
|
|
t->kcov_handle = remote_arg->common_handle;
|
|
|
|
|
}
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock, flags);
|
|
|
|
|
/* Put either in kcov_task_exit() or in KCOV_DISABLE. */
|
|
|
|
|
kcov_get(kcov);
|
|
|
|
|
return 0;
|
|
|
|
@@ -677,6 +688,7 @@ static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
|
|
|
|
|
struct kcov_remote_arg *remote_arg = NULL;
|
|
|
|
|
unsigned int remote_num_handles;
|
|
|
|
|
unsigned long remote_arg_size;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
if (cmd == KCOV_REMOTE_ENABLE) {
|
|
|
|
|
if (get_user(remote_num_handles, (unsigned __user *)(arg +
|
|
|
|
@@ -697,9 +709,9 @@ static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
kcov = filep->private_data;
|
|
|
|
|
spin_lock(&kcov->lock);
|
|
|
|
|
spin_lock_irqsave(&kcov->lock, flags);
|
|
|
|
|
res = kcov_ioctl_locked(kcov, cmd, arg);
|
|
|
|
|
spin_unlock(&kcov->lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov->lock, flags);
|
|
|
|
|
|
|
|
|
|
kfree(remote_arg);
|
|
|
|
|
|
|
|
|
@@ -716,8 +728,8 @@ static const struct file_operations kcov_fops = {
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* kcov_remote_start() and kcov_remote_stop() can be used to annotate a section
|
|
|
|
|
* of code in a kernel background thread to allow kcov to be used to collect
|
|
|
|
|
* coverage from that part of code.
|
|
|
|
|
* of code in a kernel background thread or in a softirq to allow kcov to be
|
|
|
|
|
* used to collect coverage from that part of code.
|
|
|
|
|
*
|
|
|
|
|
* The handle argument of kcov_remote_start() identifies a code section that is
|
|
|
|
|
* used for coverage collection. A userspace process passes this handle to
|
|
|
|
@@ -728,9 +740,9 @@ static const struct file_operations kcov_fops = {
|
|
|
|
|
* the type of the kernel thread whose code is being annotated.
|
|
|
|
|
*
|
|
|
|
|
* For global kernel threads that are spawned in a limited number of instances
|
|
|
|
|
* (e.g. one USB hub_event() worker thread is spawned per USB HCD), each
|
|
|
|
|
* instance must be assigned a unique 4-byte instance id. The instance id is
|
|
|
|
|
* then combined with a 1-byte subsystem id to get a handle via
|
|
|
|
|
* (e.g. one USB hub_event() worker thread is spawned per USB HCD) and for
|
|
|
|
|
* softirqs, each instance must be assigned a unique 4-byte instance id. The
|
|
|
|
|
* instance id is then combined with a 1-byte subsystem id to get a handle via
|
|
|
|
|
* kcov_remote_handle(subsystem_id, instance_id).
|
|
|
|
|
*
|
|
|
|
|
* For local kernel threads that are spawned from system calls handler when a
|
|
|
|
@@ -749,70 +761,136 @@ static const struct file_operations kcov_fops = {
|
|
|
|
|
*
|
|
|
|
|
* See Documentation/dev-tools/kcov.rst for more details.
|
|
|
|
|
*
|
|
|
|
|
* Internally, this function looks up the kcov device associated with the
|
|
|
|
|
* Internally, kcov_remote_start() looks up the kcov device associated with the
|
|
|
|
|
* provided handle, allocates an area for coverage collection, and saves the
|
|
|
|
|
* pointers to kcov and area into the current task_struct to allow coverage to
|
|
|
|
|
* be collected via __sanitizer_cov_trace_pc()
|
|
|
|
|
* In turns kcov_remote_stop() clears those pointers from task_struct to stop
|
|
|
|
|
* collecting coverage and copies all collected coverage into the kcov area.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
static inline bool kcov_mode_enabled(unsigned int mode)
|
|
|
|
|
{
|
|
|
|
|
return (mode & ~KCOV_IN_CTXSW) != KCOV_MODE_DISABLED;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void kcov_remote_softirq_start(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
|
|
|
|
|
unsigned int mode;
|
|
|
|
|
|
|
|
|
|
mode = READ_ONCE(t->kcov_mode);
|
|
|
|
|
barrier();
|
|
|
|
|
if (kcov_mode_enabled(mode)) {
|
|
|
|
|
data->saved_mode = mode;
|
|
|
|
|
data->saved_size = t->kcov_size;
|
|
|
|
|
data->saved_area = t->kcov_area;
|
|
|
|
|
data->saved_sequence = t->kcov_sequence;
|
|
|
|
|
data->saved_kcov = t->kcov;
|
|
|
|
|
kcov_stop(t);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void kcov_remote_softirq_stop(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
|
|
|
|
|
|
|
|
|
|
if (data->saved_kcov) {
|
|
|
|
|
kcov_start(t, data->saved_kcov, data->saved_size,
|
|
|
|
|
data->saved_area, data->saved_mode,
|
|
|
|
|
data->saved_sequence);
|
|
|
|
|
data->saved_mode = 0;
|
|
|
|
|
data->saved_size = 0;
|
|
|
|
|
data->saved_area = NULL;
|
|
|
|
|
data->saved_sequence = 0;
|
|
|
|
|
data->saved_kcov = NULL;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void kcov_remote_start(u64 handle)
|
|
|
|
|
{
|
|
|
|
|
struct task_struct *t = current;
|
|
|
|
|
struct kcov_remote *remote;
|
|
|
|
|
struct kcov *kcov;
|
|
|
|
|
unsigned int mode;
|
|
|
|
|
void *area;
|
|
|
|
|
struct task_struct *t;
|
|
|
|
|
unsigned int size;
|
|
|
|
|
enum kcov_mode mode;
|
|
|
|
|
int sequence;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
if (WARN_ON(!kcov_check_handle(handle, true, true, true)))
|
|
|
|
|
return;
|
|
|
|
|
if (WARN_ON(!in_task()))
|
|
|
|
|
return;
|
|
|
|
|
t = current;
|
|
|
|
|
/*
|
|
|
|
|
* Check that kcov_remote_start is not called twice
|
|
|
|
|
* nor called by user tasks (with enabled kcov).
|
|
|
|
|
*/
|
|
|
|
|
if (WARN_ON(t->kcov))
|
|
|
|
|
if (!in_task() && !in_serving_softirq())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
kcov_debug("handle = %llx\n", handle);
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Check that kcov_remote_start() is not called twice in background
|
|
|
|
|
* threads nor called by user tasks (with enabled kcov).
|
|
|
|
|
*/
|
|
|
|
|
mode = READ_ONCE(t->kcov_mode);
|
|
|
|
|
if (WARN_ON(in_task() && kcov_mode_enabled(mode))) {
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
/*
|
|
|
|
|
* Check that kcov_remote_start() is not called twice in softirqs.
|
|
|
|
|
* Note, that kcov_remote_start() can be called from a softirq that
|
|
|
|
|
* happened while collecting coverage from a background thread.
|
|
|
|
|
*/
|
|
|
|
|
if (WARN_ON(in_serving_softirq() && t->kcov_softirq)) {
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
spin_lock(&kcov_remote_lock);
|
|
|
|
|
remote = kcov_remote_find(handle);
|
|
|
|
|
if (!remote) {
|
|
|
|
|
kcov_debug("no remote found");
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock, flags);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
kcov_debug("handle = %llx, context: %s\n", handle,
|
|
|
|
|
in_task() ? "task" : "softirq");
|
|
|
|
|
kcov = remote->kcov;
|
|
|
|
|
/* Put in kcov_remote_stop(). */
|
|
|
|
|
kcov_get(remote->kcov);
|
|
|
|
|
t->kcov = remote->kcov;
|
|
|
|
|
kcov_get(kcov);
|
|
|
|
|
/*
|
|
|
|
|
* Read kcov fields before unlock to prevent races with
|
|
|
|
|
* KCOV_DISABLE / kcov_remote_reset().
|
|
|
|
|
*/
|
|
|
|
|
size = remote->kcov->remote_size;
|
|
|
|
|
mode = remote->kcov->mode;
|
|
|
|
|
sequence = remote->kcov->sequence;
|
|
|
|
|
area = kcov_remote_area_get(size);
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
mode = kcov->mode;
|
|
|
|
|
sequence = kcov->sequence;
|
|
|
|
|
if (in_task()) {
|
|
|
|
|
size = kcov->remote_size;
|
|
|
|
|
area = kcov_remote_area_get(size);
|
|
|
|
|
} else {
|
|
|
|
|
size = CONFIG_KCOV_IRQ_AREA_SIZE;
|
|
|
|
|
area = this_cpu_ptr(&kcov_percpu_data)->irq_area;
|
|
|
|
|
}
|
|
|
|
|
spin_unlock_irqrestore(&kcov_remote_lock, flags);
|
|
|
|
|
|
|
|
|
|
/* Can only happen when in_task(). */
|
|
|
|
|
if (!area) {
|
|
|
|
|
area = vmalloc(size * sizeof(unsigned long));
|
|
|
|
|
if (!area) {
|
|
|
|
|
t->kcov = NULL;
|
|
|
|
|
kcov_put(remote->kcov);
|
|
|
|
|
kcov_put(kcov);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
|
|
/* Reset coverage size. */
|
|
|
|
|
*(u64 *)area = 0;
|
|
|
|
|
|
|
|
|
|
kcov_debug("area = %px, size = %u", area, size);
|
|
|
|
|
if (in_serving_softirq()) {
|
|
|
|
|
kcov_remote_softirq_start(t);
|
|
|
|
|
t->kcov_softirq = 1;
|
|
|
|
|
}
|
|
|
|
|
kcov_start(t, kcov, size, area, mode, sequence);
|
|
|
|
|
|
|
|
|
|
kcov_start(t, size, area, mode, sequence);
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(kcov_remote_start);
|
|
|
|
@@ -876,34 +954,58 @@ static void kcov_move_area(enum kcov_mode mode, void *dst_area,
|
|
|
|
|
void kcov_remote_stop(void)
|
|
|
|
|
{
|
|
|
|
|
struct task_struct *t = current;
|
|
|
|
|
struct kcov *kcov = t->kcov;
|
|
|
|
|
void *area = t->kcov_area;
|
|
|
|
|
unsigned int size = t->kcov_size;
|
|
|
|
|
int sequence = t->kcov_sequence;
|
|
|
|
|
struct kcov *kcov;
|
|
|
|
|
unsigned int mode;
|
|
|
|
|
void *area;
|
|
|
|
|
unsigned int size;
|
|
|
|
|
int sequence;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
|
|
if (!kcov) {
|
|
|
|
|
kcov_debug("no kcov found\n");
|
|
|
|
|
if (!in_task() && !in_serving_softirq())
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
|
|
|
|
|
mode = READ_ONCE(t->kcov_mode);
|
|
|
|
|
barrier();
|
|
|
|
|
if (!kcov_mode_enabled(mode)) {
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
kcov = t->kcov;
|
|
|
|
|
area = t->kcov_area;
|
|
|
|
|
size = t->kcov_size;
|
|
|
|
|
sequence = t->kcov_sequence;
|
|
|
|
|
|
|
|
|
|
if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) {
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
kcov_stop(t);
|
|
|
|
|
t->kcov = NULL;
|
|
|
|
|
if (in_serving_softirq()) {
|
|
|
|
|
t->kcov_softirq = 0;
|
|
|
|
|
kcov_remote_softirq_stop(t);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
spin_lock(&kcov->lock);
|
|
|
|
|
/*
|
|
|
|
|
* KCOV_DISABLE could have been called between kcov_remote_start()
|
|
|
|
|
* and kcov_remote_stop(), hence the check.
|
|
|
|
|
* and kcov_remote_stop(), hence the sequence check.
|
|
|
|
|
*/
|
|
|
|
|
kcov_debug("move if: %d == %d && %d\n",
|
|
|
|
|
sequence, kcov->sequence, (int)kcov->remote);
|
|
|
|
|
if (sequence == kcov->sequence && kcov->remote)
|
|
|
|
|
kcov_move_area(kcov->mode, kcov->area, kcov->size, area);
|
|
|
|
|
spin_unlock(&kcov->lock);
|
|
|
|
|
|
|
|
|
|
spin_lock(&kcov_remote_lock);
|
|
|
|
|
kcov_remote_area_put(area, size);
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
if (in_task()) {
|
|
|
|
|
spin_lock(&kcov_remote_lock);
|
|
|
|
|
kcov_remote_area_put(area, size);
|
|
|
|
|
spin_unlock(&kcov_remote_lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
|
|
|
|
/* Get in kcov_remote_start(). */
|
|
|
|
|
kcov_put(kcov);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(kcov_remote_stop);
|
|
|
|
@@ -917,6 +1019,16 @@ EXPORT_SYMBOL(kcov_common_handle);
|
|
|
|
|
|
|
|
|
|
static int __init kcov_init(void)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
void *area = vmalloc(CONFIG_KCOV_IRQ_AREA_SIZE *
|
|
|
|
|
sizeof(unsigned long));
|
|
|
|
|
if (!area)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* The kcov debugfs file won't ever get removed and thus,
|
|
|
|
|
* there is no need to protect it against removal races. The
|
|
|
|
|