kcov.c 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080
  1. // SPDX-License-Identifier: GPL-2.0
  2. #define pr_fmt(fmt) "kcov: " fmt
  3. #define DISABLE_BRANCH_PROFILING
  4. #include <linux/atomic.h>
  5. #include <linux/compiler.h>
  6. #include <linux/errno.h>
  7. #include <linux/export.h>
  8. #include <linux/types.h>
  9. #include <linux/file.h>
  10. #include <linux/fs.h>
  11. #include <linux/hashtable.h>
  12. #include <linux/init.h>
  13. #include <linux/kmsan-checks.h>
  14. #include <linux/mm.h>
  15. #include <linux/preempt.h>
  16. #include <linux/printk.h>
  17. #include <linux/sched.h>
  18. #include <linux/slab.h>
  19. #include <linux/spinlock.h>
  20. #include <linux/vmalloc.h>
  21. #include <linux/debugfs.h>
  22. #include <linux/uaccess.h>
  23. #include <linux/kcov.h>
  24. #include <linux/refcount.h>
  25. #include <linux/log2.h>
  26. #include <asm/setup.h>
  27. #define kcov_debug(fmt, ...) pr_debug("%s: " fmt, __func__, ##__VA_ARGS__)
  28. /* Number of 64-bit words written per one comparison: */
  29. #define KCOV_WORDS_PER_CMP 4
  30. /*
  31. * kcov descriptor (one per opened debugfs file).
  32. * State transitions of the descriptor:
  33. * - initial state after open()
  34. * - then there must be a single ioctl(KCOV_INIT_TRACE) call
  35. * - then, mmap() call (several calls are allowed but not useful)
  36. * - then, ioctl(KCOV_ENABLE, arg), where arg is
  37. * KCOV_TRACE_PC - to trace only the PCs
  38. * or
  39. * KCOV_TRACE_CMP - to trace only the comparison operands
  40. * - then, ioctl(KCOV_DISABLE) to disable the task.
  41. * Enabling/disabling ioctls can be repeated (only one task a time allowed).
  42. */
  43. struct kcov {
  44. /*
  45. * Reference counter. We keep one for:
  46. * - opened file descriptor
  47. * - task with enabled coverage (we can't unwire it from another task)
  48. * - each code section for remote coverage collection
  49. */
  50. refcount_t refcount;
  51. /* The lock protects mode, size, area and t. */
  52. spinlock_t lock;
  53. enum kcov_mode mode;
  54. /* Size of arena (in long's). */
  55. unsigned int size;
  56. /* Coverage buffer shared with user space. */
  57. void *area;
  58. /* Task for which we collect coverage, or NULL. */
  59. struct task_struct *t;
  60. /* Collecting coverage from remote (background) threads. */
  61. bool remote;
  62. /* Size of remote area (in long's). */
  63. unsigned int remote_size;
  64. /*
  65. * Sequence is incremented each time kcov is reenabled, used by
  66. * kcov_remote_stop(), see the comment there.
  67. */
  68. int sequence;
  69. };
  70. struct kcov_remote_area {
  71. struct list_head list;
  72. unsigned int size;
  73. };
  74. struct kcov_remote {
  75. u64 handle;
  76. struct kcov *kcov;
  77. struct hlist_node hnode;
  78. };
  79. static DEFINE_SPINLOCK(kcov_remote_lock);
  80. static DEFINE_HASHTABLE(kcov_remote_map, 4);
  81. static struct list_head kcov_remote_areas = LIST_HEAD_INIT(kcov_remote_areas);
  82. struct kcov_percpu_data {
  83. void *irq_area;
  84. local_lock_t lock;
  85. unsigned int saved_mode;
  86. unsigned int saved_size;
  87. void *saved_area;
  88. struct kcov *saved_kcov;
  89. int saved_sequence;
  90. };
  91. static DEFINE_PER_CPU(struct kcov_percpu_data, kcov_percpu_data) = {
  92. .lock = INIT_LOCAL_LOCK(lock),
  93. };
  94. /* Must be called with kcov_remote_lock locked. */
  95. static struct kcov_remote *kcov_remote_find(u64 handle)
  96. {
  97. struct kcov_remote *remote;
  98. hash_for_each_possible(kcov_remote_map, remote, hnode, handle) {
  99. if (remote->handle == handle)
  100. return remote;
  101. }
  102. return NULL;
  103. }
  104. /* Must be called with kcov_remote_lock locked. */
  105. static struct kcov_remote *kcov_remote_add(struct kcov *kcov, u64 handle)
  106. {
  107. struct kcov_remote *remote;
  108. if (kcov_remote_find(handle))
  109. return ERR_PTR(-EEXIST);
  110. remote = kmalloc(sizeof(*remote), GFP_ATOMIC);
  111. if (!remote)
  112. return ERR_PTR(-ENOMEM);
  113. remote->handle = handle;
  114. remote->kcov = kcov;
  115. hash_add(kcov_remote_map, &remote->hnode, handle);
  116. return remote;
  117. }
  118. /* Must be called with kcov_remote_lock locked. */
  119. static struct kcov_remote_area *kcov_remote_area_get(unsigned int size)
  120. {
  121. struct kcov_remote_area *area;
  122. struct list_head *pos;
  123. list_for_each(pos, &kcov_remote_areas) {
  124. area = list_entry(pos, struct kcov_remote_area, list);
  125. if (area->size == size) {
  126. list_del(&area->list);
  127. return area;
  128. }
  129. }
  130. return NULL;
  131. }
  132. /* Must be called with kcov_remote_lock locked. */
  133. static void kcov_remote_area_put(struct kcov_remote_area *area,
  134. unsigned int size)
  135. {
  136. INIT_LIST_HEAD(&area->list);
  137. area->size = size;
  138. list_add(&area->list, &kcov_remote_areas);
  139. /*
  140. * KMSAN doesn't instrument this file, so it may not know area->list
  141. * is initialized. Unpoison it explicitly to avoid reports in
  142. * kcov_remote_area_get().
  143. */
  144. kmsan_unpoison_memory(&area->list, sizeof(area->list));
  145. }
  146. static notrace bool check_kcov_mode(enum kcov_mode needed_mode, struct task_struct *t)
  147. {
  148. unsigned int mode;
  149. /*
  150. * We are interested in code coverage as a function of a syscall inputs,
  151. * so we ignore code executed in interrupts, unless we are in a remote
  152. * coverage collection section in a softirq.
  153. */
  154. if (!in_task() && !(in_serving_softirq() && t->kcov_softirq))
  155. return false;
  156. mode = READ_ONCE(t->kcov_mode);
  157. /*
  158. * There is some code that runs in interrupts but for which
  159. * in_interrupt() returns false (e.g. preempt_schedule_irq()).
  160. * READ_ONCE()/barrier() effectively provides load-acquire wrt
  161. * interrupts, there are paired barrier()/WRITE_ONCE() in
  162. * kcov_start().
  163. */
  164. barrier();
  165. return mode == needed_mode;
  166. }
  167. static notrace unsigned long canonicalize_ip(unsigned long ip)
  168. {
  169. #ifdef CONFIG_RANDOMIZE_BASE
  170. ip -= kaslr_offset();
  171. #endif
  172. return ip;
  173. }
  174. /*
  175. * Entry point from instrumented code.
  176. * This is called once per basic-block/edge.
  177. */
  178. void notrace __sanitizer_cov_trace_pc(void)
  179. {
  180. struct task_struct *t;
  181. unsigned long *area;
  182. unsigned long ip = canonicalize_ip(_RET_IP_);
  183. unsigned long pos;
  184. t = current;
  185. if (!check_kcov_mode(KCOV_MODE_TRACE_PC, t))
  186. return;
  187. area = t->kcov_area;
  188. /* The first 64-bit word is the number of subsequent PCs. */
  189. pos = READ_ONCE(area[0]) + 1;
  190. if (likely(pos < t->kcov_size)) {
  191. /* Previously we write pc before updating pos. However, some
  192. * early interrupt code could bypass check_kcov_mode() check
  193. * and invoke __sanitizer_cov_trace_pc(). If such interrupt is
  194. * raised between writing pc and updating pos, the pc could be
  195. * overitten by the recursive __sanitizer_cov_trace_pc().
  196. * Update pos before writing pc to avoid such interleaving.
  197. */
  198. WRITE_ONCE(area[0], pos);
  199. barrier();
  200. area[pos] = ip;
  201. }
  202. }
  203. EXPORT_SYMBOL(__sanitizer_cov_trace_pc);
  204. #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
  205. static void notrace write_comp_data(u64 type, u64 arg1, u64 arg2, u64 ip)
  206. {
  207. struct task_struct *t;
  208. u64 *area;
  209. u64 count, start_index, end_pos, max_pos;
  210. t = current;
  211. if (!check_kcov_mode(KCOV_MODE_TRACE_CMP, t))
  212. return;
  213. ip = canonicalize_ip(ip);
  214. /*
  215. * We write all comparison arguments and types as u64.
  216. * The buffer was allocated for t->kcov_size unsigned longs.
  217. */
  218. area = (u64 *)t->kcov_area;
  219. max_pos = t->kcov_size * sizeof(unsigned long);
  220. count = READ_ONCE(area[0]);
  221. /* Every record is KCOV_WORDS_PER_CMP 64-bit words. */
  222. start_index = 1 + count * KCOV_WORDS_PER_CMP;
  223. end_pos = (start_index + KCOV_WORDS_PER_CMP) * sizeof(u64);
  224. if (likely(end_pos <= max_pos)) {
  225. /* See comment in __sanitizer_cov_trace_pc(). */
  226. WRITE_ONCE(area[0], count + 1);
  227. barrier();
  228. area[start_index] = type;
  229. area[start_index + 1] = arg1;
  230. area[start_index + 2] = arg2;
  231. area[start_index + 3] = ip;
  232. }
  233. }
  234. void notrace __sanitizer_cov_trace_cmp1(u8 arg1, u8 arg2)
  235. {
  236. write_comp_data(KCOV_CMP_SIZE(0), arg1, arg2, _RET_IP_);
  237. }
  238. EXPORT_SYMBOL(__sanitizer_cov_trace_cmp1);
  239. void notrace __sanitizer_cov_trace_cmp2(u16 arg1, u16 arg2)
  240. {
  241. write_comp_data(KCOV_CMP_SIZE(1), arg1, arg2, _RET_IP_);
  242. }
  243. EXPORT_SYMBOL(__sanitizer_cov_trace_cmp2);
  244. void notrace __sanitizer_cov_trace_cmp4(u32 arg1, u32 arg2)
  245. {
  246. write_comp_data(KCOV_CMP_SIZE(2), arg1, arg2, _RET_IP_);
  247. }
  248. EXPORT_SYMBOL(__sanitizer_cov_trace_cmp4);
  249. void notrace __sanitizer_cov_trace_cmp8(u64 arg1, u64 arg2)
  250. {
  251. write_comp_data(KCOV_CMP_SIZE(3), arg1, arg2, _RET_IP_);
  252. }
  253. EXPORT_SYMBOL(__sanitizer_cov_trace_cmp8);
  254. void notrace __sanitizer_cov_trace_const_cmp1(u8 arg1, u8 arg2)
  255. {
  256. write_comp_data(KCOV_CMP_SIZE(0) | KCOV_CMP_CONST, arg1, arg2,
  257. _RET_IP_);
  258. }
  259. EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp1);
  260. void notrace __sanitizer_cov_trace_const_cmp2(u16 arg1, u16 arg2)
  261. {
  262. write_comp_data(KCOV_CMP_SIZE(1) | KCOV_CMP_CONST, arg1, arg2,
  263. _RET_IP_);
  264. }
  265. EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp2);
  266. void notrace __sanitizer_cov_trace_const_cmp4(u32 arg1, u32 arg2)
  267. {
  268. write_comp_data(KCOV_CMP_SIZE(2) | KCOV_CMP_CONST, arg1, arg2,
  269. _RET_IP_);
  270. }
  271. EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp4);
  272. void notrace __sanitizer_cov_trace_const_cmp8(u64 arg1, u64 arg2)
  273. {
  274. write_comp_data(KCOV_CMP_SIZE(3) | KCOV_CMP_CONST, arg1, arg2,
  275. _RET_IP_);
  276. }
  277. EXPORT_SYMBOL(__sanitizer_cov_trace_const_cmp8);
  278. void notrace __sanitizer_cov_trace_switch(u64 val, u64 *cases)
  279. {
  280. u64 i;
  281. u64 count = cases[0];
  282. u64 size = cases[1];
  283. u64 type = KCOV_CMP_CONST;
  284. switch (size) {
  285. case 8:
  286. type |= KCOV_CMP_SIZE(0);
  287. break;
  288. case 16:
  289. type |= KCOV_CMP_SIZE(1);
  290. break;
  291. case 32:
  292. type |= KCOV_CMP_SIZE(2);
  293. break;
  294. case 64:
  295. type |= KCOV_CMP_SIZE(3);
  296. break;
  297. default:
  298. return;
  299. }
  300. for (i = 0; i < count; i++)
  301. write_comp_data(type, cases[i + 2], val, _RET_IP_);
  302. }
  303. EXPORT_SYMBOL(__sanitizer_cov_trace_switch);
  304. #endif /* ifdef CONFIG_KCOV_ENABLE_COMPARISONS */
  305. static void kcov_start(struct task_struct *t, struct kcov *kcov,
  306. unsigned int size, void *area, enum kcov_mode mode,
  307. int sequence)
  308. {
  309. kcov_debug("t = %px, size = %u, area = %px\n", t, size, area);
  310. t->kcov = kcov;
  311. /* Cache in task struct for performance. */
  312. t->kcov_size = size;
  313. t->kcov_area = area;
  314. t->kcov_sequence = sequence;
  315. /* See comment in check_kcov_mode(). */
  316. barrier();
  317. WRITE_ONCE(t->kcov_mode, mode);
  318. }
  319. static void kcov_stop(struct task_struct *t)
  320. {
  321. WRITE_ONCE(t->kcov_mode, KCOV_MODE_DISABLED);
  322. barrier();
  323. t->kcov = NULL;
  324. t->kcov_size = 0;
  325. t->kcov_area = NULL;
  326. }
  327. static void kcov_task_reset(struct task_struct *t)
  328. {
  329. kcov_stop(t);
  330. t->kcov_sequence = 0;
  331. t->kcov_handle = 0;
  332. }
  333. void kcov_task_init(struct task_struct *t)
  334. {
  335. kcov_task_reset(t);
  336. t->kcov_handle = current->kcov_handle;
  337. }
  338. static void kcov_reset(struct kcov *kcov)
  339. {
  340. kcov->t = NULL;
  341. kcov->mode = KCOV_MODE_INIT;
  342. kcov->remote = false;
  343. kcov->remote_size = 0;
  344. kcov->sequence++;
  345. }
  346. static void kcov_remote_reset(struct kcov *kcov)
  347. {
  348. int bkt;
  349. struct kcov_remote *remote;
  350. struct hlist_node *tmp;
  351. unsigned long flags;
  352. spin_lock_irqsave(&kcov_remote_lock, flags);
  353. hash_for_each_safe(kcov_remote_map, bkt, tmp, remote, hnode) {
  354. if (remote->kcov != kcov)
  355. continue;
  356. hash_del(&remote->hnode);
  357. kfree(remote);
  358. }
  359. /* Do reset before unlock to prevent races with kcov_remote_start(). */
  360. kcov_reset(kcov);
  361. spin_unlock_irqrestore(&kcov_remote_lock, flags);
  362. }
  363. static void kcov_disable(struct task_struct *t, struct kcov *kcov)
  364. {
  365. kcov_task_reset(t);
  366. if (kcov->remote)
  367. kcov_remote_reset(kcov);
  368. else
  369. kcov_reset(kcov);
  370. }
  371. static void kcov_get(struct kcov *kcov)
  372. {
  373. refcount_inc(&kcov->refcount);
  374. }
  375. static void kcov_put(struct kcov *kcov)
  376. {
  377. if (refcount_dec_and_test(&kcov->refcount)) {
  378. kcov_remote_reset(kcov);
  379. vfree(kcov->area);
  380. kfree(kcov);
  381. }
  382. }
  383. void kcov_task_exit(struct task_struct *t)
  384. {
  385. struct kcov *kcov;
  386. unsigned long flags;
  387. kcov = t->kcov;
  388. if (kcov == NULL)
  389. return;
  390. spin_lock_irqsave(&kcov->lock, flags);
  391. kcov_debug("t = %px, kcov->t = %px\n", t, kcov->t);
  392. /*
  393. * For KCOV_ENABLE devices we want to make sure that t->kcov->t == t,
  394. * which comes down to:
  395. * WARN_ON(!kcov->remote && kcov->t != t);
  396. *
  397. * For KCOV_REMOTE_ENABLE devices, the exiting task is either:
  398. *
  399. * 1. A remote task between kcov_remote_start() and kcov_remote_stop().
  400. * In this case we should print a warning right away, since a task
  401. * shouldn't be exiting when it's in a kcov coverage collection
  402. * section. Here t points to the task that is collecting remote
  403. * coverage, and t->kcov->t points to the thread that created the
  404. * kcov device. Which means that to detect this case we need to
  405. * check that t != t->kcov->t, and this gives us the following:
  406. * WARN_ON(kcov->remote && kcov->t != t);
  407. *
  408. * 2. The task that created kcov exiting without calling KCOV_DISABLE,
  409. * and then again we make sure that t->kcov->t == t:
  410. * WARN_ON(kcov->remote && kcov->t != t);
  411. *
  412. * By combining all three checks into one we get:
  413. */
  414. if (WARN_ON(kcov->t != t)) {
  415. spin_unlock_irqrestore(&kcov->lock, flags);
  416. return;
  417. }
  418. /* Just to not leave dangling references behind. */
  419. kcov_disable(t, kcov);
  420. spin_unlock_irqrestore(&kcov->lock, flags);
  421. kcov_put(kcov);
  422. }
  423. static int kcov_mmap(struct file *filep, struct vm_area_struct *vma)
  424. {
  425. int res = 0;
  426. struct kcov *kcov = vma->vm_file->private_data;
  427. unsigned long size, off;
  428. struct page *page;
  429. unsigned long flags;
  430. spin_lock_irqsave(&kcov->lock, flags);
  431. size = kcov->size * sizeof(unsigned long);
  432. if (kcov->area == NULL || vma->vm_pgoff != 0 ||
  433. vma->vm_end - vma->vm_start != size) {
  434. res = -EINVAL;
  435. goto exit;
  436. }
  437. spin_unlock_irqrestore(&kcov->lock, flags);
  438. vm_flags_set(vma, VM_DONTEXPAND);
  439. for (off = 0; off < size; off += PAGE_SIZE) {
  440. page = vmalloc_to_page(kcov->area + off);
  441. res = vm_insert_page(vma, vma->vm_start + off, page);
  442. if (res) {
  443. pr_warn_once("kcov: vm_insert_page() failed\n");
  444. return res;
  445. }
  446. }
  447. return 0;
  448. exit:
  449. spin_unlock_irqrestore(&kcov->lock, flags);
  450. return res;
  451. }
  452. static int kcov_open(struct inode *inode, struct file *filep)
  453. {
  454. struct kcov *kcov;
  455. kcov = kzalloc(sizeof(*kcov), GFP_KERNEL);
  456. if (!kcov)
  457. return -ENOMEM;
  458. kcov->mode = KCOV_MODE_DISABLED;
  459. kcov->sequence = 1;
  460. refcount_set(&kcov->refcount, 1);
  461. spin_lock_init(&kcov->lock);
  462. filep->private_data = kcov;
  463. return nonseekable_open(inode, filep);
  464. }
  465. static int kcov_close(struct inode *inode, struct file *filep)
  466. {
  467. kcov_put(filep->private_data);
  468. return 0;
  469. }
  470. static int kcov_get_mode(unsigned long arg)
  471. {
  472. if (arg == KCOV_TRACE_PC)
  473. return KCOV_MODE_TRACE_PC;
  474. else if (arg == KCOV_TRACE_CMP)
  475. #ifdef CONFIG_KCOV_ENABLE_COMPARISONS
  476. return KCOV_MODE_TRACE_CMP;
  477. #else
  478. return -ENOTSUPP;
  479. #endif
  480. else
  481. return -EINVAL;
  482. }
  483. /*
  484. * Fault in a lazily-faulted vmalloc area before it can be used by
  485. * __santizer_cov_trace_pc(), to avoid recursion issues if any code on the
  486. * vmalloc fault handling path is instrumented.
  487. */
  488. static void kcov_fault_in_area(struct kcov *kcov)
  489. {
  490. unsigned long stride = PAGE_SIZE / sizeof(unsigned long);
  491. unsigned long *area = kcov->area;
  492. unsigned long offset;
  493. for (offset = 0; offset < kcov->size; offset += stride)
  494. READ_ONCE(area[offset]);
  495. }
  496. static inline bool kcov_check_handle(u64 handle, bool common_valid,
  497. bool uncommon_valid, bool zero_valid)
  498. {
  499. if (handle & ~(KCOV_SUBSYSTEM_MASK | KCOV_INSTANCE_MASK))
  500. return false;
  501. switch (handle & KCOV_SUBSYSTEM_MASK) {
  502. case KCOV_SUBSYSTEM_COMMON:
  503. return (handle & KCOV_INSTANCE_MASK) ?
  504. common_valid : zero_valid;
  505. case KCOV_SUBSYSTEM_USB:
  506. return uncommon_valid;
  507. default:
  508. return false;
  509. }
  510. return false;
  511. }
  512. static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd,
  513. unsigned long arg)
  514. {
  515. struct task_struct *t;
  516. unsigned long flags, unused;
  517. int mode, i;
  518. struct kcov_remote_arg *remote_arg;
  519. struct kcov_remote *remote;
  520. switch (cmd) {
  521. case KCOV_ENABLE:
  522. /*
  523. * Enable coverage for the current task.
  524. * At this point user must have been enabled trace mode,
  525. * and mmapped the file. Coverage collection is disabled only
  526. * at task exit or voluntary by KCOV_DISABLE. After that it can
  527. * be enabled for another task.
  528. */
  529. if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
  530. return -EINVAL;
  531. t = current;
  532. if (kcov->t != NULL || t->kcov != NULL)
  533. return -EBUSY;
  534. mode = kcov_get_mode(arg);
  535. if (mode < 0)
  536. return mode;
  537. kcov_fault_in_area(kcov);
  538. kcov->mode = mode;
  539. kcov_start(t, kcov, kcov->size, kcov->area, kcov->mode,
  540. kcov->sequence);
  541. kcov->t = t;
  542. /* Put either in kcov_task_exit() or in KCOV_DISABLE. */
  543. kcov_get(kcov);
  544. return 0;
  545. case KCOV_DISABLE:
  546. /* Disable coverage for the current task. */
  547. unused = arg;
  548. if (unused != 0 || current->kcov != kcov)
  549. return -EINVAL;
  550. t = current;
  551. if (WARN_ON(kcov->t != t))
  552. return -EINVAL;
  553. kcov_disable(t, kcov);
  554. kcov_put(kcov);
  555. return 0;
  556. case KCOV_REMOTE_ENABLE:
  557. if (kcov->mode != KCOV_MODE_INIT || !kcov->area)
  558. return -EINVAL;
  559. t = current;
  560. if (kcov->t != NULL || t->kcov != NULL)
  561. return -EBUSY;
  562. remote_arg = (struct kcov_remote_arg *)arg;
  563. mode = kcov_get_mode(remote_arg->trace_mode);
  564. if (mode < 0)
  565. return mode;
  566. if (remote_arg->area_size > LONG_MAX / sizeof(unsigned long))
  567. return -EINVAL;
  568. kcov->mode = mode;
  569. t->kcov = kcov;
  570. kcov->t = t;
  571. kcov->remote = true;
  572. kcov->remote_size = remote_arg->area_size;
  573. spin_lock_irqsave(&kcov_remote_lock, flags);
  574. for (i = 0; i < remote_arg->num_handles; i++) {
  575. if (!kcov_check_handle(remote_arg->handles[i],
  576. false, true, false)) {
  577. spin_unlock_irqrestore(&kcov_remote_lock,
  578. flags);
  579. kcov_disable(t, kcov);
  580. return -EINVAL;
  581. }
  582. remote = kcov_remote_add(kcov, remote_arg->handles[i]);
  583. if (IS_ERR(remote)) {
  584. spin_unlock_irqrestore(&kcov_remote_lock,
  585. flags);
  586. kcov_disable(t, kcov);
  587. return PTR_ERR(remote);
  588. }
  589. }
  590. if (remote_arg->common_handle) {
  591. if (!kcov_check_handle(remote_arg->common_handle,
  592. true, false, false)) {
  593. spin_unlock_irqrestore(&kcov_remote_lock,
  594. flags);
  595. kcov_disable(t, kcov);
  596. return -EINVAL;
  597. }
  598. remote = kcov_remote_add(kcov,
  599. remote_arg->common_handle);
  600. if (IS_ERR(remote)) {
  601. spin_unlock_irqrestore(&kcov_remote_lock,
  602. flags);
  603. kcov_disable(t, kcov);
  604. return PTR_ERR(remote);
  605. }
  606. t->kcov_handle = remote_arg->common_handle;
  607. }
  608. spin_unlock_irqrestore(&kcov_remote_lock, flags);
  609. /* Put either in kcov_task_exit() or in KCOV_DISABLE. */
  610. kcov_get(kcov);
  611. return 0;
  612. default:
  613. return -ENOTTY;
  614. }
  615. }
  616. static long kcov_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
  617. {
  618. struct kcov *kcov;
  619. int res;
  620. struct kcov_remote_arg *remote_arg = NULL;
  621. unsigned int remote_num_handles;
  622. unsigned long remote_arg_size;
  623. unsigned long size, flags;
  624. void *area;
  625. kcov = filep->private_data;
  626. switch (cmd) {
  627. case KCOV_INIT_TRACE:
  628. /*
  629. * Enable kcov in trace mode and setup buffer size.
  630. * Must happen before anything else.
  631. *
  632. * First check the size argument - it must be at least 2
  633. * to hold the current position and one PC.
  634. */
  635. size = arg;
  636. if (size < 2 || size > INT_MAX / sizeof(unsigned long))
  637. return -EINVAL;
  638. area = vmalloc_user(size * sizeof(unsigned long));
  639. if (area == NULL)
  640. return -ENOMEM;
  641. spin_lock_irqsave(&kcov->lock, flags);
  642. if (kcov->mode != KCOV_MODE_DISABLED) {
  643. spin_unlock_irqrestore(&kcov->lock, flags);
  644. vfree(area);
  645. return -EBUSY;
  646. }
  647. kcov->area = area;
  648. kcov->size = size;
  649. kcov->mode = KCOV_MODE_INIT;
  650. spin_unlock_irqrestore(&kcov->lock, flags);
  651. return 0;
  652. case KCOV_REMOTE_ENABLE:
  653. if (get_user(remote_num_handles, (unsigned __user *)(arg +
  654. offsetof(struct kcov_remote_arg, num_handles))))
  655. return -EFAULT;
  656. if (remote_num_handles > KCOV_REMOTE_MAX_HANDLES)
  657. return -EINVAL;
  658. remote_arg_size = struct_size(remote_arg, handles,
  659. remote_num_handles);
  660. remote_arg = memdup_user((void __user *)arg, remote_arg_size);
  661. if (IS_ERR(remote_arg))
  662. return PTR_ERR(remote_arg);
  663. if (remote_arg->num_handles != remote_num_handles) {
  664. kfree(remote_arg);
  665. return -EINVAL;
  666. }
  667. arg = (unsigned long)remote_arg;
  668. fallthrough;
  669. default:
  670. /*
  671. * All other commands can be normally executed under a spin lock, so we
  672. * obtain and release it here in order to simplify kcov_ioctl_locked().
  673. */
  674. spin_lock_irqsave(&kcov->lock, flags);
  675. res = kcov_ioctl_locked(kcov, cmd, arg);
  676. spin_unlock_irqrestore(&kcov->lock, flags);
  677. kfree(remote_arg);
  678. return res;
  679. }
  680. }
  681. static const struct file_operations kcov_fops = {
  682. .open = kcov_open,
  683. .unlocked_ioctl = kcov_ioctl,
  684. .compat_ioctl = kcov_ioctl,
  685. .mmap = kcov_mmap,
  686. .release = kcov_close,
  687. };
  688. /*
  689. * kcov_remote_start() and kcov_remote_stop() can be used to annotate a section
  690. * of code in a kernel background thread or in a softirq to allow kcov to be
  691. * used to collect coverage from that part of code.
  692. *
  693. * The handle argument of kcov_remote_start() identifies a code section that is
  694. * used for coverage collection. A userspace process passes this handle to
  695. * KCOV_REMOTE_ENABLE ioctl to make the used kcov device start collecting
  696. * coverage for the code section identified by this handle.
  697. *
  698. * The usage of these annotations in the kernel code is different depending on
  699. * the type of the kernel thread whose code is being annotated.
  700. *
  701. * For global kernel threads that are spawned in a limited number of instances
  702. * (e.g. one USB hub_event() worker thread is spawned per USB HCD) and for
  703. * softirqs, each instance must be assigned a unique 4-byte instance id. The
  704. * instance id is then combined with a 1-byte subsystem id to get a handle via
  705. * kcov_remote_handle(subsystem_id, instance_id).
  706. *
  707. * For local kernel threads that are spawned from system calls handler when a
  708. * user interacts with some kernel interface (e.g. vhost workers), a handle is
  709. * passed from a userspace process as the common_handle field of the
  710. * kcov_remote_arg struct (note, that the user must generate a handle by using
  711. * kcov_remote_handle() with KCOV_SUBSYSTEM_COMMON as the subsystem id and an
  712. * arbitrary 4-byte non-zero number as the instance id). This common handle
  713. * then gets saved into the task_struct of the process that issued the
  714. * KCOV_REMOTE_ENABLE ioctl. When this process issues system calls that spawn
  715. * kernel threads, the common handle must be retrieved via kcov_common_handle()
  716. * and passed to the spawned threads via custom annotations. Those kernel
  717. * threads must in turn be annotated with kcov_remote_start(common_handle) and
  718. * kcov_remote_stop(). All of the threads that are spawned by the same process
  719. * obtain the same handle, hence the name "common".
  720. *
  721. * See Documentation/dev-tools/kcov.rst for more details.
  722. *
  723. * Internally, kcov_remote_start() looks up the kcov device associated with the
  724. * provided handle, allocates an area for coverage collection, and saves the
  725. * pointers to kcov and area into the current task_struct to allow coverage to
  726. * be collected via __sanitizer_cov_trace_pc().
  727. * In turns kcov_remote_stop() clears those pointers from task_struct to stop
  728. * collecting coverage and copies all collected coverage into the kcov area.
  729. */
  730. static inline bool kcov_mode_enabled(unsigned int mode)
  731. {
  732. return (mode & ~KCOV_IN_CTXSW) != KCOV_MODE_DISABLED;
  733. }
  734. static void kcov_remote_softirq_start(struct task_struct *t)
  735. {
  736. struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
  737. unsigned int mode;
  738. mode = READ_ONCE(t->kcov_mode);
  739. barrier();
  740. if (kcov_mode_enabled(mode)) {
  741. data->saved_mode = mode;
  742. data->saved_size = t->kcov_size;
  743. data->saved_area = t->kcov_area;
  744. data->saved_sequence = t->kcov_sequence;
  745. data->saved_kcov = t->kcov;
  746. kcov_stop(t);
  747. }
  748. }
  749. static void kcov_remote_softirq_stop(struct task_struct *t)
  750. {
  751. struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data);
  752. if (data->saved_kcov) {
  753. kcov_start(t, data->saved_kcov, data->saved_size,
  754. data->saved_area, data->saved_mode,
  755. data->saved_sequence);
  756. data->saved_mode = 0;
  757. data->saved_size = 0;
  758. data->saved_area = NULL;
  759. data->saved_sequence = 0;
  760. data->saved_kcov = NULL;
  761. }
  762. }
  763. void kcov_remote_start(u64 handle)
  764. {
  765. struct task_struct *t = current;
  766. struct kcov_remote *remote;
  767. struct kcov *kcov;
  768. unsigned int mode;
  769. void *area;
  770. unsigned int size;
  771. int sequence;
  772. unsigned long flags;
  773. if (WARN_ON(!kcov_check_handle(handle, true, true, true)))
  774. return;
  775. if (!in_task() && !in_serving_softirq())
  776. return;
  777. local_lock_irqsave(&kcov_percpu_data.lock, flags);
  778. /*
  779. * Check that kcov_remote_start() is not called twice in background
  780. * threads nor called by user tasks (with enabled kcov).
  781. */
  782. mode = READ_ONCE(t->kcov_mode);
  783. if (WARN_ON(in_task() && kcov_mode_enabled(mode))) {
  784. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  785. return;
  786. }
  787. /*
  788. * Check that kcov_remote_start() is not called twice in softirqs.
  789. * Note, that kcov_remote_start() can be called from a softirq that
  790. * happened while collecting coverage from a background thread.
  791. */
  792. if (WARN_ON(in_serving_softirq() && t->kcov_softirq)) {
  793. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  794. return;
  795. }
  796. spin_lock(&kcov_remote_lock);
  797. remote = kcov_remote_find(handle);
  798. if (!remote) {
  799. spin_unlock(&kcov_remote_lock);
  800. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  801. return;
  802. }
  803. kcov_debug("handle = %llx, context: %s\n", handle,
  804. in_task() ? "task" : "softirq");
  805. kcov = remote->kcov;
  806. /* Put in kcov_remote_stop(). */
  807. kcov_get(kcov);
  808. /*
  809. * Read kcov fields before unlock to prevent races with
  810. * KCOV_DISABLE / kcov_remote_reset().
  811. */
  812. mode = kcov->mode;
  813. sequence = kcov->sequence;
  814. if (in_task()) {
  815. size = kcov->remote_size;
  816. area = kcov_remote_area_get(size);
  817. } else {
  818. size = CONFIG_KCOV_IRQ_AREA_SIZE;
  819. area = this_cpu_ptr(&kcov_percpu_data)->irq_area;
  820. }
  821. spin_unlock(&kcov_remote_lock);
  822. /* Can only happen when in_task(). */
  823. if (!area) {
  824. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  825. area = vmalloc(size * sizeof(unsigned long));
  826. if (!area) {
  827. kcov_put(kcov);
  828. return;
  829. }
  830. local_lock_irqsave(&kcov_percpu_data.lock, flags);
  831. }
  832. /* Reset coverage size. */
  833. *(u64 *)area = 0;
  834. if (in_serving_softirq()) {
  835. kcov_remote_softirq_start(t);
  836. t->kcov_softirq = 1;
  837. }
  838. kcov_start(t, kcov, size, area, mode, sequence);
  839. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  840. }
  841. EXPORT_SYMBOL(kcov_remote_start);
  842. static void kcov_move_area(enum kcov_mode mode, void *dst_area,
  843. unsigned int dst_area_size, void *src_area)
  844. {
  845. u64 word_size = sizeof(unsigned long);
  846. u64 count_size, entry_size_log;
  847. u64 dst_len, src_len;
  848. void *dst_entries, *src_entries;
  849. u64 dst_occupied, dst_free, bytes_to_move, entries_moved;
  850. kcov_debug("%px %u <= %px %lu\n",
  851. dst_area, dst_area_size, src_area, *(unsigned long *)src_area);
  852. switch (mode) {
  853. case KCOV_MODE_TRACE_PC:
  854. dst_len = READ_ONCE(*(unsigned long *)dst_area);
  855. src_len = *(unsigned long *)src_area;
  856. count_size = sizeof(unsigned long);
  857. entry_size_log = __ilog2_u64(sizeof(unsigned long));
  858. break;
  859. case KCOV_MODE_TRACE_CMP:
  860. dst_len = READ_ONCE(*(u64 *)dst_area);
  861. src_len = *(u64 *)src_area;
  862. count_size = sizeof(u64);
  863. BUILD_BUG_ON(!is_power_of_2(KCOV_WORDS_PER_CMP));
  864. entry_size_log = __ilog2_u64(sizeof(u64) * KCOV_WORDS_PER_CMP);
  865. break;
  866. default:
  867. WARN_ON(1);
  868. return;
  869. }
  870. /* As arm can't divide u64 integers use log of entry size. */
  871. if (dst_len > ((dst_area_size * word_size - count_size) >>
  872. entry_size_log))
  873. return;
  874. dst_occupied = count_size + (dst_len << entry_size_log);
  875. dst_free = dst_area_size * word_size - dst_occupied;
  876. bytes_to_move = min(dst_free, src_len << entry_size_log);
  877. dst_entries = dst_area + dst_occupied;
  878. src_entries = src_area + count_size;
  879. memcpy(dst_entries, src_entries, bytes_to_move);
  880. entries_moved = bytes_to_move >> entry_size_log;
  881. switch (mode) {
  882. case KCOV_MODE_TRACE_PC:
  883. WRITE_ONCE(*(unsigned long *)dst_area, dst_len + entries_moved);
  884. break;
  885. case KCOV_MODE_TRACE_CMP:
  886. WRITE_ONCE(*(u64 *)dst_area, dst_len + entries_moved);
  887. break;
  888. default:
  889. break;
  890. }
  891. }
  892. /* See the comment before kcov_remote_start() for usage details. */
  893. void kcov_remote_stop(void)
  894. {
  895. struct task_struct *t = current;
  896. struct kcov *kcov;
  897. unsigned int mode;
  898. void *area;
  899. unsigned int size;
  900. int sequence;
  901. unsigned long flags;
  902. if (!in_task() && !in_serving_softirq())
  903. return;
  904. local_lock_irqsave(&kcov_percpu_data.lock, flags);
  905. mode = READ_ONCE(t->kcov_mode);
  906. barrier();
  907. if (!kcov_mode_enabled(mode)) {
  908. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  909. return;
  910. }
  911. /*
  912. * When in softirq, check if the corresponding kcov_remote_start()
  913. * actually found the remote handle and started collecting coverage.
  914. */
  915. if (in_serving_softirq() && !t->kcov_softirq) {
  916. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  917. return;
  918. }
  919. /* Make sure that kcov_softirq is only set when in softirq. */
  920. if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) {
  921. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  922. return;
  923. }
  924. kcov = t->kcov;
  925. area = t->kcov_area;
  926. size = t->kcov_size;
  927. sequence = t->kcov_sequence;
  928. kcov_stop(t);
  929. if (in_serving_softirq()) {
  930. t->kcov_softirq = 0;
  931. kcov_remote_softirq_stop(t);
  932. }
  933. spin_lock(&kcov->lock);
  934. /*
  935. * KCOV_DISABLE could have been called between kcov_remote_start()
  936. * and kcov_remote_stop(), hence the sequence check.
  937. */
  938. if (sequence == kcov->sequence && kcov->remote)
  939. kcov_move_area(kcov->mode, kcov->area, kcov->size, area);
  940. spin_unlock(&kcov->lock);
  941. if (in_task()) {
  942. spin_lock(&kcov_remote_lock);
  943. kcov_remote_area_put(area, size);
  944. spin_unlock(&kcov_remote_lock);
  945. }
  946. local_unlock_irqrestore(&kcov_percpu_data.lock, flags);
  947. /* Get in kcov_remote_start(). */
  948. kcov_put(kcov);
  949. }
  950. EXPORT_SYMBOL(kcov_remote_stop);
  951. /* See the comment before kcov_remote_start() for usage details. */
  952. u64 kcov_common_handle(void)
  953. {
  954. if (!in_task())
  955. return 0;
  956. return current->kcov_handle;
  957. }
  958. EXPORT_SYMBOL(kcov_common_handle);
  959. static int __init kcov_init(void)
  960. {
  961. int cpu;
  962. for_each_possible_cpu(cpu) {
  963. void *area = vmalloc_node(CONFIG_KCOV_IRQ_AREA_SIZE *
  964. sizeof(unsigned long), cpu_to_node(cpu));
  965. if (!area)
  966. return -ENOMEM;
  967. per_cpu_ptr(&kcov_percpu_data, cpu)->irq_area = area;
  968. }
  969. /*
  970. * The kcov debugfs file won't ever get removed and thus,
  971. * there is no need to protect it against removal races. The
  972. * use of debugfs_create_file_unsafe() is actually safe here.
  973. */
  974. debugfs_create_file_unsafe("kcov", 0600, NULL, NULL, &kcov_fops);
  975. return 0;
  976. }
  977. device_initcall(kcov_init);