delayacct.c 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* delayacct.c - per-task delay accounting
  3. *
  4. * Copyright (C) Shailabh Nagar, IBM Corp. 2006
  5. */
  6. #include <linux/sched.h>
  7. #include <linux/sched/task.h>
  8. #include <linux/sched/cputime.h>
  9. #include <linux/sched/clock.h>
  10. #include <linux/slab.h>
  11. #include <linux/taskstats.h>
  12. #include <linux/sysctl.h>
  13. #include <linux/delayacct.h>
  14. #include <linux/module.h>
  15. #ifdef CONFIG_TASK_DELAY_ACCT
  16. DEFINE_STATIC_KEY_FALSE(delayacct_key);
  17. int delayacct_on __read_mostly; /* Delay accounting turned on/off */
  18. struct kmem_cache *delayacct_cache;
  19. static void set_delayacct(bool enabled)
  20. {
  21. if (enabled) {
  22. static_branch_enable(&delayacct_key);
  23. delayacct_on = 1;
  24. } else {
  25. delayacct_on = 0;
  26. static_branch_disable(&delayacct_key);
  27. }
  28. }
  29. static int __init delayacct_setup_enable(char *str)
  30. {
  31. delayacct_on = 1;
  32. return 1;
  33. }
  34. __setup("delayacct", delayacct_setup_enable);
  35. void delayacct_init(void)
  36. {
  37. delayacct_cache = KMEM_CACHE(task_delay_info, SLAB_PANIC|SLAB_ACCOUNT);
  38. delayacct_tsk_init(&init_task);
  39. set_delayacct(delayacct_on);
  40. }
  41. #ifdef CONFIG_PROC_SYSCTL
  42. static int sysctl_delayacct(struct ctl_table *table, int write, void *buffer,
  43. size_t *lenp, loff_t *ppos)
  44. {
  45. int state = delayacct_on;
  46. struct ctl_table t;
  47. int err;
  48. if (write && !capable(CAP_SYS_ADMIN))
  49. return -EPERM;
  50. t = *table;
  51. t.data = &state;
  52. err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos);
  53. if (err < 0)
  54. return err;
  55. if (write)
  56. set_delayacct(state);
  57. return err;
  58. }
  59. static struct ctl_table kern_delayacct_table[] = {
  60. {
  61. .procname = "task_delayacct",
  62. .data = NULL,
  63. .maxlen = sizeof(unsigned int),
  64. .mode = 0644,
  65. .proc_handler = sysctl_delayacct,
  66. .extra1 = SYSCTL_ZERO,
  67. .extra2 = SYSCTL_ONE,
  68. },
  69. { }
  70. };
  71. static __init int kernel_delayacct_sysctls_init(void)
  72. {
  73. register_sysctl_init("kernel", kern_delayacct_table);
  74. return 0;
  75. }
  76. late_initcall(kernel_delayacct_sysctls_init);
  77. #endif
  78. void __delayacct_tsk_init(struct task_struct *tsk)
  79. {
  80. tsk->delays = kmem_cache_zalloc(delayacct_cache, GFP_KERNEL);
  81. if (tsk->delays)
  82. raw_spin_lock_init(&tsk->delays->lock);
  83. }
  84. /*
  85. * Finish delay accounting for a statistic using its timestamps (@start),
  86. * accumalator (@total) and @count
  87. */
  88. static void delayacct_end(raw_spinlock_t *lock, u64 *start, u64 *total, u32 *count)
  89. {
  90. s64 ns = local_clock() - *start;
  91. unsigned long flags;
  92. if (ns > 0) {
  93. raw_spin_lock_irqsave(lock, flags);
  94. *total += ns;
  95. (*count)++;
  96. raw_spin_unlock_irqrestore(lock, flags);
  97. }
  98. }
  99. void __delayacct_blkio_start(void)
  100. {
  101. current->delays->blkio_start = local_clock();
  102. }
  103. /*
  104. * We cannot rely on the `current` macro, as we haven't yet switched back to
  105. * the process being woken.
  106. */
  107. void __delayacct_blkio_end(struct task_struct *p)
  108. {
  109. delayacct_end(&p->delays->lock,
  110. &p->delays->blkio_start,
  111. &p->delays->blkio_delay,
  112. &p->delays->blkio_count);
  113. }
  114. int delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
  115. {
  116. u64 utime, stime, stimescaled, utimescaled;
  117. unsigned long long t2, t3;
  118. unsigned long flags, t1;
  119. s64 tmp;
  120. task_cputime(tsk, &utime, &stime);
  121. tmp = (s64)d->cpu_run_real_total;
  122. tmp += utime + stime;
  123. d->cpu_run_real_total = (tmp < (s64)d->cpu_run_real_total) ? 0 : tmp;
  124. task_cputime_scaled(tsk, &utimescaled, &stimescaled);
  125. tmp = (s64)d->cpu_scaled_run_real_total;
  126. tmp += utimescaled + stimescaled;
  127. d->cpu_scaled_run_real_total =
  128. (tmp < (s64)d->cpu_scaled_run_real_total) ? 0 : tmp;
  129. /*
  130. * No locking available for sched_info (and too expensive to add one)
  131. * Mitigate by taking snapshot of values
  132. */
  133. t1 = tsk->sched_info.pcount;
  134. t2 = tsk->sched_info.run_delay;
  135. t3 = tsk->se.sum_exec_runtime;
  136. d->cpu_count += t1;
  137. tmp = (s64)d->cpu_delay_total + t2;
  138. d->cpu_delay_total = (tmp < (s64)d->cpu_delay_total) ? 0 : tmp;
  139. tmp = (s64)d->cpu_run_virtual_total + t3;
  140. d->cpu_run_virtual_total =
  141. (tmp < (s64)d->cpu_run_virtual_total) ? 0 : tmp;
  142. if (!tsk->delays)
  143. return 0;
  144. /* zero XXX_total, non-zero XXX_count implies XXX stat overflowed */
  145. raw_spin_lock_irqsave(&tsk->delays->lock, flags);
  146. tmp = d->blkio_delay_total + tsk->delays->blkio_delay;
  147. d->blkio_delay_total = (tmp < d->blkio_delay_total) ? 0 : tmp;
  148. tmp = d->swapin_delay_total + tsk->delays->swapin_delay;
  149. d->swapin_delay_total = (tmp < d->swapin_delay_total) ? 0 : tmp;
  150. tmp = d->freepages_delay_total + tsk->delays->freepages_delay;
  151. d->freepages_delay_total = (tmp < d->freepages_delay_total) ? 0 : tmp;
  152. tmp = d->thrashing_delay_total + tsk->delays->thrashing_delay;
  153. d->thrashing_delay_total = (tmp < d->thrashing_delay_total) ? 0 : tmp;
  154. tmp = d->compact_delay_total + tsk->delays->compact_delay;
  155. d->compact_delay_total = (tmp < d->compact_delay_total) ? 0 : tmp;
  156. tmp = d->wpcopy_delay_total + tsk->delays->wpcopy_delay;
  157. d->wpcopy_delay_total = (tmp < d->wpcopy_delay_total) ? 0 : tmp;
  158. d->blkio_count += tsk->delays->blkio_count;
  159. d->swapin_count += tsk->delays->swapin_count;
  160. d->freepages_count += tsk->delays->freepages_count;
  161. d->thrashing_count += tsk->delays->thrashing_count;
  162. d->compact_count += tsk->delays->compact_count;
  163. d->wpcopy_count += tsk->delays->wpcopy_count;
  164. raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
  165. return 0;
  166. }
  167. __u64 __delayacct_blkio_ticks(struct task_struct *tsk)
  168. {
  169. __u64 ret;
  170. unsigned long flags;
  171. raw_spin_lock_irqsave(&tsk->delays->lock, flags);
  172. ret = nsec_to_clock_t(tsk->delays->blkio_delay);
  173. raw_spin_unlock_irqrestore(&tsk->delays->lock, flags);
  174. return ret;
  175. }
  176. void __delayacct_freepages_start(void)
  177. {
  178. current->delays->freepages_start = local_clock();
  179. }
  180. void __delayacct_freepages_end(void)
  181. {
  182. delayacct_end(&current->delays->lock,
  183. &current->delays->freepages_start,
  184. &current->delays->freepages_delay,
  185. &current->delays->freepages_count);
  186. }
  187. void __delayacct_thrashing_start(bool *in_thrashing)
  188. {
  189. *in_thrashing = !!current->in_thrashing;
  190. if (*in_thrashing)
  191. return;
  192. current->in_thrashing = 1;
  193. current->delays->thrashing_start = local_clock();
  194. }
  195. void __delayacct_thrashing_end(bool *in_thrashing)
  196. {
  197. if (*in_thrashing)
  198. return;
  199. current->in_thrashing = 0;
  200. delayacct_end(&current->delays->lock,
  201. &current->delays->thrashing_start,
  202. &current->delays->thrashing_delay,
  203. &current->delays->thrashing_count);
  204. }
  205. void __delayacct_swapin_start(void)
  206. {
  207. current->delays->swapin_start = local_clock();
  208. }
  209. void __delayacct_swapin_end(void)
  210. {
  211. delayacct_end(&current->delays->lock,
  212. &current->delays->swapin_start,
  213. &current->delays->swapin_delay,
  214. &current->delays->swapin_count);
  215. }
  216. void __delayacct_compact_start(void)
  217. {
  218. current->delays->compact_start = local_clock();
  219. }
  220. void __delayacct_compact_end(void)
  221. {
  222. delayacct_end(&current->delays->lock,
  223. &current->delays->compact_start,
  224. &current->delays->compact_delay,
  225. &current->delays->compact_count);
  226. }
  227. void __delayacct_wpcopy_start(void)
  228. {
  229. current->delays->wpcopy_start = local_clock();
  230. }
  231. void __delayacct_wpcopy_end(void)
  232. {
  233. delayacct_end(&current->delays->lock,
  234. &current->delays->wpcopy_start,
  235. &current->delays->wpcopy_delay,
  236. &current->delays->wpcopy_count);
  237. }
  238. #else
  239. #include <trace/hooks/delayacct.h>
  240. int delayacct_enabled __read_mostly; /* Delay accounting turned on/off */
  241. bool get_delayacct_enabled(void)
  242. {
  243. return delayacct_enabled;
  244. }
  245. void set_delayacct_enabled(bool enabled)
  246. {
  247. delayacct_enabled = enabled;
  248. }
  249. EXPORT_SYMBOL_GPL(set_delayacct_enabled);
  250. void _trace_android_rvh_delayacct_init(void)
  251. {
  252. trace_android_rvh_delayacct_init(NULL);
  253. }
  254. void _trace_android_rvh_delayacct_tsk_init(struct task_struct *tsk)
  255. {
  256. trace_android_rvh_delayacct_tsk_init(tsk);
  257. }
  258. void _trace_android_rvh_delayacct_tsk_free(struct task_struct *tsk)
  259. {
  260. trace_android_rvh_delayacct_tsk_free(tsk);
  261. }
  262. void _trace_android_vh_delayacct_blkio_start(void)
  263. {
  264. trace_android_vh_delayacct_blkio_start(NULL);
  265. }
  266. void _trace_android_vh_delayacct_blkio_end(struct task_struct *p)
  267. {
  268. trace_android_vh_delayacct_blkio_end(p);
  269. }
  270. void _trace_android_vh_delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk, int *ret)
  271. {
  272. trace_android_vh_delayacct_add_tsk(d, tsk, ret);
  273. }
  274. void _trace_android_vh_delayacct_blkio_ticks(struct task_struct *tsk, __u64 *ret)
  275. {
  276. trace_android_vh_delayacct_blkio_ticks(tsk, ret);
  277. }
  278. void _trace_android_vh_delayacct_freepages_start(void)
  279. {
  280. trace_android_vh_delayacct_freepages_start(NULL);
  281. }
  282. void _trace_android_vh_delayacct_freepages_end(void)
  283. {
  284. trace_android_vh_delayacct_freepages_end(NULL);
  285. }
  286. void _trace_android_vh_delayacct_thrashing_start(bool *in_thrashing)
  287. {
  288. trace_android_vh_delayacct_thrashing_start(in_thrashing);
  289. }
  290. void _trace_android_vh_delayacct_thrashing_end(bool *in_thrashing)
  291. {
  292. trace_android_vh_delayacct_thrashing_end(in_thrashing);
  293. }
  294. void _trace_android_vh_delayacct_swapin_start(void)
  295. {
  296. trace_android_vh_delayacct_swapin_start(NULL);
  297. }
  298. void _trace_android_vh_delayacct_swapin_end(void)
  299. {
  300. trace_android_vh_delayacct_swapin_end(NULL);
  301. }
  302. void _trace_android_vh_delayacct_compact_start(void)
  303. {
  304. trace_android_vh_delayacct_compact_start(NULL);
  305. }
  306. void _trace_android_vh_delayacct_compact_end(void)
  307. {
  308. trace_android_vh_delayacct_compact_end(NULL);
  309. }
  310. void _trace_android_vh_delayacct_wpcopy_start(void)
  311. {
  312. trace_android_vh_delayacct_wpcopy_start(NULL);
  313. }
  314. void _trace_android_vh_delayacct_wpcopy_end(void)
  315. {
  316. trace_android_vh_delayacct_wpcopy_end(NULL);
  317. }
  318. #endif