trace_sched_wakeup.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * trace task wakeup timings
  4. *
  5. * Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
  6. * Copyright (C) 2008 Ingo Molnar <[email protected]>
  7. *
  8. * Based on code from the latency_tracer, that is:
  9. *
  10. * Copyright (C) 2004-2006 Ingo Molnar
  11. * Copyright (C) 2004 Nadia Yvette Chambers
  12. */
  13. #include <linux/module.h>
  14. #include <linux/kallsyms.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/ftrace.h>
  17. #include <linux/sched/rt.h>
  18. #include <linux/sched/deadline.h>
  19. #include <trace/events/sched.h>
  20. #include "trace.h"
  21. static struct trace_array *wakeup_trace;
  22. static int __read_mostly tracer_enabled;
  23. static struct task_struct *wakeup_task;
  24. static int wakeup_cpu;
  25. static int wakeup_current_cpu;
  26. static unsigned wakeup_prio = -1;
  27. static bool wakeup_rt;
  28. static bool wakeup_dl;
  29. static bool tracing_dl;
  30. static arch_spinlock_t wakeup_lock =
  31. (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
  32. static void wakeup_reset(struct trace_array *tr);
  33. static void __wakeup_reset(struct trace_array *tr);
  34. static int start_func_tracer(struct trace_array *tr, int graph);
  35. static void stop_func_tracer(struct trace_array *tr, int graph);
  36. static int save_flags;
  37. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  38. # define is_graph(tr) ((tr)->trace_flags & TRACE_ITER_DISPLAY_GRAPH)
  39. #else
  40. # define is_graph(tr) false
  41. #endif
  42. #ifdef CONFIG_FUNCTION_TRACER
  43. static bool function_enabled;
  44. /*
  45. * Prologue for the wakeup function tracers.
  46. *
  47. * Returns 1 if it is OK to continue, and preemption
  48. * is disabled and data->disabled is incremented.
  49. * 0 if the trace is to be ignored, and preemption
  50. * is not disabled and data->disabled is
  51. * kept the same.
  52. *
  53. * Note, this function is also used outside this ifdef but
  54. * inside the #ifdef of the function graph tracer below.
  55. * This is OK, since the function graph tracer is
  56. * dependent on the function tracer.
  57. */
  58. static int
  59. func_prolog_preempt_disable(struct trace_array *tr,
  60. struct trace_array_cpu **data,
  61. unsigned int *trace_ctx)
  62. {
  63. long disabled;
  64. int cpu;
  65. if (likely(!wakeup_task))
  66. return 0;
  67. *trace_ctx = tracing_gen_ctx();
  68. preempt_disable_notrace();
  69. cpu = raw_smp_processor_id();
  70. if (cpu != wakeup_current_cpu)
  71. goto out_enable;
  72. *data = per_cpu_ptr(tr->array_buffer.data, cpu);
  73. disabled = atomic_inc_return(&(*data)->disabled);
  74. if (unlikely(disabled != 1))
  75. goto out;
  76. return 1;
  77. out:
  78. atomic_dec(&(*data)->disabled);
  79. out_enable:
  80. preempt_enable_notrace();
  81. return 0;
  82. }
  83. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  84. static int wakeup_display_graph(struct trace_array *tr, int set)
  85. {
  86. if (!(is_graph(tr) ^ set))
  87. return 0;
  88. stop_func_tracer(tr, !set);
  89. wakeup_reset(wakeup_trace);
  90. tr->max_latency = 0;
  91. return start_func_tracer(tr, set);
  92. }
  93. static int wakeup_graph_entry(struct ftrace_graph_ent *trace)
  94. {
  95. struct trace_array *tr = wakeup_trace;
  96. struct trace_array_cpu *data;
  97. unsigned int trace_ctx;
  98. int ret = 0;
  99. if (ftrace_graph_ignore_func(trace))
  100. return 0;
  101. /*
  102. * Do not trace a function if it's filtered by set_graph_notrace.
  103. * Make the index of ret stack negative to indicate that it should
  104. * ignore further functions. But it needs its own ret stack entry
  105. * to recover the original index in order to continue tracing after
  106. * returning from the function.
  107. */
  108. if (ftrace_graph_notrace_addr(trace->func))
  109. return 1;
  110. if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))
  111. return 0;
  112. ret = __trace_graph_entry(tr, trace, trace_ctx);
  113. atomic_dec(&data->disabled);
  114. preempt_enable_notrace();
  115. return ret;
  116. }
  117. static void wakeup_graph_return(struct ftrace_graph_ret *trace)
  118. {
  119. struct trace_array *tr = wakeup_trace;
  120. struct trace_array_cpu *data;
  121. unsigned int trace_ctx;
  122. ftrace_graph_addr_finish(trace);
  123. if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))
  124. return;
  125. __trace_graph_return(tr, trace, trace_ctx);
  126. atomic_dec(&data->disabled);
  127. preempt_enable_notrace();
  128. return;
  129. }
  130. static struct fgraph_ops fgraph_wakeup_ops = {
  131. .entryfunc = &wakeup_graph_entry,
  132. .retfunc = &wakeup_graph_return,
  133. };
  134. static void wakeup_trace_open(struct trace_iterator *iter)
  135. {
  136. if (is_graph(iter->tr))
  137. graph_trace_open(iter);
  138. else
  139. iter->private = NULL;
  140. }
  141. static void wakeup_trace_close(struct trace_iterator *iter)
  142. {
  143. if (iter->private)
  144. graph_trace_close(iter);
  145. }
  146. #define GRAPH_TRACER_FLAGS (TRACE_GRAPH_PRINT_PROC | \
  147. TRACE_GRAPH_PRINT_CPU | \
  148. TRACE_GRAPH_PRINT_REL_TIME | \
  149. TRACE_GRAPH_PRINT_DURATION | \
  150. TRACE_GRAPH_PRINT_OVERHEAD | \
  151. TRACE_GRAPH_PRINT_IRQS)
  152. static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
  153. {
  154. /*
  155. * In graph mode call the graph tracer output function,
  156. * otherwise go with the TRACE_FN event handler
  157. */
  158. if (is_graph(iter->tr))
  159. return print_graph_function_flags(iter, GRAPH_TRACER_FLAGS);
  160. return TRACE_TYPE_UNHANDLED;
  161. }
  162. static void wakeup_print_header(struct seq_file *s)
  163. {
  164. if (is_graph(wakeup_trace))
  165. print_graph_headers_flags(s, GRAPH_TRACER_FLAGS);
  166. else
  167. trace_default_header(s);
  168. }
  169. #endif /* else CONFIG_FUNCTION_GRAPH_TRACER */
  170. /*
  171. * wakeup uses its own tracer function to keep the overhead down:
  172. */
  173. static void
  174. wakeup_tracer_call(unsigned long ip, unsigned long parent_ip,
  175. struct ftrace_ops *op, struct ftrace_regs *fregs)
  176. {
  177. struct trace_array *tr = wakeup_trace;
  178. struct trace_array_cpu *data;
  179. unsigned long flags;
  180. unsigned int trace_ctx;
  181. if (!func_prolog_preempt_disable(tr, &data, &trace_ctx))
  182. return;
  183. local_irq_save(flags);
  184. trace_function(tr, ip, parent_ip, trace_ctx);
  185. local_irq_restore(flags);
  186. atomic_dec(&data->disabled);
  187. preempt_enable_notrace();
  188. }
  189. static int register_wakeup_function(struct trace_array *tr, int graph, int set)
  190. {
  191. int ret;
  192. /* 'set' is set if TRACE_ITER_FUNCTION is about to be set */
  193. if (function_enabled || (!set && !(tr->trace_flags & TRACE_ITER_FUNCTION)))
  194. return 0;
  195. if (graph)
  196. ret = register_ftrace_graph(&fgraph_wakeup_ops);
  197. else
  198. ret = register_ftrace_function(tr->ops);
  199. if (!ret)
  200. function_enabled = true;
  201. return ret;
  202. }
  203. static void unregister_wakeup_function(struct trace_array *tr, int graph)
  204. {
  205. if (!function_enabled)
  206. return;
  207. if (graph)
  208. unregister_ftrace_graph(&fgraph_wakeup_ops);
  209. else
  210. unregister_ftrace_function(tr->ops);
  211. function_enabled = false;
  212. }
  213. static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
  214. {
  215. if (!(mask & TRACE_ITER_FUNCTION))
  216. return 0;
  217. if (set)
  218. register_wakeup_function(tr, is_graph(tr), 1);
  219. else
  220. unregister_wakeup_function(tr, is_graph(tr));
  221. return 1;
  222. }
  223. #else /* CONFIG_FUNCTION_TRACER */
  224. static int register_wakeup_function(struct trace_array *tr, int graph, int set)
  225. {
  226. return 0;
  227. }
  228. static void unregister_wakeup_function(struct trace_array *tr, int graph) { }
  229. static int wakeup_function_set(struct trace_array *tr, u32 mask, int set)
  230. {
  231. return 0;
  232. }
  233. #endif /* else CONFIG_FUNCTION_TRACER */
  234. #ifndef CONFIG_FUNCTION_GRAPH_TRACER
  235. static enum print_line_t wakeup_print_line(struct trace_iterator *iter)
  236. {
  237. return TRACE_TYPE_UNHANDLED;
  238. }
  239. static void wakeup_trace_open(struct trace_iterator *iter) { }
  240. static void wakeup_trace_close(struct trace_iterator *iter) { }
  241. static void wakeup_print_header(struct seq_file *s)
  242. {
  243. trace_default_header(s);
  244. }
  245. #endif /* !CONFIG_FUNCTION_GRAPH_TRACER */
  246. static void
  247. __trace_function(struct trace_array *tr,
  248. unsigned long ip, unsigned long parent_ip,
  249. unsigned int trace_ctx)
  250. {
  251. if (is_graph(tr))
  252. trace_graph_function(tr, ip, parent_ip, trace_ctx);
  253. else
  254. trace_function(tr, ip, parent_ip, trace_ctx);
  255. }
  256. static int wakeup_flag_changed(struct trace_array *tr, u32 mask, int set)
  257. {
  258. struct tracer *tracer = tr->current_trace;
  259. if (wakeup_function_set(tr, mask, set))
  260. return 0;
  261. #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  262. if (mask & TRACE_ITER_DISPLAY_GRAPH)
  263. return wakeup_display_graph(tr, set);
  264. #endif
  265. return trace_keep_overwrite(tracer, mask, set);
  266. }
  267. static int start_func_tracer(struct trace_array *tr, int graph)
  268. {
  269. int ret;
  270. ret = register_wakeup_function(tr, graph, 0);
  271. if (!ret && tracing_is_enabled())
  272. tracer_enabled = 1;
  273. else
  274. tracer_enabled = 0;
  275. return ret;
  276. }
  277. static void stop_func_tracer(struct trace_array *tr, int graph)
  278. {
  279. tracer_enabled = 0;
  280. unregister_wakeup_function(tr, graph);
  281. }
  282. /*
  283. * Should this new latency be reported/recorded?
  284. */
  285. static bool report_latency(struct trace_array *tr, u64 delta)
  286. {
  287. if (tracing_thresh) {
  288. if (delta < tracing_thresh)
  289. return false;
  290. } else {
  291. if (delta <= tr->max_latency)
  292. return false;
  293. }
  294. return true;
  295. }
  296. static void
  297. probe_wakeup_migrate_task(void *ignore, struct task_struct *task, int cpu)
  298. {
  299. if (task != wakeup_task)
  300. return;
  301. wakeup_current_cpu = cpu;
  302. }
  303. static void
  304. tracing_sched_switch_trace(struct trace_array *tr,
  305. struct task_struct *prev,
  306. struct task_struct *next,
  307. unsigned int trace_ctx)
  308. {
  309. struct trace_event_call *call = &event_context_switch;
  310. struct trace_buffer *buffer = tr->array_buffer.buffer;
  311. struct ring_buffer_event *event;
  312. struct ctx_switch_entry *entry;
  313. event = trace_buffer_lock_reserve(buffer, TRACE_CTX,
  314. sizeof(*entry), trace_ctx);
  315. if (!event)
  316. return;
  317. entry = ring_buffer_event_data(event);
  318. entry->prev_pid = prev->pid;
  319. entry->prev_prio = prev->prio;
  320. entry->prev_state = task_state_index(prev);
  321. entry->next_pid = next->pid;
  322. entry->next_prio = next->prio;
  323. entry->next_state = task_state_index(next);
  324. entry->next_cpu = task_cpu(next);
  325. if (!call_filter_check_discard(call, entry, buffer, event))
  326. trace_buffer_unlock_commit(tr, buffer, event, trace_ctx);
  327. }
  328. static void
  329. tracing_sched_wakeup_trace(struct trace_array *tr,
  330. struct task_struct *wakee,
  331. struct task_struct *curr,
  332. unsigned int trace_ctx)
  333. {
  334. struct trace_event_call *call = &event_wakeup;
  335. struct ring_buffer_event *event;
  336. struct ctx_switch_entry *entry;
  337. struct trace_buffer *buffer = tr->array_buffer.buffer;
  338. event = trace_buffer_lock_reserve(buffer, TRACE_WAKE,
  339. sizeof(*entry), trace_ctx);
  340. if (!event)
  341. return;
  342. entry = ring_buffer_event_data(event);
  343. entry->prev_pid = curr->pid;
  344. entry->prev_prio = curr->prio;
  345. entry->prev_state = task_state_index(curr);
  346. entry->next_pid = wakee->pid;
  347. entry->next_prio = wakee->prio;
  348. entry->next_state = task_state_index(wakee);
  349. entry->next_cpu = task_cpu(wakee);
  350. if (!call_filter_check_discard(call, entry, buffer, event))
  351. trace_buffer_unlock_commit(tr, buffer, event, trace_ctx);
  352. }
  353. static void notrace
  354. probe_wakeup_sched_switch(void *ignore, bool preempt,
  355. struct task_struct *prev, struct task_struct *next,
  356. unsigned int prev_state)
  357. {
  358. struct trace_array_cpu *data;
  359. u64 T0, T1, delta;
  360. unsigned long flags;
  361. long disabled;
  362. int cpu;
  363. unsigned int trace_ctx;
  364. tracing_record_cmdline(prev);
  365. if (unlikely(!tracer_enabled))
  366. return;
  367. /*
  368. * When we start a new trace, we set wakeup_task to NULL
  369. * and then set tracer_enabled = 1. We want to make sure
  370. * that another CPU does not see the tracer_enabled = 1
  371. * and the wakeup_task with an older task, that might
  372. * actually be the same as next.
  373. */
  374. smp_rmb();
  375. if (next != wakeup_task)
  376. return;
  377. /* disable local data, not wakeup_cpu data */
  378. cpu = raw_smp_processor_id();
  379. disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  380. if (likely(disabled != 1))
  381. goto out;
  382. local_irq_save(flags);
  383. trace_ctx = tracing_gen_ctx_flags(flags);
  384. arch_spin_lock(&wakeup_lock);
  385. /* We could race with grabbing wakeup_lock */
  386. if (unlikely(!tracer_enabled || next != wakeup_task))
  387. goto out_unlock;
  388. /* The task we are waiting for is waking up */
  389. data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
  390. __trace_function(wakeup_trace, CALLER_ADDR0, CALLER_ADDR1, trace_ctx);
  391. tracing_sched_switch_trace(wakeup_trace, prev, next, trace_ctx);
  392. __trace_stack(wakeup_trace, trace_ctx, 0);
  393. T0 = data->preempt_timestamp;
  394. T1 = ftrace_now(cpu);
  395. delta = T1-T0;
  396. if (!report_latency(wakeup_trace, delta))
  397. goto out_unlock;
  398. if (likely(!is_tracing_stopped())) {
  399. wakeup_trace->max_latency = delta;
  400. update_max_tr(wakeup_trace, wakeup_task, wakeup_cpu, NULL);
  401. }
  402. out_unlock:
  403. __wakeup_reset(wakeup_trace);
  404. arch_spin_unlock(&wakeup_lock);
  405. local_irq_restore(flags);
  406. out:
  407. atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  408. }
  409. static void __wakeup_reset(struct trace_array *tr)
  410. {
  411. wakeup_cpu = -1;
  412. wakeup_prio = -1;
  413. tracing_dl = false;
  414. if (wakeup_task)
  415. put_task_struct(wakeup_task);
  416. wakeup_task = NULL;
  417. }
  418. static void wakeup_reset(struct trace_array *tr)
  419. {
  420. unsigned long flags;
  421. tracing_reset_online_cpus(&tr->array_buffer);
  422. local_irq_save(flags);
  423. arch_spin_lock(&wakeup_lock);
  424. __wakeup_reset(tr);
  425. arch_spin_unlock(&wakeup_lock);
  426. local_irq_restore(flags);
  427. }
  428. static void
  429. probe_wakeup(void *ignore, struct task_struct *p)
  430. {
  431. struct trace_array_cpu *data;
  432. int cpu = smp_processor_id();
  433. long disabled;
  434. unsigned int trace_ctx;
  435. if (likely(!tracer_enabled))
  436. return;
  437. tracing_record_cmdline(p);
  438. tracing_record_cmdline(current);
  439. /*
  440. * Semantic is like this:
  441. * - wakeup tracer handles all tasks in the system, independently
  442. * from their scheduling class;
  443. * - wakeup_rt tracer handles tasks belonging to sched_dl and
  444. * sched_rt class;
  445. * - wakeup_dl handles tasks belonging to sched_dl class only.
  446. */
  447. if (tracing_dl || (wakeup_dl && !dl_task(p)) ||
  448. (wakeup_rt && !dl_task(p) && !rt_task(p)) ||
  449. (!dl_task(p) && (p->prio >= wakeup_prio || p->prio >= current->prio)))
  450. return;
  451. disabled = atomic_inc_return(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  452. if (unlikely(disabled != 1))
  453. goto out;
  454. trace_ctx = tracing_gen_ctx();
  455. /* interrupts should be off from try_to_wake_up */
  456. arch_spin_lock(&wakeup_lock);
  457. /* check for races. */
  458. if (!tracer_enabled || tracing_dl ||
  459. (!dl_task(p) && p->prio >= wakeup_prio))
  460. goto out_locked;
  461. /* reset the trace */
  462. __wakeup_reset(wakeup_trace);
  463. wakeup_cpu = task_cpu(p);
  464. wakeup_current_cpu = wakeup_cpu;
  465. wakeup_prio = p->prio;
  466. /*
  467. * Once you start tracing a -deadline task, don't bother tracing
  468. * another task until the first one wakes up.
  469. */
  470. if (dl_task(p))
  471. tracing_dl = true;
  472. else
  473. tracing_dl = false;
  474. wakeup_task = get_task_struct(p);
  475. data = per_cpu_ptr(wakeup_trace->array_buffer.data, wakeup_cpu);
  476. data->preempt_timestamp = ftrace_now(cpu);
  477. tracing_sched_wakeup_trace(wakeup_trace, p, current, trace_ctx);
  478. __trace_stack(wakeup_trace, trace_ctx, 0);
  479. /*
  480. * We must be careful in using CALLER_ADDR2. But since wake_up
  481. * is not called by an assembly function (where as schedule is)
  482. * it should be safe to use it here.
  483. */
  484. __trace_function(wakeup_trace, CALLER_ADDR1, CALLER_ADDR2, trace_ctx);
  485. out_locked:
  486. arch_spin_unlock(&wakeup_lock);
  487. out:
  488. atomic_dec(&per_cpu_ptr(wakeup_trace->array_buffer.data, cpu)->disabled);
  489. }
  490. static void start_wakeup_tracer(struct trace_array *tr)
  491. {
  492. int ret;
  493. ret = register_trace_sched_wakeup(probe_wakeup, NULL);
  494. if (ret) {
  495. pr_info("wakeup trace: Couldn't activate tracepoint"
  496. " probe to kernel_sched_wakeup\n");
  497. return;
  498. }
  499. ret = register_trace_sched_wakeup_new(probe_wakeup, NULL);
  500. if (ret) {
  501. pr_info("wakeup trace: Couldn't activate tracepoint"
  502. " probe to kernel_sched_wakeup_new\n");
  503. goto fail_deprobe;
  504. }
  505. ret = register_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  506. if (ret) {
  507. pr_info("sched trace: Couldn't activate tracepoint"
  508. " probe to kernel_sched_switch\n");
  509. goto fail_deprobe_wake_new;
  510. }
  511. ret = register_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
  512. if (ret) {
  513. pr_info("wakeup trace: Couldn't activate tracepoint"
  514. " probe to kernel_sched_migrate_task\n");
  515. goto fail_deprobe_sched_switch;
  516. }
  517. wakeup_reset(tr);
  518. /*
  519. * Don't let the tracer_enabled = 1 show up before
  520. * the wakeup_task is reset. This may be overkill since
  521. * wakeup_reset does a spin_unlock after setting the
  522. * wakeup_task to NULL, but I want to be safe.
  523. * This is a slow path anyway.
  524. */
  525. smp_wmb();
  526. if (start_func_tracer(tr, is_graph(tr)))
  527. printk(KERN_ERR "failed to start wakeup tracer\n");
  528. return;
  529. fail_deprobe_sched_switch:
  530. unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  531. fail_deprobe_wake_new:
  532. unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
  533. fail_deprobe:
  534. unregister_trace_sched_wakeup(probe_wakeup, NULL);
  535. }
  536. static void stop_wakeup_tracer(struct trace_array *tr)
  537. {
  538. tracer_enabled = 0;
  539. stop_func_tracer(tr, is_graph(tr));
  540. unregister_trace_sched_switch(probe_wakeup_sched_switch, NULL);
  541. unregister_trace_sched_wakeup_new(probe_wakeup, NULL);
  542. unregister_trace_sched_wakeup(probe_wakeup, NULL);
  543. unregister_trace_sched_migrate_task(probe_wakeup_migrate_task, NULL);
  544. }
  545. static bool wakeup_busy;
  546. static int __wakeup_tracer_init(struct trace_array *tr)
  547. {
  548. save_flags = tr->trace_flags;
  549. /* non overwrite screws up the latency tracers */
  550. set_tracer_flag(tr, TRACE_ITER_OVERWRITE, 1);
  551. set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, 1);
  552. tr->max_latency = 0;
  553. wakeup_trace = tr;
  554. ftrace_init_array_ops(tr, wakeup_tracer_call);
  555. start_wakeup_tracer(tr);
  556. wakeup_busy = true;
  557. return 0;
  558. }
  559. static int wakeup_tracer_init(struct trace_array *tr)
  560. {
  561. if (wakeup_busy)
  562. return -EBUSY;
  563. wakeup_dl = false;
  564. wakeup_rt = false;
  565. return __wakeup_tracer_init(tr);
  566. }
  567. static int wakeup_rt_tracer_init(struct trace_array *tr)
  568. {
  569. if (wakeup_busy)
  570. return -EBUSY;
  571. wakeup_dl = false;
  572. wakeup_rt = true;
  573. return __wakeup_tracer_init(tr);
  574. }
  575. static int wakeup_dl_tracer_init(struct trace_array *tr)
  576. {
  577. if (wakeup_busy)
  578. return -EBUSY;
  579. wakeup_dl = true;
  580. wakeup_rt = false;
  581. return __wakeup_tracer_init(tr);
  582. }
  583. static void wakeup_tracer_reset(struct trace_array *tr)
  584. {
  585. int lat_flag = save_flags & TRACE_ITER_LATENCY_FMT;
  586. int overwrite_flag = save_flags & TRACE_ITER_OVERWRITE;
  587. stop_wakeup_tracer(tr);
  588. /* make sure we put back any tasks we are tracing */
  589. wakeup_reset(tr);
  590. set_tracer_flag(tr, TRACE_ITER_LATENCY_FMT, lat_flag);
  591. set_tracer_flag(tr, TRACE_ITER_OVERWRITE, overwrite_flag);
  592. ftrace_reset_array_ops(tr);
  593. wakeup_busy = false;
  594. }
  595. static void wakeup_tracer_start(struct trace_array *tr)
  596. {
  597. wakeup_reset(tr);
  598. tracer_enabled = 1;
  599. }
  600. static void wakeup_tracer_stop(struct trace_array *tr)
  601. {
  602. tracer_enabled = 0;
  603. }
  604. static struct tracer wakeup_tracer __read_mostly =
  605. {
  606. .name = "wakeup",
  607. .init = wakeup_tracer_init,
  608. .reset = wakeup_tracer_reset,
  609. .start = wakeup_tracer_start,
  610. .stop = wakeup_tracer_stop,
  611. .print_max = true,
  612. .print_header = wakeup_print_header,
  613. .print_line = wakeup_print_line,
  614. .flag_changed = wakeup_flag_changed,
  615. #ifdef CONFIG_FTRACE_SELFTEST
  616. .selftest = trace_selftest_startup_wakeup,
  617. #endif
  618. .open = wakeup_trace_open,
  619. .close = wakeup_trace_close,
  620. .allow_instances = true,
  621. .use_max_tr = true,
  622. };
  623. static struct tracer wakeup_rt_tracer __read_mostly =
  624. {
  625. .name = "wakeup_rt",
  626. .init = wakeup_rt_tracer_init,
  627. .reset = wakeup_tracer_reset,
  628. .start = wakeup_tracer_start,
  629. .stop = wakeup_tracer_stop,
  630. .print_max = true,
  631. .print_header = wakeup_print_header,
  632. .print_line = wakeup_print_line,
  633. .flag_changed = wakeup_flag_changed,
  634. #ifdef CONFIG_FTRACE_SELFTEST
  635. .selftest = trace_selftest_startup_wakeup,
  636. #endif
  637. .open = wakeup_trace_open,
  638. .close = wakeup_trace_close,
  639. .allow_instances = true,
  640. .use_max_tr = true,
  641. };
  642. static struct tracer wakeup_dl_tracer __read_mostly =
  643. {
  644. .name = "wakeup_dl",
  645. .init = wakeup_dl_tracer_init,
  646. .reset = wakeup_tracer_reset,
  647. .start = wakeup_tracer_start,
  648. .stop = wakeup_tracer_stop,
  649. .print_max = true,
  650. .print_header = wakeup_print_header,
  651. .print_line = wakeup_print_line,
  652. .flag_changed = wakeup_flag_changed,
  653. #ifdef CONFIG_FTRACE_SELFTEST
  654. .selftest = trace_selftest_startup_wakeup,
  655. #endif
  656. .open = wakeup_trace_open,
  657. .close = wakeup_trace_close,
  658. .allow_instances = true,
  659. .use_max_tr = true,
  660. };
  661. __init static int init_wakeup_tracer(void)
  662. {
  663. int ret;
  664. ret = register_tracer(&wakeup_tracer);
  665. if (ret)
  666. return ret;
  667. ret = register_tracer(&wakeup_rt_tracer);
  668. if (ret)
  669. return ret;
  670. ret = register_tracer(&wakeup_dl_tracer);
  671. if (ret)
  672. return ret;
  673. return 0;
  674. }
  675. core_initcall(init_wakeup_tracer);