tick-common.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * This file contains the base functions to manage periodic tick
  4. * related events.
  5. *
  6. * Copyright(C) 2005-2006, Thomas Gleixner <[email protected]>
  7. * Copyright(C) 2005-2007, Red Hat, Inc., Ingo Molnar
  8. * Copyright(C) 2006-2007, Timesys Corp., Thomas Gleixner
  9. */
  10. #include <linux/cpu.h>
  11. #include <linux/err.h>
  12. #include <linux/hrtimer.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/nmi.h>
  15. #include <linux/percpu.h>
  16. #include <linux/profile.h>
  17. #include <linux/sched.h>
  18. #include <linux/module.h>
  19. #include <trace/events/power.h>
  20. #include <trace/hooks/sched.h>
  21. #include <asm/irq_regs.h>
  22. #include "tick-internal.h"
  23. /*
  24. * Tick devices
  25. */
  26. DEFINE_PER_CPU(struct tick_device, tick_cpu_device);
  27. /*
  28. * Tick next event: keeps track of the tick time. It's updated by the
  29. * CPU which handles the tick and protected by jiffies_lock. There is
  30. * no requirement to write hold the jiffies seqcount for it.
  31. */
  32. ktime_t tick_next_period;
  33. /*
  34. * tick_do_timer_cpu is a timer core internal variable which holds the CPU NR
  35. * which is responsible for calling do_timer(), i.e. the timekeeping stuff. This
  36. * variable has two functions:
  37. *
  38. * 1) Prevent a thundering herd issue of a gazillion of CPUs trying to grab the
  39. * timekeeping lock all at once. Only the CPU which is assigned to do the
  40. * update is handling it.
  41. *
  42. * 2) Hand off the duty in the NOHZ idle case by setting the value to
  43. * TICK_DO_TIMER_NONE, i.e. a non existing CPU. So the next cpu which looks
  44. * at it will take over and keep the time keeping alive. The handover
  45. * procedure also covers cpu hotplug.
  46. */
  47. int tick_do_timer_cpu __read_mostly = TICK_DO_TIMER_BOOT;
  48. #ifdef CONFIG_NO_HZ_FULL
  49. /*
  50. * tick_do_timer_boot_cpu indicates the boot CPU temporarily owns
  51. * tick_do_timer_cpu and it should be taken over by an eligible secondary
  52. * when one comes online.
  53. */
  54. static int tick_do_timer_boot_cpu __read_mostly = -1;
  55. #endif
  56. /*
  57. * Debugging: see timer_list.c
  58. */
  59. struct tick_device *tick_get_device(int cpu)
  60. {
  61. return &per_cpu(tick_cpu_device, cpu);
  62. }
  63. /**
  64. * tick_is_oneshot_available - check for a oneshot capable event device
  65. */
  66. int tick_is_oneshot_available(void)
  67. {
  68. struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
  69. if (!dev || !(dev->features & CLOCK_EVT_FEAT_ONESHOT))
  70. return 0;
  71. if (!(dev->features & CLOCK_EVT_FEAT_C3STOP))
  72. return 1;
  73. return tick_broadcast_oneshot_available();
  74. }
  75. /*
  76. * Periodic tick
  77. */
  78. static void tick_periodic(int cpu)
  79. {
  80. if (tick_do_timer_cpu == cpu) {
  81. raw_spin_lock(&jiffies_lock);
  82. write_seqcount_begin(&jiffies_seq);
  83. /* Keep track of the next tick event */
  84. tick_next_period = ktime_add_ns(tick_next_period, TICK_NSEC);
  85. do_timer(1);
  86. write_seqcount_end(&jiffies_seq);
  87. raw_spin_unlock(&jiffies_lock);
  88. update_wall_time();
  89. trace_android_vh_jiffies_update(NULL);
  90. }
  91. update_process_times(user_mode(get_irq_regs()));
  92. profile_tick(CPU_PROFILING);
  93. }
  94. /*
  95. * Event handler for periodic ticks
  96. */
  97. void tick_handle_periodic(struct clock_event_device *dev)
  98. {
  99. int cpu = smp_processor_id();
  100. ktime_t next = dev->next_event;
  101. tick_periodic(cpu);
  102. #if defined(CONFIG_HIGH_RES_TIMERS) || defined(CONFIG_NO_HZ_COMMON)
  103. /*
  104. * The cpu might have transitioned to HIGHRES or NOHZ mode via
  105. * update_process_times() -> run_local_timers() ->
  106. * hrtimer_run_queues().
  107. */
  108. if (dev->event_handler != tick_handle_periodic)
  109. return;
  110. #endif
  111. if (!clockevent_state_oneshot(dev))
  112. return;
  113. for (;;) {
  114. /*
  115. * Setup the next period for devices, which do not have
  116. * periodic mode:
  117. */
  118. next = ktime_add_ns(next, TICK_NSEC);
  119. if (!clockevents_program_event(dev, next, false))
  120. return;
  121. /*
  122. * Have to be careful here. If we're in oneshot mode,
  123. * before we call tick_periodic() in a loop, we need
  124. * to be sure we're using a real hardware clocksource.
  125. * Otherwise we could get trapped in an infinite
  126. * loop, as the tick_periodic() increments jiffies,
  127. * which then will increment time, possibly causing
  128. * the loop to trigger again and again.
  129. */
  130. if (timekeeping_valid_for_hres())
  131. tick_periodic(cpu);
  132. }
  133. }
  134. /*
  135. * Setup the device for a periodic tick
  136. */
  137. void tick_setup_periodic(struct clock_event_device *dev, int broadcast)
  138. {
  139. tick_set_periodic_handler(dev, broadcast);
  140. /* Broadcast setup ? */
  141. if (!tick_device_is_functional(dev))
  142. return;
  143. if ((dev->features & CLOCK_EVT_FEAT_PERIODIC) &&
  144. !tick_broadcast_oneshot_active()) {
  145. clockevents_switch_state(dev, CLOCK_EVT_STATE_PERIODIC);
  146. } else {
  147. unsigned int seq;
  148. ktime_t next;
  149. do {
  150. seq = read_seqcount_begin(&jiffies_seq);
  151. next = tick_next_period;
  152. } while (read_seqcount_retry(&jiffies_seq, seq));
  153. clockevents_switch_state(dev, CLOCK_EVT_STATE_ONESHOT);
  154. for (;;) {
  155. if (!clockevents_program_event(dev, next, false))
  156. return;
  157. next = ktime_add_ns(next, TICK_NSEC);
  158. }
  159. }
  160. }
  161. #ifdef CONFIG_NO_HZ_FULL
  162. static void giveup_do_timer(void *info)
  163. {
  164. int cpu = *(unsigned int *)info;
  165. WARN_ON(tick_do_timer_cpu != smp_processor_id());
  166. tick_do_timer_cpu = cpu;
  167. }
  168. static void tick_take_do_timer_from_boot(void)
  169. {
  170. int cpu = smp_processor_id();
  171. int from = tick_do_timer_boot_cpu;
  172. if (from >= 0 && from != cpu)
  173. smp_call_function_single(from, giveup_do_timer, &cpu, 1);
  174. }
  175. #endif
  176. /*
  177. * Setup the tick device
  178. */
  179. static void tick_setup_device(struct tick_device *td,
  180. struct clock_event_device *newdev, int cpu,
  181. const struct cpumask *cpumask)
  182. {
  183. void (*handler)(struct clock_event_device *) = NULL;
  184. ktime_t next_event = 0;
  185. /*
  186. * First device setup ?
  187. */
  188. if (!td->evtdev) {
  189. /*
  190. * If no cpu took the do_timer update, assign it to
  191. * this cpu:
  192. */
  193. if (tick_do_timer_cpu == TICK_DO_TIMER_BOOT) {
  194. tick_do_timer_cpu = cpu;
  195. tick_next_period = ktime_get();
  196. #ifdef CONFIG_NO_HZ_FULL
  197. /*
  198. * The boot CPU may be nohz_full, in which case set
  199. * tick_do_timer_boot_cpu so the first housekeeping
  200. * secondary that comes up will take do_timer from
  201. * us.
  202. */
  203. if (tick_nohz_full_cpu(cpu))
  204. tick_do_timer_boot_cpu = cpu;
  205. } else if (tick_do_timer_boot_cpu != -1 &&
  206. !tick_nohz_full_cpu(cpu)) {
  207. tick_take_do_timer_from_boot();
  208. tick_do_timer_boot_cpu = -1;
  209. WARN_ON(tick_do_timer_cpu != cpu);
  210. #endif
  211. }
  212. /*
  213. * Startup in periodic mode first.
  214. */
  215. td->mode = TICKDEV_MODE_PERIODIC;
  216. } else {
  217. handler = td->evtdev->event_handler;
  218. next_event = td->evtdev->next_event;
  219. td->evtdev->event_handler = clockevents_handle_noop;
  220. }
  221. td->evtdev = newdev;
  222. /*
  223. * When the device is not per cpu, pin the interrupt to the
  224. * current cpu:
  225. */
  226. if (!cpumask_equal(newdev->cpumask, cpumask))
  227. irq_set_affinity(newdev->irq, cpumask);
  228. /*
  229. * When global broadcasting is active, check if the current
  230. * device is registered as a placeholder for broadcast mode.
  231. * This allows us to handle this x86 misfeature in a generic
  232. * way. This function also returns !=0 when we keep the
  233. * current active broadcast state for this CPU.
  234. */
  235. if (tick_device_uses_broadcast(newdev, cpu))
  236. return;
  237. if (td->mode == TICKDEV_MODE_PERIODIC)
  238. tick_setup_periodic(newdev, 0);
  239. else
  240. tick_setup_oneshot(newdev, handler, next_event);
  241. }
  242. void tick_install_replacement(struct clock_event_device *newdev)
  243. {
  244. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  245. int cpu = smp_processor_id();
  246. clockevents_exchange_device(td->evtdev, newdev);
  247. tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
  248. if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
  249. tick_oneshot_notify();
  250. }
  251. static bool tick_check_percpu(struct clock_event_device *curdev,
  252. struct clock_event_device *newdev, int cpu)
  253. {
  254. if (!cpumask_test_cpu(cpu, newdev->cpumask))
  255. return false;
  256. if (cpumask_equal(newdev->cpumask, cpumask_of(cpu)))
  257. return true;
  258. /* Check if irq affinity can be set */
  259. if (newdev->irq >= 0 && !irq_can_set_affinity(newdev->irq))
  260. return false;
  261. /* Prefer an existing cpu local device */
  262. if (curdev && cpumask_equal(curdev->cpumask, cpumask_of(cpu)))
  263. return false;
  264. return true;
  265. }
  266. static bool tick_check_preferred(struct clock_event_device *curdev,
  267. struct clock_event_device *newdev)
  268. {
  269. /* Prefer oneshot capable device */
  270. if (!(newdev->features & CLOCK_EVT_FEAT_ONESHOT)) {
  271. if (curdev && (curdev->features & CLOCK_EVT_FEAT_ONESHOT))
  272. return false;
  273. if (tick_oneshot_mode_active())
  274. return false;
  275. }
  276. /*
  277. * Use the higher rated one, but prefer a CPU local device with a lower
  278. * rating than a non-CPU local device
  279. */
  280. return !curdev ||
  281. newdev->rating > curdev->rating ||
  282. !cpumask_equal(curdev->cpumask, newdev->cpumask);
  283. }
  284. /*
  285. * Check whether the new device is a better fit than curdev. curdev
  286. * can be NULL !
  287. */
  288. bool tick_check_replacement(struct clock_event_device *curdev,
  289. struct clock_event_device *newdev)
  290. {
  291. if (!tick_check_percpu(curdev, newdev, smp_processor_id()))
  292. return false;
  293. return tick_check_preferred(curdev, newdev);
  294. }
  295. /*
  296. * Check, if the new registered device should be used. Called with
  297. * clockevents_lock held and interrupts disabled.
  298. */
  299. void tick_check_new_device(struct clock_event_device *newdev)
  300. {
  301. struct clock_event_device *curdev;
  302. struct tick_device *td;
  303. int cpu;
  304. cpu = smp_processor_id();
  305. td = &per_cpu(tick_cpu_device, cpu);
  306. curdev = td->evtdev;
  307. if (!tick_check_replacement(curdev, newdev))
  308. goto out_bc;
  309. if (!try_module_get(newdev->owner))
  310. return;
  311. /*
  312. * Replace the eventually existing device by the new
  313. * device. If the current device is the broadcast device, do
  314. * not give it back to the clockevents layer !
  315. */
  316. if (tick_is_broadcast_device(curdev)) {
  317. clockevents_shutdown(curdev);
  318. curdev = NULL;
  319. }
  320. clockevents_exchange_device(curdev, newdev);
  321. tick_setup_device(td, newdev, cpu, cpumask_of(cpu));
  322. if (newdev->features & CLOCK_EVT_FEAT_ONESHOT)
  323. tick_oneshot_notify();
  324. return;
  325. out_bc:
  326. /*
  327. * Can the new device be used as a broadcast device ?
  328. */
  329. tick_install_broadcast_device(newdev, cpu);
  330. }
  331. /**
  332. * tick_broadcast_oneshot_control - Enter/exit broadcast oneshot mode
  333. * @state: The target state (enter/exit)
  334. *
  335. * The system enters/leaves a state, where affected devices might stop
  336. * Returns 0 on success, -EBUSY if the cpu is used to broadcast wakeups.
  337. *
  338. * Called with interrupts disabled, so clockevents_lock is not
  339. * required here because the local clock event device cannot go away
  340. * under us.
  341. */
  342. int tick_broadcast_oneshot_control(enum tick_broadcast_state state)
  343. {
  344. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  345. if (!(td->evtdev->features & CLOCK_EVT_FEAT_C3STOP))
  346. return 0;
  347. return __tick_broadcast_oneshot_control(state);
  348. }
  349. EXPORT_SYMBOL_GPL(tick_broadcast_oneshot_control);
  350. #ifdef CONFIG_HOTPLUG_CPU
  351. /*
  352. * Transfer the do_timer job away from a dying cpu.
  353. *
  354. * Called with interrupts disabled. No locking required. If
  355. * tick_do_timer_cpu is owned by this cpu, nothing can change it.
  356. */
  357. void tick_handover_do_timer(void)
  358. {
  359. if (tick_do_timer_cpu == smp_processor_id())
  360. tick_do_timer_cpu = cpumask_first(cpu_online_mask);
  361. }
  362. /*
  363. * Shutdown an event device on a given cpu:
  364. *
  365. * This is called on a life CPU, when a CPU is dead. So we cannot
  366. * access the hardware device itself.
  367. * We just set the mode and remove it from the lists.
  368. */
  369. void tick_shutdown(unsigned int cpu)
  370. {
  371. struct tick_device *td = &per_cpu(tick_cpu_device, cpu);
  372. struct clock_event_device *dev = td->evtdev;
  373. td->mode = TICKDEV_MODE_PERIODIC;
  374. if (dev) {
  375. /*
  376. * Prevent that the clock events layer tries to call
  377. * the set mode function!
  378. */
  379. clockevent_set_state(dev, CLOCK_EVT_STATE_DETACHED);
  380. clockevents_exchange_device(dev, NULL);
  381. dev->event_handler = clockevents_handle_noop;
  382. td->evtdev = NULL;
  383. }
  384. }
  385. #endif
  386. /**
  387. * tick_suspend_local - Suspend the local tick device
  388. *
  389. * Called from the local cpu for freeze with interrupts disabled.
  390. *
  391. * No locks required. Nothing can change the per cpu device.
  392. */
  393. void tick_suspend_local(void)
  394. {
  395. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  396. clockevents_shutdown(td->evtdev);
  397. }
  398. /**
  399. * tick_resume_local - Resume the local tick device
  400. *
  401. * Called from the local CPU for unfreeze or XEN resume magic.
  402. *
  403. * No locks required. Nothing can change the per cpu device.
  404. */
  405. void tick_resume_local(void)
  406. {
  407. struct tick_device *td = this_cpu_ptr(&tick_cpu_device);
  408. bool broadcast = tick_resume_check_broadcast();
  409. clockevents_tick_resume(td->evtdev);
  410. if (!broadcast) {
  411. if (td->mode == TICKDEV_MODE_PERIODIC)
  412. tick_setup_periodic(td->evtdev, 0);
  413. else
  414. tick_resume_oneshot();
  415. }
  416. /*
  417. * Ensure that hrtimers are up to date and the clockevents device
  418. * is reprogrammed correctly when high resolution timers are
  419. * enabled.
  420. */
  421. hrtimers_resume_local();
  422. }
  423. /**
  424. * tick_suspend - Suspend the tick and the broadcast device
  425. *
  426. * Called from syscore_suspend() via timekeeping_suspend with only one
  427. * CPU online and interrupts disabled or from tick_unfreeze() under
  428. * tick_freeze_lock.
  429. *
  430. * No locks required. Nothing can change the per cpu device.
  431. */
  432. void tick_suspend(void)
  433. {
  434. tick_suspend_local();
  435. tick_suspend_broadcast();
  436. }
  437. /**
  438. * tick_resume - Resume the tick and the broadcast device
  439. *
  440. * Called from syscore_resume() via timekeeping_resume with only one
  441. * CPU online and interrupts disabled.
  442. *
  443. * No locks required. Nothing can change the per cpu device.
  444. */
  445. void tick_resume(void)
  446. {
  447. tick_resume_broadcast();
  448. tick_resume_local();
  449. }
  450. #ifdef CONFIG_SUSPEND
  451. static DEFINE_RAW_SPINLOCK(tick_freeze_lock);
  452. static unsigned int tick_freeze_depth;
  453. /**
  454. * tick_freeze - Suspend the local tick and (possibly) timekeeping.
  455. *
  456. * Check if this is the last online CPU executing the function and if so,
  457. * suspend timekeeping. Otherwise suspend the local tick.
  458. *
  459. * Call with interrupts disabled. Must be balanced with %tick_unfreeze().
  460. * Interrupts must not be enabled before the subsequent %tick_unfreeze().
  461. */
  462. void tick_freeze(void)
  463. {
  464. raw_spin_lock(&tick_freeze_lock);
  465. tick_freeze_depth++;
  466. if (tick_freeze_depth == num_online_cpus()) {
  467. trace_suspend_resume(TPS("timekeeping_freeze"),
  468. smp_processor_id(), true);
  469. system_state = SYSTEM_SUSPEND;
  470. sched_clock_suspend();
  471. timekeeping_suspend();
  472. } else {
  473. tick_suspend_local();
  474. }
  475. raw_spin_unlock(&tick_freeze_lock);
  476. }
  477. /**
  478. * tick_unfreeze - Resume the local tick and (possibly) timekeeping.
  479. *
  480. * Check if this is the first CPU executing the function and if so, resume
  481. * timekeeping. Otherwise resume the local tick.
  482. *
  483. * Call with interrupts disabled. Must be balanced with %tick_freeze().
  484. * Interrupts must not be enabled after the preceding %tick_freeze().
  485. */
  486. void tick_unfreeze(void)
  487. {
  488. raw_spin_lock(&tick_freeze_lock);
  489. if (tick_freeze_depth == num_online_cpus()) {
  490. timekeeping_resume();
  491. sched_clock_resume();
  492. system_state = SYSTEM_RUNNING;
  493. trace_suspend_resume(TPS("timekeeping_freeze"),
  494. smp_processor_id(), false);
  495. } else {
  496. touch_softlockup_watchdog();
  497. tick_resume_local();
  498. }
  499. tick_freeze_depth--;
  500. raw_spin_unlock(&tick_freeze_lock);
  501. }
  502. #endif /* CONFIG_SUSPEND */
  503. /**
  504. * tick_init - initialize the tick control
  505. */
  506. void __init tick_init(void)
  507. {
  508. tick_broadcast_init();
  509. tick_nohz_init();
  510. }