watchdog.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Detect hard and soft lockups on a system
  4. *
  5. * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
  6. *
  7. * Note: Most of this code is borrowed heavily from the original softlockup
  8. * detector, so thanks to Ingo for the initial implementation.
  9. * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
  10. * to those contributors as well.
  11. */
  12. #define pr_fmt(fmt) "watchdog: " fmt
  13. #include <linux/mm.h>
  14. #include <linux/cpu.h>
  15. #include <linux/nmi.h>
  16. #include <linux/init.h>
  17. #include <linux/module.h>
  18. #include <linux/sysctl.h>
  19. #include <linux/tick.h>
  20. #include <linux/sched/clock.h>
  21. #include <linux/sched/debug.h>
  22. #include <linux/sched/isolation.h>
  23. #include <linux/stop_machine.h>
  24. #include <asm/irq_regs.h>
  25. #include <linux/kvm_para.h>
  26. #include <trace/hooks/softlockup.h>
  27. static DEFINE_MUTEX(watchdog_mutex);
  28. #if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HAVE_NMI_WATCHDOG)
  29. # define WATCHDOG_DEFAULT (SOFT_WATCHDOG_ENABLED | NMI_WATCHDOG_ENABLED)
  30. # define NMI_WATCHDOG_DEFAULT 1
  31. #else
  32. # define WATCHDOG_DEFAULT (SOFT_WATCHDOG_ENABLED)
  33. # define NMI_WATCHDOG_DEFAULT 0
  34. #endif
  35. unsigned long __read_mostly watchdog_enabled;
  36. int __read_mostly watchdog_user_enabled = 1;
  37. int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT;
  38. int __read_mostly soft_watchdog_user_enabled = 1;
  39. int __read_mostly watchdog_thresh = 10;
  40. static int __read_mostly nmi_watchdog_available;
  41. struct cpumask watchdog_cpumask __read_mostly;
  42. unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
  43. #ifdef CONFIG_HARDLOCKUP_DETECTOR
  44. # ifdef CONFIG_SMP
  45. int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
  46. # endif /* CONFIG_SMP */
  47. /*
  48. * Should we panic when a soft-lockup or hard-lockup occurs:
  49. */
  50. unsigned int __read_mostly hardlockup_panic =
  51. IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC);
  52. /*
  53. * We may not want to enable hard lockup detection by default in all cases,
  54. * for example when running the kernel as a guest on a hypervisor. In these
  55. * cases this function can be called to disable hard lockup detection. This
  56. * function should only be executed once by the boot processor before the
  57. * kernel command line parameters are parsed, because otherwise it is not
  58. * possible to override this in hardlockup_panic_setup().
  59. */
  60. void __init hardlockup_detector_disable(void)
  61. {
  62. nmi_watchdog_user_enabled = 0;
  63. }
  64. static int __init hardlockup_panic_setup(char *str)
  65. {
  66. if (!strncmp(str, "panic", 5))
  67. hardlockup_panic = 1;
  68. else if (!strncmp(str, "nopanic", 7))
  69. hardlockup_panic = 0;
  70. else if (!strncmp(str, "0", 1))
  71. nmi_watchdog_user_enabled = 0;
  72. else if (!strncmp(str, "1", 1))
  73. nmi_watchdog_user_enabled = 1;
  74. return 1;
  75. }
  76. __setup("nmi_watchdog=", hardlockup_panic_setup);
  77. #endif /* CONFIG_HARDLOCKUP_DETECTOR */
  78. /*
  79. * These functions can be overridden if an architecture implements its
  80. * own hardlockup detector.
  81. *
  82. * watchdog_nmi_enable/disable can be implemented to start and stop when
  83. * softlockup watchdog start and stop. The arch must select the
  84. * SOFTLOCKUP_DETECTOR Kconfig.
  85. */
  86. int __weak watchdog_nmi_enable(unsigned int cpu)
  87. {
  88. hardlockup_detector_perf_enable();
  89. return 0;
  90. }
  91. void __weak watchdog_nmi_disable(unsigned int cpu)
  92. {
  93. hardlockup_detector_perf_disable();
  94. }
  95. /* Return 0, if a NMI watchdog is available. Error code otherwise */
  96. int __weak __init watchdog_nmi_probe(void)
  97. {
  98. return hardlockup_detector_perf_init();
  99. }
  100. /**
  101. * watchdog_nmi_stop - Stop the watchdog for reconfiguration
  102. *
  103. * The reconfiguration steps are:
  104. * watchdog_nmi_stop();
  105. * update_variables();
  106. * watchdog_nmi_start();
  107. */
  108. void __weak watchdog_nmi_stop(void) { }
  109. /**
  110. * watchdog_nmi_start - Start the watchdog after reconfiguration
  111. *
  112. * Counterpart to watchdog_nmi_stop().
  113. *
  114. * The following variables have been updated in update_variables() and
  115. * contain the currently valid configuration:
  116. * - watchdog_enabled
  117. * - watchdog_thresh
  118. * - watchdog_cpumask
  119. */
  120. void __weak watchdog_nmi_start(void) { }
  121. /**
  122. * lockup_detector_update_enable - Update the sysctl enable bit
  123. *
  124. * Caller needs to make sure that the NMI/perf watchdogs are off, so this
  125. * can't race with watchdog_nmi_disable().
  126. */
  127. static void lockup_detector_update_enable(void)
  128. {
  129. watchdog_enabled = 0;
  130. if (!watchdog_user_enabled)
  131. return;
  132. if (nmi_watchdog_available && nmi_watchdog_user_enabled)
  133. watchdog_enabled |= NMI_WATCHDOG_ENABLED;
  134. if (soft_watchdog_user_enabled)
  135. watchdog_enabled |= SOFT_WATCHDOG_ENABLED;
  136. }
  137. #ifdef CONFIG_SOFTLOCKUP_DETECTOR
  138. /*
  139. * Delay the soflockup report when running a known slow code.
  140. * It does _not_ affect the timestamp of the last successdul reschedule.
  141. */
  142. #define SOFTLOCKUP_DELAY_REPORT ULONG_MAX
  143. #ifdef CONFIG_SMP
  144. int __read_mostly sysctl_softlockup_all_cpu_backtrace;
  145. #endif
  146. static struct cpumask watchdog_allowed_mask __read_mostly;
  147. /* Global variables, exported for sysctl */
  148. unsigned int __read_mostly softlockup_panic =
  149. IS_ENABLED(CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC);
  150. static bool softlockup_initialized __read_mostly;
  151. static u64 __read_mostly sample_period;
  152. /* Timestamp taken after the last successful reschedule. */
  153. static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
  154. /* Timestamp of the last softlockup report. */
  155. static DEFINE_PER_CPU(unsigned long, watchdog_report_ts);
  156. static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
  157. static DEFINE_PER_CPU(bool, softlockup_touch_sync);
  158. static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
  159. static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
  160. static unsigned long soft_lockup_nmi_warn;
  161. static int __init softlockup_panic_setup(char *str)
  162. {
  163. softlockup_panic = simple_strtoul(str, NULL, 0);
  164. return 1;
  165. }
  166. __setup("softlockup_panic=", softlockup_panic_setup);
  167. static int __init nowatchdog_setup(char *str)
  168. {
  169. watchdog_user_enabled = 0;
  170. return 1;
  171. }
  172. __setup("nowatchdog", nowatchdog_setup);
  173. static int __init nosoftlockup_setup(char *str)
  174. {
  175. soft_watchdog_user_enabled = 0;
  176. return 1;
  177. }
  178. __setup("nosoftlockup", nosoftlockup_setup);
  179. static int __init watchdog_thresh_setup(char *str)
  180. {
  181. get_option(&str, &watchdog_thresh);
  182. return 1;
  183. }
  184. __setup("watchdog_thresh=", watchdog_thresh_setup);
  185. static void __lockup_detector_cleanup(void);
  186. /*
  187. * Hard-lockup warnings should be triggered after just a few seconds. Soft-
  188. * lockups can have false positives under extreme conditions. So we generally
  189. * want a higher threshold for soft lockups than for hard lockups. So we couple
  190. * the thresholds with a factor: we make the soft threshold twice the amount of
  191. * time the hard threshold is.
  192. */
  193. static int get_softlockup_thresh(void)
  194. {
  195. return watchdog_thresh * 2;
  196. }
  197. /*
  198. * Returns seconds, approximately. We don't need nanosecond
  199. * resolution, and we don't need to waste time with a big divide when
  200. * 2^30ns == 1.074s.
  201. */
  202. static unsigned long get_timestamp(void)
  203. {
  204. return running_clock() >> 30LL; /* 2^30 ~= 10^9 */
  205. }
  206. static void set_sample_period(void)
  207. {
  208. /*
  209. * convert watchdog_thresh from seconds to ns
  210. * the divide by 5 is to give hrtimer several chances (two
  211. * or three with the current relation between the soft
  212. * and hard thresholds) to increment before the
  213. * hardlockup detector generates a warning
  214. */
  215. sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
  216. watchdog_update_hrtimer_threshold(sample_period);
  217. }
  218. static void update_report_ts(void)
  219. {
  220. __this_cpu_write(watchdog_report_ts, get_timestamp());
  221. }
  222. /* Commands for resetting the watchdog */
  223. static void update_touch_ts(void)
  224. {
  225. __this_cpu_write(watchdog_touch_ts, get_timestamp());
  226. update_report_ts();
  227. }
  228. /**
  229. * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
  230. *
  231. * Call when the scheduler may have stalled for legitimate reasons
  232. * preventing the watchdog task from executing - e.g. the scheduler
  233. * entering idle state. This should only be used for scheduler events.
  234. * Use touch_softlockup_watchdog() for everything else.
  235. */
  236. notrace void touch_softlockup_watchdog_sched(void)
  237. {
  238. /*
  239. * Preemption can be enabled. It doesn't matter which CPU's watchdog
  240. * report period gets restarted here, so use the raw_ operation.
  241. */
  242. raw_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
  243. }
  244. notrace void touch_softlockup_watchdog(void)
  245. {
  246. touch_softlockup_watchdog_sched();
  247. wq_watchdog_touch(raw_smp_processor_id());
  248. }
  249. EXPORT_SYMBOL(touch_softlockup_watchdog);
  250. void touch_all_softlockup_watchdogs(void)
  251. {
  252. int cpu;
  253. /*
  254. * watchdog_mutex cannpt be taken here, as this might be called
  255. * from (soft)interrupt context, so the access to
  256. * watchdog_allowed_cpumask might race with a concurrent update.
  257. *
  258. * The watchdog time stamp can race against a concurrent real
  259. * update as well, the only side effect might be a cycle delay for
  260. * the softlockup check.
  261. */
  262. for_each_cpu(cpu, &watchdog_allowed_mask) {
  263. per_cpu(watchdog_report_ts, cpu) = SOFTLOCKUP_DELAY_REPORT;
  264. wq_watchdog_touch(cpu);
  265. }
  266. }
  267. void touch_softlockup_watchdog_sync(void)
  268. {
  269. __this_cpu_write(softlockup_touch_sync, true);
  270. __this_cpu_write(watchdog_report_ts, SOFTLOCKUP_DELAY_REPORT);
  271. }
  272. static int is_softlockup(unsigned long touch_ts,
  273. unsigned long period_ts,
  274. unsigned long now)
  275. {
  276. if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
  277. /* Warn about unreasonable delays. */
  278. if (time_after(now, period_ts + get_softlockup_thresh()))
  279. return now - touch_ts;
  280. }
  281. return 0;
  282. }
  283. /* watchdog detector functions */
  284. bool is_hardlockup(void)
  285. {
  286. unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
  287. if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
  288. return true;
  289. __this_cpu_write(hrtimer_interrupts_saved, hrint);
  290. return false;
  291. }
  292. static void watchdog_interrupt_count(void)
  293. {
  294. __this_cpu_inc(hrtimer_interrupts);
  295. }
  296. static DEFINE_PER_CPU(struct completion, softlockup_completion);
  297. static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);
  298. /*
  299. * The watchdog feed function - touches the timestamp.
  300. *
  301. * It only runs once every sample_period seconds (4 seconds by
  302. * default) to reset the softlockup timestamp. If this gets delayed
  303. * for more than 2*watchdog_thresh seconds then the debug-printout
  304. * triggers in watchdog_timer_fn().
  305. */
  306. static int softlockup_fn(void *data)
  307. {
  308. update_touch_ts();
  309. complete(this_cpu_ptr(&softlockup_completion));
  310. return 0;
  311. }
  312. /* watchdog kicker functions */
  313. static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
  314. {
  315. unsigned long touch_ts, period_ts, now;
  316. struct pt_regs *regs = get_irq_regs();
  317. int duration;
  318. int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
  319. if (!watchdog_enabled)
  320. return HRTIMER_NORESTART;
  321. /* kick the hardlockup detector */
  322. watchdog_interrupt_count();
  323. /* kick the softlockup detector */
  324. if (completion_done(this_cpu_ptr(&softlockup_completion))) {
  325. reinit_completion(this_cpu_ptr(&softlockup_completion));
  326. stop_one_cpu_nowait(smp_processor_id(),
  327. softlockup_fn, NULL,
  328. this_cpu_ptr(&softlockup_stop_work));
  329. }
  330. /* .. and repeat */
  331. hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
  332. /*
  333. * Read the current timestamp first. It might become invalid anytime
  334. * when a virtual machine is stopped by the host or when the watchog
  335. * is touched from NMI.
  336. */
  337. now = get_timestamp();
  338. /*
  339. * If a virtual machine is stopped by the host it can look to
  340. * the watchdog like a soft lockup. This function touches the watchdog.
  341. */
  342. kvm_check_and_clear_guest_paused();
  343. /*
  344. * The stored timestamp is comparable with @now only when not touched.
  345. * It might get touched anytime from NMI. Make sure that is_softlockup()
  346. * uses the same (valid) value.
  347. */
  348. period_ts = READ_ONCE(*this_cpu_ptr(&watchdog_report_ts));
  349. /* Reset the interval when touched by known problematic code. */
  350. if (period_ts == SOFTLOCKUP_DELAY_REPORT) {
  351. if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
  352. /*
  353. * If the time stamp was touched atomically
  354. * make sure the scheduler tick is up to date.
  355. */
  356. __this_cpu_write(softlockup_touch_sync, false);
  357. sched_clock_tick();
  358. }
  359. update_report_ts();
  360. return HRTIMER_RESTART;
  361. }
  362. /* Check for a softlockup. */
  363. touch_ts = __this_cpu_read(watchdog_touch_ts);
  364. duration = is_softlockup(touch_ts, period_ts, now);
  365. if (unlikely(duration)) {
  366. /*
  367. * Prevent multiple soft-lockup reports if one cpu is already
  368. * engaged in dumping all cpu back traces.
  369. */
  370. if (softlockup_all_cpu_backtrace) {
  371. if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
  372. return HRTIMER_RESTART;
  373. }
  374. /* Start period for the next softlockup warning. */
  375. update_report_ts();
  376. pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
  377. smp_processor_id(), duration,
  378. current->comm, task_pid_nr(current));
  379. print_modules();
  380. print_irqtrace_events(current);
  381. if (regs)
  382. show_regs(regs);
  383. else
  384. dump_stack();
  385. if (softlockup_all_cpu_backtrace) {
  386. trigger_allbutself_cpu_backtrace();
  387. clear_bit_unlock(0, &soft_lockup_nmi_warn);
  388. }
  389. trace_android_vh_watchdog_timer_softlockup(duration, regs, !!softlockup_panic);
  390. add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
  391. if (softlockup_panic)
  392. panic("softlockup: hung tasks");
  393. }
  394. return HRTIMER_RESTART;
  395. }
  396. static void watchdog_enable(unsigned int cpu)
  397. {
  398. struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
  399. struct completion *done = this_cpu_ptr(&softlockup_completion);
  400. WARN_ON_ONCE(cpu != smp_processor_id());
  401. init_completion(done);
  402. complete(done);
  403. /*
  404. * Start the timer first to prevent the NMI watchdog triggering
  405. * before the timer has a chance to fire.
  406. */
  407. hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
  408. hrtimer->function = watchdog_timer_fn;
  409. hrtimer_start(hrtimer, ns_to_ktime(sample_period),
  410. HRTIMER_MODE_REL_PINNED_HARD);
  411. /* Initialize timestamp */
  412. update_touch_ts();
  413. /* Enable the perf event */
  414. if (watchdog_enabled & NMI_WATCHDOG_ENABLED)
  415. watchdog_nmi_enable(cpu);
  416. }
  417. static void watchdog_disable(unsigned int cpu)
  418. {
  419. struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
  420. WARN_ON_ONCE(cpu != smp_processor_id());
  421. /*
  422. * Disable the perf event first. That prevents that a large delay
  423. * between disabling the timer and disabling the perf event causes
  424. * the perf NMI to detect a false positive.
  425. */
  426. watchdog_nmi_disable(cpu);
  427. hrtimer_cancel(hrtimer);
  428. wait_for_completion(this_cpu_ptr(&softlockup_completion));
  429. }
  430. static int softlockup_stop_fn(void *data)
  431. {
  432. watchdog_disable(smp_processor_id());
  433. return 0;
  434. }
  435. static void softlockup_stop_all(void)
  436. {
  437. int cpu;
  438. if (!softlockup_initialized)
  439. return;
  440. for_each_cpu(cpu, &watchdog_allowed_mask)
  441. smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);
  442. cpumask_clear(&watchdog_allowed_mask);
  443. }
  444. static int softlockup_start_fn(void *data)
  445. {
  446. watchdog_enable(smp_processor_id());
  447. return 0;
  448. }
  449. static void softlockup_start_all(void)
  450. {
  451. int cpu;
  452. cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
  453. for_each_cpu(cpu, &watchdog_allowed_mask)
  454. smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
  455. }
  456. int lockup_detector_online_cpu(unsigned int cpu)
  457. {
  458. if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
  459. watchdog_enable(cpu);
  460. return 0;
  461. }
  462. int lockup_detector_offline_cpu(unsigned int cpu)
  463. {
  464. if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
  465. watchdog_disable(cpu);
  466. return 0;
  467. }
  468. static void __lockup_detector_reconfigure(void)
  469. {
  470. cpus_read_lock();
  471. watchdog_nmi_stop();
  472. softlockup_stop_all();
  473. set_sample_period();
  474. lockup_detector_update_enable();
  475. if (watchdog_enabled && watchdog_thresh)
  476. softlockup_start_all();
  477. watchdog_nmi_start();
  478. cpus_read_unlock();
  479. /*
  480. * Must be called outside the cpus locked section to prevent
  481. * recursive locking in the perf code.
  482. */
  483. __lockup_detector_cleanup();
  484. }
  485. void lockup_detector_reconfigure(void)
  486. {
  487. mutex_lock(&watchdog_mutex);
  488. __lockup_detector_reconfigure();
  489. mutex_unlock(&watchdog_mutex);
  490. }
  491. /*
  492. * Create the watchdog infrastructure and configure the detector(s).
  493. */
  494. static __init void lockup_detector_setup(void)
  495. {
  496. /*
  497. * If sysctl is off and watchdog got disabled on the command line,
  498. * nothing to do here.
  499. */
  500. lockup_detector_update_enable();
  501. if (!IS_ENABLED(CONFIG_SYSCTL) &&
  502. !(watchdog_enabled && watchdog_thresh))
  503. return;
  504. mutex_lock(&watchdog_mutex);
  505. __lockup_detector_reconfigure();
  506. softlockup_initialized = true;
  507. mutex_unlock(&watchdog_mutex);
  508. }
  509. #else /* CONFIG_SOFTLOCKUP_DETECTOR */
  510. static void __lockup_detector_reconfigure(void)
  511. {
  512. cpus_read_lock();
  513. watchdog_nmi_stop();
  514. lockup_detector_update_enable();
  515. watchdog_nmi_start();
  516. cpus_read_unlock();
  517. }
  518. void lockup_detector_reconfigure(void)
  519. {
  520. __lockup_detector_reconfigure();
  521. }
  522. static inline void lockup_detector_setup(void)
  523. {
  524. __lockup_detector_reconfigure();
  525. }
  526. #endif /* !CONFIG_SOFTLOCKUP_DETECTOR */
  527. static void __lockup_detector_cleanup(void)
  528. {
  529. lockdep_assert_held(&watchdog_mutex);
  530. hardlockup_detector_perf_cleanup();
  531. }
  532. /**
  533. * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
  534. *
  535. * Caller must not hold the cpu hotplug rwsem.
  536. */
  537. void lockup_detector_cleanup(void)
  538. {
  539. mutex_lock(&watchdog_mutex);
  540. __lockup_detector_cleanup();
  541. mutex_unlock(&watchdog_mutex);
  542. }
  543. /**
  544. * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
  545. *
  546. * Special interface for parisc. It prevents lockup detector warnings from
  547. * the default pm_poweroff() function which busy loops forever.
  548. */
  549. void lockup_detector_soft_poweroff(void)
  550. {
  551. watchdog_enabled = 0;
  552. }
  553. #ifdef CONFIG_SYSCTL
  554. /* Propagate any changes to the watchdog infrastructure */
  555. static void proc_watchdog_update(void)
  556. {
  557. /* Remove impossible cpus to keep sysctl output clean. */
  558. cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
  559. __lockup_detector_reconfigure();
  560. }
  561. /*
  562. * common function for watchdog, nmi_watchdog and soft_watchdog parameter
  563. *
  564. * caller | table->data points to | 'which'
  565. * -------------------|----------------------------|--------------------------
  566. * proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED |
  567. * | | SOFT_WATCHDOG_ENABLED
  568. * -------------------|----------------------------|--------------------------
  569. * proc_nmi_watchdog | nmi_watchdog_user_enabled | NMI_WATCHDOG_ENABLED
  570. * -------------------|----------------------------|--------------------------
  571. * proc_soft_watchdog | soft_watchdog_user_enabled | SOFT_WATCHDOG_ENABLED
  572. */
  573. static int proc_watchdog_common(int which, struct ctl_table *table, int write,
  574. void *buffer, size_t *lenp, loff_t *ppos)
  575. {
  576. int err, old, *param = table->data;
  577. mutex_lock(&watchdog_mutex);
  578. if (!write) {
  579. /*
  580. * On read synchronize the userspace interface. This is a
  581. * racy snapshot.
  582. */
  583. *param = (watchdog_enabled & which) != 0;
  584. err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  585. } else {
  586. old = READ_ONCE(*param);
  587. err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  588. if (!err && old != READ_ONCE(*param))
  589. proc_watchdog_update();
  590. }
  591. mutex_unlock(&watchdog_mutex);
  592. return err;
  593. }
  594. /*
  595. * /proc/sys/kernel/watchdog
  596. */
  597. int proc_watchdog(struct ctl_table *table, int write,
  598. void *buffer, size_t *lenp, loff_t *ppos)
  599. {
  600. return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
  601. table, write, buffer, lenp, ppos);
  602. }
  603. /*
  604. * /proc/sys/kernel/nmi_watchdog
  605. */
  606. int proc_nmi_watchdog(struct ctl_table *table, int write,
  607. void *buffer, size_t *lenp, loff_t *ppos)
  608. {
  609. if (!nmi_watchdog_available && write)
  610. return -ENOTSUPP;
  611. return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
  612. table, write, buffer, lenp, ppos);
  613. }
  614. /*
  615. * /proc/sys/kernel/soft_watchdog
  616. */
  617. int proc_soft_watchdog(struct ctl_table *table, int write,
  618. void *buffer, size_t *lenp, loff_t *ppos)
  619. {
  620. return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
  621. table, write, buffer, lenp, ppos);
  622. }
  623. /*
  624. * /proc/sys/kernel/watchdog_thresh
  625. */
  626. int proc_watchdog_thresh(struct ctl_table *table, int write,
  627. void *buffer, size_t *lenp, loff_t *ppos)
  628. {
  629. int err, old;
  630. mutex_lock(&watchdog_mutex);
  631. old = READ_ONCE(watchdog_thresh);
  632. err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
  633. if (!err && write && old != READ_ONCE(watchdog_thresh))
  634. proc_watchdog_update();
  635. mutex_unlock(&watchdog_mutex);
  636. return err;
  637. }
  638. /*
  639. * The cpumask is the mask of possible cpus that the watchdog can run
  640. * on, not the mask of cpus it is actually running on. This allows the
  641. * user to specify a mask that will include cpus that have not yet
  642. * been brought online, if desired.
  643. */
  644. int proc_watchdog_cpumask(struct ctl_table *table, int write,
  645. void *buffer, size_t *lenp, loff_t *ppos)
  646. {
  647. int err;
  648. mutex_lock(&watchdog_mutex);
  649. err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
  650. if (!err && write)
  651. proc_watchdog_update();
  652. mutex_unlock(&watchdog_mutex);
  653. return err;
  654. }
  655. static const int sixty = 60;
  656. static struct ctl_table watchdog_sysctls[] = {
  657. {
  658. .procname = "watchdog",
  659. .data = &watchdog_user_enabled,
  660. .maxlen = sizeof(int),
  661. .mode = 0644,
  662. .proc_handler = proc_watchdog,
  663. .extra1 = SYSCTL_ZERO,
  664. .extra2 = SYSCTL_ONE,
  665. },
  666. {
  667. .procname = "watchdog_thresh",
  668. .data = &watchdog_thresh,
  669. .maxlen = sizeof(int),
  670. .mode = 0644,
  671. .proc_handler = proc_watchdog_thresh,
  672. .extra1 = SYSCTL_ZERO,
  673. .extra2 = (void *)&sixty,
  674. },
  675. {
  676. .procname = "nmi_watchdog",
  677. .data = &nmi_watchdog_user_enabled,
  678. .maxlen = sizeof(int),
  679. .mode = NMI_WATCHDOG_SYSCTL_PERM,
  680. .proc_handler = proc_nmi_watchdog,
  681. .extra1 = SYSCTL_ZERO,
  682. .extra2 = SYSCTL_ONE,
  683. },
  684. {
  685. .procname = "watchdog_cpumask",
  686. .data = &watchdog_cpumask_bits,
  687. .maxlen = NR_CPUS,
  688. .mode = 0644,
  689. .proc_handler = proc_watchdog_cpumask,
  690. },
  691. #ifdef CONFIG_SOFTLOCKUP_DETECTOR
  692. {
  693. .procname = "soft_watchdog",
  694. .data = &soft_watchdog_user_enabled,
  695. .maxlen = sizeof(int),
  696. .mode = 0644,
  697. .proc_handler = proc_soft_watchdog,
  698. .extra1 = SYSCTL_ZERO,
  699. .extra2 = SYSCTL_ONE,
  700. },
  701. {
  702. .procname = "softlockup_panic",
  703. .data = &softlockup_panic,
  704. .maxlen = sizeof(int),
  705. .mode = 0644,
  706. .proc_handler = proc_dointvec_minmax,
  707. .extra1 = SYSCTL_ZERO,
  708. .extra2 = SYSCTL_ONE,
  709. },
  710. #ifdef CONFIG_SMP
  711. {
  712. .procname = "softlockup_all_cpu_backtrace",
  713. .data = &sysctl_softlockup_all_cpu_backtrace,
  714. .maxlen = sizeof(int),
  715. .mode = 0644,
  716. .proc_handler = proc_dointvec_minmax,
  717. .extra1 = SYSCTL_ZERO,
  718. .extra2 = SYSCTL_ONE,
  719. },
  720. #endif /* CONFIG_SMP */
  721. #endif
  722. #ifdef CONFIG_HARDLOCKUP_DETECTOR
  723. {
  724. .procname = "hardlockup_panic",
  725. .data = &hardlockup_panic,
  726. .maxlen = sizeof(int),
  727. .mode = 0644,
  728. .proc_handler = proc_dointvec_minmax,
  729. .extra1 = SYSCTL_ZERO,
  730. .extra2 = SYSCTL_ONE,
  731. },
  732. #ifdef CONFIG_SMP
  733. {
  734. .procname = "hardlockup_all_cpu_backtrace",
  735. .data = &sysctl_hardlockup_all_cpu_backtrace,
  736. .maxlen = sizeof(int),
  737. .mode = 0644,
  738. .proc_handler = proc_dointvec_minmax,
  739. .extra1 = SYSCTL_ZERO,
  740. .extra2 = SYSCTL_ONE,
  741. },
  742. #endif /* CONFIG_SMP */
  743. #endif
  744. {}
  745. };
  746. static void __init watchdog_sysctl_init(void)
  747. {
  748. register_sysctl_init("kernel", watchdog_sysctls);
  749. }
  750. #else
  751. #define watchdog_sysctl_init() do { } while (0)
  752. #endif /* CONFIG_SYSCTL */
  753. void __init lockup_detector_init(void)
  754. {
  755. if (tick_nohz_full_enabled())
  756. pr_info("Disabling watchdog on nohz_full cores by default\n");
  757. cpumask_copy(&watchdog_cpumask,
  758. housekeeping_cpumask(HK_TYPE_TIMER));
  759. if (!watchdog_nmi_probe())
  760. nmi_watchdog_available = true;
  761. lockup_detector_setup();
  762. watchdog_sysctl_init();
  763. }