tsc_sync.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * check TSC synchronization.
  4. *
  5. * Copyright (C) 2006, Red Hat, Inc., Ingo Molnar
  6. *
  7. * We check whether all boot CPUs have their TSC's synchronized,
  8. * print a warning if not and turn off the TSC clock-source.
  9. *
  10. * The warp-check is point-to-point between two CPUs, the CPU
  11. * initiating the bootup is the 'source CPU', the freshly booting
  12. * CPU is the 'target CPU'.
  13. *
  14. * Only two CPUs may participate - they can enter in any order.
  15. * ( The serial nature of the boot logic and the CPU hotplug lock
  16. * protects against more than 2 CPUs entering this code. )
  17. */
  18. #include <linux/topology.h>
  19. #include <linux/spinlock.h>
  20. #include <linux/kernel.h>
  21. #include <linux/smp.h>
  22. #include <linux/nmi.h>
  23. #include <asm/tsc.h>
  24. struct tsc_adjust {
  25. s64 bootval;
  26. s64 adjusted;
  27. unsigned long nextcheck;
  28. bool warned;
  29. };
  30. static DEFINE_PER_CPU(struct tsc_adjust, tsc_adjust);
  31. static struct timer_list tsc_sync_check_timer;
  32. /*
  33. * TSC's on different sockets may be reset asynchronously.
  34. * This may cause the TSC ADJUST value on socket 0 to be NOT 0.
  35. */
  36. bool __read_mostly tsc_async_resets;
  37. void mark_tsc_async_resets(char *reason)
  38. {
  39. if (tsc_async_resets)
  40. return;
  41. tsc_async_resets = true;
  42. pr_info("tsc: Marking TSC async resets true due to %s\n", reason);
  43. }
  44. void tsc_verify_tsc_adjust(bool resume)
  45. {
  46. struct tsc_adjust *adj = this_cpu_ptr(&tsc_adjust);
  47. s64 curval;
  48. if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
  49. return;
  50. /* Skip unnecessary error messages if TSC already unstable */
  51. if (check_tsc_unstable())
  52. return;
  53. /* Rate limit the MSR check */
  54. if (!resume && time_before(jiffies, adj->nextcheck))
  55. return;
  56. adj->nextcheck = jiffies + HZ;
  57. rdmsrl(MSR_IA32_TSC_ADJUST, curval);
  58. if (adj->adjusted == curval)
  59. return;
  60. /* Restore the original value */
  61. wrmsrl(MSR_IA32_TSC_ADJUST, adj->adjusted);
  62. if (!adj->warned || resume) {
  63. pr_warn(FW_BUG "TSC ADJUST differs: CPU%u %lld --> %lld. Restoring\n",
  64. smp_processor_id(), adj->adjusted, curval);
  65. adj->warned = true;
  66. }
  67. }
  68. /*
  69. * Normally the tsc_sync will be checked every time system enters idle
  70. * state, but there is still caveat that a system won't enter idle,
  71. * either because it's too busy or configured purposely to not enter
  72. * idle.
  73. *
  74. * So setup a periodic timer (every 10 minutes) to make sure the check
  75. * is always on.
  76. */
  77. #define SYNC_CHECK_INTERVAL (HZ * 600)
  78. static void tsc_sync_check_timer_fn(struct timer_list *unused)
  79. {
  80. int next_cpu;
  81. tsc_verify_tsc_adjust(false);
  82. /* Run the check for all onlined CPUs in turn */
  83. next_cpu = cpumask_next(raw_smp_processor_id(), cpu_online_mask);
  84. if (next_cpu >= nr_cpu_ids)
  85. next_cpu = cpumask_first(cpu_online_mask);
  86. tsc_sync_check_timer.expires += SYNC_CHECK_INTERVAL;
  87. add_timer_on(&tsc_sync_check_timer, next_cpu);
  88. }
  89. static int __init start_sync_check_timer(void)
  90. {
  91. if (!cpu_feature_enabled(X86_FEATURE_TSC_ADJUST) || tsc_clocksource_reliable)
  92. return 0;
  93. timer_setup(&tsc_sync_check_timer, tsc_sync_check_timer_fn, 0);
  94. tsc_sync_check_timer.expires = jiffies + SYNC_CHECK_INTERVAL;
  95. add_timer(&tsc_sync_check_timer);
  96. return 0;
  97. }
  98. late_initcall(start_sync_check_timer);
  99. static void tsc_sanitize_first_cpu(struct tsc_adjust *cur, s64 bootval,
  100. unsigned int cpu, bool bootcpu)
  101. {
  102. /*
  103. * First online CPU in a package stores the boot value in the
  104. * adjustment value. This value might change later via the sync
  105. * mechanism. If that fails we still can yell about boot values not
  106. * being consistent.
  107. *
  108. * On the boot cpu we just force set the ADJUST value to 0 if it's
  109. * non zero. We don't do that on non boot cpus because physical
  110. * hotplug should have set the ADJUST register to a value > 0 so
  111. * the TSC is in sync with the already running cpus.
  112. *
  113. * Also don't force the ADJUST value to zero if that is a valid value
  114. * for socket 0 as determined by the system arch. This is required
  115. * when multiple sockets are reset asynchronously with each other
  116. * and socket 0 may not have an TSC ADJUST value of 0.
  117. */
  118. if (bootcpu && bootval != 0) {
  119. if (likely(!tsc_async_resets)) {
  120. pr_warn(FW_BUG "TSC ADJUST: CPU%u: %lld force to 0\n",
  121. cpu, bootval);
  122. wrmsrl(MSR_IA32_TSC_ADJUST, 0);
  123. bootval = 0;
  124. } else {
  125. pr_info("TSC ADJUST: CPU%u: %lld NOT forced to 0\n",
  126. cpu, bootval);
  127. }
  128. }
  129. cur->adjusted = bootval;
  130. }
  131. #ifndef CONFIG_SMP
  132. bool __init tsc_store_and_check_tsc_adjust(bool bootcpu)
  133. {
  134. struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust);
  135. s64 bootval;
  136. if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
  137. return false;
  138. /* Skip unnecessary error messages if TSC already unstable */
  139. if (check_tsc_unstable())
  140. return false;
  141. rdmsrl(MSR_IA32_TSC_ADJUST, bootval);
  142. cur->bootval = bootval;
  143. cur->nextcheck = jiffies + HZ;
  144. tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(), bootcpu);
  145. return false;
  146. }
  147. #else /* !CONFIG_SMP */
  148. /*
  149. * Store and check the TSC ADJUST MSR if available
  150. */
  151. bool tsc_store_and_check_tsc_adjust(bool bootcpu)
  152. {
  153. struct tsc_adjust *ref, *cur = this_cpu_ptr(&tsc_adjust);
  154. unsigned int refcpu, cpu = smp_processor_id();
  155. struct cpumask *mask;
  156. s64 bootval;
  157. if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
  158. return false;
  159. rdmsrl(MSR_IA32_TSC_ADJUST, bootval);
  160. cur->bootval = bootval;
  161. cur->nextcheck = jiffies + HZ;
  162. cur->warned = false;
  163. /*
  164. * If a non-zero TSC value for socket 0 may be valid then the default
  165. * adjusted value cannot assumed to be zero either.
  166. */
  167. if (tsc_async_resets)
  168. cur->adjusted = bootval;
  169. /*
  170. * Check whether this CPU is the first in a package to come up. In
  171. * this case do not check the boot value against another package
  172. * because the new package might have been physically hotplugged,
  173. * where TSC_ADJUST is expected to be different. When called on the
  174. * boot CPU topology_core_cpumask() might not be available yet.
  175. */
  176. mask = topology_core_cpumask(cpu);
  177. refcpu = mask ? cpumask_any_but(mask, cpu) : nr_cpu_ids;
  178. if (refcpu >= nr_cpu_ids) {
  179. tsc_sanitize_first_cpu(cur, bootval, smp_processor_id(),
  180. bootcpu);
  181. return false;
  182. }
  183. ref = per_cpu_ptr(&tsc_adjust, refcpu);
  184. /*
  185. * Compare the boot value and complain if it differs in the
  186. * package.
  187. */
  188. if (bootval != ref->bootval)
  189. printk_once(FW_BUG "TSC ADJUST differs within socket(s), fixing all errors\n");
  190. /*
  191. * The TSC_ADJUST values in a package must be the same. If the boot
  192. * value on this newly upcoming CPU differs from the adjustment
  193. * value of the already online CPU in this package, set it to that
  194. * adjusted value.
  195. */
  196. if (bootval != ref->adjusted) {
  197. cur->adjusted = ref->adjusted;
  198. wrmsrl(MSR_IA32_TSC_ADJUST, ref->adjusted);
  199. }
  200. /*
  201. * We have the TSCs forced to be in sync on this package. Skip sync
  202. * test:
  203. */
  204. return true;
  205. }
  206. /*
  207. * Entry/exit counters that make sure that both CPUs
  208. * run the measurement code at once:
  209. */
  210. static atomic_t start_count;
  211. static atomic_t stop_count;
  212. static atomic_t skip_test;
  213. static atomic_t test_runs;
  214. /*
  215. * We use a raw spinlock in this exceptional case, because
  216. * we want to have the fastest, inlined, non-debug version
  217. * of a critical section, to be able to prove TSC time-warps:
  218. */
  219. static arch_spinlock_t sync_lock = __ARCH_SPIN_LOCK_UNLOCKED;
  220. static cycles_t last_tsc;
  221. static cycles_t max_warp;
  222. static int nr_warps;
  223. static int random_warps;
  224. /*
  225. * TSC-warp measurement loop running on both CPUs. This is not called
  226. * if there is no TSC.
  227. */
  228. static cycles_t check_tsc_warp(unsigned int timeout)
  229. {
  230. cycles_t start, now, prev, end, cur_max_warp = 0;
  231. int i, cur_warps = 0;
  232. start = rdtsc_ordered();
  233. /*
  234. * The measurement runs for 'timeout' msecs:
  235. */
  236. end = start + (cycles_t) tsc_khz * timeout;
  237. for (i = 0; ; i++) {
  238. /*
  239. * We take the global lock, measure TSC, save the
  240. * previous TSC that was measured (possibly on
  241. * another CPU) and update the previous TSC timestamp.
  242. */
  243. arch_spin_lock(&sync_lock);
  244. prev = last_tsc;
  245. now = rdtsc_ordered();
  246. last_tsc = now;
  247. arch_spin_unlock(&sync_lock);
  248. /*
  249. * Be nice every now and then (and also check whether
  250. * measurement is done [we also insert a 10 million
  251. * loops safety exit, so we dont lock up in case the
  252. * TSC readout is totally broken]):
  253. */
  254. if (unlikely(!(i & 7))) {
  255. if (now > end || i > 10000000)
  256. break;
  257. cpu_relax();
  258. touch_nmi_watchdog();
  259. }
  260. /*
  261. * Outside the critical section we can now see whether
  262. * we saw a time-warp of the TSC going backwards:
  263. */
  264. if (unlikely(prev > now)) {
  265. arch_spin_lock(&sync_lock);
  266. max_warp = max(max_warp, prev - now);
  267. cur_max_warp = max_warp;
  268. /*
  269. * Check whether this bounces back and forth. Only
  270. * one CPU should observe time going backwards.
  271. */
  272. if (cur_warps != nr_warps)
  273. random_warps++;
  274. nr_warps++;
  275. cur_warps = nr_warps;
  276. arch_spin_unlock(&sync_lock);
  277. }
  278. }
  279. WARN(!(now-start),
  280. "Warning: zero tsc calibration delta: %Ld [max: %Ld]\n",
  281. now-start, end-start);
  282. return cur_max_warp;
  283. }
  284. /*
  285. * If the target CPU coming online doesn't have any of its core-siblings
  286. * online, a timeout of 20msec will be used for the TSC-warp measurement
  287. * loop. Otherwise a smaller timeout of 2msec will be used, as we have some
  288. * information about this socket already (and this information grows as we
  289. * have more and more logical-siblings in that socket).
  290. *
  291. * Ideally we should be able to skip the TSC sync check on the other
  292. * core-siblings, if the first logical CPU in a socket passed the sync test.
  293. * But as the TSC is per-logical CPU and can potentially be modified wrongly
  294. * by the bios, TSC sync test for smaller duration should be able
  295. * to catch such errors. Also this will catch the condition where all the
  296. * cores in the socket don't get reset at the same time.
  297. */
  298. static inline unsigned int loop_timeout(int cpu)
  299. {
  300. return (cpumask_weight(topology_core_cpumask(cpu)) > 1) ? 2 : 20;
  301. }
  302. /*
  303. * Source CPU calls into this - it waits for the freshly booted
  304. * target CPU to arrive and then starts the measurement:
  305. */
  306. void check_tsc_sync_source(int cpu)
  307. {
  308. int cpus = 2;
  309. /*
  310. * No need to check if we already know that the TSC is not
  311. * synchronized or if we have no TSC.
  312. */
  313. if (unsynchronized_tsc())
  314. return;
  315. /*
  316. * Set the maximum number of test runs to
  317. * 1 if the CPU does not provide the TSC_ADJUST MSR
  318. * 3 if the MSR is available, so the target can try to adjust
  319. */
  320. if (!boot_cpu_has(X86_FEATURE_TSC_ADJUST))
  321. atomic_set(&test_runs, 1);
  322. else
  323. atomic_set(&test_runs, 3);
  324. retry:
  325. /*
  326. * Wait for the target to start or to skip the test:
  327. */
  328. while (atomic_read(&start_count) != cpus - 1) {
  329. if (atomic_read(&skip_test) > 0) {
  330. atomic_set(&skip_test, 0);
  331. return;
  332. }
  333. cpu_relax();
  334. }
  335. /*
  336. * Trigger the target to continue into the measurement too:
  337. */
  338. atomic_inc(&start_count);
  339. check_tsc_warp(loop_timeout(cpu));
  340. while (atomic_read(&stop_count) != cpus-1)
  341. cpu_relax();
  342. /*
  343. * If the test was successful set the number of runs to zero and
  344. * stop. If not, decrement the number of runs an check if we can
  345. * retry. In case of random warps no retry is attempted.
  346. */
  347. if (!nr_warps) {
  348. atomic_set(&test_runs, 0);
  349. pr_debug("TSC synchronization [CPU#%d -> CPU#%d]: passed\n",
  350. smp_processor_id(), cpu);
  351. } else if (atomic_dec_and_test(&test_runs) || random_warps) {
  352. /* Force it to 0 if random warps brought us here */
  353. atomic_set(&test_runs, 0);
  354. pr_warn("TSC synchronization [CPU#%d -> CPU#%d]:\n",
  355. smp_processor_id(), cpu);
  356. pr_warn("Measured %Ld cycles TSC warp between CPUs, "
  357. "turning off TSC clock.\n", max_warp);
  358. if (random_warps)
  359. pr_warn("TSC warped randomly between CPUs\n");
  360. mark_tsc_unstable("check_tsc_sync_source failed");
  361. }
  362. /*
  363. * Reset it - just in case we boot another CPU later:
  364. */
  365. atomic_set(&start_count, 0);
  366. random_warps = 0;
  367. nr_warps = 0;
  368. max_warp = 0;
  369. last_tsc = 0;
  370. /*
  371. * Let the target continue with the bootup:
  372. */
  373. atomic_inc(&stop_count);
  374. /*
  375. * Retry, if there is a chance to do so.
  376. */
  377. if (atomic_read(&test_runs) > 0)
  378. goto retry;
  379. }
  380. /*
  381. * Freshly booted CPUs call into this:
  382. */
  383. void check_tsc_sync_target(void)
  384. {
  385. struct tsc_adjust *cur = this_cpu_ptr(&tsc_adjust);
  386. unsigned int cpu = smp_processor_id();
  387. cycles_t cur_max_warp, gbl_max_warp;
  388. int cpus = 2;
  389. /* Also aborts if there is no TSC. */
  390. if (unsynchronized_tsc())
  391. return;
  392. /*
  393. * Store, verify and sanitize the TSC adjust register. If
  394. * successful skip the test.
  395. *
  396. * The test is also skipped when the TSC is marked reliable. This
  397. * is true for SoCs which have no fallback clocksource. On these
  398. * SoCs the TSC is frequency synchronized, but still the TSC ADJUST
  399. * register might have been wreckaged by the BIOS..
  400. */
  401. if (tsc_store_and_check_tsc_adjust(false) || tsc_clocksource_reliable) {
  402. atomic_inc(&skip_test);
  403. return;
  404. }
  405. retry:
  406. /*
  407. * Register this CPU's participation and wait for the
  408. * source CPU to start the measurement:
  409. */
  410. atomic_inc(&start_count);
  411. while (atomic_read(&start_count) != cpus)
  412. cpu_relax();
  413. cur_max_warp = check_tsc_warp(loop_timeout(cpu));
  414. /*
  415. * Store the maximum observed warp value for a potential retry:
  416. */
  417. gbl_max_warp = max_warp;
  418. /*
  419. * Ok, we are done:
  420. */
  421. atomic_inc(&stop_count);
  422. /*
  423. * Wait for the source CPU to print stuff:
  424. */
  425. while (atomic_read(&stop_count) != cpus)
  426. cpu_relax();
  427. /*
  428. * Reset it for the next sync test:
  429. */
  430. atomic_set(&stop_count, 0);
  431. /*
  432. * Check the number of remaining test runs. If not zero, the test
  433. * failed and a retry with adjusted TSC is possible. If zero the
  434. * test was either successful or failed terminally.
  435. */
  436. if (!atomic_read(&test_runs))
  437. return;
  438. /*
  439. * If the warp value of this CPU is 0, then the other CPU
  440. * observed time going backwards so this TSC was ahead and
  441. * needs to move backwards.
  442. */
  443. if (!cur_max_warp)
  444. cur_max_warp = -gbl_max_warp;
  445. /*
  446. * Add the result to the previous adjustment value.
  447. *
  448. * The adjustment value is slightly off by the overhead of the
  449. * sync mechanism (observed values are ~200 TSC cycles), but this
  450. * really depends on CPU, node distance and frequency. So
  451. * compensating for this is hard to get right. Experiments show
  452. * that the warp is not longer detectable when the observed warp
  453. * value is used. In the worst case the adjustment needs to go
  454. * through a 3rd run for fine tuning.
  455. */
  456. cur->adjusted += cur_max_warp;
  457. pr_warn("TSC ADJUST compensate: CPU%u observed %lld warp. Adjust: %lld\n",
  458. cpu, cur_max_warp, cur->adjusted);
  459. wrmsrl(MSR_IA32_TSC_ADJUST, cur->adjusted);
  460. goto retry;
  461. }
  462. #endif /* CONFIG_SMP */