pvclock.c 3.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* paravirtual clock -- common code used by kvm/xen
  3. */
  4. #include <linux/clocksource.h>
  5. #include <linux/kernel.h>
  6. #include <linux/percpu.h>
  7. #include <linux/notifier.h>
  8. #include <linux/sched.h>
  9. #include <linux/gfp.h>
  10. #include <linux/memblock.h>
  11. #include <linux/nmi.h>
  12. #include <asm/fixmap.h>
  13. #include <asm/pvclock.h>
  14. #include <asm/vgtod.h>
  15. static u8 valid_flags __read_mostly = 0;
  16. static struct pvclock_vsyscall_time_info *pvti_cpu0_va __read_mostly;
  17. void pvclock_set_flags(u8 flags)
  18. {
  19. valid_flags = flags;
  20. }
  21. unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src)
  22. {
  23. u64 pv_tsc_khz = 1000000ULL << 32;
  24. do_div(pv_tsc_khz, src->tsc_to_system_mul);
  25. if (src->tsc_shift < 0)
  26. pv_tsc_khz <<= -src->tsc_shift;
  27. else
  28. pv_tsc_khz >>= src->tsc_shift;
  29. return pv_tsc_khz;
  30. }
  31. void pvclock_touch_watchdogs(void)
  32. {
  33. touch_softlockup_watchdog_sync();
  34. clocksource_touch_watchdog();
  35. rcu_cpu_stall_reset();
  36. reset_hung_task_detector();
  37. }
  38. static atomic64_t last_value = ATOMIC64_INIT(0);
  39. void pvclock_resume(void)
  40. {
  41. atomic64_set(&last_value, 0);
  42. }
  43. u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
  44. {
  45. unsigned version;
  46. u8 flags;
  47. do {
  48. version = pvclock_read_begin(src);
  49. flags = src->flags;
  50. } while (pvclock_read_retry(src, version));
  51. return flags & valid_flags;
  52. }
  53. u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
  54. {
  55. unsigned version;
  56. u64 ret;
  57. u64 last;
  58. u8 flags;
  59. do {
  60. version = pvclock_read_begin(src);
  61. ret = __pvclock_read_cycles(src, rdtsc_ordered());
  62. flags = src->flags;
  63. } while (pvclock_read_retry(src, version));
  64. if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) {
  65. src->flags &= ~PVCLOCK_GUEST_STOPPED;
  66. pvclock_touch_watchdogs();
  67. }
  68. if ((valid_flags & PVCLOCK_TSC_STABLE_BIT) &&
  69. (flags & PVCLOCK_TSC_STABLE_BIT))
  70. return ret;
  71. /*
  72. * Assumption here is that last_value, a global accumulator, always goes
  73. * forward. If we are less than that, we should not be much smaller.
  74. * We assume there is an error margin we're inside, and then the correction
  75. * does not sacrifice accuracy.
  76. *
  77. * For reads: global may have changed between test and return,
  78. * but this means someone else updated poked the clock at a later time.
  79. * We just need to make sure we are not seeing a backwards event.
  80. *
  81. * For updates: last_value = ret is not enough, since two vcpus could be
  82. * updating at the same time, and one of them could be slightly behind,
  83. * making the assumption that last_value always go forward fail to hold.
  84. */
  85. last = atomic64_read(&last_value);
  86. do {
  87. if (ret < last)
  88. return last;
  89. last = atomic64_cmpxchg(&last_value, last, ret);
  90. } while (unlikely(last != ret));
  91. return ret;
  92. }
  93. void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
  94. struct pvclock_vcpu_time_info *vcpu_time,
  95. struct timespec64 *ts)
  96. {
  97. u32 version;
  98. u64 delta;
  99. struct timespec64 now;
  100. /* get wallclock at system boot */
  101. do {
  102. version = wall_clock->version;
  103. rmb(); /* fetch version before time */
  104. /*
  105. * Note: wall_clock->sec is a u32 value, so it can
  106. * only store dates between 1970 and 2106. To allow
  107. * times beyond that, we need to create a new hypercall
  108. * interface with an extended pvclock_wall_clock structure
  109. * like ARM has.
  110. */
  111. now.tv_sec = wall_clock->sec;
  112. now.tv_nsec = wall_clock->nsec;
  113. rmb(); /* fetch time before checking version */
  114. } while ((wall_clock->version & 1) || (version != wall_clock->version));
  115. delta = pvclock_clocksource_read(vcpu_time); /* time since system boot */
  116. delta += now.tv_sec * NSEC_PER_SEC + now.tv_nsec;
  117. now.tv_nsec = do_div(delta, NSEC_PER_SEC);
  118. now.tv_sec = delta;
  119. set_normalized_timespec64(ts, now.tv_sec, now.tv_nsec);
  120. }
  121. void pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info *pvti)
  122. {
  123. WARN_ON(vclock_was_used(VDSO_CLOCKMODE_PVCLOCK));
  124. pvti_cpu0_va = pvti;
  125. }
  126. struct pvclock_vsyscall_time_info *pvclock_get_pvti_cpu0_va(void)
  127. {
  128. return pvti_cpu0_va;
  129. }
  130. EXPORT_SYMBOL_GPL(pvclock_get_pvti_cpu0_va);