msr.h 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _ASM_X86_MSR_H
  3. #define _ASM_X86_MSR_H
  4. #include "msr-index.h"
  5. #ifndef __ASSEMBLY__
  6. #include <asm/asm.h>
  7. #include <asm/errno.h>
  8. #include <asm/cpumask.h>
  9. #include <uapi/asm/msr.h>
  10. #include <asm/shared/msr.h>
  11. struct msr_info {
  12. u32 msr_no;
  13. struct msr reg;
  14. struct msr *msrs;
  15. int err;
  16. };
  17. struct msr_regs_info {
  18. u32 *regs;
  19. int err;
  20. };
  21. struct saved_msr {
  22. bool valid;
  23. struct msr_info info;
  24. };
  25. struct saved_msrs {
  26. unsigned int num;
  27. struct saved_msr *array;
  28. };
  29. /*
  30. * both i386 and x86_64 returns 64-bit value in edx:eax, but gcc's "A"
  31. * constraint has different meanings. For i386, "A" means exactly
  32. * edx:eax, while for x86_64 it doesn't mean rdx:rax or edx:eax. Instead,
  33. * it means rax *or* rdx.
  34. */
  35. #ifdef CONFIG_X86_64
  36. /* Using 64-bit values saves one instruction clearing the high half of low */
  37. #define DECLARE_ARGS(val, low, high) unsigned long low, high
  38. #define EAX_EDX_VAL(val, low, high) ((low) | (high) << 32)
  39. #define EAX_EDX_RET(val, low, high) "=a" (low), "=d" (high)
  40. #else
  41. #define DECLARE_ARGS(val, low, high) unsigned long long val
  42. #define EAX_EDX_VAL(val, low, high) (val)
  43. #define EAX_EDX_RET(val, low, high) "=A" (val)
  44. #endif
  45. /*
  46. * Be very careful with includes. This header is prone to include loops.
  47. */
  48. #include <asm/atomic.h>
  49. #include <linux/tracepoint-defs.h>
  50. #ifdef CONFIG_TRACEPOINTS
  51. DECLARE_TRACEPOINT(read_msr);
  52. DECLARE_TRACEPOINT(write_msr);
  53. DECLARE_TRACEPOINT(rdpmc);
  54. extern void do_trace_write_msr(unsigned int msr, u64 val, int failed);
  55. extern void do_trace_read_msr(unsigned int msr, u64 val, int failed);
  56. extern void do_trace_rdpmc(unsigned int msr, u64 val, int failed);
  57. #else
  58. static inline void do_trace_write_msr(unsigned int msr, u64 val, int failed) {}
  59. static inline void do_trace_read_msr(unsigned int msr, u64 val, int failed) {}
  60. static inline void do_trace_rdpmc(unsigned int msr, u64 val, int failed) {}
  61. #endif
  62. /*
  63. * __rdmsr() and __wrmsr() are the two primitives which are the bare minimum MSR
  64. * accessors and should not have any tracing or other functionality piggybacking
  65. * on them - those are *purely* for accessing MSRs and nothing more. So don't even
  66. * think of extending them - you will be slapped with a stinking trout or a frozen
  67. * shark will reach you, wherever you are! You've been warned.
  68. */
  69. static __always_inline unsigned long long __rdmsr(unsigned int msr)
  70. {
  71. DECLARE_ARGS(val, low, high);
  72. asm volatile("1: rdmsr\n"
  73. "2:\n"
  74. _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_RDMSR)
  75. : EAX_EDX_RET(val, low, high) : "c" (msr));
  76. return EAX_EDX_VAL(val, low, high);
  77. }
  78. static __always_inline void __wrmsr(unsigned int msr, u32 low, u32 high)
  79. {
  80. asm volatile("1: wrmsr\n"
  81. "2:\n"
  82. _ASM_EXTABLE_TYPE(1b, 2b, EX_TYPE_WRMSR)
  83. : : "c" (msr), "a"(low), "d" (high) : "memory");
  84. }
  85. #define native_rdmsr(msr, val1, val2) \
  86. do { \
  87. u64 __val = __rdmsr((msr)); \
  88. (void)((val1) = (u32)__val); \
  89. (void)((val2) = (u32)(__val >> 32)); \
  90. } while (0)
  91. #define native_wrmsr(msr, low, high) \
  92. __wrmsr(msr, low, high)
  93. #define native_wrmsrl(msr, val) \
  94. __wrmsr((msr), (u32)((u64)(val)), \
  95. (u32)((u64)(val) >> 32))
  96. static inline unsigned long long native_read_msr(unsigned int msr)
  97. {
  98. unsigned long long val;
  99. val = __rdmsr(msr);
  100. if (tracepoint_enabled(read_msr))
  101. do_trace_read_msr(msr, val, 0);
  102. return val;
  103. }
  104. static inline unsigned long long native_read_msr_safe(unsigned int msr,
  105. int *err)
  106. {
  107. DECLARE_ARGS(val, low, high);
  108. asm volatile("1: rdmsr ; xor %[err],%[err]\n"
  109. "2:\n\t"
  110. _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_RDMSR_SAFE, %[err])
  111. : [err] "=r" (*err), EAX_EDX_RET(val, low, high)
  112. : "c" (msr));
  113. if (tracepoint_enabled(read_msr))
  114. do_trace_read_msr(msr, EAX_EDX_VAL(val, low, high), *err);
  115. return EAX_EDX_VAL(val, low, high);
  116. }
  117. /* Can be uninlined because referenced by paravirt */
  118. static inline void notrace
  119. native_write_msr(unsigned int msr, u32 low, u32 high)
  120. {
  121. __wrmsr(msr, low, high);
  122. if (tracepoint_enabled(write_msr))
  123. do_trace_write_msr(msr, ((u64)high << 32 | low), 0);
  124. }
  125. /* Can be uninlined because referenced by paravirt */
  126. static inline int notrace
  127. native_write_msr_safe(unsigned int msr, u32 low, u32 high)
  128. {
  129. int err;
  130. asm volatile("1: wrmsr ; xor %[err],%[err]\n"
  131. "2:\n\t"
  132. _ASM_EXTABLE_TYPE_REG(1b, 2b, EX_TYPE_WRMSR_SAFE, %[err])
  133. : [err] "=a" (err)
  134. : "c" (msr), "0" (low), "d" (high)
  135. : "memory");
  136. if (tracepoint_enabled(write_msr))
  137. do_trace_write_msr(msr, ((u64)high << 32 | low), err);
  138. return err;
  139. }
  140. extern int rdmsr_safe_regs(u32 regs[8]);
  141. extern int wrmsr_safe_regs(u32 regs[8]);
  142. /**
  143. * rdtsc() - returns the current TSC without ordering constraints
  144. *
  145. * rdtsc() returns the result of RDTSC as a 64-bit integer. The
  146. * only ordering constraint it supplies is the ordering implied by
  147. * "asm volatile": it will put the RDTSC in the place you expect. The
  148. * CPU can and will speculatively execute that RDTSC, though, so the
  149. * results can be non-monotonic if compared on different CPUs.
  150. */
  151. static __always_inline unsigned long long rdtsc(void)
  152. {
  153. DECLARE_ARGS(val, low, high);
  154. asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
  155. return EAX_EDX_VAL(val, low, high);
  156. }
  157. /**
  158. * rdtsc_ordered() - read the current TSC in program order
  159. *
  160. * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
  161. * It is ordered like a load to a global in-memory counter. It should
  162. * be impossible to observe non-monotonic rdtsc_unordered() behavior
  163. * across multiple CPUs as long as the TSC is synced.
  164. */
  165. static __always_inline unsigned long long rdtsc_ordered(void)
  166. {
  167. DECLARE_ARGS(val, low, high);
  168. /*
  169. * The RDTSC instruction is not ordered relative to memory
  170. * access. The Intel SDM and the AMD APM are both vague on this
  171. * point, but empirically an RDTSC instruction can be
  172. * speculatively executed before prior loads. An RDTSC
  173. * immediately after an appropriate barrier appears to be
  174. * ordered as a normal load, that is, it provides the same
  175. * ordering guarantees as reading from a global memory location
  176. * that some other imaginary CPU is updating continuously with a
  177. * time stamp.
  178. *
  179. * Thus, use the preferred barrier on the respective CPU, aiming for
  180. * RDTSCP as the default.
  181. */
  182. asm volatile(ALTERNATIVE_2("rdtsc",
  183. "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
  184. "rdtscp", X86_FEATURE_RDTSCP)
  185. : EAX_EDX_RET(val, low, high)
  186. /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
  187. :: "ecx");
  188. return EAX_EDX_VAL(val, low, high);
  189. }
  190. static inline unsigned long long native_read_pmc(int counter)
  191. {
  192. DECLARE_ARGS(val, low, high);
  193. asm volatile("rdpmc" : EAX_EDX_RET(val, low, high) : "c" (counter));
  194. if (tracepoint_enabled(rdpmc))
  195. do_trace_rdpmc(counter, EAX_EDX_VAL(val, low, high), 0);
  196. return EAX_EDX_VAL(val, low, high);
  197. }
  198. #ifdef CONFIG_PARAVIRT_XXL
  199. #include <asm/paravirt.h>
  200. #else
  201. #include <linux/errno.h>
  202. /*
  203. * Access to machine-specific registers (available on 586 and better only)
  204. * Note: the rd* operations modify the parameters directly (without using
  205. * pointer indirection), this allows gcc to optimize better
  206. */
  207. #define rdmsr(msr, low, high) \
  208. do { \
  209. u64 __val = native_read_msr((msr)); \
  210. (void)((low) = (u32)__val); \
  211. (void)((high) = (u32)(__val >> 32)); \
  212. } while (0)
  213. static inline void wrmsr(unsigned int msr, u32 low, u32 high)
  214. {
  215. native_write_msr(msr, low, high);
  216. }
  217. #define rdmsrl(msr, val) \
  218. ((val) = native_read_msr((msr)))
  219. static inline void wrmsrl(unsigned int msr, u64 val)
  220. {
  221. native_write_msr(msr, (u32)(val & 0xffffffffULL), (u32)(val >> 32));
  222. }
  223. /* wrmsr with exception handling */
  224. static inline int wrmsr_safe(unsigned int msr, u32 low, u32 high)
  225. {
  226. return native_write_msr_safe(msr, low, high);
  227. }
  228. /* rdmsr with exception handling */
  229. #define rdmsr_safe(msr, low, high) \
  230. ({ \
  231. int __err; \
  232. u64 __val = native_read_msr_safe((msr), &__err); \
  233. (*low) = (u32)__val; \
  234. (*high) = (u32)(__val >> 32); \
  235. __err; \
  236. })
  237. static inline int rdmsrl_safe(unsigned int msr, unsigned long long *p)
  238. {
  239. int err;
  240. *p = native_read_msr_safe(msr, &err);
  241. return err;
  242. }
  243. #define rdpmc(counter, low, high) \
  244. do { \
  245. u64 _l = native_read_pmc((counter)); \
  246. (low) = (u32)_l; \
  247. (high) = (u32)(_l >> 32); \
  248. } while (0)
  249. #define rdpmcl(counter, val) ((val) = native_read_pmc(counter))
  250. #endif /* !CONFIG_PARAVIRT_XXL */
  251. /*
  252. * 64-bit version of wrmsr_safe():
  253. */
  254. static inline int wrmsrl_safe(u32 msr, u64 val)
  255. {
  256. return wrmsr_safe(msr, (u32)val, (u32)(val >> 32));
  257. }
  258. struct msr *msrs_alloc(void);
  259. void msrs_free(struct msr *msrs);
  260. int msr_set_bit(u32 msr, u8 bit);
  261. int msr_clear_bit(u32 msr, u8 bit);
  262. #ifdef CONFIG_SMP
  263. int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
  264. int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
  265. int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
  266. int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
  267. void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
  268. void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs);
  269. int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h);
  270. int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h);
  271. int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q);
  272. int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q);
  273. int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
  274. int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]);
  275. #else /* CONFIG_SMP */
  276. static inline int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h)
  277. {
  278. rdmsr(msr_no, *l, *h);
  279. return 0;
  280. }
  281. static inline int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
  282. {
  283. wrmsr(msr_no, l, h);
  284. return 0;
  285. }
  286. static inline int rdmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
  287. {
  288. rdmsrl(msr_no, *q);
  289. return 0;
  290. }
  291. static inline int wrmsrl_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
  292. {
  293. wrmsrl(msr_no, q);
  294. return 0;
  295. }
  296. static inline void rdmsr_on_cpus(const struct cpumask *m, u32 msr_no,
  297. struct msr *msrs)
  298. {
  299. rdmsr_on_cpu(0, msr_no, &(msrs[0].l), &(msrs[0].h));
  300. }
  301. static inline void wrmsr_on_cpus(const struct cpumask *m, u32 msr_no,
  302. struct msr *msrs)
  303. {
  304. wrmsr_on_cpu(0, msr_no, msrs[0].l, msrs[0].h);
  305. }
  306. static inline int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no,
  307. u32 *l, u32 *h)
  308. {
  309. return rdmsr_safe(msr_no, l, h);
  310. }
  311. static inline int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h)
  312. {
  313. return wrmsr_safe(msr_no, l, h);
  314. }
  315. static inline int rdmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 *q)
  316. {
  317. return rdmsrl_safe(msr_no, q);
  318. }
  319. static inline int wrmsrl_safe_on_cpu(unsigned int cpu, u32 msr_no, u64 q)
  320. {
  321. return wrmsrl_safe(msr_no, q);
  322. }
  323. static inline int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
  324. {
  325. return rdmsr_safe_regs(regs);
  326. }
  327. static inline int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8])
  328. {
  329. return wrmsr_safe_regs(regs);
  330. }
  331. #endif /* CONFIG_SMP */
  332. #endif /* __ASSEMBLY__ */
  333. #endif /* _ASM_X86_MSR_H */