tdp_mmu.c 56 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include "mmu.h"
  3. #include "mmu_internal.h"
  4. #include "mmutrace.h"
  5. #include "tdp_iter.h"
  6. #include "tdp_mmu.h"
  7. #include "spte.h"
  8. #include <asm/cmpxchg.h>
  9. #include <trace/events/kvm.h>
  10. static bool __read_mostly tdp_mmu_enabled = true;
  11. module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644);
  12. /* Initializes the TDP MMU for the VM, if enabled. */
  13. void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
  14. {
  15. if (!tdp_enabled || !READ_ONCE(tdp_mmu_enabled))
  16. return;
  17. /* This should not be changed for the lifetime of the VM. */
  18. kvm->arch.tdp_mmu_enabled = true;
  19. INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
  20. spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
  21. INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages);
  22. }
  23. /* Arbitrarily returns true so that this may be used in if statements. */
  24. static __always_inline bool kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm,
  25. bool shared)
  26. {
  27. if (shared)
  28. lockdep_assert_held_read(&kvm->mmu_lock);
  29. else
  30. lockdep_assert_held_write(&kvm->mmu_lock);
  31. return true;
  32. }
  33. void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm)
  34. {
  35. if (!kvm->arch.tdp_mmu_enabled)
  36. return;
  37. /*
  38. * Invalidate all roots, which besides the obvious, schedules all roots
  39. * for zapping and thus puts the TDP MMU's reference to each root, i.e.
  40. * ultimately frees all roots.
  41. */
  42. kvm_tdp_mmu_invalidate_all_roots(kvm);
  43. kvm_tdp_mmu_zap_invalidated_roots(kvm);
  44. WARN_ON(!list_empty(&kvm->arch.tdp_mmu_pages));
  45. WARN_ON(!list_empty(&kvm->arch.tdp_mmu_roots));
  46. /*
  47. * Ensure that all the outstanding RCU callbacks to free shadow pages
  48. * can run before the VM is torn down. Putting the last reference to
  49. * zapped roots will create new callbacks.
  50. */
  51. rcu_barrier();
  52. }
  53. static void tdp_mmu_free_sp(struct kvm_mmu_page *sp)
  54. {
  55. free_page((unsigned long)sp->spt);
  56. kmem_cache_free(mmu_page_header_cache, sp);
  57. }
  58. /*
  59. * This is called through call_rcu in order to free TDP page table memory
  60. * safely with respect to other kernel threads that may be operating on
  61. * the memory.
  62. * By only accessing TDP MMU page table memory in an RCU read critical
  63. * section, and freeing it after a grace period, lockless access to that
  64. * memory won't use it after it is freed.
  65. */
  66. static void tdp_mmu_free_sp_rcu_callback(struct rcu_head *head)
  67. {
  68. struct kvm_mmu_page *sp = container_of(head, struct kvm_mmu_page,
  69. rcu_head);
  70. tdp_mmu_free_sp(sp);
  71. }
  72. void kvm_tdp_mmu_put_root(struct kvm *kvm, struct kvm_mmu_page *root,
  73. bool shared)
  74. {
  75. kvm_lockdep_assert_mmu_lock_held(kvm, shared);
  76. if (!refcount_dec_and_test(&root->tdp_mmu_root_count))
  77. return;
  78. /*
  79. * The TDP MMU itself holds a reference to each root until the root is
  80. * explicitly invalidated, i.e. the final reference should be never be
  81. * put for a valid root.
  82. */
  83. KVM_BUG_ON(!is_tdp_mmu_page(root) || !root->role.invalid, kvm);
  84. spin_lock(&kvm->arch.tdp_mmu_pages_lock);
  85. list_del_rcu(&root->link);
  86. spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
  87. call_rcu(&root->rcu_head, tdp_mmu_free_sp_rcu_callback);
  88. }
  89. /*
  90. * Returns the next root after @prev_root (or the first root if @prev_root is
  91. * NULL). A reference to the returned root is acquired, and the reference to
  92. * @prev_root is released (the caller obviously must hold a reference to
  93. * @prev_root if it's non-NULL).
  94. *
  95. * If @only_valid is true, invalid roots are skipped.
  96. *
  97. * Returns NULL if the end of tdp_mmu_roots was reached.
  98. */
  99. static struct kvm_mmu_page *tdp_mmu_next_root(struct kvm *kvm,
  100. struct kvm_mmu_page *prev_root,
  101. bool shared, bool only_valid)
  102. {
  103. struct kvm_mmu_page *next_root;
  104. rcu_read_lock();
  105. if (prev_root)
  106. next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,
  107. &prev_root->link,
  108. typeof(*prev_root), link);
  109. else
  110. next_root = list_first_or_null_rcu(&kvm->arch.tdp_mmu_roots,
  111. typeof(*next_root), link);
  112. while (next_root) {
  113. if ((!only_valid || !next_root->role.invalid) &&
  114. kvm_tdp_mmu_get_root(next_root))
  115. break;
  116. next_root = list_next_or_null_rcu(&kvm->arch.tdp_mmu_roots,
  117. &next_root->link, typeof(*next_root), link);
  118. }
  119. rcu_read_unlock();
  120. if (prev_root)
  121. kvm_tdp_mmu_put_root(kvm, prev_root, shared);
  122. return next_root;
  123. }
  124. /*
  125. * Note: this iterator gets and puts references to the roots it iterates over.
  126. * This makes it safe to release the MMU lock and yield within the loop, but
  127. * if exiting the loop early, the caller must drop the reference to the most
  128. * recent root. (Unless keeping a live reference is desirable.)
  129. *
  130. * If shared is set, this function is operating under the MMU lock in read
  131. * mode. In the unlikely event that this thread must free a root, the lock
  132. * will be temporarily dropped and reacquired in write mode.
  133. */
  134. #define __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, _only_valid)\
  135. for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, _only_valid); \
  136. _root; \
  137. _root = tdp_mmu_next_root(_kvm, _root, _shared, _only_valid)) \
  138. if (kvm_lockdep_assert_mmu_lock_held(_kvm, _shared) && \
  139. kvm_mmu_page_as_id(_root) != _as_id) { \
  140. } else
  141. #define for_each_valid_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared) \
  142. __for_each_tdp_mmu_root_yield_safe(_kvm, _root, _as_id, _shared, true)
  143. #define for_each_tdp_mmu_root_yield_safe(_kvm, _root, _shared) \
  144. for (_root = tdp_mmu_next_root(_kvm, NULL, _shared, false); \
  145. _root; \
  146. _root = tdp_mmu_next_root(_kvm, _root, _shared, false)) \
  147. if (!kvm_lockdep_assert_mmu_lock_held(_kvm, _shared)) { \
  148. } else
  149. /*
  150. * Iterate over all TDP MMU roots. Requires that mmu_lock be held for write,
  151. * the implication being that any flow that holds mmu_lock for read is
  152. * inherently yield-friendly and should use the yield-safe variant above.
  153. * Holding mmu_lock for write obviates the need for RCU protection as the list
  154. * is guaranteed to be stable.
  155. */
  156. #define for_each_tdp_mmu_root(_kvm, _root, _as_id) \
  157. list_for_each_entry(_root, &_kvm->arch.tdp_mmu_roots, link) \
  158. if (kvm_lockdep_assert_mmu_lock_held(_kvm, false) && \
  159. kvm_mmu_page_as_id(_root) != _as_id) { \
  160. } else
  161. static struct kvm_mmu_page *tdp_mmu_alloc_sp(struct kvm_vcpu *vcpu)
  162. {
  163. struct kvm_mmu_page *sp;
  164. sp = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_page_header_cache);
  165. sp->spt = kvm_mmu_memory_cache_alloc(&vcpu->arch.mmu_shadow_page_cache);
  166. return sp;
  167. }
  168. static void tdp_mmu_init_sp(struct kvm_mmu_page *sp, tdp_ptep_t sptep,
  169. gfn_t gfn, union kvm_mmu_page_role role)
  170. {
  171. set_page_private(virt_to_page(sp->spt), (unsigned long)sp);
  172. sp->role = role;
  173. sp->gfn = gfn;
  174. sp->ptep = sptep;
  175. sp->tdp_mmu_page = true;
  176. trace_kvm_mmu_get_page(sp, true);
  177. }
  178. static void tdp_mmu_init_child_sp(struct kvm_mmu_page *child_sp,
  179. struct tdp_iter *iter)
  180. {
  181. struct kvm_mmu_page *parent_sp;
  182. union kvm_mmu_page_role role;
  183. parent_sp = sptep_to_sp(rcu_dereference(iter->sptep));
  184. role = parent_sp->role;
  185. role.level--;
  186. tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role);
  187. }
  188. hpa_t kvm_tdp_mmu_get_vcpu_root_hpa(struct kvm_vcpu *vcpu)
  189. {
  190. union kvm_mmu_page_role role = vcpu->arch.mmu->root_role;
  191. struct kvm *kvm = vcpu->kvm;
  192. struct kvm_mmu_page *root;
  193. lockdep_assert_held_write(&kvm->mmu_lock);
  194. /*
  195. * Check for an existing root before allocating a new one. Note, the
  196. * role check prevents consuming an invalid root.
  197. */
  198. for_each_tdp_mmu_root(kvm, root, kvm_mmu_role_as_id(role)) {
  199. if (root->role.word == role.word &&
  200. kvm_tdp_mmu_get_root(root))
  201. goto out;
  202. }
  203. root = tdp_mmu_alloc_sp(vcpu);
  204. tdp_mmu_init_sp(root, NULL, 0, role);
  205. /*
  206. * TDP MMU roots are kept until they are explicitly invalidated, either
  207. * by a memslot update or by the destruction of the VM. Initialize the
  208. * refcount to two; one reference for the vCPU, and one reference for
  209. * the TDP MMU itself, which is held until the root is invalidated and
  210. * is ultimately put by kvm_tdp_mmu_zap_invalidated_roots().
  211. */
  212. refcount_set(&root->tdp_mmu_root_count, 2);
  213. spin_lock(&kvm->arch.tdp_mmu_pages_lock);
  214. list_add_rcu(&root->link, &kvm->arch.tdp_mmu_roots);
  215. spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
  216. out:
  217. return __pa(root->spt);
  218. }
  219. static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
  220. u64 old_spte, u64 new_spte, int level,
  221. bool shared);
  222. static void handle_changed_spte_acc_track(u64 old_spte, u64 new_spte, int level)
  223. {
  224. if (!is_shadow_present_pte(old_spte) || !is_last_spte(old_spte, level))
  225. return;
  226. if (is_accessed_spte(old_spte) &&
  227. (!is_shadow_present_pte(new_spte) || !is_accessed_spte(new_spte) ||
  228. spte_to_pfn(old_spte) != spte_to_pfn(new_spte)))
  229. kvm_set_pfn_accessed(spte_to_pfn(old_spte));
  230. }
  231. static void handle_changed_spte_dirty_log(struct kvm *kvm, int as_id, gfn_t gfn,
  232. u64 old_spte, u64 new_spte, int level)
  233. {
  234. bool pfn_changed;
  235. struct kvm_memory_slot *slot;
  236. if (level > PG_LEVEL_4K)
  237. return;
  238. pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
  239. if ((!is_writable_pte(old_spte) || pfn_changed) &&
  240. is_writable_pte(new_spte)) {
  241. slot = __gfn_to_memslot(__kvm_memslots(kvm, as_id), gfn);
  242. mark_page_dirty_in_slot(kvm, slot, gfn);
  243. }
  244. }
  245. static void tdp_account_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  246. {
  247. kvm_account_pgtable_pages((void *)sp->spt, +1);
  248. }
  249. static void tdp_unaccount_mmu_page(struct kvm *kvm, struct kvm_mmu_page *sp)
  250. {
  251. kvm_account_pgtable_pages((void *)sp->spt, -1);
  252. }
  253. /**
  254. * tdp_mmu_unlink_sp() - Remove a shadow page from the list of used pages
  255. *
  256. * @kvm: kvm instance
  257. * @sp: the page to be removed
  258. * @shared: This operation may not be running under the exclusive use of
  259. * the MMU lock and the operation must synchronize with other
  260. * threads that might be adding or removing pages.
  261. */
  262. static void tdp_mmu_unlink_sp(struct kvm *kvm, struct kvm_mmu_page *sp,
  263. bool shared)
  264. {
  265. tdp_unaccount_mmu_page(kvm, sp);
  266. if (shared)
  267. spin_lock(&kvm->arch.tdp_mmu_pages_lock);
  268. else
  269. lockdep_assert_held_write(&kvm->mmu_lock);
  270. list_del(&sp->link);
  271. if (sp->lpage_disallowed)
  272. unaccount_huge_nx_page(kvm, sp);
  273. if (shared)
  274. spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
  275. }
  276. /**
  277. * handle_removed_pt() - handle a page table removed from the TDP structure
  278. *
  279. * @kvm: kvm instance
  280. * @pt: the page removed from the paging structure
  281. * @shared: This operation may not be running under the exclusive use
  282. * of the MMU lock and the operation must synchronize with other
  283. * threads that might be modifying SPTEs.
  284. *
  285. * Given a page table that has been removed from the TDP paging structure,
  286. * iterates through the page table to clear SPTEs and free child page tables.
  287. *
  288. * Note that pt is passed in as a tdp_ptep_t, but it does not need RCU
  289. * protection. Since this thread removed it from the paging structure,
  290. * this thread will be responsible for ensuring the page is freed. Hence the
  291. * early rcu_dereferences in the function.
  292. */
  293. static void handle_removed_pt(struct kvm *kvm, tdp_ptep_t pt, bool shared)
  294. {
  295. struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(pt));
  296. int level = sp->role.level;
  297. gfn_t base_gfn = sp->gfn;
  298. int i;
  299. trace_kvm_mmu_prepare_zap_page(sp);
  300. tdp_mmu_unlink_sp(kvm, sp, shared);
  301. for (i = 0; i < SPTE_ENT_PER_PAGE; i++) {
  302. tdp_ptep_t sptep = pt + i;
  303. gfn_t gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level);
  304. u64 old_spte;
  305. if (shared) {
  306. /*
  307. * Set the SPTE to a nonpresent value that other
  308. * threads will not overwrite. If the SPTE was
  309. * already marked as removed then another thread
  310. * handling a page fault could overwrite it, so
  311. * set the SPTE until it is set from some other
  312. * value to the removed SPTE value.
  313. */
  314. for (;;) {
  315. old_spte = kvm_tdp_mmu_write_spte_atomic(sptep, REMOVED_SPTE);
  316. if (!is_removed_spte(old_spte))
  317. break;
  318. cpu_relax();
  319. }
  320. } else {
  321. /*
  322. * If the SPTE is not MMU-present, there is no backing
  323. * page associated with the SPTE and so no side effects
  324. * that need to be recorded, and exclusive ownership of
  325. * mmu_lock ensures the SPTE can't be made present.
  326. * Note, zapping MMIO SPTEs is also unnecessary as they
  327. * are guarded by the memslots generation, not by being
  328. * unreachable.
  329. */
  330. old_spte = kvm_tdp_mmu_read_spte(sptep);
  331. if (!is_shadow_present_pte(old_spte))
  332. continue;
  333. /*
  334. * Use the common helper instead of a raw WRITE_ONCE as
  335. * the SPTE needs to be updated atomically if it can be
  336. * modified by a different vCPU outside of mmu_lock.
  337. * Even though the parent SPTE is !PRESENT, the TLB
  338. * hasn't yet been flushed, and both Intel and AMD
  339. * document that A/D assists can use upper-level PxE
  340. * entries that are cached in the TLB, i.e. the CPU can
  341. * still access the page and mark it dirty.
  342. *
  343. * No retry is needed in the atomic update path as the
  344. * sole concern is dropping a Dirty bit, i.e. no other
  345. * task can zap/remove the SPTE as mmu_lock is held for
  346. * write. Marking the SPTE as a removed SPTE is not
  347. * strictly necessary for the same reason, but using
  348. * the remove SPTE value keeps the shared/exclusive
  349. * paths consistent and allows the handle_changed_spte()
  350. * call below to hardcode the new value to REMOVED_SPTE.
  351. *
  352. * Note, even though dropping a Dirty bit is the only
  353. * scenario where a non-atomic update could result in a
  354. * functional bug, simply checking the Dirty bit isn't
  355. * sufficient as a fast page fault could read the upper
  356. * level SPTE before it is zapped, and then make this
  357. * target SPTE writable, resume the guest, and set the
  358. * Dirty bit between reading the SPTE above and writing
  359. * it here.
  360. */
  361. old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte,
  362. REMOVED_SPTE, level);
  363. }
  364. handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn,
  365. old_spte, REMOVED_SPTE, level, shared);
  366. }
  367. call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback);
  368. }
  369. /**
  370. * __handle_changed_spte - handle bookkeeping associated with an SPTE change
  371. * @kvm: kvm instance
  372. * @as_id: the address space of the paging structure the SPTE was a part of
  373. * @gfn: the base GFN that was mapped by the SPTE
  374. * @old_spte: The value of the SPTE before the change
  375. * @new_spte: The value of the SPTE after the change
  376. * @level: the level of the PT the SPTE is part of in the paging structure
  377. * @shared: This operation may not be running under the exclusive use of
  378. * the MMU lock and the operation must synchronize with other
  379. * threads that might be modifying SPTEs.
  380. *
  381. * Handle bookkeeping that might result from the modification of a SPTE.
  382. * This function must be called for all TDP SPTE modifications.
  383. */
  384. static void __handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
  385. u64 old_spte, u64 new_spte, int level,
  386. bool shared)
  387. {
  388. bool was_present = is_shadow_present_pte(old_spte);
  389. bool is_present = is_shadow_present_pte(new_spte);
  390. bool was_leaf = was_present && is_last_spte(old_spte, level);
  391. bool is_leaf = is_present && is_last_spte(new_spte, level);
  392. bool pfn_changed = spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
  393. WARN_ON(level > PT64_ROOT_MAX_LEVEL);
  394. WARN_ON(level < PG_LEVEL_4K);
  395. WARN_ON(gfn & (KVM_PAGES_PER_HPAGE(level) - 1));
  396. /*
  397. * If this warning were to trigger it would indicate that there was a
  398. * missing MMU notifier or a race with some notifier handler.
  399. * A present, leaf SPTE should never be directly replaced with another
  400. * present leaf SPTE pointing to a different PFN. A notifier handler
  401. * should be zapping the SPTE before the main MM's page table is
  402. * changed, or the SPTE should be zeroed, and the TLBs flushed by the
  403. * thread before replacement.
  404. */
  405. if (was_leaf && is_leaf && pfn_changed) {
  406. pr_err("Invalid SPTE change: cannot replace a present leaf\n"
  407. "SPTE with another present leaf SPTE mapping a\n"
  408. "different PFN!\n"
  409. "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
  410. as_id, gfn, old_spte, new_spte, level);
  411. /*
  412. * Crash the host to prevent error propagation and guest data
  413. * corruption.
  414. */
  415. BUG();
  416. }
  417. if (old_spte == new_spte)
  418. return;
  419. trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte);
  420. if (is_leaf)
  421. check_spte_writable_invariants(new_spte);
  422. /*
  423. * The only times a SPTE should be changed from a non-present to
  424. * non-present state is when an MMIO entry is installed/modified/
  425. * removed. In that case, there is nothing to do here.
  426. */
  427. if (!was_present && !is_present) {
  428. /*
  429. * If this change does not involve a MMIO SPTE or removed SPTE,
  430. * it is unexpected. Log the change, though it should not
  431. * impact the guest since both the former and current SPTEs
  432. * are nonpresent.
  433. */
  434. if (WARN_ON(!is_mmio_spte(old_spte) &&
  435. !is_mmio_spte(new_spte) &&
  436. !is_removed_spte(new_spte)))
  437. pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"
  438. "should not be replaced with another,\n"
  439. "different nonpresent SPTE, unless one or both\n"
  440. "are MMIO SPTEs, or the new SPTE is\n"
  441. "a temporary removed SPTE.\n"
  442. "as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
  443. as_id, gfn, old_spte, new_spte, level);
  444. return;
  445. }
  446. if (is_leaf != was_leaf)
  447. kvm_update_page_stats(kvm, level, is_leaf ? 1 : -1);
  448. if (was_leaf && is_dirty_spte(old_spte) &&
  449. (!is_present || !is_dirty_spte(new_spte) || pfn_changed))
  450. kvm_set_pfn_dirty(spte_to_pfn(old_spte));
  451. /*
  452. * Recursively handle child PTs if the change removed a subtree from
  453. * the paging structure. Note the WARN on the PFN changing without the
  454. * SPTE being converted to a hugepage (leaf) or being zapped. Shadow
  455. * pages are kernel allocations and should never be migrated.
  456. */
  457. if (was_present && !was_leaf &&
  458. (is_leaf || !is_present || WARN_ON_ONCE(pfn_changed)))
  459. handle_removed_pt(kvm, spte_to_child_pt(old_spte, level), shared);
  460. }
  461. static void handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
  462. u64 old_spte, u64 new_spte, int level,
  463. bool shared)
  464. {
  465. __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level,
  466. shared);
  467. handle_changed_spte_acc_track(old_spte, new_spte, level);
  468. handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
  469. new_spte, level);
  470. }
  471. /*
  472. * tdp_mmu_set_spte_atomic - Set a TDP MMU SPTE atomically
  473. * and handle the associated bookkeeping. Do not mark the page dirty
  474. * in KVM's dirty bitmaps.
  475. *
  476. * If setting the SPTE fails because it has changed, iter->old_spte will be
  477. * refreshed to the current value of the spte.
  478. *
  479. * @kvm: kvm instance
  480. * @iter: a tdp_iter instance currently on the SPTE that should be set
  481. * @new_spte: The value the SPTE should be set to
  482. * Return:
  483. * * 0 - If the SPTE was set.
  484. * * -EBUSY - If the SPTE cannot be set. In this case this function will have
  485. * no side-effects other than setting iter->old_spte to the last
  486. * known value of the spte.
  487. */
  488. static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm,
  489. struct tdp_iter *iter,
  490. u64 new_spte)
  491. {
  492. u64 *sptep = rcu_dereference(iter->sptep);
  493. /*
  494. * The caller is responsible for ensuring the old SPTE is not a REMOVED
  495. * SPTE. KVM should never attempt to zap or manipulate a REMOVED SPTE,
  496. * and pre-checking before inserting a new SPTE is advantageous as it
  497. * avoids unnecessary work.
  498. */
  499. WARN_ON_ONCE(iter->yielded || is_removed_spte(iter->old_spte));
  500. lockdep_assert_held_read(&kvm->mmu_lock);
  501. /*
  502. * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and
  503. * does not hold the mmu_lock.
  504. */
  505. if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))
  506. return -EBUSY;
  507. __handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte,
  508. new_spte, iter->level, true);
  509. handle_changed_spte_acc_track(iter->old_spte, new_spte, iter->level);
  510. return 0;
  511. }
  512. static inline int tdp_mmu_zap_spte_atomic(struct kvm *kvm,
  513. struct tdp_iter *iter)
  514. {
  515. int ret;
  516. /*
  517. * Freeze the SPTE by setting it to a special,
  518. * non-present value. This will stop other threads from
  519. * immediately installing a present entry in its place
  520. * before the TLBs are flushed.
  521. */
  522. ret = tdp_mmu_set_spte_atomic(kvm, iter, REMOVED_SPTE);
  523. if (ret)
  524. return ret;
  525. kvm_flush_remote_tlbs_with_address(kvm, iter->gfn,
  526. KVM_PAGES_PER_HPAGE(iter->level));
  527. /*
  528. * No other thread can overwrite the removed SPTE as they must either
  529. * wait on the MMU lock or use tdp_mmu_set_spte_atomic() which will not
  530. * overwrite the special removed SPTE value. No bookkeeping is needed
  531. * here since the SPTE is going from non-present to non-present. Use
  532. * the raw write helper to avoid an unnecessary check on volatile bits.
  533. */
  534. __kvm_tdp_mmu_write_spte(iter->sptep, 0);
  535. return 0;
  536. }
  537. /*
  538. * __tdp_mmu_set_spte - Set a TDP MMU SPTE and handle the associated bookkeeping
  539. * @kvm: KVM instance
  540. * @as_id: Address space ID, i.e. regular vs. SMM
  541. * @sptep: Pointer to the SPTE
  542. * @old_spte: The current value of the SPTE
  543. * @new_spte: The new value that will be set for the SPTE
  544. * @gfn: The base GFN that was (or will be) mapped by the SPTE
  545. * @level: The level _containing_ the SPTE (its parent PT's level)
  546. * @record_acc_track: Notify the MM subsystem of changes to the accessed state
  547. * of the page. Should be set unless handling an MMU
  548. * notifier for access tracking. Leaving record_acc_track
  549. * unset in that case prevents page accesses from being
  550. * double counted.
  551. * @record_dirty_log: Record the page as dirty in the dirty bitmap if
  552. * appropriate for the change being made. Should be set
  553. * unless performing certain dirty logging operations.
  554. * Leaving record_dirty_log unset in that case prevents page
  555. * writes from being double counted.
  556. *
  557. * Returns the old SPTE value, which _may_ be different than @old_spte if the
  558. * SPTE had voldatile bits.
  559. */
  560. static u64 __tdp_mmu_set_spte(struct kvm *kvm, int as_id, tdp_ptep_t sptep,
  561. u64 old_spte, u64 new_spte, gfn_t gfn, int level,
  562. bool record_acc_track, bool record_dirty_log)
  563. {
  564. lockdep_assert_held_write(&kvm->mmu_lock);
  565. /*
  566. * No thread should be using this function to set SPTEs to or from the
  567. * temporary removed SPTE value.
  568. * If operating under the MMU lock in read mode, tdp_mmu_set_spte_atomic
  569. * should be used. If operating under the MMU lock in write mode, the
  570. * use of the removed SPTE should not be necessary.
  571. */
  572. WARN_ON(is_removed_spte(old_spte) || is_removed_spte(new_spte));
  573. old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, new_spte, level);
  574. __handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level, false);
  575. if (record_acc_track)
  576. handle_changed_spte_acc_track(old_spte, new_spte, level);
  577. if (record_dirty_log)
  578. handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
  579. new_spte, level);
  580. return old_spte;
  581. }
  582. static inline void _tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
  583. u64 new_spte, bool record_acc_track,
  584. bool record_dirty_log)
  585. {
  586. WARN_ON_ONCE(iter->yielded);
  587. iter->old_spte = __tdp_mmu_set_spte(kvm, iter->as_id, iter->sptep,
  588. iter->old_spte, new_spte,
  589. iter->gfn, iter->level,
  590. record_acc_track, record_dirty_log);
  591. }
  592. static inline void tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
  593. u64 new_spte)
  594. {
  595. _tdp_mmu_set_spte(kvm, iter, new_spte, true, true);
  596. }
  597. static inline void tdp_mmu_set_spte_no_acc_track(struct kvm *kvm,
  598. struct tdp_iter *iter,
  599. u64 new_spte)
  600. {
  601. _tdp_mmu_set_spte(kvm, iter, new_spte, false, true);
  602. }
  603. static inline void tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm,
  604. struct tdp_iter *iter,
  605. u64 new_spte)
  606. {
  607. _tdp_mmu_set_spte(kvm, iter, new_spte, true, false);
  608. }
  609. #define tdp_root_for_each_pte(_iter, _root, _start, _end) \
  610. for_each_tdp_pte(_iter, _root, _start, _end)
  611. #define tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \
  612. tdp_root_for_each_pte(_iter, _root, _start, _end) \
  613. if (!is_shadow_present_pte(_iter.old_spte) || \
  614. !is_last_spte(_iter.old_spte, _iter.level)) \
  615. continue; \
  616. else
  617. #define tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \
  618. for_each_tdp_pte(_iter, to_shadow_page(_mmu->root.hpa), _start, _end)
  619. /*
  620. * Yield if the MMU lock is contended or this thread needs to return control
  621. * to the scheduler.
  622. *
  623. * If this function should yield and flush is set, it will perform a remote
  624. * TLB flush before yielding.
  625. *
  626. * If this function yields, iter->yielded is set and the caller must skip to
  627. * the next iteration, where tdp_iter_next() will reset the tdp_iter's walk
  628. * over the paging structures to allow the iterator to continue its traversal
  629. * from the paging structure root.
  630. *
  631. * Returns true if this function yielded.
  632. */
  633. static inline bool __must_check tdp_mmu_iter_cond_resched(struct kvm *kvm,
  634. struct tdp_iter *iter,
  635. bool flush, bool shared)
  636. {
  637. WARN_ON(iter->yielded);
  638. /* Ensure forward progress has been made before yielding. */
  639. if (iter->next_last_level_gfn == iter->yielded_gfn)
  640. return false;
  641. if (need_resched() || rwlock_needbreak(&kvm->mmu_lock)) {
  642. if (flush)
  643. kvm_flush_remote_tlbs(kvm);
  644. rcu_read_unlock();
  645. if (shared)
  646. cond_resched_rwlock_read(&kvm->mmu_lock);
  647. else
  648. cond_resched_rwlock_write(&kvm->mmu_lock);
  649. rcu_read_lock();
  650. WARN_ON(iter->gfn > iter->next_last_level_gfn);
  651. iter->yielded = true;
  652. }
  653. return iter->yielded;
  654. }
  655. static inline gfn_t tdp_mmu_max_gfn_exclusive(void)
  656. {
  657. /*
  658. * Bound TDP MMU walks at host.MAXPHYADDR. KVM disallows memslots with
  659. * a gpa range that would exceed the max gfn, and KVM does not create
  660. * MMIO SPTEs for "impossible" gfns, instead sending such accesses down
  661. * the slow emulation path every time.
  662. */
  663. return kvm_mmu_max_gfn() + 1;
  664. }
  665. static void __tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
  666. bool shared, int zap_level)
  667. {
  668. struct tdp_iter iter;
  669. gfn_t end = tdp_mmu_max_gfn_exclusive();
  670. gfn_t start = 0;
  671. for_each_tdp_pte_min_level(iter, root, zap_level, start, end) {
  672. retry:
  673. if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared))
  674. continue;
  675. if (!is_shadow_present_pte(iter.old_spte))
  676. continue;
  677. if (iter.level > zap_level)
  678. continue;
  679. if (!shared)
  680. tdp_mmu_set_spte(kvm, &iter, 0);
  681. else if (tdp_mmu_set_spte_atomic(kvm, &iter, 0))
  682. goto retry;
  683. }
  684. }
  685. static void tdp_mmu_zap_root(struct kvm *kvm, struct kvm_mmu_page *root,
  686. bool shared)
  687. {
  688. /*
  689. * The root must have an elevated refcount so that it's reachable via
  690. * mmu_notifier callbacks, which allows this path to yield and drop
  691. * mmu_lock. When handling an unmap/release mmu_notifier command, KVM
  692. * must drop all references to relevant pages prior to completing the
  693. * callback. Dropping mmu_lock with an unreachable root would result
  694. * in zapping SPTEs after a relevant mmu_notifier callback completes
  695. * and lead to use-after-free as zapping a SPTE triggers "writeback" of
  696. * dirty accessed bits to the SPTE's associated struct page.
  697. */
  698. WARN_ON_ONCE(!refcount_read(&root->tdp_mmu_root_count));
  699. kvm_lockdep_assert_mmu_lock_held(kvm, shared);
  700. rcu_read_lock();
  701. /*
  702. * To avoid RCU stalls due to recursively removing huge swaths of SPs,
  703. * split the zap into two passes. On the first pass, zap at the 1gb
  704. * level, and then zap top-level SPs on the second pass. "1gb" is not
  705. * arbitrary, as KVM must be able to zap a 1gb shadow page without
  706. * inducing a stall to allow in-place replacement with a 1gb hugepage.
  707. *
  708. * Because zapping a SP recurses on its children, stepping down to
  709. * PG_LEVEL_4K in the iterator itself is unnecessary.
  710. */
  711. __tdp_mmu_zap_root(kvm, root, shared, PG_LEVEL_1G);
  712. __tdp_mmu_zap_root(kvm, root, shared, root->role.level);
  713. rcu_read_unlock();
  714. }
  715. bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp)
  716. {
  717. u64 old_spte;
  718. /*
  719. * This helper intentionally doesn't allow zapping a root shadow page,
  720. * which doesn't have a parent page table and thus no associated entry.
  721. */
  722. if (WARN_ON_ONCE(!sp->ptep))
  723. return false;
  724. old_spte = kvm_tdp_mmu_read_spte(sp->ptep);
  725. if (WARN_ON_ONCE(!is_shadow_present_pte(old_spte)))
  726. return false;
  727. __tdp_mmu_set_spte(kvm, kvm_mmu_page_as_id(sp), sp->ptep, old_spte, 0,
  728. sp->gfn, sp->role.level + 1, true, true);
  729. return true;
  730. }
  731. /*
  732. * If can_yield is true, will release the MMU lock and reschedule if the
  733. * scheduler needs the CPU or there is contention on the MMU lock. If this
  734. * function cannot yield, it will not release the MMU lock or reschedule and
  735. * the caller must ensure it does not supply too large a GFN range, or the
  736. * operation can cause a soft lockup.
  737. */
  738. static bool tdp_mmu_zap_leafs(struct kvm *kvm, struct kvm_mmu_page *root,
  739. gfn_t start, gfn_t end, bool can_yield, bool flush)
  740. {
  741. struct tdp_iter iter;
  742. end = min(end, tdp_mmu_max_gfn_exclusive());
  743. lockdep_assert_held_write(&kvm->mmu_lock);
  744. rcu_read_lock();
  745. for_each_tdp_pte_min_level(iter, root, PG_LEVEL_4K, start, end) {
  746. if (can_yield &&
  747. tdp_mmu_iter_cond_resched(kvm, &iter, flush, false)) {
  748. flush = false;
  749. continue;
  750. }
  751. if (!is_shadow_present_pte(iter.old_spte) ||
  752. !is_last_spte(iter.old_spte, iter.level))
  753. continue;
  754. tdp_mmu_set_spte(kvm, &iter, 0);
  755. flush = true;
  756. }
  757. rcu_read_unlock();
  758. /*
  759. * Because this flow zaps _only_ leaf SPTEs, the caller doesn't need
  760. * to provide RCU protection as no 'struct kvm_mmu_page' will be freed.
  761. */
  762. return flush;
  763. }
  764. /*
  765. * Zap leaf SPTEs for the range of gfns, [start, end), for all roots. Returns
  766. * true if a TLB flush is needed before releasing the MMU lock, i.e. if one or
  767. * more SPTEs were zapped since the MMU lock was last acquired.
  768. */
  769. bool kvm_tdp_mmu_zap_leafs(struct kvm *kvm, gfn_t start, gfn_t end, bool flush)
  770. {
  771. struct kvm_mmu_page *root;
  772. for_each_tdp_mmu_root_yield_safe(kvm, root, false)
  773. flush = tdp_mmu_zap_leafs(kvm, root, start, end, true, flush);
  774. return flush;
  775. }
  776. void kvm_tdp_mmu_zap_all(struct kvm *kvm)
  777. {
  778. struct kvm_mmu_page *root;
  779. /*
  780. * Zap all roots, including invalid roots, as all SPTEs must be dropped
  781. * before returning to the caller. Zap directly even if the root is
  782. * also being zapped by a worker. Walking zapped top-level SPTEs isn't
  783. * all that expensive and mmu_lock is already held, which means the
  784. * worker has yielded, i.e. flushing the work instead of zapping here
  785. * isn't guaranteed to be any faster.
  786. *
  787. * A TLB flush is unnecessary, KVM zaps everything if and only the VM
  788. * is being destroyed or the userspace VMM has exited. In both cases,
  789. * KVM_RUN is unreachable, i.e. no vCPUs will ever service the request.
  790. */
  791. for_each_tdp_mmu_root_yield_safe(kvm, root, false)
  792. tdp_mmu_zap_root(kvm, root, false);
  793. }
  794. /*
  795. * Zap all invalidated roots to ensure all SPTEs are dropped before the "fast
  796. * zap" completes.
  797. */
  798. void kvm_tdp_mmu_zap_invalidated_roots(struct kvm *kvm)
  799. {
  800. struct kvm_mmu_page *root;
  801. read_lock(&kvm->mmu_lock);
  802. for_each_tdp_mmu_root_yield_safe(kvm, root, true) {
  803. if (!root->tdp_mmu_scheduled_root_to_zap)
  804. continue;
  805. root->tdp_mmu_scheduled_root_to_zap = false;
  806. KVM_BUG_ON(!root->role.invalid, kvm);
  807. /*
  808. * A TLB flush is not necessary as KVM performs a local TLB
  809. * flush when allocating a new root (see kvm_mmu_load()), and
  810. * when migrating a vCPU to a different pCPU. Note, the local
  811. * TLB flush on reuse also invalidates paging-structure-cache
  812. * entries, i.e. TLB entries for intermediate paging structures,
  813. * that may be zapped, as such entries are associated with the
  814. * ASID on both VMX and SVM.
  815. */
  816. tdp_mmu_zap_root(kvm, root, true);
  817. /*
  818. * The referenced needs to be put *after* zapping the root, as
  819. * the root must be reachable by mmu_notifiers while it's being
  820. * zapped
  821. */
  822. kvm_tdp_mmu_put_root(kvm, root, true);
  823. }
  824. read_unlock(&kvm->mmu_lock);
  825. }
  826. /*
  827. * Mark each TDP MMU root as invalid to prevent vCPUs from reusing a root that
  828. * is about to be zapped, e.g. in response to a memslots update. The actual
  829. * zapping is done separately so that it happens with mmu_lock with read,
  830. * whereas invalidating roots must be done with mmu_lock held for write (unless
  831. * the VM is being destroyed).
  832. *
  833. * Note, kvm_tdp_mmu_zap_invalidated_roots() is gifted the TDP MMU's reference.
  834. * See kvm_tdp_mmu_get_vcpu_root_hpa().
  835. */
  836. void kvm_tdp_mmu_invalidate_all_roots(struct kvm *kvm)
  837. {
  838. struct kvm_mmu_page *root;
  839. /*
  840. * mmu_lock must be held for write to ensure that a root doesn't become
  841. * invalid while there are active readers (invalidating a root while
  842. * there are active readers may or may not be problematic in practice,
  843. * but it's uncharted territory and not supported).
  844. *
  845. * Waive the assertion if there are no users of @kvm, i.e. the VM is
  846. * being destroyed after all references have been put, or if no vCPUs
  847. * have been created (which means there are no roots), i.e. the VM is
  848. * being destroyed in an error path of KVM_CREATE_VM.
  849. */
  850. if (IS_ENABLED(CONFIG_PROVE_LOCKING) &&
  851. refcount_read(&kvm->users_count) && kvm->created_vcpus)
  852. lockdep_assert_held_write(&kvm->mmu_lock);
  853. /*
  854. * As above, mmu_lock isn't held when destroying the VM! There can't
  855. * be other references to @kvm, i.e. nothing else can invalidate roots
  856. * or get/put references to roots.
  857. */
  858. list_for_each_entry(root, &kvm->arch.tdp_mmu_roots, link) {
  859. /*
  860. * Note, invalid roots can outlive a memslot update! Invalid
  861. * roots must be *zapped* before the memslot update completes,
  862. * but a different task can acquire a reference and keep the
  863. * root alive after its been zapped.
  864. */
  865. if (!root->role.invalid) {
  866. root->tdp_mmu_scheduled_root_to_zap = true;
  867. root->role.invalid = true;
  868. }
  869. }
  870. }
  871. /*
  872. * Installs a last-level SPTE to handle a TDP page fault.
  873. * (NPT/EPT violation/misconfiguration)
  874. */
  875. static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
  876. struct kvm_page_fault *fault,
  877. struct tdp_iter *iter)
  878. {
  879. struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep));
  880. u64 new_spte;
  881. int ret = RET_PF_FIXED;
  882. bool wrprot = false;
  883. WARN_ON(sp->role.level != fault->goal_level);
  884. if (unlikely(!fault->slot))
  885. new_spte = make_mmio_spte(vcpu, iter->gfn, ACC_ALL);
  886. else
  887. wrprot = make_spte(vcpu, sp, fault->slot, ACC_ALL, iter->gfn,
  888. fault->pfn, iter->old_spte, fault->prefetch, true,
  889. fault->map_writable, &new_spte);
  890. if (new_spte == iter->old_spte)
  891. ret = RET_PF_SPURIOUS;
  892. else if (tdp_mmu_set_spte_atomic(vcpu->kvm, iter, new_spte))
  893. return RET_PF_RETRY;
  894. else if (is_shadow_present_pte(iter->old_spte) &&
  895. !is_last_spte(iter->old_spte, iter->level))
  896. kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
  897. KVM_PAGES_PER_HPAGE(iter->level + 1));
  898. /*
  899. * If the page fault was caused by a write but the page is write
  900. * protected, emulation is needed. If the emulation was skipped,
  901. * the vCPU would have the same fault again.
  902. */
  903. if (wrprot) {
  904. if (fault->write)
  905. ret = RET_PF_EMULATE;
  906. }
  907. /* If a MMIO SPTE is installed, the MMIO will need to be emulated. */
  908. if (unlikely(is_mmio_spte(new_spte))) {
  909. vcpu->stat.pf_mmio_spte_created++;
  910. trace_mark_mmio_spte(rcu_dereference(iter->sptep), iter->gfn,
  911. new_spte);
  912. ret = RET_PF_EMULATE;
  913. } else {
  914. trace_kvm_mmu_set_spte(iter->level, iter->gfn,
  915. rcu_dereference(iter->sptep));
  916. }
  917. return ret;
  918. }
  919. /*
  920. * tdp_mmu_link_sp - Replace the given spte with an spte pointing to the
  921. * provided page table.
  922. *
  923. * @kvm: kvm instance
  924. * @iter: a tdp_iter instance currently on the SPTE that should be set
  925. * @sp: The new TDP page table to install.
  926. * @account_nx: True if this page table is being installed to split a
  927. * non-executable huge page.
  928. * @shared: This operation is running under the MMU lock in read mode.
  929. *
  930. * Returns: 0 if the new page table was installed. Non-0 if the page table
  931. * could not be installed (e.g. the atomic compare-exchange failed).
  932. */
  933. static int tdp_mmu_link_sp(struct kvm *kvm, struct tdp_iter *iter,
  934. struct kvm_mmu_page *sp, bool account_nx,
  935. bool shared)
  936. {
  937. u64 spte = make_nonleaf_spte(sp->spt, !kvm_ad_enabled());
  938. int ret = 0;
  939. if (shared) {
  940. ret = tdp_mmu_set_spte_atomic(kvm, iter, spte);
  941. if (ret)
  942. return ret;
  943. } else {
  944. tdp_mmu_set_spte(kvm, iter, spte);
  945. }
  946. spin_lock(&kvm->arch.tdp_mmu_pages_lock);
  947. list_add(&sp->link, &kvm->arch.tdp_mmu_pages);
  948. if (account_nx)
  949. account_huge_nx_page(kvm, sp);
  950. spin_unlock(&kvm->arch.tdp_mmu_pages_lock);
  951. tdp_account_mmu_page(kvm, sp);
  952. return 0;
  953. }
  954. /*
  955. * Handle a TDP page fault (NPT/EPT violation/misconfiguration) by installing
  956. * page tables and SPTEs to translate the faulting guest physical address.
  957. */
  958. int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault)
  959. {
  960. struct kvm_mmu *mmu = vcpu->arch.mmu;
  961. struct tdp_iter iter;
  962. struct kvm_mmu_page *sp;
  963. int ret;
  964. kvm_mmu_hugepage_adjust(vcpu, fault);
  965. trace_kvm_mmu_spte_requested(fault);
  966. rcu_read_lock();
  967. tdp_mmu_for_each_pte(iter, mmu, fault->gfn, fault->gfn + 1) {
  968. if (fault->nx_huge_page_workaround_enabled)
  969. disallowed_hugepage_adjust(fault, iter.old_spte, iter.level);
  970. if (iter.level == fault->goal_level)
  971. break;
  972. /*
  973. * If there is an SPTE mapping a large page at a higher level
  974. * than the target, that SPTE must be cleared and replaced
  975. * with a non-leaf SPTE.
  976. */
  977. if (is_shadow_present_pte(iter.old_spte) &&
  978. is_large_pte(iter.old_spte)) {
  979. if (tdp_mmu_zap_spte_atomic(vcpu->kvm, &iter))
  980. break;
  981. /*
  982. * The iter must explicitly re-read the spte here
  983. * because the new value informs the !present
  984. * path below.
  985. */
  986. iter.old_spte = kvm_tdp_mmu_read_spte(iter.sptep);
  987. }
  988. if (!is_shadow_present_pte(iter.old_spte)) {
  989. bool account_nx = fault->huge_page_disallowed &&
  990. fault->req_level >= iter.level;
  991. /*
  992. * If SPTE has been frozen by another thread, just
  993. * give up and retry, avoiding unnecessary page table
  994. * allocation and free.
  995. */
  996. if (is_removed_spte(iter.old_spte))
  997. break;
  998. sp = tdp_mmu_alloc_sp(vcpu);
  999. tdp_mmu_init_child_sp(sp, &iter);
  1000. if (tdp_mmu_link_sp(vcpu->kvm, &iter, sp, account_nx, true)) {
  1001. tdp_mmu_free_sp(sp);
  1002. break;
  1003. }
  1004. }
  1005. }
  1006. /*
  1007. * Force the guest to retry the access if the upper level SPTEs aren't
  1008. * in place, or if the target leaf SPTE is frozen by another CPU.
  1009. */
  1010. if (iter.level != fault->goal_level || is_removed_spte(iter.old_spte)) {
  1011. rcu_read_unlock();
  1012. return RET_PF_RETRY;
  1013. }
  1014. ret = tdp_mmu_map_handle_target_level(vcpu, fault, &iter);
  1015. rcu_read_unlock();
  1016. return ret;
  1017. }
  1018. bool kvm_tdp_mmu_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range,
  1019. bool flush)
  1020. {
  1021. struct kvm_mmu_page *root;
  1022. __for_each_tdp_mmu_root_yield_safe(kvm, root, range->slot->as_id, false, false)
  1023. flush = tdp_mmu_zap_leafs(kvm, root, range->start, range->end,
  1024. range->may_block, flush);
  1025. return flush;
  1026. }
  1027. typedef bool (*tdp_handler_t)(struct kvm *kvm, struct tdp_iter *iter,
  1028. struct kvm_gfn_range *range);
  1029. static __always_inline bool kvm_tdp_mmu_handle_gfn(struct kvm *kvm,
  1030. struct kvm_gfn_range *range,
  1031. tdp_handler_t handler)
  1032. {
  1033. struct kvm_mmu_page *root;
  1034. struct tdp_iter iter;
  1035. bool ret = false;
  1036. /*
  1037. * Don't support rescheduling, none of the MMU notifiers that funnel
  1038. * into this helper allow blocking; it'd be dead, wasteful code.
  1039. */
  1040. for_each_tdp_mmu_root(kvm, root, range->slot->as_id) {
  1041. rcu_read_lock();
  1042. tdp_root_for_each_leaf_pte(iter, root, range->start, range->end)
  1043. ret |= handler(kvm, &iter, range);
  1044. rcu_read_unlock();
  1045. }
  1046. return ret;
  1047. }
  1048. /*
  1049. * Mark the SPTEs range of GFNs [start, end) unaccessed and return non-zero
  1050. * if any of the GFNs in the range have been accessed.
  1051. */
  1052. static bool age_gfn_range(struct kvm *kvm, struct tdp_iter *iter,
  1053. struct kvm_gfn_range *range)
  1054. {
  1055. u64 new_spte = 0;
  1056. /* If we have a non-accessed entry we don't need to change the pte. */
  1057. if (!is_accessed_spte(iter->old_spte))
  1058. return false;
  1059. new_spte = iter->old_spte;
  1060. if (spte_ad_enabled(new_spte)) {
  1061. new_spte &= ~shadow_accessed_mask;
  1062. } else {
  1063. /*
  1064. * Capture the dirty status of the page, so that it doesn't get
  1065. * lost when the SPTE is marked for access tracking.
  1066. */
  1067. if (is_writable_pte(new_spte))
  1068. kvm_set_pfn_dirty(spte_to_pfn(new_spte));
  1069. new_spte = mark_spte_for_access_track(new_spte);
  1070. }
  1071. tdp_mmu_set_spte_no_acc_track(kvm, iter, new_spte);
  1072. return true;
  1073. }
  1074. bool kvm_tdp_mmu_age_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
  1075. {
  1076. return kvm_tdp_mmu_handle_gfn(kvm, range, age_gfn_range);
  1077. }
  1078. static bool test_age_gfn(struct kvm *kvm, struct tdp_iter *iter,
  1079. struct kvm_gfn_range *range)
  1080. {
  1081. return is_accessed_spte(iter->old_spte);
  1082. }
  1083. bool kvm_tdp_mmu_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
  1084. {
  1085. return kvm_tdp_mmu_handle_gfn(kvm, range, test_age_gfn);
  1086. }
  1087. static bool set_spte_gfn(struct kvm *kvm, struct tdp_iter *iter,
  1088. struct kvm_gfn_range *range)
  1089. {
  1090. u64 new_spte;
  1091. /* Huge pages aren't expected to be modified without first being zapped. */
  1092. WARN_ON(pte_huge(range->pte) || range->start + 1 != range->end);
  1093. if (iter->level != PG_LEVEL_4K ||
  1094. !is_shadow_present_pte(iter->old_spte))
  1095. return false;
  1096. /*
  1097. * Note, when changing a read-only SPTE, it's not strictly necessary to
  1098. * zero the SPTE before setting the new PFN, but doing so preserves the
  1099. * invariant that the PFN of a present * leaf SPTE can never change.
  1100. * See __handle_changed_spte().
  1101. */
  1102. tdp_mmu_set_spte(kvm, iter, 0);
  1103. if (!pte_write(range->pte)) {
  1104. new_spte = kvm_mmu_changed_pte_notifier_make_spte(iter->old_spte,
  1105. pte_pfn(range->pte));
  1106. tdp_mmu_set_spte(kvm, iter, new_spte);
  1107. }
  1108. return true;
  1109. }
  1110. /*
  1111. * Handle the changed_pte MMU notifier for the TDP MMU.
  1112. * data is a pointer to the new pte_t mapping the HVA specified by the MMU
  1113. * notifier.
  1114. * Returns non-zero if a flush is needed before releasing the MMU lock.
  1115. */
  1116. bool kvm_tdp_mmu_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
  1117. {
  1118. /*
  1119. * No need to handle the remote TLB flush under RCU protection, the
  1120. * target SPTE _must_ be a leaf SPTE, i.e. cannot result in freeing a
  1121. * shadow page. See the WARN on pfn_changed in __handle_changed_spte().
  1122. */
  1123. return kvm_tdp_mmu_handle_gfn(kvm, range, set_spte_gfn);
  1124. }
  1125. /*
  1126. * Remove write access from all SPTEs at or above min_level that map GFNs
  1127. * [start, end). Returns true if an SPTE has been changed and the TLBs need to
  1128. * be flushed.
  1129. */
  1130. static bool wrprot_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
  1131. gfn_t start, gfn_t end, int min_level)
  1132. {
  1133. struct tdp_iter iter;
  1134. u64 new_spte;
  1135. bool spte_set = false;
  1136. rcu_read_lock();
  1137. BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
  1138. for_each_tdp_pte_min_level(iter, root, min_level, start, end) {
  1139. retry:
  1140. if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
  1141. continue;
  1142. if (!is_shadow_present_pte(iter.old_spte) ||
  1143. !is_last_spte(iter.old_spte, iter.level) ||
  1144. !(iter.old_spte & PT_WRITABLE_MASK))
  1145. continue;
  1146. new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
  1147. if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte))
  1148. goto retry;
  1149. spte_set = true;
  1150. }
  1151. rcu_read_unlock();
  1152. return spte_set;
  1153. }
  1154. /*
  1155. * Remove write access from all the SPTEs mapping GFNs in the memslot. Will
  1156. * only affect leaf SPTEs down to min_level.
  1157. * Returns true if an SPTE has been changed and the TLBs need to be flushed.
  1158. */
  1159. bool kvm_tdp_mmu_wrprot_slot(struct kvm *kvm,
  1160. const struct kvm_memory_slot *slot, int min_level)
  1161. {
  1162. struct kvm_mmu_page *root;
  1163. bool spte_set = false;
  1164. lockdep_assert_held_read(&kvm->mmu_lock);
  1165. for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
  1166. spte_set |= wrprot_gfn_range(kvm, root, slot->base_gfn,
  1167. slot->base_gfn + slot->npages, min_level);
  1168. return spte_set;
  1169. }
  1170. static struct kvm_mmu_page *__tdp_mmu_alloc_sp_for_split(gfp_t gfp)
  1171. {
  1172. struct kvm_mmu_page *sp;
  1173. gfp |= __GFP_ZERO;
  1174. sp = kmem_cache_alloc(mmu_page_header_cache, gfp);
  1175. if (!sp)
  1176. return NULL;
  1177. sp->spt = (void *)__get_free_page(gfp);
  1178. if (!sp->spt) {
  1179. kmem_cache_free(mmu_page_header_cache, sp);
  1180. return NULL;
  1181. }
  1182. return sp;
  1183. }
  1184. static struct kvm_mmu_page *tdp_mmu_alloc_sp_for_split(struct kvm *kvm,
  1185. struct tdp_iter *iter,
  1186. bool shared)
  1187. {
  1188. struct kvm_mmu_page *sp;
  1189. /*
  1190. * Since we are allocating while under the MMU lock we have to be
  1191. * careful about GFP flags. Use GFP_NOWAIT to avoid blocking on direct
  1192. * reclaim and to avoid making any filesystem callbacks (which can end
  1193. * up invoking KVM MMU notifiers, resulting in a deadlock).
  1194. *
  1195. * If this allocation fails we drop the lock and retry with reclaim
  1196. * allowed.
  1197. */
  1198. sp = __tdp_mmu_alloc_sp_for_split(GFP_NOWAIT | __GFP_ACCOUNT);
  1199. if (sp)
  1200. return sp;
  1201. rcu_read_unlock();
  1202. if (shared)
  1203. read_unlock(&kvm->mmu_lock);
  1204. else
  1205. write_unlock(&kvm->mmu_lock);
  1206. iter->yielded = true;
  1207. sp = __tdp_mmu_alloc_sp_for_split(GFP_KERNEL_ACCOUNT);
  1208. if (shared)
  1209. read_lock(&kvm->mmu_lock);
  1210. else
  1211. write_lock(&kvm->mmu_lock);
  1212. rcu_read_lock();
  1213. return sp;
  1214. }
  1215. static int tdp_mmu_split_huge_page(struct kvm *kvm, struct tdp_iter *iter,
  1216. struct kvm_mmu_page *sp, bool shared)
  1217. {
  1218. const u64 huge_spte = iter->old_spte;
  1219. const int level = iter->level;
  1220. int ret, i;
  1221. tdp_mmu_init_child_sp(sp, iter);
  1222. /*
  1223. * No need for atomics when writing to sp->spt since the page table has
  1224. * not been linked in yet and thus is not reachable from any other CPU.
  1225. */
  1226. for (i = 0; i < SPTE_ENT_PER_PAGE; i++)
  1227. sp->spt[i] = make_huge_page_split_spte(kvm, huge_spte, sp->role, i);
  1228. /*
  1229. * Replace the huge spte with a pointer to the populated lower level
  1230. * page table. Since we are making this change without a TLB flush vCPUs
  1231. * will see a mix of the split mappings and the original huge mapping,
  1232. * depending on what's currently in their TLB. This is fine from a
  1233. * correctness standpoint since the translation will be the same either
  1234. * way.
  1235. */
  1236. ret = tdp_mmu_link_sp(kvm, iter, sp, false, shared);
  1237. if (ret)
  1238. goto out;
  1239. /*
  1240. * tdp_mmu_link_sp_atomic() will handle subtracting the huge page we
  1241. * are overwriting from the page stats. But we have to manually update
  1242. * the page stats with the new present child pages.
  1243. */
  1244. kvm_update_page_stats(kvm, level - 1, SPTE_ENT_PER_PAGE);
  1245. out:
  1246. trace_kvm_mmu_split_huge_page(iter->gfn, huge_spte, level, ret);
  1247. return ret;
  1248. }
  1249. static int tdp_mmu_split_huge_pages_root(struct kvm *kvm,
  1250. struct kvm_mmu_page *root,
  1251. gfn_t start, gfn_t end,
  1252. int target_level, bool shared)
  1253. {
  1254. struct kvm_mmu_page *sp = NULL;
  1255. struct tdp_iter iter;
  1256. int ret = 0;
  1257. rcu_read_lock();
  1258. /*
  1259. * Traverse the page table splitting all huge pages above the target
  1260. * level into one lower level. For example, if we encounter a 1GB page
  1261. * we split it into 512 2MB pages.
  1262. *
  1263. * Since the TDP iterator uses a pre-order traversal, we are guaranteed
  1264. * to visit an SPTE before ever visiting its children, which means we
  1265. * will correctly recursively split huge pages that are more than one
  1266. * level above the target level (e.g. splitting a 1GB to 512 2MB pages,
  1267. * and then splitting each of those to 512 4KB pages).
  1268. */
  1269. for_each_tdp_pte_min_level(iter, root, target_level + 1, start, end) {
  1270. retry:
  1271. if (tdp_mmu_iter_cond_resched(kvm, &iter, false, shared))
  1272. continue;
  1273. if (!is_shadow_present_pte(iter.old_spte) || !is_large_pte(iter.old_spte))
  1274. continue;
  1275. if (!sp) {
  1276. sp = tdp_mmu_alloc_sp_for_split(kvm, &iter, shared);
  1277. if (!sp) {
  1278. ret = -ENOMEM;
  1279. trace_kvm_mmu_split_huge_page(iter.gfn,
  1280. iter.old_spte,
  1281. iter.level, ret);
  1282. break;
  1283. }
  1284. if (iter.yielded)
  1285. continue;
  1286. }
  1287. if (tdp_mmu_split_huge_page(kvm, &iter, sp, shared))
  1288. goto retry;
  1289. sp = NULL;
  1290. }
  1291. rcu_read_unlock();
  1292. /*
  1293. * It's possible to exit the loop having never used the last sp if, for
  1294. * example, a vCPU doing HugePage NX splitting wins the race and
  1295. * installs its own sp in place of the last sp we tried to split.
  1296. */
  1297. if (sp)
  1298. tdp_mmu_free_sp(sp);
  1299. return ret;
  1300. }
  1301. /*
  1302. * Try to split all huge pages mapped by the TDP MMU down to the target level.
  1303. */
  1304. void kvm_tdp_mmu_try_split_huge_pages(struct kvm *kvm,
  1305. const struct kvm_memory_slot *slot,
  1306. gfn_t start, gfn_t end,
  1307. int target_level, bool shared)
  1308. {
  1309. struct kvm_mmu_page *root;
  1310. int r = 0;
  1311. kvm_lockdep_assert_mmu_lock_held(kvm, shared);
  1312. for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, shared) {
  1313. r = tdp_mmu_split_huge_pages_root(kvm, root, start, end, target_level, shared);
  1314. if (r) {
  1315. kvm_tdp_mmu_put_root(kvm, root, shared);
  1316. break;
  1317. }
  1318. }
  1319. }
  1320. /*
  1321. * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If
  1322. * AD bits are enabled, this will involve clearing the dirty bit on each SPTE.
  1323. * If AD bits are not enabled, this will require clearing the writable bit on
  1324. * each SPTE. Returns true if an SPTE has been changed and the TLBs need to
  1325. * be flushed.
  1326. */
  1327. static bool clear_dirty_gfn_range(struct kvm *kvm, struct kvm_mmu_page *root,
  1328. gfn_t start, gfn_t end)
  1329. {
  1330. struct tdp_iter iter;
  1331. u64 new_spte;
  1332. bool spte_set = false;
  1333. rcu_read_lock();
  1334. tdp_root_for_each_leaf_pte(iter, root, start, end) {
  1335. retry:
  1336. if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
  1337. continue;
  1338. if (!is_shadow_present_pte(iter.old_spte))
  1339. continue;
  1340. if (spte_ad_need_write_protect(iter.old_spte)) {
  1341. if (is_writable_pte(iter.old_spte))
  1342. new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
  1343. else
  1344. continue;
  1345. } else {
  1346. if (iter.old_spte & shadow_dirty_mask)
  1347. new_spte = iter.old_spte & ~shadow_dirty_mask;
  1348. else
  1349. continue;
  1350. }
  1351. if (tdp_mmu_set_spte_atomic(kvm, &iter, new_spte))
  1352. goto retry;
  1353. spte_set = true;
  1354. }
  1355. rcu_read_unlock();
  1356. return spte_set;
  1357. }
  1358. /*
  1359. * Clear the dirty status of all the SPTEs mapping GFNs in the memslot. If
  1360. * AD bits are enabled, this will involve clearing the dirty bit on each SPTE.
  1361. * If AD bits are not enabled, this will require clearing the writable bit on
  1362. * each SPTE. Returns true if an SPTE has been changed and the TLBs need to
  1363. * be flushed.
  1364. */
  1365. bool kvm_tdp_mmu_clear_dirty_slot(struct kvm *kvm,
  1366. const struct kvm_memory_slot *slot)
  1367. {
  1368. struct kvm_mmu_page *root;
  1369. bool spte_set = false;
  1370. lockdep_assert_held_read(&kvm->mmu_lock);
  1371. for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
  1372. spte_set |= clear_dirty_gfn_range(kvm, root, slot->base_gfn,
  1373. slot->base_gfn + slot->npages);
  1374. return spte_set;
  1375. }
  1376. /*
  1377. * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is
  1378. * set in mask, starting at gfn. The given memslot is expected to contain all
  1379. * the GFNs represented by set bits in the mask. If AD bits are enabled,
  1380. * clearing the dirty status will involve clearing the dirty bit on each SPTE
  1381. * or, if AD bits are not enabled, clearing the writable bit on each SPTE.
  1382. */
  1383. static void clear_dirty_pt_masked(struct kvm *kvm, struct kvm_mmu_page *root,
  1384. gfn_t gfn, unsigned long mask, bool wrprot)
  1385. {
  1386. struct tdp_iter iter;
  1387. u64 new_spte;
  1388. rcu_read_lock();
  1389. tdp_root_for_each_leaf_pte(iter, root, gfn + __ffs(mask),
  1390. gfn + BITS_PER_LONG) {
  1391. if (!mask)
  1392. break;
  1393. if (iter.level > PG_LEVEL_4K ||
  1394. !(mask & (1UL << (iter.gfn - gfn))))
  1395. continue;
  1396. mask &= ~(1UL << (iter.gfn - gfn));
  1397. if (wrprot || spte_ad_need_write_protect(iter.old_spte)) {
  1398. if (is_writable_pte(iter.old_spte))
  1399. new_spte = iter.old_spte & ~PT_WRITABLE_MASK;
  1400. else
  1401. continue;
  1402. } else {
  1403. if (iter.old_spte & shadow_dirty_mask)
  1404. new_spte = iter.old_spte & ~shadow_dirty_mask;
  1405. else
  1406. continue;
  1407. }
  1408. tdp_mmu_set_spte_no_dirty_log(kvm, &iter, new_spte);
  1409. }
  1410. rcu_read_unlock();
  1411. }
  1412. /*
  1413. * Clears the dirty status of all the 4k SPTEs mapping GFNs for which a bit is
  1414. * set in mask, starting at gfn. The given memslot is expected to contain all
  1415. * the GFNs represented by set bits in the mask. If AD bits are enabled,
  1416. * clearing the dirty status will involve clearing the dirty bit on each SPTE
  1417. * or, if AD bits are not enabled, clearing the writable bit on each SPTE.
  1418. */
  1419. void kvm_tdp_mmu_clear_dirty_pt_masked(struct kvm *kvm,
  1420. struct kvm_memory_slot *slot,
  1421. gfn_t gfn, unsigned long mask,
  1422. bool wrprot)
  1423. {
  1424. struct kvm_mmu_page *root;
  1425. lockdep_assert_held_write(&kvm->mmu_lock);
  1426. for_each_tdp_mmu_root(kvm, root, slot->as_id)
  1427. clear_dirty_pt_masked(kvm, root, gfn, mask, wrprot);
  1428. }
  1429. static void zap_collapsible_spte_range(struct kvm *kvm,
  1430. struct kvm_mmu_page *root,
  1431. const struct kvm_memory_slot *slot)
  1432. {
  1433. gfn_t start = slot->base_gfn;
  1434. gfn_t end = start + slot->npages;
  1435. struct tdp_iter iter;
  1436. int max_mapping_level;
  1437. rcu_read_lock();
  1438. for_each_tdp_pte_min_level(iter, root, PG_LEVEL_2M, start, end) {
  1439. retry:
  1440. if (tdp_mmu_iter_cond_resched(kvm, &iter, false, true))
  1441. continue;
  1442. if (iter.level > KVM_MAX_HUGEPAGE_LEVEL ||
  1443. !is_shadow_present_pte(iter.old_spte))
  1444. continue;
  1445. /*
  1446. * Don't zap leaf SPTEs, if a leaf SPTE could be replaced with
  1447. * a large page size, then its parent would have been zapped
  1448. * instead of stepping down.
  1449. */
  1450. if (is_last_spte(iter.old_spte, iter.level))
  1451. continue;
  1452. /*
  1453. * If iter.gfn resides outside of the slot, i.e. the page for
  1454. * the current level overlaps but is not contained by the slot,
  1455. * then the SPTE can't be made huge. More importantly, trying
  1456. * to query that info from slot->arch.lpage_info will cause an
  1457. * out-of-bounds access.
  1458. */
  1459. if (iter.gfn < start || iter.gfn >= end)
  1460. continue;
  1461. max_mapping_level = kvm_mmu_max_mapping_level(kvm, slot,
  1462. iter.gfn, PG_LEVEL_NUM);
  1463. if (max_mapping_level < iter.level)
  1464. continue;
  1465. /* Note, a successful atomic zap also does a remote TLB flush. */
  1466. if (tdp_mmu_zap_spte_atomic(kvm, &iter))
  1467. goto retry;
  1468. }
  1469. rcu_read_unlock();
  1470. }
  1471. /*
  1472. * Zap non-leaf SPTEs (and free their associated page tables) which could
  1473. * be replaced by huge pages, for GFNs within the slot.
  1474. */
  1475. void kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm,
  1476. const struct kvm_memory_slot *slot)
  1477. {
  1478. struct kvm_mmu_page *root;
  1479. lockdep_assert_held_read(&kvm->mmu_lock);
  1480. for_each_valid_tdp_mmu_root_yield_safe(kvm, root, slot->as_id, true)
  1481. zap_collapsible_spte_range(kvm, root, slot);
  1482. }
  1483. /*
  1484. * Removes write access on the last level SPTE mapping this GFN and unsets the
  1485. * MMU-writable bit to ensure future writes continue to be intercepted.
  1486. * Returns true if an SPTE was set and a TLB flush is needed.
  1487. */
  1488. static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root,
  1489. gfn_t gfn, int min_level)
  1490. {
  1491. struct tdp_iter iter;
  1492. u64 new_spte;
  1493. bool spte_set = false;
  1494. BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL);
  1495. rcu_read_lock();
  1496. for_each_tdp_pte_min_level(iter, root, min_level, gfn, gfn + 1) {
  1497. if (!is_shadow_present_pte(iter.old_spte) ||
  1498. !is_last_spte(iter.old_spte, iter.level))
  1499. continue;
  1500. new_spte = iter.old_spte &
  1501. ~(PT_WRITABLE_MASK | shadow_mmu_writable_mask);
  1502. if (new_spte == iter.old_spte)
  1503. break;
  1504. tdp_mmu_set_spte(kvm, &iter, new_spte);
  1505. spte_set = true;
  1506. }
  1507. rcu_read_unlock();
  1508. return spte_set;
  1509. }
  1510. /*
  1511. * Removes write access on the last level SPTE mapping this GFN and unsets the
  1512. * MMU-writable bit to ensure future writes continue to be intercepted.
  1513. * Returns true if an SPTE was set and a TLB flush is needed.
  1514. */
  1515. bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm,
  1516. struct kvm_memory_slot *slot, gfn_t gfn,
  1517. int min_level)
  1518. {
  1519. struct kvm_mmu_page *root;
  1520. bool spte_set = false;
  1521. lockdep_assert_held_write(&kvm->mmu_lock);
  1522. for_each_tdp_mmu_root(kvm, root, slot->as_id)
  1523. spte_set |= write_protect_gfn(kvm, root, gfn, min_level);
  1524. return spte_set;
  1525. }
  1526. /*
  1527. * Return the level of the lowest level SPTE added to sptes.
  1528. * That SPTE may be non-present.
  1529. *
  1530. * Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
  1531. */
  1532. int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
  1533. int *root_level)
  1534. {
  1535. struct tdp_iter iter;
  1536. struct kvm_mmu *mmu = vcpu->arch.mmu;
  1537. gfn_t gfn = addr >> PAGE_SHIFT;
  1538. int leaf = -1;
  1539. *root_level = vcpu->arch.mmu->root_role.level;
  1540. tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
  1541. leaf = iter.level;
  1542. sptes[leaf] = iter.old_spte;
  1543. }
  1544. return leaf;
  1545. }
  1546. /*
  1547. * Returns the last level spte pointer of the shadow page walk for the given
  1548. * gpa, and sets *spte to the spte value. This spte may be non-preset. If no
  1549. * walk could be performed, returns NULL and *spte does not contain valid data.
  1550. *
  1551. * Contract:
  1552. * - Must be called between kvm_tdp_mmu_walk_lockless_{begin,end}.
  1553. * - The returned sptep must not be used after kvm_tdp_mmu_walk_lockless_end.
  1554. *
  1555. * WARNING: This function is only intended to be called during fast_page_fault.
  1556. */
  1557. u64 *kvm_tdp_mmu_fast_pf_get_last_sptep(struct kvm_vcpu *vcpu, u64 addr,
  1558. u64 *spte)
  1559. {
  1560. struct tdp_iter iter;
  1561. struct kvm_mmu *mmu = vcpu->arch.mmu;
  1562. gfn_t gfn = addr >> PAGE_SHIFT;
  1563. tdp_ptep_t sptep = NULL;
  1564. tdp_mmu_for_each_pte(iter, mmu, gfn, gfn + 1) {
  1565. *spte = iter.old_spte;
  1566. sptep = iter.sptep;
  1567. }
  1568. /*
  1569. * Perform the rcu_dereference to get the raw spte pointer value since
  1570. * we are passing it up to fast_page_fault, which is shared with the
  1571. * legacy MMU and thus does not retain the TDP MMU-specific __rcu
  1572. * annotation.
  1573. *
  1574. * This is safe since fast_page_fault obeys the contracts of this
  1575. * function as well as all TDP MMU contracts around modifying SPTEs
  1576. * outside of mmu_lock.
  1577. */
  1578. return rcu_dereference(sptep);
  1579. }