booke.c 57 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. *
  4. * Copyright IBM Corp. 2007
  5. * Copyright 2010-2011 Freescale Semiconductor, Inc.
  6. *
  7. * Authors: Hollis Blanchard <[email protected]>
  8. * Christian Ehrhardt <[email protected]>
  9. * Scott Wood <[email protected]>
  10. * Varun Sethi <[email protected]>
  11. */
  12. #include <linux/errno.h>
  13. #include <linux/err.h>
  14. #include <linux/kvm_host.h>
  15. #include <linux/gfp.h>
  16. #include <linux/module.h>
  17. #include <linux/vmalloc.h>
  18. #include <linux/fs.h>
  19. #include <asm/cputable.h>
  20. #include <linux/uaccess.h>
  21. #include <asm/interrupt.h>
  22. #include <asm/kvm_ppc.h>
  23. #include <asm/cacheflush.h>
  24. #include <asm/dbell.h>
  25. #include <asm/hw_irq.h>
  26. #include <asm/irq.h>
  27. #include <asm/time.h>
  28. #include "timing.h"
  29. #include "booke.h"
  30. #define CREATE_TRACE_POINTS
  31. #include "trace_booke.h"
  32. unsigned long kvmppc_booke_handlers;
  33. const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
  34. KVM_GENERIC_VM_STATS(),
  35. STATS_DESC_ICOUNTER(VM, num_2M_pages),
  36. STATS_DESC_ICOUNTER(VM, num_1G_pages)
  37. };
  38. const struct kvm_stats_header kvm_vm_stats_header = {
  39. .name_size = KVM_STATS_NAME_SIZE,
  40. .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
  41. .id_offset = sizeof(struct kvm_stats_header),
  42. .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  43. .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  44. sizeof(kvm_vm_stats_desc),
  45. };
  46. const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
  47. KVM_GENERIC_VCPU_STATS(),
  48. STATS_DESC_COUNTER(VCPU, sum_exits),
  49. STATS_DESC_COUNTER(VCPU, mmio_exits),
  50. STATS_DESC_COUNTER(VCPU, signal_exits),
  51. STATS_DESC_COUNTER(VCPU, light_exits),
  52. STATS_DESC_COUNTER(VCPU, itlb_real_miss_exits),
  53. STATS_DESC_COUNTER(VCPU, itlb_virt_miss_exits),
  54. STATS_DESC_COUNTER(VCPU, dtlb_real_miss_exits),
  55. STATS_DESC_COUNTER(VCPU, dtlb_virt_miss_exits),
  56. STATS_DESC_COUNTER(VCPU, syscall_exits),
  57. STATS_DESC_COUNTER(VCPU, isi_exits),
  58. STATS_DESC_COUNTER(VCPU, dsi_exits),
  59. STATS_DESC_COUNTER(VCPU, emulated_inst_exits),
  60. STATS_DESC_COUNTER(VCPU, dec_exits),
  61. STATS_DESC_COUNTER(VCPU, ext_intr_exits),
  62. STATS_DESC_COUNTER(VCPU, halt_successful_wait),
  63. STATS_DESC_COUNTER(VCPU, dbell_exits),
  64. STATS_DESC_COUNTER(VCPU, gdbell_exits),
  65. STATS_DESC_COUNTER(VCPU, ld),
  66. STATS_DESC_COUNTER(VCPU, st),
  67. STATS_DESC_COUNTER(VCPU, pthru_all),
  68. STATS_DESC_COUNTER(VCPU, pthru_host),
  69. STATS_DESC_COUNTER(VCPU, pthru_bad_aff)
  70. };
  71. const struct kvm_stats_header kvm_vcpu_stats_header = {
  72. .name_size = KVM_STATS_NAME_SIZE,
  73. .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
  74. .id_offset = sizeof(struct kvm_stats_header),
  75. .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  76. .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  77. sizeof(kvm_vcpu_stats_desc),
  78. };
  79. /* TODO: use vcpu_printf() */
  80. void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu)
  81. {
  82. int i;
  83. printk("pc: %08lx msr: %08llx\n", vcpu->arch.regs.nip,
  84. vcpu->arch.shared->msr);
  85. printk("lr: %08lx ctr: %08lx\n", vcpu->arch.regs.link,
  86. vcpu->arch.regs.ctr);
  87. printk("srr0: %08llx srr1: %08llx\n", vcpu->arch.shared->srr0,
  88. vcpu->arch.shared->srr1);
  89. printk("exceptions: %08lx\n", vcpu->arch.pending_exceptions);
  90. for (i = 0; i < 32; i += 4) {
  91. printk("gpr%02d: %08lx %08lx %08lx %08lx\n", i,
  92. kvmppc_get_gpr(vcpu, i),
  93. kvmppc_get_gpr(vcpu, i+1),
  94. kvmppc_get_gpr(vcpu, i+2),
  95. kvmppc_get_gpr(vcpu, i+3));
  96. }
  97. }
  98. #ifdef CONFIG_SPE
  99. void kvmppc_vcpu_disable_spe(struct kvm_vcpu *vcpu)
  100. {
  101. preempt_disable();
  102. enable_kernel_spe();
  103. kvmppc_save_guest_spe(vcpu);
  104. disable_kernel_spe();
  105. vcpu->arch.shadow_msr &= ~MSR_SPE;
  106. preempt_enable();
  107. }
  108. static void kvmppc_vcpu_enable_spe(struct kvm_vcpu *vcpu)
  109. {
  110. preempt_disable();
  111. enable_kernel_spe();
  112. kvmppc_load_guest_spe(vcpu);
  113. disable_kernel_spe();
  114. vcpu->arch.shadow_msr |= MSR_SPE;
  115. preempt_enable();
  116. }
  117. static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
  118. {
  119. if (vcpu->arch.shared->msr & MSR_SPE) {
  120. if (!(vcpu->arch.shadow_msr & MSR_SPE))
  121. kvmppc_vcpu_enable_spe(vcpu);
  122. } else if (vcpu->arch.shadow_msr & MSR_SPE) {
  123. kvmppc_vcpu_disable_spe(vcpu);
  124. }
  125. }
  126. #else
  127. static void kvmppc_vcpu_sync_spe(struct kvm_vcpu *vcpu)
  128. {
  129. }
  130. #endif
  131. /*
  132. * Load up guest vcpu FP state if it's needed.
  133. * It also set the MSR_FP in thread so that host know
  134. * we're holding FPU, and then host can help to save
  135. * guest vcpu FP state if other threads require to use FPU.
  136. * This simulates an FP unavailable fault.
  137. *
  138. * It requires to be called with preemption disabled.
  139. */
  140. static inline void kvmppc_load_guest_fp(struct kvm_vcpu *vcpu)
  141. {
  142. #ifdef CONFIG_PPC_FPU
  143. if (!(current->thread.regs->msr & MSR_FP)) {
  144. enable_kernel_fp();
  145. load_fp_state(&vcpu->arch.fp);
  146. disable_kernel_fp();
  147. current->thread.fp_save_area = &vcpu->arch.fp;
  148. current->thread.regs->msr |= MSR_FP;
  149. }
  150. #endif
  151. }
  152. /*
  153. * Save guest vcpu FP state into thread.
  154. * It requires to be called with preemption disabled.
  155. */
  156. static inline void kvmppc_save_guest_fp(struct kvm_vcpu *vcpu)
  157. {
  158. #ifdef CONFIG_PPC_FPU
  159. if (current->thread.regs->msr & MSR_FP)
  160. giveup_fpu(current);
  161. current->thread.fp_save_area = NULL;
  162. #endif
  163. }
  164. static void kvmppc_vcpu_sync_fpu(struct kvm_vcpu *vcpu)
  165. {
  166. #if defined(CONFIG_PPC_FPU) && !defined(CONFIG_KVM_BOOKE_HV)
  167. /* We always treat the FP bit as enabled from the host
  168. perspective, so only need to adjust the shadow MSR */
  169. vcpu->arch.shadow_msr &= ~MSR_FP;
  170. vcpu->arch.shadow_msr |= vcpu->arch.shared->msr & MSR_FP;
  171. #endif
  172. }
  173. /*
  174. * Simulate AltiVec unavailable fault to load guest state
  175. * from thread to AltiVec unit.
  176. * It requires to be called with preemption disabled.
  177. */
  178. static inline void kvmppc_load_guest_altivec(struct kvm_vcpu *vcpu)
  179. {
  180. #ifdef CONFIG_ALTIVEC
  181. if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
  182. if (!(current->thread.regs->msr & MSR_VEC)) {
  183. enable_kernel_altivec();
  184. load_vr_state(&vcpu->arch.vr);
  185. disable_kernel_altivec();
  186. current->thread.vr_save_area = &vcpu->arch.vr;
  187. current->thread.regs->msr |= MSR_VEC;
  188. }
  189. }
  190. #endif
  191. }
  192. /*
  193. * Save guest vcpu AltiVec state into thread.
  194. * It requires to be called with preemption disabled.
  195. */
  196. static inline void kvmppc_save_guest_altivec(struct kvm_vcpu *vcpu)
  197. {
  198. #ifdef CONFIG_ALTIVEC
  199. if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
  200. if (current->thread.regs->msr & MSR_VEC)
  201. giveup_altivec(current);
  202. current->thread.vr_save_area = NULL;
  203. }
  204. #endif
  205. }
  206. static void kvmppc_vcpu_sync_debug(struct kvm_vcpu *vcpu)
  207. {
  208. /* Synchronize guest's desire to get debug interrupts into shadow MSR */
  209. #ifndef CONFIG_KVM_BOOKE_HV
  210. vcpu->arch.shadow_msr &= ~MSR_DE;
  211. vcpu->arch.shadow_msr |= vcpu->arch.shared->msr & MSR_DE;
  212. #endif
  213. /* Force enable debug interrupts when user space wants to debug */
  214. if (vcpu->guest_debug) {
  215. #ifdef CONFIG_KVM_BOOKE_HV
  216. /*
  217. * Since there is no shadow MSR, sync MSR_DE into the guest
  218. * visible MSR.
  219. */
  220. vcpu->arch.shared->msr |= MSR_DE;
  221. #else
  222. vcpu->arch.shadow_msr |= MSR_DE;
  223. vcpu->arch.shared->msr &= ~MSR_DE;
  224. #endif
  225. }
  226. }
  227. /*
  228. * Helper function for "full" MSR writes. No need to call this if only
  229. * EE/CE/ME/DE/RI are changing.
  230. */
  231. void kvmppc_set_msr(struct kvm_vcpu *vcpu, u32 new_msr)
  232. {
  233. u32 old_msr = vcpu->arch.shared->msr;
  234. #ifdef CONFIG_KVM_BOOKE_HV
  235. new_msr |= MSR_GS;
  236. #endif
  237. vcpu->arch.shared->msr = new_msr;
  238. kvmppc_mmu_msr_notify(vcpu, old_msr);
  239. kvmppc_vcpu_sync_spe(vcpu);
  240. kvmppc_vcpu_sync_fpu(vcpu);
  241. kvmppc_vcpu_sync_debug(vcpu);
  242. }
  243. static void kvmppc_booke_queue_irqprio(struct kvm_vcpu *vcpu,
  244. unsigned int priority)
  245. {
  246. trace_kvm_booke_queue_irqprio(vcpu, priority);
  247. set_bit(priority, &vcpu->arch.pending_exceptions);
  248. }
  249. void kvmppc_core_queue_dtlb_miss(struct kvm_vcpu *vcpu,
  250. ulong dear_flags, ulong esr_flags)
  251. {
  252. vcpu->arch.queued_dear = dear_flags;
  253. vcpu->arch.queued_esr = esr_flags;
  254. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DTLB_MISS);
  255. }
  256. void kvmppc_core_queue_data_storage(struct kvm_vcpu *vcpu,
  257. ulong dear_flags, ulong esr_flags)
  258. {
  259. vcpu->arch.queued_dear = dear_flags;
  260. vcpu->arch.queued_esr = esr_flags;
  261. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DATA_STORAGE);
  262. }
  263. void kvmppc_core_queue_itlb_miss(struct kvm_vcpu *vcpu)
  264. {
  265. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
  266. }
  267. void kvmppc_core_queue_inst_storage(struct kvm_vcpu *vcpu, ulong esr_flags)
  268. {
  269. vcpu->arch.queued_esr = esr_flags;
  270. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_INST_STORAGE);
  271. }
  272. static void kvmppc_core_queue_alignment(struct kvm_vcpu *vcpu, ulong dear_flags,
  273. ulong esr_flags)
  274. {
  275. vcpu->arch.queued_dear = dear_flags;
  276. vcpu->arch.queued_esr = esr_flags;
  277. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALIGNMENT);
  278. }
  279. void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong esr_flags)
  280. {
  281. vcpu->arch.queued_esr = esr_flags;
  282. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_PROGRAM);
  283. }
  284. void kvmppc_core_queue_fpunavail(struct kvm_vcpu *vcpu)
  285. {
  286. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
  287. }
  288. #ifdef CONFIG_ALTIVEC
  289. void kvmppc_core_queue_vec_unavail(struct kvm_vcpu *vcpu)
  290. {
  291. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_UNAVAIL);
  292. }
  293. #endif
  294. void kvmppc_core_queue_dec(struct kvm_vcpu *vcpu)
  295. {
  296. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DECREMENTER);
  297. }
  298. int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu)
  299. {
  300. return test_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
  301. }
  302. void kvmppc_core_dequeue_dec(struct kvm_vcpu *vcpu)
  303. {
  304. clear_bit(BOOKE_IRQPRIO_DECREMENTER, &vcpu->arch.pending_exceptions);
  305. }
  306. void kvmppc_core_queue_external(struct kvm_vcpu *vcpu,
  307. struct kvm_interrupt *irq)
  308. {
  309. unsigned int prio = BOOKE_IRQPRIO_EXTERNAL;
  310. if (irq->irq == KVM_INTERRUPT_SET_LEVEL)
  311. prio = BOOKE_IRQPRIO_EXTERNAL_LEVEL;
  312. kvmppc_booke_queue_irqprio(vcpu, prio);
  313. }
  314. void kvmppc_core_dequeue_external(struct kvm_vcpu *vcpu)
  315. {
  316. clear_bit(BOOKE_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions);
  317. clear_bit(BOOKE_IRQPRIO_EXTERNAL_LEVEL, &vcpu->arch.pending_exceptions);
  318. }
  319. static void kvmppc_core_queue_watchdog(struct kvm_vcpu *vcpu)
  320. {
  321. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_WATCHDOG);
  322. }
  323. static void kvmppc_core_dequeue_watchdog(struct kvm_vcpu *vcpu)
  324. {
  325. clear_bit(BOOKE_IRQPRIO_WATCHDOG, &vcpu->arch.pending_exceptions);
  326. }
  327. void kvmppc_core_queue_debug(struct kvm_vcpu *vcpu)
  328. {
  329. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_DEBUG);
  330. }
  331. void kvmppc_core_dequeue_debug(struct kvm_vcpu *vcpu)
  332. {
  333. clear_bit(BOOKE_IRQPRIO_DEBUG, &vcpu->arch.pending_exceptions);
  334. }
  335. static void set_guest_srr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  336. {
  337. kvmppc_set_srr0(vcpu, srr0);
  338. kvmppc_set_srr1(vcpu, srr1);
  339. }
  340. static void set_guest_csrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  341. {
  342. vcpu->arch.csrr0 = srr0;
  343. vcpu->arch.csrr1 = srr1;
  344. }
  345. static void set_guest_dsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  346. {
  347. if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC)) {
  348. vcpu->arch.dsrr0 = srr0;
  349. vcpu->arch.dsrr1 = srr1;
  350. } else {
  351. set_guest_csrr(vcpu, srr0, srr1);
  352. }
  353. }
  354. static void set_guest_mcsrr(struct kvm_vcpu *vcpu, unsigned long srr0, u32 srr1)
  355. {
  356. vcpu->arch.mcsrr0 = srr0;
  357. vcpu->arch.mcsrr1 = srr1;
  358. }
  359. /* Deliver the interrupt of the corresponding priority, if possible. */
  360. static int kvmppc_booke_irqprio_deliver(struct kvm_vcpu *vcpu,
  361. unsigned int priority)
  362. {
  363. int allowed = 0;
  364. ulong msr_mask = 0;
  365. bool update_esr = false, update_dear = false, update_epr = false;
  366. ulong crit_raw = vcpu->arch.shared->critical;
  367. ulong crit_r1 = kvmppc_get_gpr(vcpu, 1);
  368. bool crit;
  369. bool keep_irq = false;
  370. enum int_class int_class;
  371. ulong new_msr = vcpu->arch.shared->msr;
  372. /* Truncate crit indicators in 32 bit mode */
  373. if (!(vcpu->arch.shared->msr & MSR_SF)) {
  374. crit_raw &= 0xffffffff;
  375. crit_r1 &= 0xffffffff;
  376. }
  377. /* Critical section when crit == r1 */
  378. crit = (crit_raw == crit_r1);
  379. /* ... and we're in supervisor mode */
  380. crit = crit && !(vcpu->arch.shared->msr & MSR_PR);
  381. if (priority == BOOKE_IRQPRIO_EXTERNAL_LEVEL) {
  382. priority = BOOKE_IRQPRIO_EXTERNAL;
  383. keep_irq = true;
  384. }
  385. if ((priority == BOOKE_IRQPRIO_EXTERNAL) && vcpu->arch.epr_flags)
  386. update_epr = true;
  387. switch (priority) {
  388. case BOOKE_IRQPRIO_DTLB_MISS:
  389. case BOOKE_IRQPRIO_DATA_STORAGE:
  390. case BOOKE_IRQPRIO_ALIGNMENT:
  391. update_dear = true;
  392. fallthrough;
  393. case BOOKE_IRQPRIO_INST_STORAGE:
  394. case BOOKE_IRQPRIO_PROGRAM:
  395. update_esr = true;
  396. fallthrough;
  397. case BOOKE_IRQPRIO_ITLB_MISS:
  398. case BOOKE_IRQPRIO_SYSCALL:
  399. case BOOKE_IRQPRIO_FP_UNAVAIL:
  400. #ifdef CONFIG_SPE_POSSIBLE
  401. case BOOKE_IRQPRIO_SPE_UNAVAIL:
  402. case BOOKE_IRQPRIO_SPE_FP_DATA:
  403. case BOOKE_IRQPRIO_SPE_FP_ROUND:
  404. #endif
  405. #ifdef CONFIG_ALTIVEC
  406. case BOOKE_IRQPRIO_ALTIVEC_UNAVAIL:
  407. case BOOKE_IRQPRIO_ALTIVEC_ASSIST:
  408. #endif
  409. case BOOKE_IRQPRIO_AP_UNAVAIL:
  410. allowed = 1;
  411. msr_mask = MSR_CE | MSR_ME | MSR_DE;
  412. int_class = INT_CLASS_NONCRIT;
  413. break;
  414. case BOOKE_IRQPRIO_WATCHDOG:
  415. case BOOKE_IRQPRIO_CRITICAL:
  416. case BOOKE_IRQPRIO_DBELL_CRIT:
  417. allowed = vcpu->arch.shared->msr & MSR_CE;
  418. allowed = allowed && !crit;
  419. msr_mask = MSR_ME;
  420. int_class = INT_CLASS_CRIT;
  421. break;
  422. case BOOKE_IRQPRIO_MACHINE_CHECK:
  423. allowed = vcpu->arch.shared->msr & MSR_ME;
  424. allowed = allowed && !crit;
  425. int_class = INT_CLASS_MC;
  426. break;
  427. case BOOKE_IRQPRIO_DECREMENTER:
  428. case BOOKE_IRQPRIO_FIT:
  429. keep_irq = true;
  430. fallthrough;
  431. case BOOKE_IRQPRIO_EXTERNAL:
  432. case BOOKE_IRQPRIO_DBELL:
  433. allowed = vcpu->arch.shared->msr & MSR_EE;
  434. allowed = allowed && !crit;
  435. msr_mask = MSR_CE | MSR_ME | MSR_DE;
  436. int_class = INT_CLASS_NONCRIT;
  437. break;
  438. case BOOKE_IRQPRIO_DEBUG:
  439. allowed = vcpu->arch.shared->msr & MSR_DE;
  440. allowed = allowed && !crit;
  441. msr_mask = MSR_ME;
  442. if (cpu_has_feature(CPU_FTR_DEBUG_LVL_EXC))
  443. int_class = INT_CLASS_DBG;
  444. else
  445. int_class = INT_CLASS_CRIT;
  446. break;
  447. }
  448. if (allowed) {
  449. switch (int_class) {
  450. case INT_CLASS_NONCRIT:
  451. set_guest_srr(vcpu, vcpu->arch.regs.nip,
  452. vcpu->arch.shared->msr);
  453. break;
  454. case INT_CLASS_CRIT:
  455. set_guest_csrr(vcpu, vcpu->arch.regs.nip,
  456. vcpu->arch.shared->msr);
  457. break;
  458. case INT_CLASS_DBG:
  459. set_guest_dsrr(vcpu, vcpu->arch.regs.nip,
  460. vcpu->arch.shared->msr);
  461. break;
  462. case INT_CLASS_MC:
  463. set_guest_mcsrr(vcpu, vcpu->arch.regs.nip,
  464. vcpu->arch.shared->msr);
  465. break;
  466. }
  467. vcpu->arch.regs.nip = vcpu->arch.ivpr |
  468. vcpu->arch.ivor[priority];
  469. if (update_esr)
  470. kvmppc_set_esr(vcpu, vcpu->arch.queued_esr);
  471. if (update_dear)
  472. kvmppc_set_dar(vcpu, vcpu->arch.queued_dear);
  473. if (update_epr) {
  474. if (vcpu->arch.epr_flags & KVMPPC_EPR_USER)
  475. kvm_make_request(KVM_REQ_EPR_EXIT, vcpu);
  476. else if (vcpu->arch.epr_flags & KVMPPC_EPR_KERNEL) {
  477. BUG_ON(vcpu->arch.irq_type != KVMPPC_IRQ_MPIC);
  478. kvmppc_mpic_set_epr(vcpu);
  479. }
  480. }
  481. new_msr &= msr_mask;
  482. #if defined(CONFIG_64BIT)
  483. if (vcpu->arch.epcr & SPRN_EPCR_ICM)
  484. new_msr |= MSR_CM;
  485. #endif
  486. kvmppc_set_msr(vcpu, new_msr);
  487. if (!keep_irq)
  488. clear_bit(priority, &vcpu->arch.pending_exceptions);
  489. }
  490. #ifdef CONFIG_KVM_BOOKE_HV
  491. /*
  492. * If an interrupt is pending but masked, raise a guest doorbell
  493. * so that we are notified when the guest enables the relevant
  494. * MSR bit.
  495. */
  496. if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_EE)
  497. kvmppc_set_pending_interrupt(vcpu, INT_CLASS_NONCRIT);
  498. if (vcpu->arch.pending_exceptions & BOOKE_IRQMASK_CE)
  499. kvmppc_set_pending_interrupt(vcpu, INT_CLASS_CRIT);
  500. if (vcpu->arch.pending_exceptions & BOOKE_IRQPRIO_MACHINE_CHECK)
  501. kvmppc_set_pending_interrupt(vcpu, INT_CLASS_MC);
  502. #endif
  503. return allowed;
  504. }
  505. /*
  506. * Return the number of jiffies until the next timeout. If the timeout is
  507. * longer than the NEXT_TIMER_MAX_DELTA, then return NEXT_TIMER_MAX_DELTA
  508. * because the larger value can break the timer APIs.
  509. */
  510. static unsigned long watchdog_next_timeout(struct kvm_vcpu *vcpu)
  511. {
  512. u64 tb, wdt_tb, wdt_ticks = 0;
  513. u64 nr_jiffies = 0;
  514. u32 period = TCR_GET_WP(vcpu->arch.tcr);
  515. wdt_tb = 1ULL << (63 - period);
  516. tb = get_tb();
  517. /*
  518. * The watchdog timeout will hapeen when TB bit corresponding
  519. * to watchdog will toggle from 0 to 1.
  520. */
  521. if (tb & wdt_tb)
  522. wdt_ticks = wdt_tb;
  523. wdt_ticks += wdt_tb - (tb & (wdt_tb - 1));
  524. /* Convert timebase ticks to jiffies */
  525. nr_jiffies = wdt_ticks;
  526. if (do_div(nr_jiffies, tb_ticks_per_jiffy))
  527. nr_jiffies++;
  528. return min_t(unsigned long long, nr_jiffies, NEXT_TIMER_MAX_DELTA);
  529. }
  530. static void arm_next_watchdog(struct kvm_vcpu *vcpu)
  531. {
  532. unsigned long nr_jiffies;
  533. unsigned long flags;
  534. /*
  535. * If TSR_ENW and TSR_WIS are not set then no need to exit to
  536. * userspace, so clear the KVM_REQ_WATCHDOG request.
  537. */
  538. if ((vcpu->arch.tsr & (TSR_ENW | TSR_WIS)) != (TSR_ENW | TSR_WIS))
  539. kvm_clear_request(KVM_REQ_WATCHDOG, vcpu);
  540. spin_lock_irqsave(&vcpu->arch.wdt_lock, flags);
  541. nr_jiffies = watchdog_next_timeout(vcpu);
  542. /*
  543. * If the number of jiffies of watchdog timer >= NEXT_TIMER_MAX_DELTA
  544. * then do not run the watchdog timer as this can break timer APIs.
  545. */
  546. if (nr_jiffies < NEXT_TIMER_MAX_DELTA)
  547. mod_timer(&vcpu->arch.wdt_timer, jiffies + nr_jiffies);
  548. else
  549. del_timer(&vcpu->arch.wdt_timer);
  550. spin_unlock_irqrestore(&vcpu->arch.wdt_lock, flags);
  551. }
  552. void kvmppc_watchdog_func(struct timer_list *t)
  553. {
  554. struct kvm_vcpu *vcpu = from_timer(vcpu, t, arch.wdt_timer);
  555. u32 tsr, new_tsr;
  556. int final;
  557. do {
  558. new_tsr = tsr = vcpu->arch.tsr;
  559. final = 0;
  560. /* Time out event */
  561. if (tsr & TSR_ENW) {
  562. if (tsr & TSR_WIS)
  563. final = 1;
  564. else
  565. new_tsr = tsr | TSR_WIS;
  566. } else {
  567. new_tsr = tsr | TSR_ENW;
  568. }
  569. } while (cmpxchg(&vcpu->arch.tsr, tsr, new_tsr) != tsr);
  570. if (new_tsr & TSR_WIS) {
  571. smp_wmb();
  572. kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
  573. kvm_vcpu_kick(vcpu);
  574. }
  575. /*
  576. * If this is final watchdog expiry and some action is required
  577. * then exit to userspace.
  578. */
  579. if (final && (vcpu->arch.tcr & TCR_WRC_MASK) &&
  580. vcpu->arch.watchdog_enabled) {
  581. smp_wmb();
  582. kvm_make_request(KVM_REQ_WATCHDOG, vcpu);
  583. kvm_vcpu_kick(vcpu);
  584. }
  585. /*
  586. * Stop running the watchdog timer after final expiration to
  587. * prevent the host from being flooded with timers if the
  588. * guest sets a short period.
  589. * Timers will resume when TSR/TCR is updated next time.
  590. */
  591. if (!final)
  592. arm_next_watchdog(vcpu);
  593. }
  594. static void update_timer_ints(struct kvm_vcpu *vcpu)
  595. {
  596. if ((vcpu->arch.tcr & TCR_DIE) && (vcpu->arch.tsr & TSR_DIS))
  597. kvmppc_core_queue_dec(vcpu);
  598. else
  599. kvmppc_core_dequeue_dec(vcpu);
  600. if ((vcpu->arch.tcr & TCR_WIE) && (vcpu->arch.tsr & TSR_WIS))
  601. kvmppc_core_queue_watchdog(vcpu);
  602. else
  603. kvmppc_core_dequeue_watchdog(vcpu);
  604. }
  605. static void kvmppc_core_check_exceptions(struct kvm_vcpu *vcpu)
  606. {
  607. unsigned long *pending = &vcpu->arch.pending_exceptions;
  608. unsigned int priority;
  609. priority = __ffs(*pending);
  610. while (priority < BOOKE_IRQPRIO_MAX) {
  611. if (kvmppc_booke_irqprio_deliver(vcpu, priority))
  612. break;
  613. priority = find_next_bit(pending,
  614. BITS_PER_BYTE * sizeof(*pending),
  615. priority + 1);
  616. }
  617. /* Tell the guest about our interrupt status */
  618. vcpu->arch.shared->int_pending = !!*pending;
  619. }
  620. /* Check pending exceptions and deliver one, if possible. */
  621. int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu)
  622. {
  623. int r = 0;
  624. WARN_ON_ONCE(!irqs_disabled());
  625. kvmppc_core_check_exceptions(vcpu);
  626. if (kvm_request_pending(vcpu)) {
  627. /* Exception delivery raised request; start over */
  628. return 1;
  629. }
  630. if (vcpu->arch.shared->msr & MSR_WE) {
  631. local_irq_enable();
  632. kvm_vcpu_halt(vcpu);
  633. hard_irq_disable();
  634. kvmppc_set_exit_type(vcpu, EMULATED_MTMSRWE_EXITS);
  635. r = 1;
  636. }
  637. return r;
  638. }
  639. int kvmppc_core_check_requests(struct kvm_vcpu *vcpu)
  640. {
  641. int r = 1; /* Indicate we want to get back into the guest */
  642. if (kvm_check_request(KVM_REQ_PENDING_TIMER, vcpu))
  643. update_timer_ints(vcpu);
  644. #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
  645. if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu))
  646. kvmppc_core_flush_tlb(vcpu);
  647. #endif
  648. if (kvm_check_request(KVM_REQ_WATCHDOG, vcpu)) {
  649. vcpu->run->exit_reason = KVM_EXIT_WATCHDOG;
  650. r = 0;
  651. }
  652. if (kvm_check_request(KVM_REQ_EPR_EXIT, vcpu)) {
  653. vcpu->run->epr.epr = 0;
  654. vcpu->arch.epr_needed = true;
  655. vcpu->run->exit_reason = KVM_EXIT_EPR;
  656. r = 0;
  657. }
  658. return r;
  659. }
  660. int kvmppc_vcpu_run(struct kvm_vcpu *vcpu)
  661. {
  662. int ret, s;
  663. struct debug_reg debug;
  664. if (!vcpu->arch.sane) {
  665. vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
  666. return -EINVAL;
  667. }
  668. s = kvmppc_prepare_to_enter(vcpu);
  669. if (s <= 0) {
  670. ret = s;
  671. goto out;
  672. }
  673. /* interrupts now hard-disabled */
  674. #ifdef CONFIG_PPC_FPU
  675. /* Save userspace FPU state in stack */
  676. enable_kernel_fp();
  677. /*
  678. * Since we can't trap on MSR_FP in GS-mode, we consider the guest
  679. * as always using the FPU.
  680. */
  681. kvmppc_load_guest_fp(vcpu);
  682. #endif
  683. #ifdef CONFIG_ALTIVEC
  684. /* Save userspace AltiVec state in stack */
  685. if (cpu_has_feature(CPU_FTR_ALTIVEC))
  686. enable_kernel_altivec();
  687. /*
  688. * Since we can't trap on MSR_VEC in GS-mode, we consider the guest
  689. * as always using the AltiVec.
  690. */
  691. kvmppc_load_guest_altivec(vcpu);
  692. #endif
  693. /* Switch to guest debug context */
  694. debug = vcpu->arch.dbg_reg;
  695. switch_booke_debug_regs(&debug);
  696. debug = current->thread.debug;
  697. current->thread.debug = vcpu->arch.dbg_reg;
  698. vcpu->arch.pgdir = vcpu->kvm->mm->pgd;
  699. kvmppc_fix_ee_before_entry();
  700. ret = __kvmppc_vcpu_run(vcpu);
  701. /* No need for guest_exit. It's done in handle_exit.
  702. We also get here with interrupts enabled. */
  703. /* Switch back to user space debug context */
  704. switch_booke_debug_regs(&debug);
  705. current->thread.debug = debug;
  706. #ifdef CONFIG_PPC_FPU
  707. kvmppc_save_guest_fp(vcpu);
  708. #endif
  709. #ifdef CONFIG_ALTIVEC
  710. kvmppc_save_guest_altivec(vcpu);
  711. #endif
  712. out:
  713. vcpu->mode = OUTSIDE_GUEST_MODE;
  714. return ret;
  715. }
  716. static int emulation_exit(struct kvm_vcpu *vcpu)
  717. {
  718. enum emulation_result er;
  719. er = kvmppc_emulate_instruction(vcpu);
  720. switch (er) {
  721. case EMULATE_DONE:
  722. /* don't overwrite subtypes, just account kvm_stats */
  723. kvmppc_account_exit_stat(vcpu, EMULATED_INST_EXITS);
  724. /* Future optimization: only reload non-volatiles if
  725. * they were actually modified by emulation. */
  726. return RESUME_GUEST_NV;
  727. case EMULATE_AGAIN:
  728. return RESUME_GUEST;
  729. case EMULATE_FAIL:
  730. printk(KERN_CRIT "%s: emulation at %lx failed (%08x)\n",
  731. __func__, vcpu->arch.regs.nip, vcpu->arch.last_inst);
  732. /* For debugging, encode the failing instruction and
  733. * report it to userspace. */
  734. vcpu->run->hw.hardware_exit_reason = ~0ULL << 32;
  735. vcpu->run->hw.hardware_exit_reason |= vcpu->arch.last_inst;
  736. kvmppc_core_queue_program(vcpu, ESR_PIL);
  737. return RESUME_HOST;
  738. case EMULATE_EXIT_USER:
  739. return RESUME_HOST;
  740. default:
  741. BUG();
  742. }
  743. }
  744. static int kvmppc_handle_debug(struct kvm_vcpu *vcpu)
  745. {
  746. struct kvm_run *run = vcpu->run;
  747. struct debug_reg *dbg_reg = &(vcpu->arch.dbg_reg);
  748. u32 dbsr = vcpu->arch.dbsr;
  749. if (vcpu->guest_debug == 0) {
  750. /*
  751. * Debug resources belong to Guest.
  752. * Imprecise debug event is not injected
  753. */
  754. if (dbsr & DBSR_IDE) {
  755. dbsr &= ~DBSR_IDE;
  756. if (!dbsr)
  757. return RESUME_GUEST;
  758. }
  759. if (dbsr && (vcpu->arch.shared->msr & MSR_DE) &&
  760. (vcpu->arch.dbg_reg.dbcr0 & DBCR0_IDM))
  761. kvmppc_core_queue_debug(vcpu);
  762. /* Inject a program interrupt if trap debug is not allowed */
  763. if ((dbsr & DBSR_TIE) && !(vcpu->arch.shared->msr & MSR_DE))
  764. kvmppc_core_queue_program(vcpu, ESR_PTR);
  765. return RESUME_GUEST;
  766. }
  767. /*
  768. * Debug resource owned by userspace.
  769. * Clear guest dbsr (vcpu->arch.dbsr)
  770. */
  771. vcpu->arch.dbsr = 0;
  772. run->debug.arch.status = 0;
  773. run->debug.arch.address = vcpu->arch.regs.nip;
  774. if (dbsr & (DBSR_IAC1 | DBSR_IAC2 | DBSR_IAC3 | DBSR_IAC4)) {
  775. run->debug.arch.status |= KVMPPC_DEBUG_BREAKPOINT;
  776. } else {
  777. if (dbsr & (DBSR_DAC1W | DBSR_DAC2W))
  778. run->debug.arch.status |= KVMPPC_DEBUG_WATCH_WRITE;
  779. else if (dbsr & (DBSR_DAC1R | DBSR_DAC2R))
  780. run->debug.arch.status |= KVMPPC_DEBUG_WATCH_READ;
  781. if (dbsr & (DBSR_DAC1R | DBSR_DAC1W))
  782. run->debug.arch.address = dbg_reg->dac1;
  783. else if (dbsr & (DBSR_DAC2R | DBSR_DAC2W))
  784. run->debug.arch.address = dbg_reg->dac2;
  785. }
  786. return RESUME_HOST;
  787. }
  788. static void kvmppc_fill_pt_regs(struct pt_regs *regs)
  789. {
  790. ulong r1, ip, msr, lr;
  791. asm("mr %0, 1" : "=r"(r1));
  792. asm("mflr %0" : "=r"(lr));
  793. asm("mfmsr %0" : "=r"(msr));
  794. asm("bl 1f; 1: mflr %0" : "=r"(ip));
  795. memset(regs, 0, sizeof(*regs));
  796. regs->gpr[1] = r1;
  797. regs->nip = ip;
  798. regs->msr = msr;
  799. regs->link = lr;
  800. }
  801. /*
  802. * For interrupts needed to be handled by host interrupt handlers,
  803. * corresponding host handler are called from here in similar way
  804. * (but not exact) as they are called from low level handler
  805. * (such as from arch/powerpc/kernel/head_fsl_booke.S).
  806. */
  807. static void kvmppc_restart_interrupt(struct kvm_vcpu *vcpu,
  808. unsigned int exit_nr)
  809. {
  810. struct pt_regs regs;
  811. switch (exit_nr) {
  812. case BOOKE_INTERRUPT_EXTERNAL:
  813. kvmppc_fill_pt_regs(&regs);
  814. do_IRQ(&regs);
  815. break;
  816. case BOOKE_INTERRUPT_DECREMENTER:
  817. kvmppc_fill_pt_regs(&regs);
  818. timer_interrupt(&regs);
  819. break;
  820. #if defined(CONFIG_PPC_DOORBELL)
  821. case BOOKE_INTERRUPT_DOORBELL:
  822. kvmppc_fill_pt_regs(&regs);
  823. doorbell_exception(&regs);
  824. break;
  825. #endif
  826. case BOOKE_INTERRUPT_MACHINE_CHECK:
  827. /* FIXME */
  828. break;
  829. case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
  830. kvmppc_fill_pt_regs(&regs);
  831. performance_monitor_exception(&regs);
  832. break;
  833. case BOOKE_INTERRUPT_WATCHDOG:
  834. kvmppc_fill_pt_regs(&regs);
  835. #ifdef CONFIG_BOOKE_WDT
  836. WatchdogException(&regs);
  837. #else
  838. unknown_exception(&regs);
  839. #endif
  840. break;
  841. case BOOKE_INTERRUPT_CRITICAL:
  842. kvmppc_fill_pt_regs(&regs);
  843. unknown_exception(&regs);
  844. break;
  845. case BOOKE_INTERRUPT_DEBUG:
  846. /* Save DBSR before preemption is enabled */
  847. vcpu->arch.dbsr = mfspr(SPRN_DBSR);
  848. kvmppc_clear_dbsr();
  849. break;
  850. }
  851. }
  852. static int kvmppc_resume_inst_load(struct kvm_vcpu *vcpu,
  853. enum emulation_result emulated, u32 last_inst)
  854. {
  855. switch (emulated) {
  856. case EMULATE_AGAIN:
  857. return RESUME_GUEST;
  858. case EMULATE_FAIL:
  859. pr_debug("%s: load instruction from guest address %lx failed\n",
  860. __func__, vcpu->arch.regs.nip);
  861. /* For debugging, encode the failing instruction and
  862. * report it to userspace. */
  863. vcpu->run->hw.hardware_exit_reason = ~0ULL << 32;
  864. vcpu->run->hw.hardware_exit_reason |= last_inst;
  865. kvmppc_core_queue_program(vcpu, ESR_PIL);
  866. return RESUME_HOST;
  867. default:
  868. BUG();
  869. }
  870. }
  871. /**
  872. * kvmppc_handle_exit
  873. *
  874. * Return value is in the form (errcode<<2 | RESUME_FLAG_HOST | RESUME_FLAG_NV)
  875. */
  876. int kvmppc_handle_exit(struct kvm_vcpu *vcpu, unsigned int exit_nr)
  877. {
  878. struct kvm_run *run = vcpu->run;
  879. int r = RESUME_HOST;
  880. int s;
  881. int idx;
  882. u32 last_inst = KVM_INST_FETCH_FAILED;
  883. enum emulation_result emulated = EMULATE_DONE;
  884. /* update before a new last_exit_type is rewritten */
  885. kvmppc_update_timing_stats(vcpu);
  886. /* restart interrupts if they were meant for the host */
  887. kvmppc_restart_interrupt(vcpu, exit_nr);
  888. /*
  889. * get last instruction before being preempted
  890. * TODO: for e6500 check also BOOKE_INTERRUPT_LRAT_ERROR & ESR_DATA
  891. */
  892. switch (exit_nr) {
  893. case BOOKE_INTERRUPT_DATA_STORAGE:
  894. case BOOKE_INTERRUPT_DTLB_MISS:
  895. case BOOKE_INTERRUPT_HV_PRIV:
  896. emulated = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
  897. break;
  898. case BOOKE_INTERRUPT_PROGRAM:
  899. /* SW breakpoints arrive as illegal instructions on HV */
  900. if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
  901. emulated = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
  902. break;
  903. default:
  904. break;
  905. }
  906. trace_kvm_exit(exit_nr, vcpu);
  907. context_tracking_guest_exit();
  908. if (!vtime_accounting_enabled_this_cpu()) {
  909. local_irq_enable();
  910. /*
  911. * Service IRQs here before vtime_account_guest_exit() so any
  912. * ticks that occurred while running the guest are accounted to
  913. * the guest. If vtime accounting is enabled, accounting uses
  914. * TB rather than ticks, so it can be done without enabling
  915. * interrupts here, which has the problem that it accounts
  916. * interrupt processing overhead to the host.
  917. */
  918. local_irq_disable();
  919. }
  920. vtime_account_guest_exit();
  921. local_irq_enable();
  922. run->exit_reason = KVM_EXIT_UNKNOWN;
  923. run->ready_for_interrupt_injection = 1;
  924. if (emulated != EMULATE_DONE) {
  925. r = kvmppc_resume_inst_load(vcpu, emulated, last_inst);
  926. goto out;
  927. }
  928. switch (exit_nr) {
  929. case BOOKE_INTERRUPT_MACHINE_CHECK:
  930. printk("MACHINE CHECK: %lx\n", mfspr(SPRN_MCSR));
  931. kvmppc_dump_vcpu(vcpu);
  932. /* For debugging, send invalid exit reason to user space */
  933. run->hw.hardware_exit_reason = ~1ULL << 32;
  934. run->hw.hardware_exit_reason |= mfspr(SPRN_MCSR);
  935. r = RESUME_HOST;
  936. break;
  937. case BOOKE_INTERRUPT_EXTERNAL:
  938. kvmppc_account_exit(vcpu, EXT_INTR_EXITS);
  939. r = RESUME_GUEST;
  940. break;
  941. case BOOKE_INTERRUPT_DECREMENTER:
  942. kvmppc_account_exit(vcpu, DEC_EXITS);
  943. r = RESUME_GUEST;
  944. break;
  945. case BOOKE_INTERRUPT_WATCHDOG:
  946. r = RESUME_GUEST;
  947. break;
  948. case BOOKE_INTERRUPT_DOORBELL:
  949. kvmppc_account_exit(vcpu, DBELL_EXITS);
  950. r = RESUME_GUEST;
  951. break;
  952. case BOOKE_INTERRUPT_GUEST_DBELL_CRIT:
  953. kvmppc_account_exit(vcpu, GDBELL_EXITS);
  954. /*
  955. * We are here because there is a pending guest interrupt
  956. * which could not be delivered as MSR_CE or MSR_ME was not
  957. * set. Once we break from here we will retry delivery.
  958. */
  959. r = RESUME_GUEST;
  960. break;
  961. case BOOKE_INTERRUPT_GUEST_DBELL:
  962. kvmppc_account_exit(vcpu, GDBELL_EXITS);
  963. /*
  964. * We are here because there is a pending guest interrupt
  965. * which could not be delivered as MSR_EE was not set. Once
  966. * we break from here we will retry delivery.
  967. */
  968. r = RESUME_GUEST;
  969. break;
  970. case BOOKE_INTERRUPT_PERFORMANCE_MONITOR:
  971. r = RESUME_GUEST;
  972. break;
  973. case BOOKE_INTERRUPT_HV_PRIV:
  974. r = emulation_exit(vcpu);
  975. break;
  976. case BOOKE_INTERRUPT_PROGRAM:
  977. if ((vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) &&
  978. (last_inst == KVMPPC_INST_SW_BREAKPOINT)) {
  979. /*
  980. * We are here because of an SW breakpoint instr,
  981. * so lets return to host to handle.
  982. */
  983. r = kvmppc_handle_debug(vcpu);
  984. run->exit_reason = KVM_EXIT_DEBUG;
  985. kvmppc_account_exit(vcpu, DEBUG_EXITS);
  986. break;
  987. }
  988. if (vcpu->arch.shared->msr & (MSR_PR | MSR_GS)) {
  989. /*
  990. * Program traps generated by user-level software must
  991. * be handled by the guest kernel.
  992. *
  993. * In GS mode, hypervisor privileged instructions trap
  994. * on BOOKE_INTERRUPT_HV_PRIV, not here, so these are
  995. * actual program interrupts, handled by the guest.
  996. */
  997. kvmppc_core_queue_program(vcpu, vcpu->arch.fault_esr);
  998. r = RESUME_GUEST;
  999. kvmppc_account_exit(vcpu, USR_PR_INST);
  1000. break;
  1001. }
  1002. r = emulation_exit(vcpu);
  1003. break;
  1004. case BOOKE_INTERRUPT_FP_UNAVAIL:
  1005. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_FP_UNAVAIL);
  1006. kvmppc_account_exit(vcpu, FP_UNAVAIL);
  1007. r = RESUME_GUEST;
  1008. break;
  1009. #ifdef CONFIG_SPE
  1010. case BOOKE_INTERRUPT_SPE_UNAVAIL: {
  1011. if (vcpu->arch.shared->msr & MSR_SPE)
  1012. kvmppc_vcpu_enable_spe(vcpu);
  1013. else
  1014. kvmppc_booke_queue_irqprio(vcpu,
  1015. BOOKE_IRQPRIO_SPE_UNAVAIL);
  1016. r = RESUME_GUEST;
  1017. break;
  1018. }
  1019. case BOOKE_INTERRUPT_SPE_FP_DATA:
  1020. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_DATA);
  1021. r = RESUME_GUEST;
  1022. break;
  1023. case BOOKE_INTERRUPT_SPE_FP_ROUND:
  1024. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SPE_FP_ROUND);
  1025. r = RESUME_GUEST;
  1026. break;
  1027. #elif defined(CONFIG_SPE_POSSIBLE)
  1028. case BOOKE_INTERRUPT_SPE_UNAVAIL:
  1029. /*
  1030. * Guest wants SPE, but host kernel doesn't support it. Send
  1031. * an "unimplemented operation" program check to the guest.
  1032. */
  1033. kvmppc_core_queue_program(vcpu, ESR_PUO | ESR_SPV);
  1034. r = RESUME_GUEST;
  1035. break;
  1036. /*
  1037. * These really should never happen without CONFIG_SPE,
  1038. * as we should never enable the real MSR[SPE] in the guest.
  1039. */
  1040. case BOOKE_INTERRUPT_SPE_FP_DATA:
  1041. case BOOKE_INTERRUPT_SPE_FP_ROUND:
  1042. printk(KERN_CRIT "%s: unexpected SPE interrupt %u at %08lx\n",
  1043. __func__, exit_nr, vcpu->arch.regs.nip);
  1044. run->hw.hardware_exit_reason = exit_nr;
  1045. r = RESUME_HOST;
  1046. break;
  1047. #endif /* CONFIG_SPE_POSSIBLE */
  1048. /*
  1049. * On cores with Vector category, KVM is loaded only if CONFIG_ALTIVEC,
  1050. * see kvmppc_core_check_processor_compat().
  1051. */
  1052. #ifdef CONFIG_ALTIVEC
  1053. case BOOKE_INTERRUPT_ALTIVEC_UNAVAIL:
  1054. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_UNAVAIL);
  1055. r = RESUME_GUEST;
  1056. break;
  1057. case BOOKE_INTERRUPT_ALTIVEC_ASSIST:
  1058. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ALTIVEC_ASSIST);
  1059. r = RESUME_GUEST;
  1060. break;
  1061. #endif
  1062. case BOOKE_INTERRUPT_DATA_STORAGE:
  1063. kvmppc_core_queue_data_storage(vcpu, vcpu->arch.fault_dear,
  1064. vcpu->arch.fault_esr);
  1065. kvmppc_account_exit(vcpu, DSI_EXITS);
  1066. r = RESUME_GUEST;
  1067. break;
  1068. case BOOKE_INTERRUPT_INST_STORAGE:
  1069. kvmppc_core_queue_inst_storage(vcpu, vcpu->arch.fault_esr);
  1070. kvmppc_account_exit(vcpu, ISI_EXITS);
  1071. r = RESUME_GUEST;
  1072. break;
  1073. case BOOKE_INTERRUPT_ALIGNMENT:
  1074. kvmppc_core_queue_alignment(vcpu, vcpu->arch.fault_dear,
  1075. vcpu->arch.fault_esr);
  1076. r = RESUME_GUEST;
  1077. break;
  1078. #ifdef CONFIG_KVM_BOOKE_HV
  1079. case BOOKE_INTERRUPT_HV_SYSCALL:
  1080. if (!(vcpu->arch.shared->msr & MSR_PR)) {
  1081. kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
  1082. } else {
  1083. /*
  1084. * hcall from guest userspace -- send privileged
  1085. * instruction program check.
  1086. */
  1087. kvmppc_core_queue_program(vcpu, ESR_PPR);
  1088. }
  1089. r = RESUME_GUEST;
  1090. break;
  1091. #else
  1092. case BOOKE_INTERRUPT_SYSCALL:
  1093. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  1094. (((u32)kvmppc_get_gpr(vcpu, 0)) == KVM_SC_MAGIC_R0)) {
  1095. /* KVM PV hypercalls */
  1096. kvmppc_set_gpr(vcpu, 3, kvmppc_kvm_pv(vcpu));
  1097. r = RESUME_GUEST;
  1098. } else {
  1099. /* Guest syscalls */
  1100. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_SYSCALL);
  1101. }
  1102. kvmppc_account_exit(vcpu, SYSCALL_EXITS);
  1103. r = RESUME_GUEST;
  1104. break;
  1105. #endif
  1106. case BOOKE_INTERRUPT_DTLB_MISS: {
  1107. unsigned long eaddr = vcpu->arch.fault_dear;
  1108. int gtlb_index;
  1109. gpa_t gpaddr;
  1110. gfn_t gfn;
  1111. #ifdef CONFIG_KVM_E500V2
  1112. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  1113. (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
  1114. kvmppc_map_magic(vcpu);
  1115. kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
  1116. r = RESUME_GUEST;
  1117. break;
  1118. }
  1119. #endif
  1120. /* Check the guest TLB. */
  1121. gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
  1122. if (gtlb_index < 0) {
  1123. /* The guest didn't have a mapping for it. */
  1124. kvmppc_core_queue_dtlb_miss(vcpu,
  1125. vcpu->arch.fault_dear,
  1126. vcpu->arch.fault_esr);
  1127. kvmppc_mmu_dtlb_miss(vcpu);
  1128. kvmppc_account_exit(vcpu, DTLB_REAL_MISS_EXITS);
  1129. r = RESUME_GUEST;
  1130. break;
  1131. }
  1132. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1133. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  1134. gfn = gpaddr >> PAGE_SHIFT;
  1135. if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  1136. /* The guest TLB had a mapping, but the shadow TLB
  1137. * didn't, and it is RAM. This could be because:
  1138. * a) the entry is mapping the host kernel, or
  1139. * b) the guest used a large mapping which we're faking
  1140. * Either way, we need to satisfy the fault without
  1141. * invoking the guest. */
  1142. kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
  1143. kvmppc_account_exit(vcpu, DTLB_VIRT_MISS_EXITS);
  1144. r = RESUME_GUEST;
  1145. } else {
  1146. /* Guest has mapped and accessed a page which is not
  1147. * actually RAM. */
  1148. vcpu->arch.paddr_accessed = gpaddr;
  1149. vcpu->arch.vaddr_accessed = eaddr;
  1150. r = kvmppc_emulate_mmio(vcpu);
  1151. kvmppc_account_exit(vcpu, MMIO_EXITS);
  1152. }
  1153. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1154. break;
  1155. }
  1156. case BOOKE_INTERRUPT_ITLB_MISS: {
  1157. unsigned long eaddr = vcpu->arch.regs.nip;
  1158. gpa_t gpaddr;
  1159. gfn_t gfn;
  1160. int gtlb_index;
  1161. r = RESUME_GUEST;
  1162. /* Check the guest TLB. */
  1163. gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
  1164. if (gtlb_index < 0) {
  1165. /* The guest didn't have a mapping for it. */
  1166. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_ITLB_MISS);
  1167. kvmppc_mmu_itlb_miss(vcpu);
  1168. kvmppc_account_exit(vcpu, ITLB_REAL_MISS_EXITS);
  1169. break;
  1170. }
  1171. kvmppc_account_exit(vcpu, ITLB_VIRT_MISS_EXITS);
  1172. idx = srcu_read_lock(&vcpu->kvm->srcu);
  1173. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  1174. gfn = gpaddr >> PAGE_SHIFT;
  1175. if (kvm_is_visible_gfn(vcpu->kvm, gfn)) {
  1176. /* The guest TLB had a mapping, but the shadow TLB
  1177. * didn't. This could be because:
  1178. * a) the entry is mapping the host kernel, or
  1179. * b) the guest used a large mapping which we're faking
  1180. * Either way, we need to satisfy the fault without
  1181. * invoking the guest. */
  1182. kvmppc_mmu_map(vcpu, eaddr, gpaddr, gtlb_index);
  1183. } else {
  1184. /* Guest mapped and leaped at non-RAM! */
  1185. kvmppc_booke_queue_irqprio(vcpu, BOOKE_IRQPRIO_MACHINE_CHECK);
  1186. }
  1187. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  1188. break;
  1189. }
  1190. case BOOKE_INTERRUPT_DEBUG: {
  1191. r = kvmppc_handle_debug(vcpu);
  1192. if (r == RESUME_HOST)
  1193. run->exit_reason = KVM_EXIT_DEBUG;
  1194. kvmppc_account_exit(vcpu, DEBUG_EXITS);
  1195. break;
  1196. }
  1197. default:
  1198. printk(KERN_EMERG "exit_nr %d\n", exit_nr);
  1199. BUG();
  1200. }
  1201. out:
  1202. /*
  1203. * To avoid clobbering exit_reason, only check for signals if we
  1204. * aren't already exiting to userspace for some other reason.
  1205. */
  1206. if (!(r & RESUME_HOST)) {
  1207. s = kvmppc_prepare_to_enter(vcpu);
  1208. if (s <= 0)
  1209. r = (s << 2) | RESUME_HOST | (r & RESUME_FLAG_NV);
  1210. else {
  1211. /* interrupts now hard-disabled */
  1212. kvmppc_fix_ee_before_entry();
  1213. kvmppc_load_guest_fp(vcpu);
  1214. kvmppc_load_guest_altivec(vcpu);
  1215. }
  1216. }
  1217. return r;
  1218. }
  1219. static void kvmppc_set_tsr(struct kvm_vcpu *vcpu, u32 new_tsr)
  1220. {
  1221. u32 old_tsr = vcpu->arch.tsr;
  1222. vcpu->arch.tsr = new_tsr;
  1223. if ((old_tsr ^ vcpu->arch.tsr) & (TSR_ENW | TSR_WIS))
  1224. arm_next_watchdog(vcpu);
  1225. update_timer_ints(vcpu);
  1226. }
  1227. int kvmppc_subarch_vcpu_init(struct kvm_vcpu *vcpu)
  1228. {
  1229. /* setup watchdog timer once */
  1230. spin_lock_init(&vcpu->arch.wdt_lock);
  1231. timer_setup(&vcpu->arch.wdt_timer, kvmppc_watchdog_func, 0);
  1232. /*
  1233. * Clear DBSR.MRR to avoid guest debug interrupt as
  1234. * this is of host interest
  1235. */
  1236. mtspr(SPRN_DBSR, DBSR_MRR);
  1237. return 0;
  1238. }
  1239. void kvmppc_subarch_vcpu_uninit(struct kvm_vcpu *vcpu)
  1240. {
  1241. del_timer_sync(&vcpu->arch.wdt_timer);
  1242. }
  1243. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  1244. {
  1245. int i;
  1246. vcpu_load(vcpu);
  1247. regs->pc = vcpu->arch.regs.nip;
  1248. regs->cr = kvmppc_get_cr(vcpu);
  1249. regs->ctr = vcpu->arch.regs.ctr;
  1250. regs->lr = vcpu->arch.regs.link;
  1251. regs->xer = kvmppc_get_xer(vcpu);
  1252. regs->msr = vcpu->arch.shared->msr;
  1253. regs->srr0 = kvmppc_get_srr0(vcpu);
  1254. regs->srr1 = kvmppc_get_srr1(vcpu);
  1255. regs->pid = vcpu->arch.pid;
  1256. regs->sprg0 = kvmppc_get_sprg0(vcpu);
  1257. regs->sprg1 = kvmppc_get_sprg1(vcpu);
  1258. regs->sprg2 = kvmppc_get_sprg2(vcpu);
  1259. regs->sprg3 = kvmppc_get_sprg3(vcpu);
  1260. regs->sprg4 = kvmppc_get_sprg4(vcpu);
  1261. regs->sprg5 = kvmppc_get_sprg5(vcpu);
  1262. regs->sprg6 = kvmppc_get_sprg6(vcpu);
  1263. regs->sprg7 = kvmppc_get_sprg7(vcpu);
  1264. for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
  1265. regs->gpr[i] = kvmppc_get_gpr(vcpu, i);
  1266. vcpu_put(vcpu);
  1267. return 0;
  1268. }
  1269. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  1270. {
  1271. int i;
  1272. vcpu_load(vcpu);
  1273. vcpu->arch.regs.nip = regs->pc;
  1274. kvmppc_set_cr(vcpu, regs->cr);
  1275. vcpu->arch.regs.ctr = regs->ctr;
  1276. vcpu->arch.regs.link = regs->lr;
  1277. kvmppc_set_xer(vcpu, regs->xer);
  1278. kvmppc_set_msr(vcpu, regs->msr);
  1279. kvmppc_set_srr0(vcpu, regs->srr0);
  1280. kvmppc_set_srr1(vcpu, regs->srr1);
  1281. kvmppc_set_pid(vcpu, regs->pid);
  1282. kvmppc_set_sprg0(vcpu, regs->sprg0);
  1283. kvmppc_set_sprg1(vcpu, regs->sprg1);
  1284. kvmppc_set_sprg2(vcpu, regs->sprg2);
  1285. kvmppc_set_sprg3(vcpu, regs->sprg3);
  1286. kvmppc_set_sprg4(vcpu, regs->sprg4);
  1287. kvmppc_set_sprg5(vcpu, regs->sprg5);
  1288. kvmppc_set_sprg6(vcpu, regs->sprg6);
  1289. kvmppc_set_sprg7(vcpu, regs->sprg7);
  1290. for (i = 0; i < ARRAY_SIZE(regs->gpr); i++)
  1291. kvmppc_set_gpr(vcpu, i, regs->gpr[i]);
  1292. vcpu_put(vcpu);
  1293. return 0;
  1294. }
  1295. static void get_sregs_base(struct kvm_vcpu *vcpu,
  1296. struct kvm_sregs *sregs)
  1297. {
  1298. u64 tb = get_tb();
  1299. sregs->u.e.features |= KVM_SREGS_E_BASE;
  1300. sregs->u.e.csrr0 = vcpu->arch.csrr0;
  1301. sregs->u.e.csrr1 = vcpu->arch.csrr1;
  1302. sregs->u.e.mcsr = vcpu->arch.mcsr;
  1303. sregs->u.e.esr = kvmppc_get_esr(vcpu);
  1304. sregs->u.e.dear = kvmppc_get_dar(vcpu);
  1305. sregs->u.e.tsr = vcpu->arch.tsr;
  1306. sregs->u.e.tcr = vcpu->arch.tcr;
  1307. sregs->u.e.dec = kvmppc_get_dec(vcpu, tb);
  1308. sregs->u.e.tb = tb;
  1309. sregs->u.e.vrsave = vcpu->arch.vrsave;
  1310. }
  1311. static int set_sregs_base(struct kvm_vcpu *vcpu,
  1312. struct kvm_sregs *sregs)
  1313. {
  1314. if (!(sregs->u.e.features & KVM_SREGS_E_BASE))
  1315. return 0;
  1316. vcpu->arch.csrr0 = sregs->u.e.csrr0;
  1317. vcpu->arch.csrr1 = sregs->u.e.csrr1;
  1318. vcpu->arch.mcsr = sregs->u.e.mcsr;
  1319. kvmppc_set_esr(vcpu, sregs->u.e.esr);
  1320. kvmppc_set_dar(vcpu, sregs->u.e.dear);
  1321. vcpu->arch.vrsave = sregs->u.e.vrsave;
  1322. kvmppc_set_tcr(vcpu, sregs->u.e.tcr);
  1323. if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_DEC) {
  1324. vcpu->arch.dec = sregs->u.e.dec;
  1325. kvmppc_emulate_dec(vcpu);
  1326. }
  1327. if (sregs->u.e.update_special & KVM_SREGS_E_UPDATE_TSR)
  1328. kvmppc_set_tsr(vcpu, sregs->u.e.tsr);
  1329. return 0;
  1330. }
  1331. static void get_sregs_arch206(struct kvm_vcpu *vcpu,
  1332. struct kvm_sregs *sregs)
  1333. {
  1334. sregs->u.e.features |= KVM_SREGS_E_ARCH206;
  1335. sregs->u.e.pir = vcpu->vcpu_id;
  1336. sregs->u.e.mcsrr0 = vcpu->arch.mcsrr0;
  1337. sregs->u.e.mcsrr1 = vcpu->arch.mcsrr1;
  1338. sregs->u.e.decar = vcpu->arch.decar;
  1339. sregs->u.e.ivpr = vcpu->arch.ivpr;
  1340. }
  1341. static int set_sregs_arch206(struct kvm_vcpu *vcpu,
  1342. struct kvm_sregs *sregs)
  1343. {
  1344. if (!(sregs->u.e.features & KVM_SREGS_E_ARCH206))
  1345. return 0;
  1346. if (sregs->u.e.pir != vcpu->vcpu_id)
  1347. return -EINVAL;
  1348. vcpu->arch.mcsrr0 = sregs->u.e.mcsrr0;
  1349. vcpu->arch.mcsrr1 = sregs->u.e.mcsrr1;
  1350. vcpu->arch.decar = sregs->u.e.decar;
  1351. vcpu->arch.ivpr = sregs->u.e.ivpr;
  1352. return 0;
  1353. }
  1354. int kvmppc_get_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  1355. {
  1356. sregs->u.e.features |= KVM_SREGS_E_IVOR;
  1357. sregs->u.e.ivor_low[0] = vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL];
  1358. sregs->u.e.ivor_low[1] = vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK];
  1359. sregs->u.e.ivor_low[2] = vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE];
  1360. sregs->u.e.ivor_low[3] = vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE];
  1361. sregs->u.e.ivor_low[4] = vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL];
  1362. sregs->u.e.ivor_low[5] = vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT];
  1363. sregs->u.e.ivor_low[6] = vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM];
  1364. sregs->u.e.ivor_low[7] = vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL];
  1365. sregs->u.e.ivor_low[8] = vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL];
  1366. sregs->u.e.ivor_low[9] = vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL];
  1367. sregs->u.e.ivor_low[10] = vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER];
  1368. sregs->u.e.ivor_low[11] = vcpu->arch.ivor[BOOKE_IRQPRIO_FIT];
  1369. sregs->u.e.ivor_low[12] = vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG];
  1370. sregs->u.e.ivor_low[13] = vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS];
  1371. sregs->u.e.ivor_low[14] = vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS];
  1372. sregs->u.e.ivor_low[15] = vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG];
  1373. return 0;
  1374. }
  1375. int kvmppc_set_sregs_ivor(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs)
  1376. {
  1377. if (!(sregs->u.e.features & KVM_SREGS_E_IVOR))
  1378. return 0;
  1379. vcpu->arch.ivor[BOOKE_IRQPRIO_CRITICAL] = sregs->u.e.ivor_low[0];
  1380. vcpu->arch.ivor[BOOKE_IRQPRIO_MACHINE_CHECK] = sregs->u.e.ivor_low[1];
  1381. vcpu->arch.ivor[BOOKE_IRQPRIO_DATA_STORAGE] = sregs->u.e.ivor_low[2];
  1382. vcpu->arch.ivor[BOOKE_IRQPRIO_INST_STORAGE] = sregs->u.e.ivor_low[3];
  1383. vcpu->arch.ivor[BOOKE_IRQPRIO_EXTERNAL] = sregs->u.e.ivor_low[4];
  1384. vcpu->arch.ivor[BOOKE_IRQPRIO_ALIGNMENT] = sregs->u.e.ivor_low[5];
  1385. vcpu->arch.ivor[BOOKE_IRQPRIO_PROGRAM] = sregs->u.e.ivor_low[6];
  1386. vcpu->arch.ivor[BOOKE_IRQPRIO_FP_UNAVAIL] = sregs->u.e.ivor_low[7];
  1387. vcpu->arch.ivor[BOOKE_IRQPRIO_SYSCALL] = sregs->u.e.ivor_low[8];
  1388. vcpu->arch.ivor[BOOKE_IRQPRIO_AP_UNAVAIL] = sregs->u.e.ivor_low[9];
  1389. vcpu->arch.ivor[BOOKE_IRQPRIO_DECREMENTER] = sregs->u.e.ivor_low[10];
  1390. vcpu->arch.ivor[BOOKE_IRQPRIO_FIT] = sregs->u.e.ivor_low[11];
  1391. vcpu->arch.ivor[BOOKE_IRQPRIO_WATCHDOG] = sregs->u.e.ivor_low[12];
  1392. vcpu->arch.ivor[BOOKE_IRQPRIO_DTLB_MISS] = sregs->u.e.ivor_low[13];
  1393. vcpu->arch.ivor[BOOKE_IRQPRIO_ITLB_MISS] = sregs->u.e.ivor_low[14];
  1394. vcpu->arch.ivor[BOOKE_IRQPRIO_DEBUG] = sregs->u.e.ivor_low[15];
  1395. return 0;
  1396. }
  1397. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  1398. struct kvm_sregs *sregs)
  1399. {
  1400. int ret;
  1401. vcpu_load(vcpu);
  1402. sregs->pvr = vcpu->arch.pvr;
  1403. get_sregs_base(vcpu, sregs);
  1404. get_sregs_arch206(vcpu, sregs);
  1405. ret = vcpu->kvm->arch.kvm_ops->get_sregs(vcpu, sregs);
  1406. vcpu_put(vcpu);
  1407. return ret;
  1408. }
  1409. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  1410. struct kvm_sregs *sregs)
  1411. {
  1412. int ret = -EINVAL;
  1413. vcpu_load(vcpu);
  1414. if (vcpu->arch.pvr != sregs->pvr)
  1415. goto out;
  1416. ret = set_sregs_base(vcpu, sregs);
  1417. if (ret < 0)
  1418. goto out;
  1419. ret = set_sregs_arch206(vcpu, sregs);
  1420. if (ret < 0)
  1421. goto out;
  1422. ret = vcpu->kvm->arch.kvm_ops->set_sregs(vcpu, sregs);
  1423. out:
  1424. vcpu_put(vcpu);
  1425. return ret;
  1426. }
  1427. int kvmppc_get_one_reg(struct kvm_vcpu *vcpu, u64 id,
  1428. union kvmppc_one_reg *val)
  1429. {
  1430. int r = 0;
  1431. switch (id) {
  1432. case KVM_REG_PPC_IAC1:
  1433. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac1);
  1434. break;
  1435. case KVM_REG_PPC_IAC2:
  1436. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac2);
  1437. break;
  1438. #if CONFIG_PPC_ADV_DEBUG_IACS > 2
  1439. case KVM_REG_PPC_IAC3:
  1440. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac3);
  1441. break;
  1442. case KVM_REG_PPC_IAC4:
  1443. *val = get_reg_val(id, vcpu->arch.dbg_reg.iac4);
  1444. break;
  1445. #endif
  1446. case KVM_REG_PPC_DAC1:
  1447. *val = get_reg_val(id, vcpu->arch.dbg_reg.dac1);
  1448. break;
  1449. case KVM_REG_PPC_DAC2:
  1450. *val = get_reg_val(id, vcpu->arch.dbg_reg.dac2);
  1451. break;
  1452. case KVM_REG_PPC_EPR: {
  1453. u32 epr = kvmppc_get_epr(vcpu);
  1454. *val = get_reg_val(id, epr);
  1455. break;
  1456. }
  1457. #if defined(CONFIG_64BIT)
  1458. case KVM_REG_PPC_EPCR:
  1459. *val = get_reg_val(id, vcpu->arch.epcr);
  1460. break;
  1461. #endif
  1462. case KVM_REG_PPC_TCR:
  1463. *val = get_reg_val(id, vcpu->arch.tcr);
  1464. break;
  1465. case KVM_REG_PPC_TSR:
  1466. *val = get_reg_val(id, vcpu->arch.tsr);
  1467. break;
  1468. case KVM_REG_PPC_DEBUG_INST:
  1469. *val = get_reg_val(id, KVMPPC_INST_SW_BREAKPOINT);
  1470. break;
  1471. case KVM_REG_PPC_VRSAVE:
  1472. *val = get_reg_val(id, vcpu->arch.vrsave);
  1473. break;
  1474. default:
  1475. r = vcpu->kvm->arch.kvm_ops->get_one_reg(vcpu, id, val);
  1476. break;
  1477. }
  1478. return r;
  1479. }
  1480. int kvmppc_set_one_reg(struct kvm_vcpu *vcpu, u64 id,
  1481. union kvmppc_one_reg *val)
  1482. {
  1483. int r = 0;
  1484. switch (id) {
  1485. case KVM_REG_PPC_IAC1:
  1486. vcpu->arch.dbg_reg.iac1 = set_reg_val(id, *val);
  1487. break;
  1488. case KVM_REG_PPC_IAC2:
  1489. vcpu->arch.dbg_reg.iac2 = set_reg_val(id, *val);
  1490. break;
  1491. #if CONFIG_PPC_ADV_DEBUG_IACS > 2
  1492. case KVM_REG_PPC_IAC3:
  1493. vcpu->arch.dbg_reg.iac3 = set_reg_val(id, *val);
  1494. break;
  1495. case KVM_REG_PPC_IAC4:
  1496. vcpu->arch.dbg_reg.iac4 = set_reg_val(id, *val);
  1497. break;
  1498. #endif
  1499. case KVM_REG_PPC_DAC1:
  1500. vcpu->arch.dbg_reg.dac1 = set_reg_val(id, *val);
  1501. break;
  1502. case KVM_REG_PPC_DAC2:
  1503. vcpu->arch.dbg_reg.dac2 = set_reg_val(id, *val);
  1504. break;
  1505. case KVM_REG_PPC_EPR: {
  1506. u32 new_epr = set_reg_val(id, *val);
  1507. kvmppc_set_epr(vcpu, new_epr);
  1508. break;
  1509. }
  1510. #if defined(CONFIG_64BIT)
  1511. case KVM_REG_PPC_EPCR: {
  1512. u32 new_epcr = set_reg_val(id, *val);
  1513. kvmppc_set_epcr(vcpu, new_epcr);
  1514. break;
  1515. }
  1516. #endif
  1517. case KVM_REG_PPC_OR_TSR: {
  1518. u32 tsr_bits = set_reg_val(id, *val);
  1519. kvmppc_set_tsr_bits(vcpu, tsr_bits);
  1520. break;
  1521. }
  1522. case KVM_REG_PPC_CLEAR_TSR: {
  1523. u32 tsr_bits = set_reg_val(id, *val);
  1524. kvmppc_clr_tsr_bits(vcpu, tsr_bits);
  1525. break;
  1526. }
  1527. case KVM_REG_PPC_TSR: {
  1528. u32 tsr = set_reg_val(id, *val);
  1529. kvmppc_set_tsr(vcpu, tsr);
  1530. break;
  1531. }
  1532. case KVM_REG_PPC_TCR: {
  1533. u32 tcr = set_reg_val(id, *val);
  1534. kvmppc_set_tcr(vcpu, tcr);
  1535. break;
  1536. }
  1537. case KVM_REG_PPC_VRSAVE:
  1538. vcpu->arch.vrsave = set_reg_val(id, *val);
  1539. break;
  1540. default:
  1541. r = vcpu->kvm->arch.kvm_ops->set_one_reg(vcpu, id, val);
  1542. break;
  1543. }
  1544. return r;
  1545. }
  1546. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  1547. {
  1548. return -EOPNOTSUPP;
  1549. }
  1550. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  1551. {
  1552. return -EOPNOTSUPP;
  1553. }
  1554. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  1555. struct kvm_translation *tr)
  1556. {
  1557. int r;
  1558. vcpu_load(vcpu);
  1559. r = kvmppc_core_vcpu_translate(vcpu, tr);
  1560. vcpu_put(vcpu);
  1561. return r;
  1562. }
  1563. void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
  1564. {
  1565. }
  1566. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
  1567. {
  1568. return -EOPNOTSUPP;
  1569. }
  1570. void kvmppc_core_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
  1571. {
  1572. }
  1573. int kvmppc_core_prepare_memory_region(struct kvm *kvm,
  1574. const struct kvm_memory_slot *old,
  1575. struct kvm_memory_slot *new,
  1576. enum kvm_mr_change change)
  1577. {
  1578. return 0;
  1579. }
  1580. void kvmppc_core_commit_memory_region(struct kvm *kvm,
  1581. struct kvm_memory_slot *old,
  1582. const struct kvm_memory_slot *new,
  1583. enum kvm_mr_change change)
  1584. {
  1585. }
  1586. void kvmppc_core_flush_memslot(struct kvm *kvm, struct kvm_memory_slot *memslot)
  1587. {
  1588. }
  1589. void kvmppc_set_epcr(struct kvm_vcpu *vcpu, u32 new_epcr)
  1590. {
  1591. #if defined(CONFIG_64BIT)
  1592. vcpu->arch.epcr = new_epcr;
  1593. #ifdef CONFIG_KVM_BOOKE_HV
  1594. vcpu->arch.shadow_epcr &= ~SPRN_EPCR_GICM;
  1595. if (vcpu->arch.epcr & SPRN_EPCR_ICM)
  1596. vcpu->arch.shadow_epcr |= SPRN_EPCR_GICM;
  1597. #endif
  1598. #endif
  1599. }
  1600. void kvmppc_set_tcr(struct kvm_vcpu *vcpu, u32 new_tcr)
  1601. {
  1602. vcpu->arch.tcr = new_tcr;
  1603. arm_next_watchdog(vcpu);
  1604. update_timer_ints(vcpu);
  1605. }
  1606. void kvmppc_set_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
  1607. {
  1608. set_bits(tsr_bits, &vcpu->arch.tsr);
  1609. smp_wmb();
  1610. kvm_make_request(KVM_REQ_PENDING_TIMER, vcpu);
  1611. kvm_vcpu_kick(vcpu);
  1612. }
  1613. void kvmppc_clr_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits)
  1614. {
  1615. clear_bits(tsr_bits, &vcpu->arch.tsr);
  1616. /*
  1617. * We may have stopped the watchdog due to
  1618. * being stuck on final expiration.
  1619. */
  1620. if (tsr_bits & (TSR_ENW | TSR_WIS))
  1621. arm_next_watchdog(vcpu);
  1622. update_timer_ints(vcpu);
  1623. }
  1624. void kvmppc_decrementer_func(struct kvm_vcpu *vcpu)
  1625. {
  1626. if (vcpu->arch.tcr & TCR_ARE) {
  1627. vcpu->arch.dec = vcpu->arch.decar;
  1628. kvmppc_emulate_dec(vcpu);
  1629. }
  1630. kvmppc_set_tsr_bits(vcpu, TSR_DIS);
  1631. }
  1632. static int kvmppc_booke_add_breakpoint(struct debug_reg *dbg_reg,
  1633. uint64_t addr, int index)
  1634. {
  1635. switch (index) {
  1636. case 0:
  1637. dbg_reg->dbcr0 |= DBCR0_IAC1;
  1638. dbg_reg->iac1 = addr;
  1639. break;
  1640. case 1:
  1641. dbg_reg->dbcr0 |= DBCR0_IAC2;
  1642. dbg_reg->iac2 = addr;
  1643. break;
  1644. #if CONFIG_PPC_ADV_DEBUG_IACS > 2
  1645. case 2:
  1646. dbg_reg->dbcr0 |= DBCR0_IAC3;
  1647. dbg_reg->iac3 = addr;
  1648. break;
  1649. case 3:
  1650. dbg_reg->dbcr0 |= DBCR0_IAC4;
  1651. dbg_reg->iac4 = addr;
  1652. break;
  1653. #endif
  1654. default:
  1655. return -EINVAL;
  1656. }
  1657. dbg_reg->dbcr0 |= DBCR0_IDM;
  1658. return 0;
  1659. }
  1660. static int kvmppc_booke_add_watchpoint(struct debug_reg *dbg_reg, uint64_t addr,
  1661. int type, int index)
  1662. {
  1663. switch (index) {
  1664. case 0:
  1665. if (type & KVMPPC_DEBUG_WATCH_READ)
  1666. dbg_reg->dbcr0 |= DBCR0_DAC1R;
  1667. if (type & KVMPPC_DEBUG_WATCH_WRITE)
  1668. dbg_reg->dbcr0 |= DBCR0_DAC1W;
  1669. dbg_reg->dac1 = addr;
  1670. break;
  1671. case 1:
  1672. if (type & KVMPPC_DEBUG_WATCH_READ)
  1673. dbg_reg->dbcr0 |= DBCR0_DAC2R;
  1674. if (type & KVMPPC_DEBUG_WATCH_WRITE)
  1675. dbg_reg->dbcr0 |= DBCR0_DAC2W;
  1676. dbg_reg->dac2 = addr;
  1677. break;
  1678. default:
  1679. return -EINVAL;
  1680. }
  1681. dbg_reg->dbcr0 |= DBCR0_IDM;
  1682. return 0;
  1683. }
  1684. void kvm_guest_protect_msr(struct kvm_vcpu *vcpu, ulong prot_bitmap, bool set)
  1685. {
  1686. /* XXX: Add similar MSR protection for BookE-PR */
  1687. #ifdef CONFIG_KVM_BOOKE_HV
  1688. BUG_ON(prot_bitmap & ~(MSRP_UCLEP | MSRP_DEP | MSRP_PMMP));
  1689. if (set) {
  1690. if (prot_bitmap & MSR_UCLE)
  1691. vcpu->arch.shadow_msrp |= MSRP_UCLEP;
  1692. if (prot_bitmap & MSR_DE)
  1693. vcpu->arch.shadow_msrp |= MSRP_DEP;
  1694. if (prot_bitmap & MSR_PMM)
  1695. vcpu->arch.shadow_msrp |= MSRP_PMMP;
  1696. } else {
  1697. if (prot_bitmap & MSR_UCLE)
  1698. vcpu->arch.shadow_msrp &= ~MSRP_UCLEP;
  1699. if (prot_bitmap & MSR_DE)
  1700. vcpu->arch.shadow_msrp &= ~MSRP_DEP;
  1701. if (prot_bitmap & MSR_PMM)
  1702. vcpu->arch.shadow_msrp &= ~MSRP_PMMP;
  1703. }
  1704. #endif
  1705. }
  1706. int kvmppc_xlate(struct kvm_vcpu *vcpu, ulong eaddr, enum xlate_instdata xlid,
  1707. enum xlate_readwrite xlrw, struct kvmppc_pte *pte)
  1708. {
  1709. int gtlb_index;
  1710. gpa_t gpaddr;
  1711. #ifdef CONFIG_KVM_E500V2
  1712. if (!(vcpu->arch.shared->msr & MSR_PR) &&
  1713. (eaddr & PAGE_MASK) == vcpu->arch.magic_page_ea) {
  1714. pte->eaddr = eaddr;
  1715. pte->raddr = (vcpu->arch.magic_page_pa & PAGE_MASK) |
  1716. (eaddr & ~PAGE_MASK);
  1717. pte->vpage = eaddr >> PAGE_SHIFT;
  1718. pte->may_read = true;
  1719. pte->may_write = true;
  1720. pte->may_execute = true;
  1721. return 0;
  1722. }
  1723. #endif
  1724. /* Check the guest TLB. */
  1725. switch (xlid) {
  1726. case XLATE_INST:
  1727. gtlb_index = kvmppc_mmu_itlb_index(vcpu, eaddr);
  1728. break;
  1729. case XLATE_DATA:
  1730. gtlb_index = kvmppc_mmu_dtlb_index(vcpu, eaddr);
  1731. break;
  1732. default:
  1733. BUG();
  1734. }
  1735. /* Do we have a TLB entry at all? */
  1736. if (gtlb_index < 0)
  1737. return -ENOENT;
  1738. gpaddr = kvmppc_mmu_xlate(vcpu, gtlb_index, eaddr);
  1739. pte->eaddr = eaddr;
  1740. pte->raddr = (gpaddr & PAGE_MASK) | (eaddr & ~PAGE_MASK);
  1741. pte->vpage = eaddr >> PAGE_SHIFT;
  1742. /* XXX read permissions from the guest TLB */
  1743. pte->may_read = true;
  1744. pte->may_write = true;
  1745. pte->may_execute = true;
  1746. return 0;
  1747. }
  1748. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  1749. struct kvm_guest_debug *dbg)
  1750. {
  1751. struct debug_reg *dbg_reg;
  1752. int n, b = 0, w = 0;
  1753. int ret = 0;
  1754. vcpu_load(vcpu);
  1755. if (!(dbg->control & KVM_GUESTDBG_ENABLE)) {
  1756. vcpu->arch.dbg_reg.dbcr0 = 0;
  1757. vcpu->guest_debug = 0;
  1758. kvm_guest_protect_msr(vcpu, MSR_DE, false);
  1759. goto out;
  1760. }
  1761. kvm_guest_protect_msr(vcpu, MSR_DE, true);
  1762. vcpu->guest_debug = dbg->control;
  1763. vcpu->arch.dbg_reg.dbcr0 = 0;
  1764. if (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP)
  1765. vcpu->arch.dbg_reg.dbcr0 |= DBCR0_IDM | DBCR0_IC;
  1766. /* Code below handles only HW breakpoints */
  1767. dbg_reg = &(vcpu->arch.dbg_reg);
  1768. #ifdef CONFIG_KVM_BOOKE_HV
  1769. /*
  1770. * On BookE-HV (e500mc) the guest is always executed with MSR.GS=1
  1771. * DBCR1 and DBCR2 are set to trigger debug events when MSR.PR is 0
  1772. */
  1773. dbg_reg->dbcr1 = 0;
  1774. dbg_reg->dbcr2 = 0;
  1775. #else
  1776. /*
  1777. * On BookE-PR (e500v2) the guest is always executed with MSR.PR=1
  1778. * We set DBCR1 and DBCR2 to only trigger debug events when MSR.PR
  1779. * is set.
  1780. */
  1781. dbg_reg->dbcr1 = DBCR1_IAC1US | DBCR1_IAC2US | DBCR1_IAC3US |
  1782. DBCR1_IAC4US;
  1783. dbg_reg->dbcr2 = DBCR2_DAC1US | DBCR2_DAC2US;
  1784. #endif
  1785. if (!(vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP))
  1786. goto out;
  1787. ret = -EINVAL;
  1788. for (n = 0; n < (KVMPPC_BOOKE_IAC_NUM + KVMPPC_BOOKE_DAC_NUM); n++) {
  1789. uint64_t addr = dbg->arch.bp[n].addr;
  1790. uint32_t type = dbg->arch.bp[n].type;
  1791. if (type == KVMPPC_DEBUG_NONE)
  1792. continue;
  1793. if (type & ~(KVMPPC_DEBUG_WATCH_READ |
  1794. KVMPPC_DEBUG_WATCH_WRITE |
  1795. KVMPPC_DEBUG_BREAKPOINT))
  1796. goto out;
  1797. if (type & KVMPPC_DEBUG_BREAKPOINT) {
  1798. /* Setting H/W breakpoint */
  1799. if (kvmppc_booke_add_breakpoint(dbg_reg, addr, b++))
  1800. goto out;
  1801. } else {
  1802. /* Setting H/W watchpoint */
  1803. if (kvmppc_booke_add_watchpoint(dbg_reg, addr,
  1804. type, w++))
  1805. goto out;
  1806. }
  1807. }
  1808. ret = 0;
  1809. out:
  1810. vcpu_put(vcpu);
  1811. return ret;
  1812. }
  1813. void kvmppc_booke_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1814. {
  1815. vcpu->cpu = smp_processor_id();
  1816. current->thread.kvm_vcpu = vcpu;
  1817. }
  1818. void kvmppc_booke_vcpu_put(struct kvm_vcpu *vcpu)
  1819. {
  1820. current->thread.kvm_vcpu = NULL;
  1821. vcpu->cpu = -1;
  1822. /* Clear pending debug event in DBSR */
  1823. kvmppc_clear_dbsr();
  1824. }
  1825. int kvmppc_core_init_vm(struct kvm *kvm)
  1826. {
  1827. return kvm->arch.kvm_ops->init_vm(kvm);
  1828. }
  1829. int kvmppc_core_vcpu_create(struct kvm_vcpu *vcpu)
  1830. {
  1831. int i;
  1832. int r;
  1833. r = vcpu->kvm->arch.kvm_ops->vcpu_create(vcpu);
  1834. if (r)
  1835. return r;
  1836. /* Initial guest state: 16MB mapping 0 -> 0, PC = 0, MSR = 0, R1 = 16MB */
  1837. vcpu->arch.regs.nip = 0;
  1838. vcpu->arch.shared->pir = vcpu->vcpu_id;
  1839. kvmppc_set_gpr(vcpu, 1, (16<<20) - 8); /* -8 for the callee-save LR slot */
  1840. kvmppc_set_msr(vcpu, 0);
  1841. #ifndef CONFIG_KVM_BOOKE_HV
  1842. vcpu->arch.shadow_msr = MSR_USER | MSR_IS | MSR_DS;
  1843. vcpu->arch.shadow_pid = 1;
  1844. vcpu->arch.shared->msr = 0;
  1845. #endif
  1846. /* Eye-catching numbers so we know if the guest takes an interrupt
  1847. * before it's programmed its own IVPR/IVORs. */
  1848. vcpu->arch.ivpr = 0x55550000;
  1849. for (i = 0; i < BOOKE_IRQPRIO_MAX; i++)
  1850. vcpu->arch.ivor[i] = 0x7700 | i * 4;
  1851. kvmppc_init_timing_stats(vcpu);
  1852. r = kvmppc_core_vcpu_setup(vcpu);
  1853. if (r)
  1854. vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
  1855. kvmppc_sanity_check(vcpu);
  1856. return r;
  1857. }
  1858. void kvmppc_core_vcpu_free(struct kvm_vcpu *vcpu)
  1859. {
  1860. vcpu->kvm->arch.kvm_ops->vcpu_free(vcpu);
  1861. }
  1862. void kvmppc_core_destroy_vm(struct kvm *kvm)
  1863. {
  1864. kvm->arch.kvm_ops->destroy_vm(kvm);
  1865. }
  1866. void kvmppc_core_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  1867. {
  1868. vcpu->kvm->arch.kvm_ops->vcpu_load(vcpu, cpu);
  1869. }
  1870. void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu)
  1871. {
  1872. vcpu->kvm->arch.kvm_ops->vcpu_put(vcpu);
  1873. }
  1874. int __init kvmppc_booke_init(void)
  1875. {
  1876. #ifndef CONFIG_KVM_BOOKE_HV
  1877. unsigned long ivor[16];
  1878. unsigned long *handler = kvmppc_booke_handler_addr;
  1879. unsigned long max_ivor = 0;
  1880. unsigned long handler_len;
  1881. int i;
  1882. /* We install our own exception handlers by hijacking IVPR. IVPR must
  1883. * be 16-bit aligned, so we need a 64KB allocation. */
  1884. kvmppc_booke_handlers = __get_free_pages(GFP_KERNEL | __GFP_ZERO,
  1885. VCPU_SIZE_ORDER);
  1886. if (!kvmppc_booke_handlers)
  1887. return -ENOMEM;
  1888. /* XXX make sure our handlers are smaller than Linux's */
  1889. /* Copy our interrupt handlers to match host IVORs. That way we don't
  1890. * have to swap the IVORs on every guest/host transition. */
  1891. ivor[0] = mfspr(SPRN_IVOR0);
  1892. ivor[1] = mfspr(SPRN_IVOR1);
  1893. ivor[2] = mfspr(SPRN_IVOR2);
  1894. ivor[3] = mfspr(SPRN_IVOR3);
  1895. ivor[4] = mfspr(SPRN_IVOR4);
  1896. ivor[5] = mfspr(SPRN_IVOR5);
  1897. ivor[6] = mfspr(SPRN_IVOR6);
  1898. ivor[7] = mfspr(SPRN_IVOR7);
  1899. ivor[8] = mfspr(SPRN_IVOR8);
  1900. ivor[9] = mfspr(SPRN_IVOR9);
  1901. ivor[10] = mfspr(SPRN_IVOR10);
  1902. ivor[11] = mfspr(SPRN_IVOR11);
  1903. ivor[12] = mfspr(SPRN_IVOR12);
  1904. ivor[13] = mfspr(SPRN_IVOR13);
  1905. ivor[14] = mfspr(SPRN_IVOR14);
  1906. ivor[15] = mfspr(SPRN_IVOR15);
  1907. for (i = 0; i < 16; i++) {
  1908. if (ivor[i] > max_ivor)
  1909. max_ivor = i;
  1910. handler_len = handler[i + 1] - handler[i];
  1911. memcpy((void *)kvmppc_booke_handlers + ivor[i],
  1912. (void *)handler[i], handler_len);
  1913. }
  1914. handler_len = handler[max_ivor + 1] - handler[max_ivor];
  1915. flush_icache_range(kvmppc_booke_handlers, kvmppc_booke_handlers +
  1916. ivor[max_ivor] + handler_len);
  1917. #endif /* !BOOKE_HV */
  1918. return 0;
  1919. }
  1920. void __exit kvmppc_booke_exit(void)
  1921. {
  1922. free_pages(kvmppc_booke_handlers, VCPU_SIZE_ORDER);
  1923. kvm_exit();
  1924. }