kvm-s390.c 155 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * hosting IBM Z kernel virtual machines (s390x)
  4. *
  5. * Copyright IBM Corp. 2008, 2020
  6. *
  7. * Author(s): Carsten Otte <[email protected]>
  8. * Christian Borntraeger <[email protected]>
  9. * Christian Ehrhardt <[email protected]>
  10. * Jason J. Herne <[email protected]>
  11. */
  12. #define KMSG_COMPONENT "kvm-s390"
  13. #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  14. #include <linux/compiler.h>
  15. #include <linux/err.h>
  16. #include <linux/fs.h>
  17. #include <linux/hrtimer.h>
  18. #include <linux/init.h>
  19. #include <linux/kvm.h>
  20. #include <linux/kvm_host.h>
  21. #include <linux/mman.h>
  22. #include <linux/module.h>
  23. #include <linux/moduleparam.h>
  24. #include <linux/random.h>
  25. #include <linux/slab.h>
  26. #include <linux/timer.h>
  27. #include <linux/vmalloc.h>
  28. #include <linux/bitmap.h>
  29. #include <linux/sched/signal.h>
  30. #include <linux/string.h>
  31. #include <linux/pgtable.h>
  32. #include <linux/mmu_notifier.h>
  33. #include <asm/asm-offsets.h>
  34. #include <asm/lowcore.h>
  35. #include <asm/stp.h>
  36. #include <asm/gmap.h>
  37. #include <asm/nmi.h>
  38. #include <asm/switch_to.h>
  39. #include <asm/isc.h>
  40. #include <asm/sclp.h>
  41. #include <asm/cpacf.h>
  42. #include <asm/timex.h>
  43. #include <asm/ap.h>
  44. #include <asm/uv.h>
  45. #include <asm/fpu/api.h>
  46. #include "kvm-s390.h"
  47. #include "gaccess.h"
  48. #include "pci.h"
  49. #define CREATE_TRACE_POINTS
  50. #include "trace.h"
  51. #include "trace-s390.h"
  52. #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
  53. #define LOCAL_IRQS 32
  54. #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
  55. (KVM_MAX_VCPUS + LOCAL_IRQS))
  56. const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
  57. KVM_GENERIC_VM_STATS(),
  58. STATS_DESC_COUNTER(VM, inject_io),
  59. STATS_DESC_COUNTER(VM, inject_float_mchk),
  60. STATS_DESC_COUNTER(VM, inject_pfault_done),
  61. STATS_DESC_COUNTER(VM, inject_service_signal),
  62. STATS_DESC_COUNTER(VM, inject_virtio),
  63. STATS_DESC_COUNTER(VM, aen_forward)
  64. };
  65. const struct kvm_stats_header kvm_vm_stats_header = {
  66. .name_size = KVM_STATS_NAME_SIZE,
  67. .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
  68. .id_offset = sizeof(struct kvm_stats_header),
  69. .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  70. .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  71. sizeof(kvm_vm_stats_desc),
  72. };
  73. const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
  74. KVM_GENERIC_VCPU_STATS(),
  75. STATS_DESC_COUNTER(VCPU, exit_userspace),
  76. STATS_DESC_COUNTER(VCPU, exit_null),
  77. STATS_DESC_COUNTER(VCPU, exit_external_request),
  78. STATS_DESC_COUNTER(VCPU, exit_io_request),
  79. STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
  80. STATS_DESC_COUNTER(VCPU, exit_stop_request),
  81. STATS_DESC_COUNTER(VCPU, exit_validity),
  82. STATS_DESC_COUNTER(VCPU, exit_instruction),
  83. STATS_DESC_COUNTER(VCPU, exit_pei),
  84. STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
  85. STATS_DESC_COUNTER(VCPU, instruction_lctl),
  86. STATS_DESC_COUNTER(VCPU, instruction_lctlg),
  87. STATS_DESC_COUNTER(VCPU, instruction_stctl),
  88. STATS_DESC_COUNTER(VCPU, instruction_stctg),
  89. STATS_DESC_COUNTER(VCPU, exit_program_interruption),
  90. STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
  91. STATS_DESC_COUNTER(VCPU, exit_operation_exception),
  92. STATS_DESC_COUNTER(VCPU, deliver_ckc),
  93. STATS_DESC_COUNTER(VCPU, deliver_cputm),
  94. STATS_DESC_COUNTER(VCPU, deliver_external_call),
  95. STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
  96. STATS_DESC_COUNTER(VCPU, deliver_service_signal),
  97. STATS_DESC_COUNTER(VCPU, deliver_virtio),
  98. STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
  99. STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
  100. STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
  101. STATS_DESC_COUNTER(VCPU, deliver_program),
  102. STATS_DESC_COUNTER(VCPU, deliver_io),
  103. STATS_DESC_COUNTER(VCPU, deliver_machine_check),
  104. STATS_DESC_COUNTER(VCPU, exit_wait_state),
  105. STATS_DESC_COUNTER(VCPU, inject_ckc),
  106. STATS_DESC_COUNTER(VCPU, inject_cputm),
  107. STATS_DESC_COUNTER(VCPU, inject_external_call),
  108. STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
  109. STATS_DESC_COUNTER(VCPU, inject_mchk),
  110. STATS_DESC_COUNTER(VCPU, inject_pfault_init),
  111. STATS_DESC_COUNTER(VCPU, inject_program),
  112. STATS_DESC_COUNTER(VCPU, inject_restart),
  113. STATS_DESC_COUNTER(VCPU, inject_set_prefix),
  114. STATS_DESC_COUNTER(VCPU, inject_stop_signal),
  115. STATS_DESC_COUNTER(VCPU, instruction_epsw),
  116. STATS_DESC_COUNTER(VCPU, instruction_gs),
  117. STATS_DESC_COUNTER(VCPU, instruction_io_other),
  118. STATS_DESC_COUNTER(VCPU, instruction_lpsw),
  119. STATS_DESC_COUNTER(VCPU, instruction_lpswe),
  120. STATS_DESC_COUNTER(VCPU, instruction_pfmf),
  121. STATS_DESC_COUNTER(VCPU, instruction_ptff),
  122. STATS_DESC_COUNTER(VCPU, instruction_sck),
  123. STATS_DESC_COUNTER(VCPU, instruction_sckpf),
  124. STATS_DESC_COUNTER(VCPU, instruction_stidp),
  125. STATS_DESC_COUNTER(VCPU, instruction_spx),
  126. STATS_DESC_COUNTER(VCPU, instruction_stpx),
  127. STATS_DESC_COUNTER(VCPU, instruction_stap),
  128. STATS_DESC_COUNTER(VCPU, instruction_iske),
  129. STATS_DESC_COUNTER(VCPU, instruction_ri),
  130. STATS_DESC_COUNTER(VCPU, instruction_rrbe),
  131. STATS_DESC_COUNTER(VCPU, instruction_sske),
  132. STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
  133. STATS_DESC_COUNTER(VCPU, instruction_stsi),
  134. STATS_DESC_COUNTER(VCPU, instruction_stfl),
  135. STATS_DESC_COUNTER(VCPU, instruction_tb),
  136. STATS_DESC_COUNTER(VCPU, instruction_tpi),
  137. STATS_DESC_COUNTER(VCPU, instruction_tprot),
  138. STATS_DESC_COUNTER(VCPU, instruction_tsch),
  139. STATS_DESC_COUNTER(VCPU, instruction_sie),
  140. STATS_DESC_COUNTER(VCPU, instruction_essa),
  141. STATS_DESC_COUNTER(VCPU, instruction_sthyi),
  142. STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
  143. STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
  144. STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
  145. STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
  146. STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
  147. STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
  148. STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
  149. STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
  150. STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
  151. STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
  152. STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
  153. STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
  154. STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
  155. STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
  156. STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
  157. STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
  158. STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
  159. STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
  160. STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
  161. STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
  162. STATS_DESC_COUNTER(VCPU, diag_9c_forward),
  163. STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
  164. STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
  165. STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
  166. STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
  167. STATS_DESC_COUNTER(VCPU, pfault_sync)
  168. };
  169. const struct kvm_stats_header kvm_vcpu_stats_header = {
  170. .name_size = KVM_STATS_NAME_SIZE,
  171. .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
  172. .id_offset = sizeof(struct kvm_stats_header),
  173. .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
  174. .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
  175. sizeof(kvm_vcpu_stats_desc),
  176. };
  177. /* allow nested virtualization in KVM (if enabled by user space) */
  178. static int nested;
  179. module_param(nested, int, S_IRUGO);
  180. MODULE_PARM_DESC(nested, "Nested virtualization support");
  181. /* allow 1m huge page guest backing, if !nested */
  182. static int hpage;
  183. module_param(hpage, int, 0444);
  184. MODULE_PARM_DESC(hpage, "1m huge page backing support");
  185. /* maximum percentage of steal time for polling. >100 is treated like 100 */
  186. static u8 halt_poll_max_steal = 10;
  187. module_param(halt_poll_max_steal, byte, 0644);
  188. MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
  189. /* if set to true, the GISA will be initialized and used if available */
  190. static bool use_gisa = true;
  191. module_param(use_gisa, bool, 0644);
  192. MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
  193. /* maximum diag9c forwarding per second */
  194. unsigned int diag9c_forwarding_hz;
  195. module_param(diag9c_forwarding_hz, uint, 0644);
  196. MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
  197. /*
  198. * For now we handle at most 16 double words as this is what the s390 base
  199. * kernel handles and stores in the prefix page. If we ever need to go beyond
  200. * this, this requires changes to code, but the external uapi can stay.
  201. */
  202. #define SIZE_INTERNAL 16
  203. /*
  204. * Base feature mask that defines default mask for facilities. Consists of the
  205. * defines in FACILITIES_KVM and the non-hypervisor managed bits.
  206. */
  207. static unsigned long kvm_s390_fac_base[SIZE_INTERNAL] = { FACILITIES_KVM };
  208. /*
  209. * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
  210. * and defines the facilities that can be enabled via a cpu model.
  211. */
  212. static unsigned long kvm_s390_fac_ext[SIZE_INTERNAL] = { FACILITIES_KVM_CPUMODEL };
  213. static unsigned long kvm_s390_fac_size(void)
  214. {
  215. BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_MASK_SIZE_U64);
  216. BUILD_BUG_ON(SIZE_INTERNAL > S390_ARCH_FAC_LIST_SIZE_U64);
  217. BUILD_BUG_ON(SIZE_INTERNAL * sizeof(unsigned long) >
  218. sizeof(stfle_fac_list));
  219. return SIZE_INTERNAL;
  220. }
  221. /* available cpu features supported by kvm */
  222. static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
  223. /* available subfunctions indicated via query / "test bit" */
  224. static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
  225. static struct gmap_notifier gmap_notifier;
  226. static struct gmap_notifier vsie_gmap_notifier;
  227. debug_info_t *kvm_s390_dbf;
  228. debug_info_t *kvm_s390_dbf_uv;
  229. /* Section: not file related */
  230. int kvm_arch_hardware_enable(void)
  231. {
  232. /* every s390 is virtualization enabled ;-) */
  233. return 0;
  234. }
  235. int kvm_arch_check_processor_compat(void *opaque)
  236. {
  237. return 0;
  238. }
  239. /* forward declarations */
  240. static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
  241. unsigned long end);
  242. static int sca_switch_to_extended(struct kvm *kvm);
  243. static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
  244. {
  245. u8 delta_idx = 0;
  246. /*
  247. * The TOD jumps by delta, we have to compensate this by adding
  248. * -delta to the epoch.
  249. */
  250. delta = -delta;
  251. /* sign-extension - we're adding to signed values below */
  252. if ((s64)delta < 0)
  253. delta_idx = -1;
  254. scb->epoch += delta;
  255. if (scb->ecd & ECD_MEF) {
  256. scb->epdx += delta_idx;
  257. if (scb->epoch < delta)
  258. scb->epdx += 1;
  259. }
  260. }
  261. /*
  262. * This callback is executed during stop_machine(). All CPUs are therefore
  263. * temporarily stopped. In order not to change guest behavior, we have to
  264. * disable preemption whenever we touch the epoch of kvm and the VCPUs,
  265. * so a CPU won't be stopped while calculating with the epoch.
  266. */
  267. static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
  268. void *v)
  269. {
  270. struct kvm *kvm;
  271. struct kvm_vcpu *vcpu;
  272. unsigned long i;
  273. unsigned long long *delta = v;
  274. list_for_each_entry(kvm, &vm_list, vm_list) {
  275. kvm_for_each_vcpu(i, vcpu, kvm) {
  276. kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
  277. if (i == 0) {
  278. kvm->arch.epoch = vcpu->arch.sie_block->epoch;
  279. kvm->arch.epdx = vcpu->arch.sie_block->epdx;
  280. }
  281. if (vcpu->arch.cputm_enabled)
  282. vcpu->arch.cputm_start += *delta;
  283. if (vcpu->arch.vsie_block)
  284. kvm_clock_sync_scb(vcpu->arch.vsie_block,
  285. *delta);
  286. }
  287. }
  288. return NOTIFY_OK;
  289. }
  290. static struct notifier_block kvm_clock_notifier = {
  291. .notifier_call = kvm_clock_sync,
  292. };
  293. int kvm_arch_hardware_setup(void *opaque)
  294. {
  295. gmap_notifier.notifier_call = kvm_gmap_notifier;
  296. gmap_register_pte_notifier(&gmap_notifier);
  297. vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier;
  298. gmap_register_pte_notifier(&vsie_gmap_notifier);
  299. atomic_notifier_chain_register(&s390_epoch_delta_notifier,
  300. &kvm_clock_notifier);
  301. return 0;
  302. }
  303. void kvm_arch_hardware_unsetup(void)
  304. {
  305. gmap_unregister_pte_notifier(&gmap_notifier);
  306. gmap_unregister_pte_notifier(&vsie_gmap_notifier);
  307. atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
  308. &kvm_clock_notifier);
  309. }
  310. static void allow_cpu_feat(unsigned long nr)
  311. {
  312. set_bit_inv(nr, kvm_s390_available_cpu_feat);
  313. }
  314. static inline int plo_test_bit(unsigned char nr)
  315. {
  316. unsigned long function = (unsigned long)nr | 0x100;
  317. int cc;
  318. asm volatile(
  319. " lgr 0,%[function]\n"
  320. /* Parameter registers are ignored for "test bit" */
  321. " plo 0,0,0,0(0)\n"
  322. " ipm %0\n"
  323. " srl %0,28\n"
  324. : "=d" (cc)
  325. : [function] "d" (function)
  326. : "cc", "0");
  327. return cc == 0;
  328. }
  329. static __always_inline void __insn32_query(unsigned int opcode, u8 *query)
  330. {
  331. asm volatile(
  332. " lghi 0,0\n"
  333. " lgr 1,%[query]\n"
  334. /* Parameter registers are ignored */
  335. " .insn rrf,%[opc] << 16,2,4,6,0\n"
  336. :
  337. : [query] "d" ((unsigned long)query), [opc] "i" (opcode)
  338. : "cc", "memory", "0", "1");
  339. }
  340. #define INSN_SORTL 0xb938
  341. #define INSN_DFLTCC 0xb939
  342. static void kvm_s390_cpu_feat_init(void)
  343. {
  344. int i;
  345. for (i = 0; i < 256; ++i) {
  346. if (plo_test_bit(i))
  347. kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
  348. }
  349. if (test_facility(28)) /* TOD-clock steering */
  350. ptff(kvm_s390_available_subfunc.ptff,
  351. sizeof(kvm_s390_available_subfunc.ptff),
  352. PTFF_QAF);
  353. if (test_facility(17)) { /* MSA */
  354. __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
  355. kvm_s390_available_subfunc.kmac);
  356. __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
  357. kvm_s390_available_subfunc.kmc);
  358. __cpacf_query(CPACF_KM, (cpacf_mask_t *)
  359. kvm_s390_available_subfunc.km);
  360. __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
  361. kvm_s390_available_subfunc.kimd);
  362. __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
  363. kvm_s390_available_subfunc.klmd);
  364. }
  365. if (test_facility(76)) /* MSA3 */
  366. __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
  367. kvm_s390_available_subfunc.pckmo);
  368. if (test_facility(77)) { /* MSA4 */
  369. __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
  370. kvm_s390_available_subfunc.kmctr);
  371. __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
  372. kvm_s390_available_subfunc.kmf);
  373. __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
  374. kvm_s390_available_subfunc.kmo);
  375. __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
  376. kvm_s390_available_subfunc.pcc);
  377. }
  378. if (test_facility(57)) /* MSA5 */
  379. __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
  380. kvm_s390_available_subfunc.ppno);
  381. if (test_facility(146)) /* MSA8 */
  382. __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
  383. kvm_s390_available_subfunc.kma);
  384. if (test_facility(155)) /* MSA9 */
  385. __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
  386. kvm_s390_available_subfunc.kdsa);
  387. if (test_facility(150)) /* SORTL */
  388. __insn32_query(INSN_SORTL, kvm_s390_available_subfunc.sortl);
  389. if (test_facility(151)) /* DFLTCC */
  390. __insn32_query(INSN_DFLTCC, kvm_s390_available_subfunc.dfltcc);
  391. if (MACHINE_HAS_ESOP)
  392. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
  393. /*
  394. * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
  395. * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
  396. */
  397. if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao ||
  398. !test_facility(3) || !nested)
  399. return;
  400. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
  401. if (sclp.has_64bscao)
  402. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
  403. if (sclp.has_siif)
  404. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
  405. if (sclp.has_gpere)
  406. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
  407. if (sclp.has_gsls)
  408. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
  409. if (sclp.has_ib)
  410. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
  411. if (sclp.has_cei)
  412. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
  413. if (sclp.has_ibs)
  414. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
  415. if (sclp.has_kss)
  416. allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
  417. /*
  418. * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
  419. * all skey handling functions read/set the skey from the PGSTE
  420. * instead of the real storage key.
  421. *
  422. * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
  423. * pages being detected as preserved although they are resident.
  424. *
  425. * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
  426. * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
  427. *
  428. * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
  429. * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
  430. * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
  431. *
  432. * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
  433. * cannot easily shadow the SCA because of the ipte lock.
  434. */
  435. }
  436. int kvm_arch_init(void *opaque)
  437. {
  438. int rc = -ENOMEM;
  439. kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
  440. if (!kvm_s390_dbf)
  441. return -ENOMEM;
  442. kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
  443. if (!kvm_s390_dbf_uv)
  444. goto out;
  445. if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
  446. debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
  447. goto out;
  448. kvm_s390_cpu_feat_init();
  449. /* Register floating interrupt controller interface. */
  450. rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
  451. if (rc) {
  452. pr_err("A FLIC registration call failed with rc=%d\n", rc);
  453. goto out;
  454. }
  455. if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
  456. rc = kvm_s390_pci_init();
  457. if (rc) {
  458. pr_err("Unable to allocate AIFT for PCI\n");
  459. goto out;
  460. }
  461. }
  462. rc = kvm_s390_gib_init(GAL_ISC);
  463. if (rc)
  464. goto out;
  465. return 0;
  466. out:
  467. kvm_arch_exit();
  468. return rc;
  469. }
  470. void kvm_arch_exit(void)
  471. {
  472. kvm_s390_gib_destroy();
  473. if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
  474. kvm_s390_pci_exit();
  475. debug_unregister(kvm_s390_dbf);
  476. debug_unregister(kvm_s390_dbf_uv);
  477. }
  478. /* Section: device related */
  479. long kvm_arch_dev_ioctl(struct file *filp,
  480. unsigned int ioctl, unsigned long arg)
  481. {
  482. if (ioctl == KVM_S390_ENABLE_SIE)
  483. return s390_enable_sie();
  484. return -EINVAL;
  485. }
  486. int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
  487. {
  488. int r;
  489. switch (ext) {
  490. case KVM_CAP_S390_PSW:
  491. case KVM_CAP_S390_GMAP:
  492. case KVM_CAP_SYNC_MMU:
  493. #ifdef CONFIG_KVM_S390_UCONTROL
  494. case KVM_CAP_S390_UCONTROL:
  495. #endif
  496. case KVM_CAP_ASYNC_PF:
  497. case KVM_CAP_SYNC_REGS:
  498. case KVM_CAP_ONE_REG:
  499. case KVM_CAP_ENABLE_CAP:
  500. case KVM_CAP_S390_CSS_SUPPORT:
  501. case KVM_CAP_IOEVENTFD:
  502. case KVM_CAP_DEVICE_CTRL:
  503. case KVM_CAP_S390_IRQCHIP:
  504. case KVM_CAP_VM_ATTRIBUTES:
  505. case KVM_CAP_MP_STATE:
  506. case KVM_CAP_IMMEDIATE_EXIT:
  507. case KVM_CAP_S390_INJECT_IRQ:
  508. case KVM_CAP_S390_USER_SIGP:
  509. case KVM_CAP_S390_USER_STSI:
  510. case KVM_CAP_S390_SKEYS:
  511. case KVM_CAP_S390_IRQ_STATE:
  512. case KVM_CAP_S390_USER_INSTR0:
  513. case KVM_CAP_S390_CMMA_MIGRATION:
  514. case KVM_CAP_S390_AIS:
  515. case KVM_CAP_S390_AIS_MIGRATION:
  516. case KVM_CAP_S390_VCPU_RESETS:
  517. case KVM_CAP_SET_GUEST_DEBUG:
  518. case KVM_CAP_S390_DIAG318:
  519. case KVM_CAP_S390_MEM_OP_EXTENSION:
  520. r = 1;
  521. break;
  522. case KVM_CAP_SET_GUEST_DEBUG2:
  523. r = KVM_GUESTDBG_VALID_MASK;
  524. break;
  525. case KVM_CAP_S390_HPAGE_1M:
  526. r = 0;
  527. if (hpage && !kvm_is_ucontrol(kvm))
  528. r = 1;
  529. break;
  530. case KVM_CAP_S390_MEM_OP:
  531. r = MEM_OP_MAX_SIZE;
  532. break;
  533. case KVM_CAP_NR_VCPUS:
  534. case KVM_CAP_MAX_VCPUS:
  535. case KVM_CAP_MAX_VCPU_ID:
  536. r = KVM_S390_BSCA_CPU_SLOTS;
  537. if (!kvm_s390_use_sca_entries())
  538. r = KVM_MAX_VCPUS;
  539. else if (sclp.has_esca && sclp.has_64bscao)
  540. r = KVM_S390_ESCA_CPU_SLOTS;
  541. if (ext == KVM_CAP_NR_VCPUS)
  542. r = min_t(unsigned int, num_online_cpus(), r);
  543. break;
  544. case KVM_CAP_S390_COW:
  545. r = MACHINE_HAS_ESOP;
  546. break;
  547. case KVM_CAP_S390_VECTOR_REGISTERS:
  548. r = MACHINE_HAS_VX;
  549. break;
  550. case KVM_CAP_S390_RI:
  551. r = test_facility(64);
  552. break;
  553. case KVM_CAP_S390_GS:
  554. r = test_facility(133);
  555. break;
  556. case KVM_CAP_S390_BPB:
  557. r = test_facility(82);
  558. break;
  559. case KVM_CAP_S390_PROTECTED:
  560. r = is_prot_virt_host();
  561. break;
  562. case KVM_CAP_S390_PROTECTED_DUMP: {
  563. u64 pv_cmds_dump[] = {
  564. BIT_UVC_CMD_DUMP_INIT,
  565. BIT_UVC_CMD_DUMP_CONFIG_STOR_STATE,
  566. BIT_UVC_CMD_DUMP_CPU,
  567. BIT_UVC_CMD_DUMP_COMPLETE,
  568. };
  569. int i;
  570. r = is_prot_virt_host();
  571. for (i = 0; i < ARRAY_SIZE(pv_cmds_dump); i++) {
  572. if (!test_bit_inv(pv_cmds_dump[i],
  573. (unsigned long *)&uv_info.inst_calls_list)) {
  574. r = 0;
  575. break;
  576. }
  577. }
  578. break;
  579. }
  580. case KVM_CAP_S390_ZPCI_OP:
  581. r = kvm_s390_pci_interp_allowed();
  582. break;
  583. case KVM_CAP_S390_CPU_TOPOLOGY:
  584. r = test_facility(11);
  585. break;
  586. default:
  587. r = 0;
  588. }
  589. return r;
  590. }
  591. void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
  592. {
  593. int i;
  594. gfn_t cur_gfn, last_gfn;
  595. unsigned long gaddr, vmaddr;
  596. struct gmap *gmap = kvm->arch.gmap;
  597. DECLARE_BITMAP(bitmap, _PAGE_ENTRIES);
  598. /* Loop over all guest segments */
  599. cur_gfn = memslot->base_gfn;
  600. last_gfn = memslot->base_gfn + memslot->npages;
  601. for (; cur_gfn <= last_gfn; cur_gfn += _PAGE_ENTRIES) {
  602. gaddr = gfn_to_gpa(cur_gfn);
  603. vmaddr = gfn_to_hva_memslot(memslot, cur_gfn);
  604. if (kvm_is_error_hva(vmaddr))
  605. continue;
  606. bitmap_zero(bitmap, _PAGE_ENTRIES);
  607. gmap_sync_dirty_log_pmd(gmap, bitmap, gaddr, vmaddr);
  608. for (i = 0; i < _PAGE_ENTRIES; i++) {
  609. if (test_bit(i, bitmap))
  610. mark_page_dirty(kvm, cur_gfn + i);
  611. }
  612. if (fatal_signal_pending(current))
  613. return;
  614. cond_resched();
  615. }
  616. }
  617. /* Section: vm related */
  618. static void sca_del_vcpu(struct kvm_vcpu *vcpu);
  619. /*
  620. * Get (and clear) the dirty memory log for a memory slot.
  621. */
  622. int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
  623. struct kvm_dirty_log *log)
  624. {
  625. int r;
  626. unsigned long n;
  627. struct kvm_memory_slot *memslot;
  628. int is_dirty;
  629. if (kvm_is_ucontrol(kvm))
  630. return -EINVAL;
  631. mutex_lock(&kvm->slots_lock);
  632. r = -EINVAL;
  633. if (log->slot >= KVM_USER_MEM_SLOTS)
  634. goto out;
  635. r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
  636. if (r)
  637. goto out;
  638. /* Clear the dirty log */
  639. if (is_dirty) {
  640. n = kvm_dirty_bitmap_bytes(memslot);
  641. memset(memslot->dirty_bitmap, 0, n);
  642. }
  643. r = 0;
  644. out:
  645. mutex_unlock(&kvm->slots_lock);
  646. return r;
  647. }
  648. static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
  649. {
  650. unsigned long i;
  651. struct kvm_vcpu *vcpu;
  652. kvm_for_each_vcpu(i, vcpu, kvm) {
  653. kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
  654. }
  655. }
  656. int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
  657. {
  658. int r;
  659. if (cap->flags)
  660. return -EINVAL;
  661. switch (cap->cap) {
  662. case KVM_CAP_S390_IRQCHIP:
  663. VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
  664. kvm->arch.use_irqchip = 1;
  665. r = 0;
  666. break;
  667. case KVM_CAP_S390_USER_SIGP:
  668. VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
  669. kvm->arch.user_sigp = 1;
  670. r = 0;
  671. break;
  672. case KVM_CAP_S390_VECTOR_REGISTERS:
  673. mutex_lock(&kvm->lock);
  674. if (kvm->created_vcpus) {
  675. r = -EBUSY;
  676. } else if (MACHINE_HAS_VX) {
  677. set_kvm_facility(kvm->arch.model.fac_mask, 129);
  678. set_kvm_facility(kvm->arch.model.fac_list, 129);
  679. if (test_facility(134)) {
  680. set_kvm_facility(kvm->arch.model.fac_mask, 134);
  681. set_kvm_facility(kvm->arch.model.fac_list, 134);
  682. }
  683. if (test_facility(135)) {
  684. set_kvm_facility(kvm->arch.model.fac_mask, 135);
  685. set_kvm_facility(kvm->arch.model.fac_list, 135);
  686. }
  687. if (test_facility(148)) {
  688. set_kvm_facility(kvm->arch.model.fac_mask, 148);
  689. set_kvm_facility(kvm->arch.model.fac_list, 148);
  690. }
  691. if (test_facility(152)) {
  692. set_kvm_facility(kvm->arch.model.fac_mask, 152);
  693. set_kvm_facility(kvm->arch.model.fac_list, 152);
  694. }
  695. if (test_facility(192)) {
  696. set_kvm_facility(kvm->arch.model.fac_mask, 192);
  697. set_kvm_facility(kvm->arch.model.fac_list, 192);
  698. }
  699. r = 0;
  700. } else
  701. r = -EINVAL;
  702. mutex_unlock(&kvm->lock);
  703. VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
  704. r ? "(not available)" : "(success)");
  705. break;
  706. case KVM_CAP_S390_RI:
  707. r = -EINVAL;
  708. mutex_lock(&kvm->lock);
  709. if (kvm->created_vcpus) {
  710. r = -EBUSY;
  711. } else if (test_facility(64)) {
  712. set_kvm_facility(kvm->arch.model.fac_mask, 64);
  713. set_kvm_facility(kvm->arch.model.fac_list, 64);
  714. r = 0;
  715. }
  716. mutex_unlock(&kvm->lock);
  717. VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
  718. r ? "(not available)" : "(success)");
  719. break;
  720. case KVM_CAP_S390_AIS:
  721. mutex_lock(&kvm->lock);
  722. if (kvm->created_vcpus) {
  723. r = -EBUSY;
  724. } else {
  725. set_kvm_facility(kvm->arch.model.fac_mask, 72);
  726. set_kvm_facility(kvm->arch.model.fac_list, 72);
  727. r = 0;
  728. }
  729. mutex_unlock(&kvm->lock);
  730. VM_EVENT(kvm, 3, "ENABLE: AIS %s",
  731. r ? "(not available)" : "(success)");
  732. break;
  733. case KVM_CAP_S390_GS:
  734. r = -EINVAL;
  735. mutex_lock(&kvm->lock);
  736. if (kvm->created_vcpus) {
  737. r = -EBUSY;
  738. } else if (test_facility(133)) {
  739. set_kvm_facility(kvm->arch.model.fac_mask, 133);
  740. set_kvm_facility(kvm->arch.model.fac_list, 133);
  741. r = 0;
  742. }
  743. mutex_unlock(&kvm->lock);
  744. VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
  745. r ? "(not available)" : "(success)");
  746. break;
  747. case KVM_CAP_S390_HPAGE_1M:
  748. mutex_lock(&kvm->lock);
  749. if (kvm->created_vcpus)
  750. r = -EBUSY;
  751. else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
  752. r = -EINVAL;
  753. else {
  754. r = 0;
  755. mmap_write_lock(kvm->mm);
  756. kvm->mm->context.allow_gmap_hpage_1m = 1;
  757. mmap_write_unlock(kvm->mm);
  758. /*
  759. * We might have to create fake 4k page
  760. * tables. To avoid that the hardware works on
  761. * stale PGSTEs, we emulate these instructions.
  762. */
  763. kvm->arch.use_skf = 0;
  764. kvm->arch.use_pfmfi = 0;
  765. }
  766. mutex_unlock(&kvm->lock);
  767. VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
  768. r ? "(not available)" : "(success)");
  769. break;
  770. case KVM_CAP_S390_USER_STSI:
  771. VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
  772. kvm->arch.user_stsi = 1;
  773. r = 0;
  774. break;
  775. case KVM_CAP_S390_USER_INSTR0:
  776. VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
  777. kvm->arch.user_instr0 = 1;
  778. icpt_operexc_on_all_vcpus(kvm);
  779. r = 0;
  780. break;
  781. case KVM_CAP_S390_CPU_TOPOLOGY:
  782. r = -EINVAL;
  783. mutex_lock(&kvm->lock);
  784. if (kvm->created_vcpus) {
  785. r = -EBUSY;
  786. } else if (test_facility(11)) {
  787. set_kvm_facility(kvm->arch.model.fac_mask, 11);
  788. set_kvm_facility(kvm->arch.model.fac_list, 11);
  789. r = 0;
  790. }
  791. mutex_unlock(&kvm->lock);
  792. VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
  793. r ? "(not available)" : "(success)");
  794. break;
  795. default:
  796. r = -EINVAL;
  797. break;
  798. }
  799. return r;
  800. }
  801. static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
  802. {
  803. int ret;
  804. switch (attr->attr) {
  805. case KVM_S390_VM_MEM_LIMIT_SIZE:
  806. ret = 0;
  807. VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
  808. kvm->arch.mem_limit);
  809. if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
  810. ret = -EFAULT;
  811. break;
  812. default:
  813. ret = -ENXIO;
  814. break;
  815. }
  816. return ret;
  817. }
  818. static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
  819. {
  820. int ret;
  821. unsigned int idx;
  822. switch (attr->attr) {
  823. case KVM_S390_VM_MEM_ENABLE_CMMA:
  824. ret = -ENXIO;
  825. if (!sclp.has_cmma)
  826. break;
  827. VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
  828. mutex_lock(&kvm->lock);
  829. if (kvm->created_vcpus)
  830. ret = -EBUSY;
  831. else if (kvm->mm->context.allow_gmap_hpage_1m)
  832. ret = -EINVAL;
  833. else {
  834. kvm->arch.use_cmma = 1;
  835. /* Not compatible with cmma. */
  836. kvm->arch.use_pfmfi = 0;
  837. ret = 0;
  838. }
  839. mutex_unlock(&kvm->lock);
  840. break;
  841. case KVM_S390_VM_MEM_CLR_CMMA:
  842. ret = -ENXIO;
  843. if (!sclp.has_cmma)
  844. break;
  845. ret = -EINVAL;
  846. if (!kvm->arch.use_cmma)
  847. break;
  848. VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
  849. mutex_lock(&kvm->lock);
  850. idx = srcu_read_lock(&kvm->srcu);
  851. s390_reset_cmma(kvm->arch.gmap->mm);
  852. srcu_read_unlock(&kvm->srcu, idx);
  853. mutex_unlock(&kvm->lock);
  854. ret = 0;
  855. break;
  856. case KVM_S390_VM_MEM_LIMIT_SIZE: {
  857. unsigned long new_limit;
  858. if (kvm_is_ucontrol(kvm))
  859. return -EINVAL;
  860. if (get_user(new_limit, (u64 __user *)attr->addr))
  861. return -EFAULT;
  862. if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
  863. new_limit > kvm->arch.mem_limit)
  864. return -E2BIG;
  865. if (!new_limit)
  866. return -EINVAL;
  867. /* gmap_create takes last usable address */
  868. if (new_limit != KVM_S390_NO_MEM_LIMIT)
  869. new_limit -= 1;
  870. ret = -EBUSY;
  871. mutex_lock(&kvm->lock);
  872. if (!kvm->created_vcpus) {
  873. /* gmap_create will round the limit up */
  874. struct gmap *new = gmap_create(current->mm, new_limit);
  875. if (!new) {
  876. ret = -ENOMEM;
  877. } else {
  878. gmap_remove(kvm->arch.gmap);
  879. new->private = kvm;
  880. kvm->arch.gmap = new;
  881. ret = 0;
  882. }
  883. }
  884. mutex_unlock(&kvm->lock);
  885. VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
  886. VM_EVENT(kvm, 3, "New guest asce: 0x%pK",
  887. (void *) kvm->arch.gmap->asce);
  888. break;
  889. }
  890. default:
  891. ret = -ENXIO;
  892. break;
  893. }
  894. return ret;
  895. }
  896. static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
  897. void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
  898. {
  899. struct kvm_vcpu *vcpu;
  900. unsigned long i;
  901. kvm_s390_vcpu_block_all(kvm);
  902. kvm_for_each_vcpu(i, vcpu, kvm) {
  903. kvm_s390_vcpu_crypto_setup(vcpu);
  904. /* recreate the shadow crycb by leaving the VSIE handler */
  905. kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
  906. }
  907. kvm_s390_vcpu_unblock_all(kvm);
  908. }
  909. static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
  910. {
  911. mutex_lock(&kvm->lock);
  912. switch (attr->attr) {
  913. case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
  914. if (!test_kvm_facility(kvm, 76)) {
  915. mutex_unlock(&kvm->lock);
  916. return -EINVAL;
  917. }
  918. get_random_bytes(
  919. kvm->arch.crypto.crycb->aes_wrapping_key_mask,
  920. sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
  921. kvm->arch.crypto.aes_kw = 1;
  922. VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
  923. break;
  924. case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
  925. if (!test_kvm_facility(kvm, 76)) {
  926. mutex_unlock(&kvm->lock);
  927. return -EINVAL;
  928. }
  929. get_random_bytes(
  930. kvm->arch.crypto.crycb->dea_wrapping_key_mask,
  931. sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
  932. kvm->arch.crypto.dea_kw = 1;
  933. VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
  934. break;
  935. case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
  936. if (!test_kvm_facility(kvm, 76)) {
  937. mutex_unlock(&kvm->lock);
  938. return -EINVAL;
  939. }
  940. kvm->arch.crypto.aes_kw = 0;
  941. memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
  942. sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
  943. VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
  944. break;
  945. case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
  946. if (!test_kvm_facility(kvm, 76)) {
  947. mutex_unlock(&kvm->lock);
  948. return -EINVAL;
  949. }
  950. kvm->arch.crypto.dea_kw = 0;
  951. memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
  952. sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
  953. VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
  954. break;
  955. case KVM_S390_VM_CRYPTO_ENABLE_APIE:
  956. if (!ap_instructions_available()) {
  957. mutex_unlock(&kvm->lock);
  958. return -EOPNOTSUPP;
  959. }
  960. kvm->arch.crypto.apie = 1;
  961. break;
  962. case KVM_S390_VM_CRYPTO_DISABLE_APIE:
  963. if (!ap_instructions_available()) {
  964. mutex_unlock(&kvm->lock);
  965. return -EOPNOTSUPP;
  966. }
  967. kvm->arch.crypto.apie = 0;
  968. break;
  969. default:
  970. mutex_unlock(&kvm->lock);
  971. return -ENXIO;
  972. }
  973. kvm_s390_vcpu_crypto_reset_all(kvm);
  974. mutex_unlock(&kvm->lock);
  975. return 0;
  976. }
  977. static void kvm_s390_vcpu_pci_setup(struct kvm_vcpu *vcpu)
  978. {
  979. /* Only set the ECB bits after guest requests zPCI interpretation */
  980. if (!vcpu->kvm->arch.use_zpci_interp)
  981. return;
  982. vcpu->arch.sie_block->ecb2 |= ECB2_ZPCI_LSI;
  983. vcpu->arch.sie_block->ecb3 |= ECB3_AISII + ECB3_AISI;
  984. }
  985. void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
  986. {
  987. struct kvm_vcpu *vcpu;
  988. unsigned long i;
  989. lockdep_assert_held(&kvm->lock);
  990. if (!kvm_s390_pci_interp_allowed())
  991. return;
  992. /*
  993. * If host is configured for PCI and the necessary facilities are
  994. * available, turn on interpretation for the life of this guest
  995. */
  996. kvm->arch.use_zpci_interp = 1;
  997. kvm_s390_vcpu_block_all(kvm);
  998. kvm_for_each_vcpu(i, vcpu, kvm) {
  999. kvm_s390_vcpu_pci_setup(vcpu);
  1000. kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
  1001. }
  1002. kvm_s390_vcpu_unblock_all(kvm);
  1003. }
  1004. static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
  1005. {
  1006. unsigned long cx;
  1007. struct kvm_vcpu *vcpu;
  1008. kvm_for_each_vcpu(cx, vcpu, kvm)
  1009. kvm_s390_sync_request(req, vcpu);
  1010. }
  1011. /*
  1012. * Must be called with kvm->srcu held to avoid races on memslots, and with
  1013. * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration.
  1014. */
  1015. static int kvm_s390_vm_start_migration(struct kvm *kvm)
  1016. {
  1017. struct kvm_memory_slot *ms;
  1018. struct kvm_memslots *slots;
  1019. unsigned long ram_pages = 0;
  1020. int bkt;
  1021. /* migration mode already enabled */
  1022. if (kvm->arch.migration_mode)
  1023. return 0;
  1024. slots = kvm_memslots(kvm);
  1025. if (!slots || kvm_memslots_empty(slots))
  1026. return -EINVAL;
  1027. if (!kvm->arch.use_cmma) {
  1028. kvm->arch.migration_mode = 1;
  1029. return 0;
  1030. }
  1031. /* mark all the pages in active slots as dirty */
  1032. kvm_for_each_memslot(ms, bkt, slots) {
  1033. if (!ms->dirty_bitmap)
  1034. return -EINVAL;
  1035. /*
  1036. * The second half of the bitmap is only used on x86,
  1037. * and would be wasted otherwise, so we put it to good
  1038. * use here to keep track of the state of the storage
  1039. * attributes.
  1040. */
  1041. memset(kvm_second_dirty_bitmap(ms), 0xff, kvm_dirty_bitmap_bytes(ms));
  1042. ram_pages += ms->npages;
  1043. }
  1044. atomic64_set(&kvm->arch.cmma_dirty_pages, ram_pages);
  1045. kvm->arch.migration_mode = 1;
  1046. kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
  1047. return 0;
  1048. }
  1049. /*
  1050. * Must be called with kvm->slots_lock to avoid races with ourselves and
  1051. * kvm_s390_vm_start_migration.
  1052. */
  1053. static int kvm_s390_vm_stop_migration(struct kvm *kvm)
  1054. {
  1055. /* migration mode already disabled */
  1056. if (!kvm->arch.migration_mode)
  1057. return 0;
  1058. kvm->arch.migration_mode = 0;
  1059. if (kvm->arch.use_cmma)
  1060. kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
  1061. return 0;
  1062. }
  1063. static int kvm_s390_vm_set_migration(struct kvm *kvm,
  1064. struct kvm_device_attr *attr)
  1065. {
  1066. int res = -ENXIO;
  1067. mutex_lock(&kvm->slots_lock);
  1068. switch (attr->attr) {
  1069. case KVM_S390_VM_MIGRATION_START:
  1070. res = kvm_s390_vm_start_migration(kvm);
  1071. break;
  1072. case KVM_S390_VM_MIGRATION_STOP:
  1073. res = kvm_s390_vm_stop_migration(kvm);
  1074. break;
  1075. default:
  1076. break;
  1077. }
  1078. mutex_unlock(&kvm->slots_lock);
  1079. return res;
  1080. }
  1081. static int kvm_s390_vm_get_migration(struct kvm *kvm,
  1082. struct kvm_device_attr *attr)
  1083. {
  1084. u64 mig = kvm->arch.migration_mode;
  1085. if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
  1086. return -ENXIO;
  1087. if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
  1088. return -EFAULT;
  1089. return 0;
  1090. }
  1091. static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
  1092. static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
  1093. {
  1094. struct kvm_s390_vm_tod_clock gtod;
  1095. if (copy_from_user(&gtod, (void __user *)attr->addr, sizeof(gtod)))
  1096. return -EFAULT;
  1097. if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
  1098. return -EINVAL;
  1099. __kvm_s390_set_tod_clock(kvm, &gtod);
  1100. VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
  1101. gtod.epoch_idx, gtod.tod);
  1102. return 0;
  1103. }
  1104. static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
  1105. {
  1106. u8 gtod_high;
  1107. if (copy_from_user(&gtod_high, (void __user *)attr->addr,
  1108. sizeof(gtod_high)))
  1109. return -EFAULT;
  1110. if (gtod_high != 0)
  1111. return -EINVAL;
  1112. VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
  1113. return 0;
  1114. }
  1115. static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
  1116. {
  1117. struct kvm_s390_vm_tod_clock gtod = { 0 };
  1118. if (copy_from_user(&gtod.tod, (void __user *)attr->addr,
  1119. sizeof(gtod.tod)))
  1120. return -EFAULT;
  1121. __kvm_s390_set_tod_clock(kvm, &gtod);
  1122. VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
  1123. return 0;
  1124. }
  1125. static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
  1126. {
  1127. int ret;
  1128. if (attr->flags)
  1129. return -EINVAL;
  1130. mutex_lock(&kvm->lock);
  1131. /*
  1132. * For protected guests, the TOD is managed by the ultravisor, so trying
  1133. * to change it will never bring the expected results.
  1134. */
  1135. if (kvm_s390_pv_is_protected(kvm)) {
  1136. ret = -EOPNOTSUPP;
  1137. goto out_unlock;
  1138. }
  1139. switch (attr->attr) {
  1140. case KVM_S390_VM_TOD_EXT:
  1141. ret = kvm_s390_set_tod_ext(kvm, attr);
  1142. break;
  1143. case KVM_S390_VM_TOD_HIGH:
  1144. ret = kvm_s390_set_tod_high(kvm, attr);
  1145. break;
  1146. case KVM_S390_VM_TOD_LOW:
  1147. ret = kvm_s390_set_tod_low(kvm, attr);
  1148. break;
  1149. default:
  1150. ret = -ENXIO;
  1151. break;
  1152. }
  1153. out_unlock:
  1154. mutex_unlock(&kvm->lock);
  1155. return ret;
  1156. }
  1157. static void kvm_s390_get_tod_clock(struct kvm *kvm,
  1158. struct kvm_s390_vm_tod_clock *gtod)
  1159. {
  1160. union tod_clock clk;
  1161. preempt_disable();
  1162. store_tod_clock_ext(&clk);
  1163. gtod->tod = clk.tod + kvm->arch.epoch;
  1164. gtod->epoch_idx = 0;
  1165. if (test_kvm_facility(kvm, 139)) {
  1166. gtod->epoch_idx = clk.ei + kvm->arch.epdx;
  1167. if (gtod->tod < clk.tod)
  1168. gtod->epoch_idx += 1;
  1169. }
  1170. preempt_enable();
  1171. }
  1172. static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
  1173. {
  1174. struct kvm_s390_vm_tod_clock gtod;
  1175. memset(&gtod, 0, sizeof(gtod));
  1176. kvm_s390_get_tod_clock(kvm, &gtod);
  1177. if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
  1178. return -EFAULT;
  1179. VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
  1180. gtod.epoch_idx, gtod.tod);
  1181. return 0;
  1182. }
  1183. static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
  1184. {
  1185. u8 gtod_high = 0;
  1186. if (copy_to_user((void __user *)attr->addr, &gtod_high,
  1187. sizeof(gtod_high)))
  1188. return -EFAULT;
  1189. VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
  1190. return 0;
  1191. }
  1192. static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
  1193. {
  1194. u64 gtod;
  1195. gtod = kvm_s390_get_tod_clock_fast(kvm);
  1196. if (copy_to_user((void __user *)attr->addr, &gtod, sizeof(gtod)))
  1197. return -EFAULT;
  1198. VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
  1199. return 0;
  1200. }
  1201. static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
  1202. {
  1203. int ret;
  1204. if (attr->flags)
  1205. return -EINVAL;
  1206. switch (attr->attr) {
  1207. case KVM_S390_VM_TOD_EXT:
  1208. ret = kvm_s390_get_tod_ext(kvm, attr);
  1209. break;
  1210. case KVM_S390_VM_TOD_HIGH:
  1211. ret = kvm_s390_get_tod_high(kvm, attr);
  1212. break;
  1213. case KVM_S390_VM_TOD_LOW:
  1214. ret = kvm_s390_get_tod_low(kvm, attr);
  1215. break;
  1216. default:
  1217. ret = -ENXIO;
  1218. break;
  1219. }
  1220. return ret;
  1221. }
  1222. static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
  1223. {
  1224. struct kvm_s390_vm_cpu_processor *proc;
  1225. u16 lowest_ibc, unblocked_ibc;
  1226. int ret = 0;
  1227. mutex_lock(&kvm->lock);
  1228. if (kvm->created_vcpus) {
  1229. ret = -EBUSY;
  1230. goto out;
  1231. }
  1232. proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
  1233. if (!proc) {
  1234. ret = -ENOMEM;
  1235. goto out;
  1236. }
  1237. if (!copy_from_user(proc, (void __user *)attr->addr,
  1238. sizeof(*proc))) {
  1239. kvm->arch.model.cpuid = proc->cpuid;
  1240. lowest_ibc = sclp.ibc >> 16 & 0xfff;
  1241. unblocked_ibc = sclp.ibc & 0xfff;
  1242. if (lowest_ibc && proc->ibc) {
  1243. if (proc->ibc > unblocked_ibc)
  1244. kvm->arch.model.ibc = unblocked_ibc;
  1245. else if (proc->ibc < lowest_ibc)
  1246. kvm->arch.model.ibc = lowest_ibc;
  1247. else
  1248. kvm->arch.model.ibc = proc->ibc;
  1249. }
  1250. memcpy(kvm->arch.model.fac_list, proc->fac_list,
  1251. S390_ARCH_FAC_LIST_SIZE_BYTE);
  1252. VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
  1253. kvm->arch.model.ibc,
  1254. kvm->arch.model.cpuid);
  1255. VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
  1256. kvm->arch.model.fac_list[0],
  1257. kvm->arch.model.fac_list[1],
  1258. kvm->arch.model.fac_list[2]);
  1259. } else
  1260. ret = -EFAULT;
  1261. kfree(proc);
  1262. out:
  1263. mutex_unlock(&kvm->lock);
  1264. return ret;
  1265. }
  1266. static int kvm_s390_set_processor_feat(struct kvm *kvm,
  1267. struct kvm_device_attr *attr)
  1268. {
  1269. struct kvm_s390_vm_cpu_feat data;
  1270. if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
  1271. return -EFAULT;
  1272. if (!bitmap_subset((unsigned long *) data.feat,
  1273. kvm_s390_available_cpu_feat,
  1274. KVM_S390_VM_CPU_FEAT_NR_BITS))
  1275. return -EINVAL;
  1276. mutex_lock(&kvm->lock);
  1277. if (kvm->created_vcpus) {
  1278. mutex_unlock(&kvm->lock);
  1279. return -EBUSY;
  1280. }
  1281. bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
  1282. mutex_unlock(&kvm->lock);
  1283. VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
  1284. data.feat[0],
  1285. data.feat[1],
  1286. data.feat[2]);
  1287. return 0;
  1288. }
  1289. static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
  1290. struct kvm_device_attr *attr)
  1291. {
  1292. mutex_lock(&kvm->lock);
  1293. if (kvm->created_vcpus) {
  1294. mutex_unlock(&kvm->lock);
  1295. return -EBUSY;
  1296. }
  1297. if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
  1298. sizeof(struct kvm_s390_vm_cpu_subfunc))) {
  1299. mutex_unlock(&kvm->lock);
  1300. return -EFAULT;
  1301. }
  1302. mutex_unlock(&kvm->lock);
  1303. VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1304. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
  1305. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
  1306. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
  1307. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
  1308. VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
  1309. ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
  1310. ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
  1311. VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
  1312. ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
  1313. ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
  1314. VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
  1315. ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
  1316. ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
  1317. VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
  1318. ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
  1319. ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
  1320. VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
  1321. ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
  1322. ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
  1323. VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
  1324. ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
  1325. ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
  1326. VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
  1327. ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
  1328. ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
  1329. VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
  1330. ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
  1331. ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
  1332. VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
  1333. ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
  1334. ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
  1335. VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
  1336. ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
  1337. ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
  1338. VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
  1339. ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
  1340. ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
  1341. VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
  1342. ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
  1343. ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
  1344. VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
  1345. ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
  1346. ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
  1347. VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
  1348. ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
  1349. ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
  1350. VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1351. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
  1352. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
  1353. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
  1354. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
  1355. VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1356. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
  1357. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
  1358. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
  1359. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
  1360. return 0;
  1361. }
  1362. static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
  1363. {
  1364. int ret = -ENXIO;
  1365. switch (attr->attr) {
  1366. case KVM_S390_VM_CPU_PROCESSOR:
  1367. ret = kvm_s390_set_processor(kvm, attr);
  1368. break;
  1369. case KVM_S390_VM_CPU_PROCESSOR_FEAT:
  1370. ret = kvm_s390_set_processor_feat(kvm, attr);
  1371. break;
  1372. case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
  1373. ret = kvm_s390_set_processor_subfunc(kvm, attr);
  1374. break;
  1375. }
  1376. return ret;
  1377. }
  1378. static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
  1379. {
  1380. struct kvm_s390_vm_cpu_processor *proc;
  1381. int ret = 0;
  1382. proc = kzalloc(sizeof(*proc), GFP_KERNEL_ACCOUNT);
  1383. if (!proc) {
  1384. ret = -ENOMEM;
  1385. goto out;
  1386. }
  1387. proc->cpuid = kvm->arch.model.cpuid;
  1388. proc->ibc = kvm->arch.model.ibc;
  1389. memcpy(&proc->fac_list, kvm->arch.model.fac_list,
  1390. S390_ARCH_FAC_LIST_SIZE_BYTE);
  1391. VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
  1392. kvm->arch.model.ibc,
  1393. kvm->arch.model.cpuid);
  1394. VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
  1395. kvm->arch.model.fac_list[0],
  1396. kvm->arch.model.fac_list[1],
  1397. kvm->arch.model.fac_list[2]);
  1398. if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
  1399. ret = -EFAULT;
  1400. kfree(proc);
  1401. out:
  1402. return ret;
  1403. }
  1404. static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
  1405. {
  1406. struct kvm_s390_vm_cpu_machine *mach;
  1407. int ret = 0;
  1408. mach = kzalloc(sizeof(*mach), GFP_KERNEL_ACCOUNT);
  1409. if (!mach) {
  1410. ret = -ENOMEM;
  1411. goto out;
  1412. }
  1413. get_cpu_id((struct cpuid *) &mach->cpuid);
  1414. mach->ibc = sclp.ibc;
  1415. memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
  1416. S390_ARCH_FAC_LIST_SIZE_BYTE);
  1417. memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
  1418. sizeof(stfle_fac_list));
  1419. VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
  1420. kvm->arch.model.ibc,
  1421. kvm->arch.model.cpuid);
  1422. VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
  1423. mach->fac_mask[0],
  1424. mach->fac_mask[1],
  1425. mach->fac_mask[2]);
  1426. VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
  1427. mach->fac_list[0],
  1428. mach->fac_list[1],
  1429. mach->fac_list[2]);
  1430. if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
  1431. ret = -EFAULT;
  1432. kfree(mach);
  1433. out:
  1434. return ret;
  1435. }
  1436. static int kvm_s390_get_processor_feat(struct kvm *kvm,
  1437. struct kvm_device_attr *attr)
  1438. {
  1439. struct kvm_s390_vm_cpu_feat data;
  1440. bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
  1441. if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
  1442. return -EFAULT;
  1443. VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
  1444. data.feat[0],
  1445. data.feat[1],
  1446. data.feat[2]);
  1447. return 0;
  1448. }
  1449. static int kvm_s390_get_machine_feat(struct kvm *kvm,
  1450. struct kvm_device_attr *attr)
  1451. {
  1452. struct kvm_s390_vm_cpu_feat data;
  1453. bitmap_to_arr64(data.feat, kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
  1454. if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
  1455. return -EFAULT;
  1456. VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
  1457. data.feat[0],
  1458. data.feat[1],
  1459. data.feat[2]);
  1460. return 0;
  1461. }
  1462. static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
  1463. struct kvm_device_attr *attr)
  1464. {
  1465. if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
  1466. sizeof(struct kvm_s390_vm_cpu_subfunc)))
  1467. return -EFAULT;
  1468. VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1469. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
  1470. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
  1471. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
  1472. ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
  1473. VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
  1474. ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
  1475. ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
  1476. VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
  1477. ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
  1478. ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
  1479. VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
  1480. ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
  1481. ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
  1482. VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
  1483. ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
  1484. ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
  1485. VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
  1486. ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
  1487. ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
  1488. VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
  1489. ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
  1490. ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
  1491. VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
  1492. ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
  1493. ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
  1494. VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
  1495. ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
  1496. ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
  1497. VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
  1498. ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
  1499. ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
  1500. VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
  1501. ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
  1502. ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
  1503. VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
  1504. ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
  1505. ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
  1506. VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
  1507. ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
  1508. ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
  1509. VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
  1510. ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
  1511. ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
  1512. VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
  1513. ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
  1514. ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
  1515. VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1516. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
  1517. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
  1518. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
  1519. ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
  1520. VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1521. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
  1522. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
  1523. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
  1524. ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
  1525. return 0;
  1526. }
  1527. static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
  1528. struct kvm_device_attr *attr)
  1529. {
  1530. if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
  1531. sizeof(struct kvm_s390_vm_cpu_subfunc)))
  1532. return -EFAULT;
  1533. VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1534. ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
  1535. ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
  1536. ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
  1537. ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
  1538. VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
  1539. ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
  1540. ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
  1541. VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
  1542. ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
  1543. ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
  1544. VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
  1545. ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
  1546. ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
  1547. VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
  1548. ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
  1549. ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
  1550. VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
  1551. ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
  1552. ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
  1553. VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
  1554. ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
  1555. ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
  1556. VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
  1557. ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
  1558. ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
  1559. VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
  1560. ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
  1561. ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
  1562. VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
  1563. ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
  1564. ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
  1565. VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
  1566. ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
  1567. ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
  1568. VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
  1569. ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
  1570. ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
  1571. VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
  1572. ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
  1573. ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
  1574. VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
  1575. ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
  1576. ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
  1577. VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
  1578. ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
  1579. ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
  1580. VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1581. ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
  1582. ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
  1583. ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
  1584. ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
  1585. VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
  1586. ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
  1587. ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
  1588. ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
  1589. ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
  1590. return 0;
  1591. }
  1592. static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
  1593. {
  1594. int ret = -ENXIO;
  1595. switch (attr->attr) {
  1596. case KVM_S390_VM_CPU_PROCESSOR:
  1597. ret = kvm_s390_get_processor(kvm, attr);
  1598. break;
  1599. case KVM_S390_VM_CPU_MACHINE:
  1600. ret = kvm_s390_get_machine(kvm, attr);
  1601. break;
  1602. case KVM_S390_VM_CPU_PROCESSOR_FEAT:
  1603. ret = kvm_s390_get_processor_feat(kvm, attr);
  1604. break;
  1605. case KVM_S390_VM_CPU_MACHINE_FEAT:
  1606. ret = kvm_s390_get_machine_feat(kvm, attr);
  1607. break;
  1608. case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
  1609. ret = kvm_s390_get_processor_subfunc(kvm, attr);
  1610. break;
  1611. case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
  1612. ret = kvm_s390_get_machine_subfunc(kvm, attr);
  1613. break;
  1614. }
  1615. return ret;
  1616. }
  1617. /**
  1618. * kvm_s390_update_topology_change_report - update CPU topology change report
  1619. * @kvm: guest KVM description
  1620. * @val: set or clear the MTCR bit
  1621. *
  1622. * Updates the Multiprocessor Topology-Change-Report bit to signal
  1623. * the guest with a topology change.
  1624. * This is only relevant if the topology facility is present.
  1625. *
  1626. * The SCA version, bsca or esca, doesn't matter as offset is the same.
  1627. */
  1628. static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
  1629. {
  1630. union sca_utility new, old;
  1631. struct bsca_block *sca;
  1632. read_lock(&kvm->arch.sca_lock);
  1633. sca = kvm->arch.sca;
  1634. do {
  1635. old = READ_ONCE(sca->utility);
  1636. new = old;
  1637. new.mtcr = val;
  1638. } while (cmpxchg(&sca->utility.val, old.val, new.val) != old.val);
  1639. read_unlock(&kvm->arch.sca_lock);
  1640. }
  1641. static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
  1642. struct kvm_device_attr *attr)
  1643. {
  1644. if (!test_kvm_facility(kvm, 11))
  1645. return -ENXIO;
  1646. kvm_s390_update_topology_change_report(kvm, !!attr->attr);
  1647. return 0;
  1648. }
  1649. static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
  1650. struct kvm_device_attr *attr)
  1651. {
  1652. u8 topo;
  1653. if (!test_kvm_facility(kvm, 11))
  1654. return -ENXIO;
  1655. read_lock(&kvm->arch.sca_lock);
  1656. topo = ((struct bsca_block *)kvm->arch.sca)->utility.mtcr;
  1657. read_unlock(&kvm->arch.sca_lock);
  1658. return put_user(topo, (u8 __user *)attr->addr);
  1659. }
  1660. static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
  1661. {
  1662. int ret;
  1663. switch (attr->group) {
  1664. case KVM_S390_VM_MEM_CTRL:
  1665. ret = kvm_s390_set_mem_control(kvm, attr);
  1666. break;
  1667. case KVM_S390_VM_TOD:
  1668. ret = kvm_s390_set_tod(kvm, attr);
  1669. break;
  1670. case KVM_S390_VM_CPU_MODEL:
  1671. ret = kvm_s390_set_cpu_model(kvm, attr);
  1672. break;
  1673. case KVM_S390_VM_CRYPTO:
  1674. ret = kvm_s390_vm_set_crypto(kvm, attr);
  1675. break;
  1676. case KVM_S390_VM_MIGRATION:
  1677. ret = kvm_s390_vm_set_migration(kvm, attr);
  1678. break;
  1679. case KVM_S390_VM_CPU_TOPOLOGY:
  1680. ret = kvm_s390_set_topo_change_indication(kvm, attr);
  1681. break;
  1682. default:
  1683. ret = -ENXIO;
  1684. break;
  1685. }
  1686. return ret;
  1687. }
  1688. static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
  1689. {
  1690. int ret;
  1691. switch (attr->group) {
  1692. case KVM_S390_VM_MEM_CTRL:
  1693. ret = kvm_s390_get_mem_control(kvm, attr);
  1694. break;
  1695. case KVM_S390_VM_TOD:
  1696. ret = kvm_s390_get_tod(kvm, attr);
  1697. break;
  1698. case KVM_S390_VM_CPU_MODEL:
  1699. ret = kvm_s390_get_cpu_model(kvm, attr);
  1700. break;
  1701. case KVM_S390_VM_MIGRATION:
  1702. ret = kvm_s390_vm_get_migration(kvm, attr);
  1703. break;
  1704. case KVM_S390_VM_CPU_TOPOLOGY:
  1705. ret = kvm_s390_get_topo_change_indication(kvm, attr);
  1706. break;
  1707. default:
  1708. ret = -ENXIO;
  1709. break;
  1710. }
  1711. return ret;
  1712. }
  1713. static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
  1714. {
  1715. int ret;
  1716. switch (attr->group) {
  1717. case KVM_S390_VM_MEM_CTRL:
  1718. switch (attr->attr) {
  1719. case KVM_S390_VM_MEM_ENABLE_CMMA:
  1720. case KVM_S390_VM_MEM_CLR_CMMA:
  1721. ret = sclp.has_cmma ? 0 : -ENXIO;
  1722. break;
  1723. case KVM_S390_VM_MEM_LIMIT_SIZE:
  1724. ret = 0;
  1725. break;
  1726. default:
  1727. ret = -ENXIO;
  1728. break;
  1729. }
  1730. break;
  1731. case KVM_S390_VM_TOD:
  1732. switch (attr->attr) {
  1733. case KVM_S390_VM_TOD_LOW:
  1734. case KVM_S390_VM_TOD_HIGH:
  1735. ret = 0;
  1736. break;
  1737. default:
  1738. ret = -ENXIO;
  1739. break;
  1740. }
  1741. break;
  1742. case KVM_S390_VM_CPU_MODEL:
  1743. switch (attr->attr) {
  1744. case KVM_S390_VM_CPU_PROCESSOR:
  1745. case KVM_S390_VM_CPU_MACHINE:
  1746. case KVM_S390_VM_CPU_PROCESSOR_FEAT:
  1747. case KVM_S390_VM_CPU_MACHINE_FEAT:
  1748. case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
  1749. case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
  1750. ret = 0;
  1751. break;
  1752. default:
  1753. ret = -ENXIO;
  1754. break;
  1755. }
  1756. break;
  1757. case KVM_S390_VM_CRYPTO:
  1758. switch (attr->attr) {
  1759. case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
  1760. case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
  1761. case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
  1762. case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
  1763. ret = 0;
  1764. break;
  1765. case KVM_S390_VM_CRYPTO_ENABLE_APIE:
  1766. case KVM_S390_VM_CRYPTO_DISABLE_APIE:
  1767. ret = ap_instructions_available() ? 0 : -ENXIO;
  1768. break;
  1769. default:
  1770. ret = -ENXIO;
  1771. break;
  1772. }
  1773. break;
  1774. case KVM_S390_VM_MIGRATION:
  1775. ret = 0;
  1776. break;
  1777. case KVM_S390_VM_CPU_TOPOLOGY:
  1778. ret = test_kvm_facility(kvm, 11) ? 0 : -ENXIO;
  1779. break;
  1780. default:
  1781. ret = -ENXIO;
  1782. break;
  1783. }
  1784. return ret;
  1785. }
  1786. static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
  1787. {
  1788. uint8_t *keys;
  1789. uint64_t hva;
  1790. int srcu_idx, i, r = 0;
  1791. if (args->flags != 0)
  1792. return -EINVAL;
  1793. /* Is this guest using storage keys? */
  1794. if (!mm_uses_skeys(current->mm))
  1795. return KVM_S390_GET_SKEYS_NONE;
  1796. /* Enforce sane limit on memory allocation */
  1797. if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
  1798. return -EINVAL;
  1799. keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
  1800. if (!keys)
  1801. return -ENOMEM;
  1802. mmap_read_lock(current->mm);
  1803. srcu_idx = srcu_read_lock(&kvm->srcu);
  1804. for (i = 0; i < args->count; i++) {
  1805. hva = gfn_to_hva(kvm, args->start_gfn + i);
  1806. if (kvm_is_error_hva(hva)) {
  1807. r = -EFAULT;
  1808. break;
  1809. }
  1810. r = get_guest_storage_key(current->mm, hva, &keys[i]);
  1811. if (r)
  1812. break;
  1813. }
  1814. srcu_read_unlock(&kvm->srcu, srcu_idx);
  1815. mmap_read_unlock(current->mm);
  1816. if (!r) {
  1817. r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
  1818. sizeof(uint8_t) * args->count);
  1819. if (r)
  1820. r = -EFAULT;
  1821. }
  1822. kvfree(keys);
  1823. return r;
  1824. }
  1825. static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
  1826. {
  1827. uint8_t *keys;
  1828. uint64_t hva;
  1829. int srcu_idx, i, r = 0;
  1830. bool unlocked;
  1831. if (args->flags != 0)
  1832. return -EINVAL;
  1833. /* Enforce sane limit on memory allocation */
  1834. if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
  1835. return -EINVAL;
  1836. keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL_ACCOUNT);
  1837. if (!keys)
  1838. return -ENOMEM;
  1839. r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
  1840. sizeof(uint8_t) * args->count);
  1841. if (r) {
  1842. r = -EFAULT;
  1843. goto out;
  1844. }
  1845. /* Enable storage key handling for the guest */
  1846. r = s390_enable_skey();
  1847. if (r)
  1848. goto out;
  1849. i = 0;
  1850. mmap_read_lock(current->mm);
  1851. srcu_idx = srcu_read_lock(&kvm->srcu);
  1852. while (i < args->count) {
  1853. unlocked = false;
  1854. hva = gfn_to_hva(kvm, args->start_gfn + i);
  1855. if (kvm_is_error_hva(hva)) {
  1856. r = -EFAULT;
  1857. break;
  1858. }
  1859. /* Lowest order bit is reserved */
  1860. if (keys[i] & 0x01) {
  1861. r = -EINVAL;
  1862. break;
  1863. }
  1864. r = set_guest_storage_key(current->mm, hva, keys[i], 0);
  1865. if (r) {
  1866. r = fixup_user_fault(current->mm, hva,
  1867. FAULT_FLAG_WRITE, &unlocked);
  1868. if (r)
  1869. break;
  1870. }
  1871. if (!r)
  1872. i++;
  1873. }
  1874. srcu_read_unlock(&kvm->srcu, srcu_idx);
  1875. mmap_read_unlock(current->mm);
  1876. out:
  1877. kvfree(keys);
  1878. return r;
  1879. }
  1880. /*
  1881. * Base address and length must be sent at the start of each block, therefore
  1882. * it's cheaper to send some clean data, as long as it's less than the size of
  1883. * two longs.
  1884. */
  1885. #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *))
  1886. /* for consistency */
  1887. #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX)
  1888. static int kvm_s390_peek_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
  1889. u8 *res, unsigned long bufsize)
  1890. {
  1891. unsigned long pgstev, hva, cur_gfn = args->start_gfn;
  1892. args->count = 0;
  1893. while (args->count < bufsize) {
  1894. hva = gfn_to_hva(kvm, cur_gfn);
  1895. /*
  1896. * We return an error if the first value was invalid, but we
  1897. * return successfully if at least one value was copied.
  1898. */
  1899. if (kvm_is_error_hva(hva))
  1900. return args->count ? 0 : -EFAULT;
  1901. if (get_pgste(kvm->mm, hva, &pgstev) < 0)
  1902. pgstev = 0;
  1903. res[args->count++] = (pgstev >> 24) & 0x43;
  1904. cur_gfn++;
  1905. }
  1906. return 0;
  1907. }
  1908. static struct kvm_memory_slot *gfn_to_memslot_approx(struct kvm_memslots *slots,
  1909. gfn_t gfn)
  1910. {
  1911. return ____gfn_to_memslot(slots, gfn, true);
  1912. }
  1913. static unsigned long kvm_s390_next_dirty_cmma(struct kvm_memslots *slots,
  1914. unsigned long cur_gfn)
  1915. {
  1916. struct kvm_memory_slot *ms = gfn_to_memslot_approx(slots, cur_gfn);
  1917. unsigned long ofs = cur_gfn - ms->base_gfn;
  1918. struct rb_node *mnode = &ms->gfn_node[slots->node_idx];
  1919. if (ms->base_gfn + ms->npages <= cur_gfn) {
  1920. mnode = rb_next(mnode);
  1921. /* If we are above the highest slot, wrap around */
  1922. if (!mnode)
  1923. mnode = rb_first(&slots->gfn_tree);
  1924. ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]);
  1925. ofs = 0;
  1926. }
  1927. if (cur_gfn < ms->base_gfn)
  1928. ofs = 0;
  1929. ofs = find_next_bit(kvm_second_dirty_bitmap(ms), ms->npages, ofs);
  1930. while (ofs >= ms->npages && (mnode = rb_next(mnode))) {
  1931. ms = container_of(mnode, struct kvm_memory_slot, gfn_node[slots->node_idx]);
  1932. ofs = find_first_bit(kvm_second_dirty_bitmap(ms), ms->npages);
  1933. }
  1934. return ms->base_gfn + ofs;
  1935. }
  1936. static int kvm_s390_get_cmma(struct kvm *kvm, struct kvm_s390_cmma_log *args,
  1937. u8 *res, unsigned long bufsize)
  1938. {
  1939. unsigned long mem_end, cur_gfn, next_gfn, hva, pgstev;
  1940. struct kvm_memslots *slots = kvm_memslots(kvm);
  1941. struct kvm_memory_slot *ms;
  1942. if (unlikely(kvm_memslots_empty(slots)))
  1943. return 0;
  1944. cur_gfn = kvm_s390_next_dirty_cmma(slots, args->start_gfn);
  1945. ms = gfn_to_memslot(kvm, cur_gfn);
  1946. args->count = 0;
  1947. args->start_gfn = cur_gfn;
  1948. if (!ms)
  1949. return 0;
  1950. next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
  1951. mem_end = kvm_s390_get_gfn_end(slots);
  1952. while (args->count < bufsize) {
  1953. hva = gfn_to_hva(kvm, cur_gfn);
  1954. if (kvm_is_error_hva(hva))
  1955. return 0;
  1956. /* Decrement only if we actually flipped the bit to 0 */
  1957. if (test_and_clear_bit(cur_gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms)))
  1958. atomic64_dec(&kvm->arch.cmma_dirty_pages);
  1959. if (get_pgste(kvm->mm, hva, &pgstev) < 0)
  1960. pgstev = 0;
  1961. /* Save the value */
  1962. res[args->count++] = (pgstev >> 24) & 0x43;
  1963. /* If the next bit is too far away, stop. */
  1964. if (next_gfn > cur_gfn + KVM_S390_MAX_BIT_DISTANCE)
  1965. return 0;
  1966. /* If we reached the previous "next", find the next one */
  1967. if (cur_gfn == next_gfn)
  1968. next_gfn = kvm_s390_next_dirty_cmma(slots, cur_gfn + 1);
  1969. /* Reached the end of memory or of the buffer, stop */
  1970. if ((next_gfn >= mem_end) ||
  1971. (next_gfn - args->start_gfn >= bufsize))
  1972. return 0;
  1973. cur_gfn++;
  1974. /* Reached the end of the current memslot, take the next one. */
  1975. if (cur_gfn - ms->base_gfn >= ms->npages) {
  1976. ms = gfn_to_memslot(kvm, cur_gfn);
  1977. if (!ms)
  1978. return 0;
  1979. }
  1980. }
  1981. return 0;
  1982. }
  1983. /*
  1984. * This function searches for the next page with dirty CMMA attributes, and
  1985. * saves the attributes in the buffer up to either the end of the buffer or
  1986. * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
  1987. * no trailing clean bytes are saved.
  1988. * In case no dirty bits were found, or if CMMA was not enabled or used, the
  1989. * output buffer will indicate 0 as length.
  1990. */
  1991. static int kvm_s390_get_cmma_bits(struct kvm *kvm,
  1992. struct kvm_s390_cmma_log *args)
  1993. {
  1994. unsigned long bufsize;
  1995. int srcu_idx, peek, ret;
  1996. u8 *values;
  1997. if (!kvm->arch.use_cmma)
  1998. return -ENXIO;
  1999. /* Invalid/unsupported flags were specified */
  2000. if (args->flags & ~KVM_S390_CMMA_PEEK)
  2001. return -EINVAL;
  2002. /* Migration mode query, and we are not doing a migration */
  2003. peek = !!(args->flags & KVM_S390_CMMA_PEEK);
  2004. if (!peek && !kvm->arch.migration_mode)
  2005. return -EINVAL;
  2006. /* CMMA is disabled or was not used, or the buffer has length zero */
  2007. bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX);
  2008. if (!bufsize || !kvm->mm->context.uses_cmm) {
  2009. memset(args, 0, sizeof(*args));
  2010. return 0;
  2011. }
  2012. /* We are not peeking, and there are no dirty pages */
  2013. if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
  2014. memset(args, 0, sizeof(*args));
  2015. return 0;
  2016. }
  2017. values = vmalloc(bufsize);
  2018. if (!values)
  2019. return -ENOMEM;
  2020. mmap_read_lock(kvm->mm);
  2021. srcu_idx = srcu_read_lock(&kvm->srcu);
  2022. if (peek)
  2023. ret = kvm_s390_peek_cmma(kvm, args, values, bufsize);
  2024. else
  2025. ret = kvm_s390_get_cmma(kvm, args, values, bufsize);
  2026. srcu_read_unlock(&kvm->srcu, srcu_idx);
  2027. mmap_read_unlock(kvm->mm);
  2028. if (kvm->arch.migration_mode)
  2029. args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
  2030. else
  2031. args->remaining = 0;
  2032. if (copy_to_user((void __user *)args->values, values, args->count))
  2033. ret = -EFAULT;
  2034. vfree(values);
  2035. return ret;
  2036. }
  2037. /*
  2038. * This function sets the CMMA attributes for the given pages. If the input
  2039. * buffer has zero length, no action is taken, otherwise the attributes are
  2040. * set and the mm->context.uses_cmm flag is set.
  2041. */
  2042. static int kvm_s390_set_cmma_bits(struct kvm *kvm,
  2043. const struct kvm_s390_cmma_log *args)
  2044. {
  2045. unsigned long hva, mask, pgstev, i;
  2046. uint8_t *bits;
  2047. int srcu_idx, r = 0;
  2048. mask = args->mask;
  2049. if (!kvm->arch.use_cmma)
  2050. return -ENXIO;
  2051. /* invalid/unsupported flags */
  2052. if (args->flags != 0)
  2053. return -EINVAL;
  2054. /* Enforce sane limit on memory allocation */
  2055. if (args->count > KVM_S390_CMMA_SIZE_MAX)
  2056. return -EINVAL;
  2057. /* Nothing to do */
  2058. if (args->count == 0)
  2059. return 0;
  2060. bits = vmalloc(array_size(sizeof(*bits), args->count));
  2061. if (!bits)
  2062. return -ENOMEM;
  2063. r = copy_from_user(bits, (void __user *)args->values, args->count);
  2064. if (r) {
  2065. r = -EFAULT;
  2066. goto out;
  2067. }
  2068. mmap_read_lock(kvm->mm);
  2069. srcu_idx = srcu_read_lock(&kvm->srcu);
  2070. for (i = 0; i < args->count; i++) {
  2071. hva = gfn_to_hva(kvm, args->start_gfn + i);
  2072. if (kvm_is_error_hva(hva)) {
  2073. r = -EFAULT;
  2074. break;
  2075. }
  2076. pgstev = bits[i];
  2077. pgstev = pgstev << 24;
  2078. mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT;
  2079. set_pgste_bits(kvm->mm, hva, mask, pgstev);
  2080. }
  2081. srcu_read_unlock(&kvm->srcu, srcu_idx);
  2082. mmap_read_unlock(kvm->mm);
  2083. if (!kvm->mm->context.uses_cmm) {
  2084. mmap_write_lock(kvm->mm);
  2085. kvm->mm->context.uses_cmm = 1;
  2086. mmap_write_unlock(kvm->mm);
  2087. }
  2088. out:
  2089. vfree(bits);
  2090. return r;
  2091. }
  2092. /**
  2093. * kvm_s390_cpus_from_pv - Convert all protected vCPUs in a protected VM to
  2094. * non protected.
  2095. * @kvm: the VM whose protected vCPUs are to be converted
  2096. * @rc: return value for the RC field of the UVC (in case of error)
  2097. * @rrc: return value for the RRC field of the UVC (in case of error)
  2098. *
  2099. * Does not stop in case of error, tries to convert as many
  2100. * CPUs as possible. In case of error, the RC and RRC of the last error are
  2101. * returned.
  2102. *
  2103. * Return: 0 in case of success, otherwise -EIO
  2104. */
  2105. int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
  2106. {
  2107. struct kvm_vcpu *vcpu;
  2108. unsigned long i;
  2109. u16 _rc, _rrc;
  2110. int ret = 0;
  2111. /*
  2112. * We ignore failures and try to destroy as many CPUs as possible.
  2113. * At the same time we must not free the assigned resources when
  2114. * this fails, as the ultravisor has still access to that memory.
  2115. * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
  2116. * behind.
  2117. * We want to return the first failure rc and rrc, though.
  2118. */
  2119. kvm_for_each_vcpu(i, vcpu, kvm) {
  2120. mutex_lock(&vcpu->mutex);
  2121. if (kvm_s390_pv_destroy_cpu(vcpu, &_rc, &_rrc) && !ret) {
  2122. *rc = _rc;
  2123. *rrc = _rrc;
  2124. ret = -EIO;
  2125. }
  2126. mutex_unlock(&vcpu->mutex);
  2127. }
  2128. /* Ensure that we re-enable gisa if the non-PV guest used it but the PV guest did not. */
  2129. if (use_gisa)
  2130. kvm_s390_gisa_enable(kvm);
  2131. return ret;
  2132. }
  2133. /**
  2134. * kvm_s390_cpus_to_pv - Convert all non-protected vCPUs in a protected VM
  2135. * to protected.
  2136. * @kvm: the VM whose protected vCPUs are to be converted
  2137. * @rc: return value for the RC field of the UVC (in case of error)
  2138. * @rrc: return value for the RRC field of the UVC (in case of error)
  2139. *
  2140. * Tries to undo the conversion in case of error.
  2141. *
  2142. * Return: 0 in case of success, otherwise -EIO
  2143. */
  2144. static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
  2145. {
  2146. unsigned long i;
  2147. int r = 0;
  2148. u16 dummy;
  2149. struct kvm_vcpu *vcpu;
  2150. /* Disable the GISA if the ultravisor does not support AIV. */
  2151. if (!test_bit_inv(BIT_UV_FEAT_AIV, &uv_info.uv_feature_indications))
  2152. kvm_s390_gisa_disable(kvm);
  2153. kvm_for_each_vcpu(i, vcpu, kvm) {
  2154. mutex_lock(&vcpu->mutex);
  2155. r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
  2156. mutex_unlock(&vcpu->mutex);
  2157. if (r)
  2158. break;
  2159. }
  2160. if (r)
  2161. kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
  2162. return r;
  2163. }
  2164. /*
  2165. * Here we provide user space with a direct interface to query UV
  2166. * related data like UV maxima and available features as well as
  2167. * feature specific data.
  2168. *
  2169. * To facilitate future extension of the data structures we'll try to
  2170. * write data up to the maximum requested length.
  2171. */
  2172. static ssize_t kvm_s390_handle_pv_info(struct kvm_s390_pv_info *info)
  2173. {
  2174. ssize_t len_min;
  2175. switch (info->header.id) {
  2176. case KVM_PV_INFO_VM: {
  2177. len_min = sizeof(info->header) + sizeof(info->vm);
  2178. if (info->header.len_max < len_min)
  2179. return -EINVAL;
  2180. memcpy(info->vm.inst_calls_list,
  2181. uv_info.inst_calls_list,
  2182. sizeof(uv_info.inst_calls_list));
  2183. /* It's max cpuid not max cpus, so it's off by one */
  2184. info->vm.max_cpus = uv_info.max_guest_cpu_id + 1;
  2185. info->vm.max_guests = uv_info.max_num_sec_conf;
  2186. info->vm.max_guest_addr = uv_info.max_sec_stor_addr;
  2187. info->vm.feature_indication = uv_info.uv_feature_indications;
  2188. return len_min;
  2189. }
  2190. case KVM_PV_INFO_DUMP: {
  2191. len_min = sizeof(info->header) + sizeof(info->dump);
  2192. if (info->header.len_max < len_min)
  2193. return -EINVAL;
  2194. info->dump.dump_cpu_buffer_len = uv_info.guest_cpu_stor_len;
  2195. info->dump.dump_config_mem_buffer_per_1m = uv_info.conf_dump_storage_state_len;
  2196. info->dump.dump_config_finalize_len = uv_info.conf_dump_finalize_len;
  2197. return len_min;
  2198. }
  2199. default:
  2200. return -EINVAL;
  2201. }
  2202. }
  2203. static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
  2204. struct kvm_s390_pv_dmp dmp)
  2205. {
  2206. int r = -EINVAL;
  2207. void __user *result_buff = (void __user *)dmp.buff_addr;
  2208. switch (dmp.subcmd) {
  2209. case KVM_PV_DUMP_INIT: {
  2210. if (kvm->arch.pv.dumping)
  2211. break;
  2212. /*
  2213. * Block SIE entry as concurrent dump UVCs could lead
  2214. * to validities.
  2215. */
  2216. kvm_s390_vcpu_block_all(kvm);
  2217. r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
  2218. UVC_CMD_DUMP_INIT, &cmd->rc, &cmd->rrc);
  2219. KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
  2220. cmd->rc, cmd->rrc);
  2221. if (!r) {
  2222. kvm->arch.pv.dumping = true;
  2223. } else {
  2224. kvm_s390_vcpu_unblock_all(kvm);
  2225. r = -EINVAL;
  2226. }
  2227. break;
  2228. }
  2229. case KVM_PV_DUMP_CONFIG_STOR_STATE: {
  2230. if (!kvm->arch.pv.dumping)
  2231. break;
  2232. /*
  2233. * gaddr is an output parameter since we might stop
  2234. * early. As dmp will be copied back in our caller, we
  2235. * don't need to do it ourselves.
  2236. */
  2237. r = kvm_s390_pv_dump_stor_state(kvm, result_buff, &dmp.gaddr, dmp.buff_len,
  2238. &cmd->rc, &cmd->rrc);
  2239. break;
  2240. }
  2241. case KVM_PV_DUMP_COMPLETE: {
  2242. if (!kvm->arch.pv.dumping)
  2243. break;
  2244. r = -EINVAL;
  2245. if (dmp.buff_len < uv_info.conf_dump_finalize_len)
  2246. break;
  2247. r = kvm_s390_pv_dump_complete(kvm, result_buff,
  2248. &cmd->rc, &cmd->rrc);
  2249. break;
  2250. }
  2251. default:
  2252. r = -ENOTTY;
  2253. break;
  2254. }
  2255. return r;
  2256. }
  2257. static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
  2258. {
  2259. int r = 0;
  2260. u16 dummy;
  2261. void __user *argp = (void __user *)cmd->data;
  2262. switch (cmd->cmd) {
  2263. case KVM_PV_ENABLE: {
  2264. r = -EINVAL;
  2265. if (kvm_s390_pv_is_protected(kvm))
  2266. break;
  2267. /*
  2268. * FMT 4 SIE needs esca. As we never switch back to bsca from
  2269. * esca, we need no cleanup in the error cases below
  2270. */
  2271. r = sca_switch_to_extended(kvm);
  2272. if (r)
  2273. break;
  2274. mmap_write_lock(current->mm);
  2275. r = gmap_mark_unmergeable();
  2276. mmap_write_unlock(current->mm);
  2277. if (r)
  2278. break;
  2279. r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
  2280. if (r)
  2281. break;
  2282. r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
  2283. if (r)
  2284. kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
  2285. /* we need to block service interrupts from now on */
  2286. set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
  2287. break;
  2288. }
  2289. case KVM_PV_DISABLE: {
  2290. r = -EINVAL;
  2291. if (!kvm_s390_pv_is_protected(kvm))
  2292. break;
  2293. r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
  2294. /*
  2295. * If a CPU could not be destroyed, destroy VM will also fail.
  2296. * There is no point in trying to destroy it. Instead return
  2297. * the rc and rrc from the first CPU that failed destroying.
  2298. */
  2299. if (r)
  2300. break;
  2301. r = kvm_s390_pv_deinit_vm(kvm, &cmd->rc, &cmd->rrc);
  2302. /* no need to block service interrupts any more */
  2303. clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
  2304. break;
  2305. }
  2306. case KVM_PV_SET_SEC_PARMS: {
  2307. struct kvm_s390_pv_sec_parm parms = {};
  2308. void *hdr;
  2309. r = -EINVAL;
  2310. if (!kvm_s390_pv_is_protected(kvm))
  2311. break;
  2312. r = -EFAULT;
  2313. if (copy_from_user(&parms, argp, sizeof(parms)))
  2314. break;
  2315. /* Currently restricted to 8KB */
  2316. r = -EINVAL;
  2317. if (parms.length > PAGE_SIZE * 2)
  2318. break;
  2319. r = -ENOMEM;
  2320. hdr = vmalloc(parms.length);
  2321. if (!hdr)
  2322. break;
  2323. r = -EFAULT;
  2324. if (!copy_from_user(hdr, (void __user *)parms.origin,
  2325. parms.length))
  2326. r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
  2327. &cmd->rc, &cmd->rrc);
  2328. vfree(hdr);
  2329. break;
  2330. }
  2331. case KVM_PV_UNPACK: {
  2332. struct kvm_s390_pv_unp unp = {};
  2333. r = -EINVAL;
  2334. if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
  2335. break;
  2336. r = -EFAULT;
  2337. if (copy_from_user(&unp, argp, sizeof(unp)))
  2338. break;
  2339. r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
  2340. &cmd->rc, &cmd->rrc);
  2341. break;
  2342. }
  2343. case KVM_PV_VERIFY: {
  2344. r = -EINVAL;
  2345. if (!kvm_s390_pv_is_protected(kvm))
  2346. break;
  2347. r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
  2348. UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
  2349. KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
  2350. cmd->rrc);
  2351. break;
  2352. }
  2353. case KVM_PV_PREP_RESET: {
  2354. r = -EINVAL;
  2355. if (!kvm_s390_pv_is_protected(kvm))
  2356. break;
  2357. r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
  2358. UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
  2359. KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
  2360. cmd->rc, cmd->rrc);
  2361. break;
  2362. }
  2363. case KVM_PV_UNSHARE_ALL: {
  2364. r = -EINVAL;
  2365. if (!kvm_s390_pv_is_protected(kvm))
  2366. break;
  2367. r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
  2368. UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
  2369. KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
  2370. cmd->rc, cmd->rrc);
  2371. break;
  2372. }
  2373. case KVM_PV_INFO: {
  2374. struct kvm_s390_pv_info info = {};
  2375. ssize_t data_len;
  2376. /*
  2377. * No need to check the VM protection here.
  2378. *
  2379. * Maybe user space wants to query some of the data
  2380. * when the VM is still unprotected. If we see the
  2381. * need to fence a new data command we can still
  2382. * return an error in the info handler.
  2383. */
  2384. r = -EFAULT;
  2385. if (copy_from_user(&info, argp, sizeof(info.header)))
  2386. break;
  2387. r = -EINVAL;
  2388. if (info.header.len_max < sizeof(info.header))
  2389. break;
  2390. data_len = kvm_s390_handle_pv_info(&info);
  2391. if (data_len < 0) {
  2392. r = data_len;
  2393. break;
  2394. }
  2395. /*
  2396. * If a data command struct is extended (multiple
  2397. * times) this can be used to determine how much of it
  2398. * is valid.
  2399. */
  2400. info.header.len_written = data_len;
  2401. r = -EFAULT;
  2402. if (copy_to_user(argp, &info, data_len))
  2403. break;
  2404. r = 0;
  2405. break;
  2406. }
  2407. case KVM_PV_DUMP: {
  2408. struct kvm_s390_pv_dmp dmp;
  2409. r = -EINVAL;
  2410. if (!kvm_s390_pv_is_protected(kvm))
  2411. break;
  2412. r = -EFAULT;
  2413. if (copy_from_user(&dmp, argp, sizeof(dmp)))
  2414. break;
  2415. r = kvm_s390_pv_dmp(kvm, cmd, dmp);
  2416. if (r)
  2417. break;
  2418. if (copy_to_user(argp, &dmp, sizeof(dmp))) {
  2419. r = -EFAULT;
  2420. break;
  2421. }
  2422. break;
  2423. }
  2424. default:
  2425. r = -ENOTTY;
  2426. }
  2427. return r;
  2428. }
  2429. static bool access_key_invalid(u8 access_key)
  2430. {
  2431. return access_key > 0xf;
  2432. }
  2433. static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
  2434. {
  2435. void __user *uaddr = (void __user *)mop->buf;
  2436. u64 supported_flags;
  2437. void *tmpbuf = NULL;
  2438. int r, srcu_idx;
  2439. supported_flags = KVM_S390_MEMOP_F_SKEY_PROTECTION
  2440. | KVM_S390_MEMOP_F_CHECK_ONLY;
  2441. if (mop->flags & ~supported_flags || !mop->size)
  2442. return -EINVAL;
  2443. if (mop->size > MEM_OP_MAX_SIZE)
  2444. return -E2BIG;
  2445. /*
  2446. * This is technically a heuristic only, if the kvm->lock is not
  2447. * taken, it is not guaranteed that the vm is/remains non-protected.
  2448. * This is ok from a kernel perspective, wrongdoing is detected
  2449. * on the access, -EFAULT is returned and the vm may crash the
  2450. * next time it accesses the memory in question.
  2451. * There is no sane usecase to do switching and a memop on two
  2452. * different CPUs at the same time.
  2453. */
  2454. if (kvm_s390_pv_get_handle(kvm))
  2455. return -EINVAL;
  2456. if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
  2457. if (access_key_invalid(mop->key))
  2458. return -EINVAL;
  2459. } else {
  2460. mop->key = 0;
  2461. }
  2462. if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
  2463. tmpbuf = vmalloc(mop->size);
  2464. if (!tmpbuf)
  2465. return -ENOMEM;
  2466. }
  2467. srcu_idx = srcu_read_lock(&kvm->srcu);
  2468. if (kvm_is_error_gpa(kvm, mop->gaddr)) {
  2469. r = PGM_ADDRESSING;
  2470. goto out_unlock;
  2471. }
  2472. switch (mop->op) {
  2473. case KVM_S390_MEMOP_ABSOLUTE_READ: {
  2474. if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
  2475. r = check_gpa_range(kvm, mop->gaddr, mop->size, GACC_FETCH, mop->key);
  2476. } else {
  2477. r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
  2478. mop->size, GACC_FETCH, mop->key);
  2479. if (r == 0) {
  2480. if (copy_to_user(uaddr, tmpbuf, mop->size))
  2481. r = -EFAULT;
  2482. }
  2483. }
  2484. break;
  2485. }
  2486. case KVM_S390_MEMOP_ABSOLUTE_WRITE: {
  2487. if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
  2488. r = check_gpa_range(kvm, mop->gaddr, mop->size, GACC_STORE, mop->key);
  2489. } else {
  2490. if (copy_from_user(tmpbuf, uaddr, mop->size)) {
  2491. r = -EFAULT;
  2492. break;
  2493. }
  2494. r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
  2495. mop->size, GACC_STORE, mop->key);
  2496. }
  2497. break;
  2498. }
  2499. default:
  2500. r = -EINVAL;
  2501. }
  2502. out_unlock:
  2503. srcu_read_unlock(&kvm->srcu, srcu_idx);
  2504. vfree(tmpbuf);
  2505. return r;
  2506. }
  2507. long kvm_arch_vm_ioctl(struct file *filp,
  2508. unsigned int ioctl, unsigned long arg)
  2509. {
  2510. struct kvm *kvm = filp->private_data;
  2511. void __user *argp = (void __user *)arg;
  2512. struct kvm_device_attr attr;
  2513. int r;
  2514. switch (ioctl) {
  2515. case KVM_S390_INTERRUPT: {
  2516. struct kvm_s390_interrupt s390int;
  2517. r = -EFAULT;
  2518. if (copy_from_user(&s390int, argp, sizeof(s390int)))
  2519. break;
  2520. r = kvm_s390_inject_vm(kvm, &s390int);
  2521. break;
  2522. }
  2523. case KVM_CREATE_IRQCHIP: {
  2524. struct kvm_irq_routing_entry routing;
  2525. r = -EINVAL;
  2526. if (kvm->arch.use_irqchip) {
  2527. /* Set up dummy routing. */
  2528. memset(&routing, 0, sizeof(routing));
  2529. r = kvm_set_irq_routing(kvm, &routing, 0, 0);
  2530. }
  2531. break;
  2532. }
  2533. case KVM_SET_DEVICE_ATTR: {
  2534. r = -EFAULT;
  2535. if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
  2536. break;
  2537. r = kvm_s390_vm_set_attr(kvm, &attr);
  2538. break;
  2539. }
  2540. case KVM_GET_DEVICE_ATTR: {
  2541. r = -EFAULT;
  2542. if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
  2543. break;
  2544. r = kvm_s390_vm_get_attr(kvm, &attr);
  2545. break;
  2546. }
  2547. case KVM_HAS_DEVICE_ATTR: {
  2548. r = -EFAULT;
  2549. if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
  2550. break;
  2551. r = kvm_s390_vm_has_attr(kvm, &attr);
  2552. break;
  2553. }
  2554. case KVM_S390_GET_SKEYS: {
  2555. struct kvm_s390_skeys args;
  2556. r = -EFAULT;
  2557. if (copy_from_user(&args, argp,
  2558. sizeof(struct kvm_s390_skeys)))
  2559. break;
  2560. r = kvm_s390_get_skeys(kvm, &args);
  2561. break;
  2562. }
  2563. case KVM_S390_SET_SKEYS: {
  2564. struct kvm_s390_skeys args;
  2565. r = -EFAULT;
  2566. if (copy_from_user(&args, argp,
  2567. sizeof(struct kvm_s390_skeys)))
  2568. break;
  2569. r = kvm_s390_set_skeys(kvm, &args);
  2570. break;
  2571. }
  2572. case KVM_S390_GET_CMMA_BITS: {
  2573. struct kvm_s390_cmma_log args;
  2574. r = -EFAULT;
  2575. if (copy_from_user(&args, argp, sizeof(args)))
  2576. break;
  2577. mutex_lock(&kvm->slots_lock);
  2578. r = kvm_s390_get_cmma_bits(kvm, &args);
  2579. mutex_unlock(&kvm->slots_lock);
  2580. if (!r) {
  2581. r = copy_to_user(argp, &args, sizeof(args));
  2582. if (r)
  2583. r = -EFAULT;
  2584. }
  2585. break;
  2586. }
  2587. case KVM_S390_SET_CMMA_BITS: {
  2588. struct kvm_s390_cmma_log args;
  2589. r = -EFAULT;
  2590. if (copy_from_user(&args, argp, sizeof(args)))
  2591. break;
  2592. mutex_lock(&kvm->slots_lock);
  2593. r = kvm_s390_set_cmma_bits(kvm, &args);
  2594. mutex_unlock(&kvm->slots_lock);
  2595. break;
  2596. }
  2597. case KVM_S390_PV_COMMAND: {
  2598. struct kvm_pv_cmd args;
  2599. /* protvirt means user cpu state */
  2600. kvm_s390_set_user_cpu_state_ctrl(kvm);
  2601. r = 0;
  2602. if (!is_prot_virt_host()) {
  2603. r = -EINVAL;
  2604. break;
  2605. }
  2606. if (copy_from_user(&args, argp, sizeof(args))) {
  2607. r = -EFAULT;
  2608. break;
  2609. }
  2610. if (args.flags) {
  2611. r = -EINVAL;
  2612. break;
  2613. }
  2614. mutex_lock(&kvm->lock);
  2615. r = kvm_s390_handle_pv(kvm, &args);
  2616. mutex_unlock(&kvm->lock);
  2617. if (copy_to_user(argp, &args, sizeof(args))) {
  2618. r = -EFAULT;
  2619. break;
  2620. }
  2621. break;
  2622. }
  2623. case KVM_S390_MEM_OP: {
  2624. struct kvm_s390_mem_op mem_op;
  2625. if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
  2626. r = kvm_s390_vm_mem_op(kvm, &mem_op);
  2627. else
  2628. r = -EFAULT;
  2629. break;
  2630. }
  2631. case KVM_S390_ZPCI_OP: {
  2632. struct kvm_s390_zpci_op args;
  2633. r = -EINVAL;
  2634. if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
  2635. break;
  2636. if (copy_from_user(&args, argp, sizeof(args))) {
  2637. r = -EFAULT;
  2638. break;
  2639. }
  2640. r = kvm_s390_pci_zpci_op(kvm, &args);
  2641. break;
  2642. }
  2643. default:
  2644. r = -ENOTTY;
  2645. }
  2646. return r;
  2647. }
  2648. static int kvm_s390_apxa_installed(void)
  2649. {
  2650. struct ap_config_info info;
  2651. if (ap_instructions_available()) {
  2652. if (ap_qci(&info) == 0)
  2653. return info.apxa;
  2654. }
  2655. return 0;
  2656. }
  2657. /*
  2658. * The format of the crypto control block (CRYCB) is specified in the 3 low
  2659. * order bits of the CRYCB designation (CRYCBD) field as follows:
  2660. * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
  2661. * AP extended addressing (APXA) facility are installed.
  2662. * Format 1: The APXA facility is not installed but the MSAX3 facility is.
  2663. * Format 2: Both the APXA and MSAX3 facilities are installed
  2664. */
  2665. static void kvm_s390_set_crycb_format(struct kvm *kvm)
  2666. {
  2667. kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb;
  2668. /* Clear the CRYCB format bits - i.e., set format 0 by default */
  2669. kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
  2670. /* Check whether MSAX3 is installed */
  2671. if (!test_kvm_facility(kvm, 76))
  2672. return;
  2673. if (kvm_s390_apxa_installed())
  2674. kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
  2675. else
  2676. kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
  2677. }
  2678. /*
  2679. * kvm_arch_crypto_set_masks
  2680. *
  2681. * @kvm: pointer to the target guest's KVM struct containing the crypto masks
  2682. * to be set.
  2683. * @apm: the mask identifying the accessible AP adapters
  2684. * @aqm: the mask identifying the accessible AP domains
  2685. * @adm: the mask identifying the accessible AP control domains
  2686. *
  2687. * Set the masks that identify the adapters, domains and control domains to
  2688. * which the KVM guest is granted access.
  2689. *
  2690. * Note: The kvm->lock mutex must be locked by the caller before invoking this
  2691. * function.
  2692. */
  2693. void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
  2694. unsigned long *aqm, unsigned long *adm)
  2695. {
  2696. struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
  2697. kvm_s390_vcpu_block_all(kvm);
  2698. switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
  2699. case CRYCB_FORMAT2: /* APCB1 use 256 bits */
  2700. memcpy(crycb->apcb1.apm, apm, 32);
  2701. VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
  2702. apm[0], apm[1], apm[2], apm[3]);
  2703. memcpy(crycb->apcb1.aqm, aqm, 32);
  2704. VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
  2705. aqm[0], aqm[1], aqm[2], aqm[3]);
  2706. memcpy(crycb->apcb1.adm, adm, 32);
  2707. VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
  2708. adm[0], adm[1], adm[2], adm[3]);
  2709. break;
  2710. case CRYCB_FORMAT1:
  2711. case CRYCB_FORMAT0: /* Fall through both use APCB0 */
  2712. memcpy(crycb->apcb0.apm, apm, 8);
  2713. memcpy(crycb->apcb0.aqm, aqm, 2);
  2714. memcpy(crycb->apcb0.adm, adm, 2);
  2715. VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
  2716. apm[0], *((unsigned short *)aqm),
  2717. *((unsigned short *)adm));
  2718. break;
  2719. default: /* Can not happen */
  2720. break;
  2721. }
  2722. /* recreate the shadow crycb for each vcpu */
  2723. kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
  2724. kvm_s390_vcpu_unblock_all(kvm);
  2725. }
  2726. EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
  2727. /*
  2728. * kvm_arch_crypto_clear_masks
  2729. *
  2730. * @kvm: pointer to the target guest's KVM struct containing the crypto masks
  2731. * to be cleared.
  2732. *
  2733. * Clear the masks that identify the adapters, domains and control domains to
  2734. * which the KVM guest is granted access.
  2735. *
  2736. * Note: The kvm->lock mutex must be locked by the caller before invoking this
  2737. * function.
  2738. */
  2739. void kvm_arch_crypto_clear_masks(struct kvm *kvm)
  2740. {
  2741. kvm_s390_vcpu_block_all(kvm);
  2742. memset(&kvm->arch.crypto.crycb->apcb0, 0,
  2743. sizeof(kvm->arch.crypto.crycb->apcb0));
  2744. memset(&kvm->arch.crypto.crycb->apcb1, 0,
  2745. sizeof(kvm->arch.crypto.crycb->apcb1));
  2746. VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
  2747. /* recreate the shadow crycb for each vcpu */
  2748. kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
  2749. kvm_s390_vcpu_unblock_all(kvm);
  2750. }
  2751. EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
  2752. static u64 kvm_s390_get_initial_cpuid(void)
  2753. {
  2754. struct cpuid cpuid;
  2755. get_cpu_id(&cpuid);
  2756. cpuid.version = 0xff;
  2757. return *((u64 *) &cpuid);
  2758. }
  2759. static void kvm_s390_crypto_init(struct kvm *kvm)
  2760. {
  2761. kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
  2762. kvm_s390_set_crycb_format(kvm);
  2763. init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
  2764. if (!test_kvm_facility(kvm, 76))
  2765. return;
  2766. /* Enable AES/DEA protected key functions by default */
  2767. kvm->arch.crypto.aes_kw = 1;
  2768. kvm->arch.crypto.dea_kw = 1;
  2769. get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
  2770. sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
  2771. get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
  2772. sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
  2773. }
  2774. static void sca_dispose(struct kvm *kvm)
  2775. {
  2776. if (kvm->arch.use_esca)
  2777. free_pages_exact(kvm->arch.sca, sizeof(struct esca_block));
  2778. else
  2779. free_page((unsigned long)(kvm->arch.sca));
  2780. kvm->arch.sca = NULL;
  2781. }
  2782. void kvm_arch_free_vm(struct kvm *kvm)
  2783. {
  2784. if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
  2785. kvm_s390_pci_clear_list(kvm);
  2786. __kvm_arch_free_vm(kvm);
  2787. }
  2788. int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
  2789. {
  2790. gfp_t alloc_flags = GFP_KERNEL_ACCOUNT;
  2791. int i, rc;
  2792. char debug_name[16];
  2793. static unsigned long sca_offset;
  2794. rc = -EINVAL;
  2795. #ifdef CONFIG_KVM_S390_UCONTROL
  2796. if (type & ~KVM_VM_S390_UCONTROL)
  2797. goto out_err;
  2798. if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
  2799. goto out_err;
  2800. #else
  2801. if (type)
  2802. goto out_err;
  2803. #endif
  2804. rc = s390_enable_sie();
  2805. if (rc)
  2806. goto out_err;
  2807. rc = -ENOMEM;
  2808. if (!sclp.has_64bscao)
  2809. alloc_flags |= GFP_DMA;
  2810. rwlock_init(&kvm->arch.sca_lock);
  2811. /* start with basic SCA */
  2812. kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags);
  2813. if (!kvm->arch.sca)
  2814. goto out_err;
  2815. mutex_lock(&kvm_lock);
  2816. sca_offset += 16;
  2817. if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE)
  2818. sca_offset = 0;
  2819. kvm->arch.sca = (struct bsca_block *)
  2820. ((char *) kvm->arch.sca + sca_offset);
  2821. mutex_unlock(&kvm_lock);
  2822. sprintf(debug_name, "kvm-%u", current->pid);
  2823. kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
  2824. if (!kvm->arch.dbf)
  2825. goto out_err;
  2826. BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
  2827. kvm->arch.sie_page2 =
  2828. (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
  2829. if (!kvm->arch.sie_page2)
  2830. goto out_err;
  2831. kvm->arch.sie_page2->kvm = kvm;
  2832. kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
  2833. for (i = 0; i < kvm_s390_fac_size(); i++) {
  2834. kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
  2835. (kvm_s390_fac_base[i] |
  2836. kvm_s390_fac_ext[i]);
  2837. kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
  2838. kvm_s390_fac_base[i];
  2839. }
  2840. kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
  2841. /* we are always in czam mode - even on pre z14 machines */
  2842. set_kvm_facility(kvm->arch.model.fac_mask, 138);
  2843. set_kvm_facility(kvm->arch.model.fac_list, 138);
  2844. /* we emulate STHYI in kvm */
  2845. set_kvm_facility(kvm->arch.model.fac_mask, 74);
  2846. set_kvm_facility(kvm->arch.model.fac_list, 74);
  2847. if (MACHINE_HAS_TLB_GUEST) {
  2848. set_kvm_facility(kvm->arch.model.fac_mask, 147);
  2849. set_kvm_facility(kvm->arch.model.fac_list, 147);
  2850. }
  2851. if (css_general_characteristics.aiv && test_facility(65))
  2852. set_kvm_facility(kvm->arch.model.fac_mask, 65);
  2853. kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
  2854. kvm->arch.model.ibc = sclp.ibc & 0x0fff;
  2855. kvm_s390_crypto_init(kvm);
  2856. if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
  2857. mutex_lock(&kvm->lock);
  2858. kvm_s390_pci_init_list(kvm);
  2859. kvm_s390_vcpu_pci_enable_interp(kvm);
  2860. mutex_unlock(&kvm->lock);
  2861. }
  2862. mutex_init(&kvm->arch.float_int.ais_lock);
  2863. spin_lock_init(&kvm->arch.float_int.lock);
  2864. for (i = 0; i < FIRQ_LIST_COUNT; i++)
  2865. INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
  2866. init_waitqueue_head(&kvm->arch.ipte_wq);
  2867. mutex_init(&kvm->arch.ipte_mutex);
  2868. debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
  2869. VM_EVENT(kvm, 3, "vm created with type %lu", type);
  2870. if (type & KVM_VM_S390_UCONTROL) {
  2871. kvm->arch.gmap = NULL;
  2872. kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT;
  2873. } else {
  2874. if (sclp.hamax == U64_MAX)
  2875. kvm->arch.mem_limit = TASK_SIZE_MAX;
  2876. else
  2877. kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX,
  2878. sclp.hamax + 1);
  2879. kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1);
  2880. if (!kvm->arch.gmap)
  2881. goto out_err;
  2882. kvm->arch.gmap->private = kvm;
  2883. kvm->arch.gmap->pfault_enabled = 0;
  2884. }
  2885. kvm->arch.use_pfmfi = sclp.has_pfmfi;
  2886. kvm->arch.use_skf = sclp.has_skey;
  2887. spin_lock_init(&kvm->arch.start_stop_lock);
  2888. kvm_s390_vsie_init(kvm);
  2889. if (use_gisa)
  2890. kvm_s390_gisa_init(kvm);
  2891. KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid);
  2892. return 0;
  2893. out_err:
  2894. free_page((unsigned long)kvm->arch.sie_page2);
  2895. debug_unregister(kvm->arch.dbf);
  2896. sca_dispose(kvm);
  2897. KVM_EVENT(3, "creation of vm failed: %d", rc);
  2898. return rc;
  2899. }
  2900. void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
  2901. {
  2902. u16 rc, rrc;
  2903. VCPU_EVENT(vcpu, 3, "%s", "free cpu");
  2904. trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
  2905. kvm_s390_clear_local_irqs(vcpu);
  2906. kvm_clear_async_pf_completion_queue(vcpu);
  2907. if (!kvm_is_ucontrol(vcpu->kvm))
  2908. sca_del_vcpu(vcpu);
  2909. kvm_s390_update_topology_change_report(vcpu->kvm, 1);
  2910. if (kvm_is_ucontrol(vcpu->kvm))
  2911. gmap_remove(vcpu->arch.gmap);
  2912. if (vcpu->kvm->arch.use_cmma)
  2913. kvm_s390_vcpu_unsetup_cmma(vcpu);
  2914. /* We can not hold the vcpu mutex here, we are already dying */
  2915. if (kvm_s390_pv_cpu_get_handle(vcpu))
  2916. kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
  2917. free_page((unsigned long)(vcpu->arch.sie_block));
  2918. }
  2919. void kvm_arch_destroy_vm(struct kvm *kvm)
  2920. {
  2921. u16 rc, rrc;
  2922. kvm_destroy_vcpus(kvm);
  2923. sca_dispose(kvm);
  2924. kvm_s390_gisa_destroy(kvm);
  2925. /*
  2926. * We are already at the end of life and kvm->lock is not taken.
  2927. * This is ok as the file descriptor is closed by now and nobody
  2928. * can mess with the pv state. To avoid lockdep_assert_held from
  2929. * complaining we do not use kvm_s390_pv_is_protected.
  2930. */
  2931. if (kvm_s390_pv_get_handle(kvm))
  2932. kvm_s390_pv_deinit_vm(kvm, &rc, &rrc);
  2933. /*
  2934. * Remove the mmu notifier only when the whole KVM VM is torn down,
  2935. * and only if one was registered to begin with. If the VM is
  2936. * currently not protected, but has been previously been protected,
  2937. * then it's possible that the notifier is still registered.
  2938. */
  2939. if (kvm->arch.pv.mmu_notifier.ops)
  2940. mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
  2941. debug_unregister(kvm->arch.dbf);
  2942. free_page((unsigned long)kvm->arch.sie_page2);
  2943. if (!kvm_is_ucontrol(kvm))
  2944. gmap_remove(kvm->arch.gmap);
  2945. kvm_s390_destroy_adapters(kvm);
  2946. kvm_s390_clear_float_irqs(kvm);
  2947. kvm_s390_vsie_destroy(kvm);
  2948. KVM_EVENT(3, "vm 0x%pK destroyed", kvm);
  2949. }
  2950. /* Section: vcpu related */
  2951. static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu)
  2952. {
  2953. vcpu->arch.gmap = gmap_create(current->mm, -1UL);
  2954. if (!vcpu->arch.gmap)
  2955. return -ENOMEM;
  2956. vcpu->arch.gmap->private = vcpu->kvm;
  2957. return 0;
  2958. }
  2959. static void sca_del_vcpu(struct kvm_vcpu *vcpu)
  2960. {
  2961. if (!kvm_s390_use_sca_entries())
  2962. return;
  2963. read_lock(&vcpu->kvm->arch.sca_lock);
  2964. if (vcpu->kvm->arch.use_esca) {
  2965. struct esca_block *sca = vcpu->kvm->arch.sca;
  2966. clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
  2967. sca->cpu[vcpu->vcpu_id].sda = 0;
  2968. } else {
  2969. struct bsca_block *sca = vcpu->kvm->arch.sca;
  2970. clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
  2971. sca->cpu[vcpu->vcpu_id].sda = 0;
  2972. }
  2973. read_unlock(&vcpu->kvm->arch.sca_lock);
  2974. }
  2975. static void sca_add_vcpu(struct kvm_vcpu *vcpu)
  2976. {
  2977. if (!kvm_s390_use_sca_entries()) {
  2978. struct bsca_block *sca = vcpu->kvm->arch.sca;
  2979. /* we still need the basic sca for the ipte control */
  2980. vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
  2981. vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
  2982. return;
  2983. }
  2984. read_lock(&vcpu->kvm->arch.sca_lock);
  2985. if (vcpu->kvm->arch.use_esca) {
  2986. struct esca_block *sca = vcpu->kvm->arch.sca;
  2987. sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
  2988. vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
  2989. vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU;
  2990. vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
  2991. set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn);
  2992. } else {
  2993. struct bsca_block *sca = vcpu->kvm->arch.sca;
  2994. sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block;
  2995. vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32);
  2996. vcpu->arch.sie_block->scaol = (__u32)(__u64)sca;
  2997. set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn);
  2998. }
  2999. read_unlock(&vcpu->kvm->arch.sca_lock);
  3000. }
  3001. /* Basic SCA to Extended SCA data copy routines */
  3002. static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s)
  3003. {
  3004. d->sda = s->sda;
  3005. d->sigp_ctrl.c = s->sigp_ctrl.c;
  3006. d->sigp_ctrl.scn = s->sigp_ctrl.scn;
  3007. }
  3008. static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s)
  3009. {
  3010. int i;
  3011. d->ipte_control = s->ipte_control;
  3012. d->mcn[0] = s->mcn;
  3013. for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++)
  3014. sca_copy_entry(&d->cpu[i], &s->cpu[i]);
  3015. }
  3016. static int sca_switch_to_extended(struct kvm *kvm)
  3017. {
  3018. struct bsca_block *old_sca = kvm->arch.sca;
  3019. struct esca_block *new_sca;
  3020. struct kvm_vcpu *vcpu;
  3021. unsigned long vcpu_idx;
  3022. u32 scaol, scaoh;
  3023. if (kvm->arch.use_esca)
  3024. return 0;
  3025. new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL_ACCOUNT | __GFP_ZERO);
  3026. if (!new_sca)
  3027. return -ENOMEM;
  3028. scaoh = (u32)((u64)(new_sca) >> 32);
  3029. scaol = (u32)(u64)(new_sca) & ~0x3fU;
  3030. kvm_s390_vcpu_block_all(kvm);
  3031. write_lock(&kvm->arch.sca_lock);
  3032. sca_copy_b_to_e(new_sca, old_sca);
  3033. kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) {
  3034. vcpu->arch.sie_block->scaoh = scaoh;
  3035. vcpu->arch.sie_block->scaol = scaol;
  3036. vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
  3037. }
  3038. kvm->arch.sca = new_sca;
  3039. kvm->arch.use_esca = 1;
  3040. write_unlock(&kvm->arch.sca_lock);
  3041. kvm_s390_vcpu_unblock_all(kvm);
  3042. free_page((unsigned long)old_sca);
  3043. VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)",
  3044. old_sca, kvm->arch.sca);
  3045. return 0;
  3046. }
  3047. static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
  3048. {
  3049. int rc;
  3050. if (!kvm_s390_use_sca_entries()) {
  3051. if (id < KVM_MAX_VCPUS)
  3052. return true;
  3053. return false;
  3054. }
  3055. if (id < KVM_S390_BSCA_CPU_SLOTS)
  3056. return true;
  3057. if (!sclp.has_esca || !sclp.has_64bscao)
  3058. return false;
  3059. rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm);
  3060. return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS;
  3061. }
  3062. /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
  3063. static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
  3064. {
  3065. WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
  3066. raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
  3067. vcpu->arch.cputm_start = get_tod_clock_fast();
  3068. raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
  3069. }
  3070. /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
  3071. static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
  3072. {
  3073. WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
  3074. raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
  3075. vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
  3076. vcpu->arch.cputm_start = 0;
  3077. raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
  3078. }
  3079. /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
  3080. static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
  3081. {
  3082. WARN_ON_ONCE(vcpu->arch.cputm_enabled);
  3083. vcpu->arch.cputm_enabled = true;
  3084. __start_cpu_timer_accounting(vcpu);
  3085. }
  3086. /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
  3087. static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
  3088. {
  3089. WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
  3090. __stop_cpu_timer_accounting(vcpu);
  3091. vcpu->arch.cputm_enabled = false;
  3092. }
  3093. static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
  3094. {
  3095. preempt_disable(); /* protect from TOD sync and vcpu_load/put */
  3096. __enable_cpu_timer_accounting(vcpu);
  3097. preempt_enable();
  3098. }
  3099. static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
  3100. {
  3101. preempt_disable(); /* protect from TOD sync and vcpu_load/put */
  3102. __disable_cpu_timer_accounting(vcpu);
  3103. preempt_enable();
  3104. }
  3105. /* set the cpu timer - may only be called from the VCPU thread itself */
  3106. void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
  3107. {
  3108. preempt_disable(); /* protect from TOD sync and vcpu_load/put */
  3109. raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
  3110. if (vcpu->arch.cputm_enabled)
  3111. vcpu->arch.cputm_start = get_tod_clock_fast();
  3112. vcpu->arch.sie_block->cputm = cputm;
  3113. raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
  3114. preempt_enable();
  3115. }
  3116. /* update and get the cpu timer - can also be called from other VCPU threads */
  3117. __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
  3118. {
  3119. unsigned int seq;
  3120. __u64 value;
  3121. if (unlikely(!vcpu->arch.cputm_enabled))
  3122. return vcpu->arch.sie_block->cputm;
  3123. preempt_disable(); /* protect from TOD sync and vcpu_load/put */
  3124. do {
  3125. seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
  3126. /*
  3127. * If the writer would ever execute a read in the critical
  3128. * section, e.g. in irq context, we have a deadlock.
  3129. */
  3130. WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
  3131. value = vcpu->arch.sie_block->cputm;
  3132. /* if cputm_start is 0, accounting is being started/stopped */
  3133. if (likely(vcpu->arch.cputm_start))
  3134. value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
  3135. } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
  3136. preempt_enable();
  3137. return value;
  3138. }
  3139. void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
  3140. {
  3141. gmap_enable(vcpu->arch.enabled_gmap);
  3142. kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
  3143. if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
  3144. __start_cpu_timer_accounting(vcpu);
  3145. vcpu->cpu = cpu;
  3146. }
  3147. void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
  3148. {
  3149. vcpu->cpu = -1;
  3150. if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
  3151. __stop_cpu_timer_accounting(vcpu);
  3152. kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
  3153. vcpu->arch.enabled_gmap = gmap_get_enabled();
  3154. gmap_disable(vcpu->arch.enabled_gmap);
  3155. }
  3156. void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
  3157. {
  3158. mutex_lock(&vcpu->kvm->lock);
  3159. preempt_disable();
  3160. vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
  3161. vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
  3162. preempt_enable();
  3163. mutex_unlock(&vcpu->kvm->lock);
  3164. if (!kvm_is_ucontrol(vcpu->kvm)) {
  3165. vcpu->arch.gmap = vcpu->kvm->arch.gmap;
  3166. sca_add_vcpu(vcpu);
  3167. }
  3168. if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0)
  3169. vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
  3170. /* make vcpu_load load the right gmap on the first trigger */
  3171. vcpu->arch.enabled_gmap = vcpu->arch.gmap;
  3172. }
  3173. static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
  3174. {
  3175. if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
  3176. test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
  3177. return true;
  3178. return false;
  3179. }
  3180. static bool kvm_has_pckmo_ecc(struct kvm *kvm)
  3181. {
  3182. /* At least one ECC subfunction must be present */
  3183. return kvm_has_pckmo_subfunc(kvm, 32) ||
  3184. kvm_has_pckmo_subfunc(kvm, 33) ||
  3185. kvm_has_pckmo_subfunc(kvm, 34) ||
  3186. kvm_has_pckmo_subfunc(kvm, 40) ||
  3187. kvm_has_pckmo_subfunc(kvm, 41);
  3188. }
  3189. static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
  3190. {
  3191. /*
  3192. * If the AP instructions are not being interpreted and the MSAX3
  3193. * facility is not configured for the guest, there is nothing to set up.
  3194. */
  3195. if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
  3196. return;
  3197. vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
  3198. vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
  3199. vcpu->arch.sie_block->eca &= ~ECA_APIE;
  3200. vcpu->arch.sie_block->ecd &= ~ECD_ECC;
  3201. if (vcpu->kvm->arch.crypto.apie)
  3202. vcpu->arch.sie_block->eca |= ECA_APIE;
  3203. /* Set up protected key support */
  3204. if (vcpu->kvm->arch.crypto.aes_kw) {
  3205. vcpu->arch.sie_block->ecb3 |= ECB3_AES;
  3206. /* ecc is also wrapped with AES key */
  3207. if (kvm_has_pckmo_ecc(vcpu->kvm))
  3208. vcpu->arch.sie_block->ecd |= ECD_ECC;
  3209. }
  3210. if (vcpu->kvm->arch.crypto.dea_kw)
  3211. vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
  3212. }
  3213. void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
  3214. {
  3215. free_page(vcpu->arch.sie_block->cbrlo);
  3216. vcpu->arch.sie_block->cbrlo = 0;
  3217. }
  3218. int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
  3219. {
  3220. vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL_ACCOUNT);
  3221. if (!vcpu->arch.sie_block->cbrlo)
  3222. return -ENOMEM;
  3223. return 0;
  3224. }
  3225. static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
  3226. {
  3227. struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
  3228. vcpu->arch.sie_block->ibc = model->ibc;
  3229. if (test_kvm_facility(vcpu->kvm, 7))
  3230. vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list;
  3231. }
  3232. static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
  3233. {
  3234. int rc = 0;
  3235. u16 uvrc, uvrrc;
  3236. atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
  3237. CPUSTAT_SM |
  3238. CPUSTAT_STOPPED);
  3239. if (test_kvm_facility(vcpu->kvm, 78))
  3240. kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
  3241. else if (test_kvm_facility(vcpu->kvm, 8))
  3242. kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
  3243. kvm_s390_vcpu_setup_model(vcpu);
  3244. /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */
  3245. if (MACHINE_HAS_ESOP)
  3246. vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
  3247. if (test_kvm_facility(vcpu->kvm, 9))
  3248. vcpu->arch.sie_block->ecb |= ECB_SRSI;
  3249. if (test_kvm_facility(vcpu->kvm, 11))
  3250. vcpu->arch.sie_block->ecb |= ECB_PTF;
  3251. if (test_kvm_facility(vcpu->kvm, 73))
  3252. vcpu->arch.sie_block->ecb |= ECB_TE;
  3253. if (!kvm_is_ucontrol(vcpu->kvm))
  3254. vcpu->arch.sie_block->ecb |= ECB_SPECI;
  3255. if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
  3256. vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
  3257. if (test_kvm_facility(vcpu->kvm, 130))
  3258. vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
  3259. vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
  3260. if (sclp.has_cei)
  3261. vcpu->arch.sie_block->eca |= ECA_CEI;
  3262. if (sclp.has_ib)
  3263. vcpu->arch.sie_block->eca |= ECA_IB;
  3264. if (sclp.has_siif)
  3265. vcpu->arch.sie_block->eca |= ECA_SII;
  3266. if (sclp.has_sigpif)
  3267. vcpu->arch.sie_block->eca |= ECA_SIGPI;
  3268. if (test_kvm_facility(vcpu->kvm, 129)) {
  3269. vcpu->arch.sie_block->eca |= ECA_VX;
  3270. vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
  3271. }
  3272. if (test_kvm_facility(vcpu->kvm, 139))
  3273. vcpu->arch.sie_block->ecd |= ECD_MEF;
  3274. if (test_kvm_facility(vcpu->kvm, 156))
  3275. vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
  3276. if (vcpu->arch.sie_block->gd) {
  3277. vcpu->arch.sie_block->eca |= ECA_AIV;
  3278. VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
  3279. vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
  3280. }
  3281. vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx)
  3282. | SDNXC;
  3283. vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb;
  3284. if (sclp.has_kss)
  3285. kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
  3286. else
  3287. vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
  3288. if (vcpu->kvm->arch.use_cmma) {
  3289. rc = kvm_s390_vcpu_setup_cmma(vcpu);
  3290. if (rc)
  3291. return rc;
  3292. }
  3293. hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
  3294. vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup;
  3295. vcpu->arch.sie_block->hpid = HPID_KVM;
  3296. kvm_s390_vcpu_crypto_setup(vcpu);
  3297. kvm_s390_vcpu_pci_setup(vcpu);
  3298. mutex_lock(&vcpu->kvm->lock);
  3299. if (kvm_s390_pv_is_protected(vcpu->kvm)) {
  3300. rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
  3301. if (rc)
  3302. kvm_s390_vcpu_unsetup_cmma(vcpu);
  3303. }
  3304. mutex_unlock(&vcpu->kvm->lock);
  3305. return rc;
  3306. }
  3307. int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
  3308. {
  3309. if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
  3310. return -EINVAL;
  3311. return 0;
  3312. }
  3313. int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
  3314. {
  3315. struct sie_page *sie_page;
  3316. int rc;
  3317. BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
  3318. sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
  3319. if (!sie_page)
  3320. return -ENOMEM;
  3321. vcpu->arch.sie_block = &sie_page->sie_block;
  3322. vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb;
  3323. /* the real guest size will always be smaller than msl */
  3324. vcpu->arch.sie_block->mso = 0;
  3325. vcpu->arch.sie_block->msl = sclp.hamax;
  3326. vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
  3327. spin_lock_init(&vcpu->arch.local_int.lock);
  3328. vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
  3329. seqcount_init(&vcpu->arch.cputm_seqcount);
  3330. vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
  3331. kvm_clear_async_pf_completion_queue(vcpu);
  3332. vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
  3333. KVM_SYNC_GPRS |
  3334. KVM_SYNC_ACRS |
  3335. KVM_SYNC_CRS |
  3336. KVM_SYNC_ARCH0 |
  3337. KVM_SYNC_PFAULT |
  3338. KVM_SYNC_DIAG318;
  3339. kvm_s390_set_prefix(vcpu, 0);
  3340. if (test_kvm_facility(vcpu->kvm, 64))
  3341. vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
  3342. if (test_kvm_facility(vcpu->kvm, 82))
  3343. vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
  3344. if (test_kvm_facility(vcpu->kvm, 133))
  3345. vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
  3346. if (test_kvm_facility(vcpu->kvm, 156))
  3347. vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
  3348. /* fprs can be synchronized via vrs, even if the guest has no vx. With
  3349. * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format.
  3350. */
  3351. if (MACHINE_HAS_VX)
  3352. vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
  3353. else
  3354. vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
  3355. if (kvm_is_ucontrol(vcpu->kvm)) {
  3356. rc = __kvm_ucontrol_vcpu_init(vcpu);
  3357. if (rc)
  3358. goto out_free_sie_block;
  3359. }
  3360. VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK",
  3361. vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
  3362. trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
  3363. rc = kvm_s390_vcpu_setup(vcpu);
  3364. if (rc)
  3365. goto out_ucontrol_uninit;
  3366. kvm_s390_update_topology_change_report(vcpu->kvm, 1);
  3367. return 0;
  3368. out_ucontrol_uninit:
  3369. if (kvm_is_ucontrol(vcpu->kvm))
  3370. gmap_remove(vcpu->arch.gmap);
  3371. out_free_sie_block:
  3372. free_page((unsigned long)(vcpu->arch.sie_block));
  3373. return rc;
  3374. }
  3375. int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
  3376. {
  3377. clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
  3378. return kvm_s390_vcpu_has_irq(vcpu, 0);
  3379. }
  3380. bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
  3381. {
  3382. return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
  3383. }
  3384. void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
  3385. {
  3386. atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
  3387. exit_sie(vcpu);
  3388. }
  3389. void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
  3390. {
  3391. atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
  3392. }
  3393. static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
  3394. {
  3395. atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
  3396. exit_sie(vcpu);
  3397. }
  3398. bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
  3399. {
  3400. return atomic_read(&vcpu->arch.sie_block->prog20) &
  3401. (PROG_BLOCK_SIE | PROG_REQUEST);
  3402. }
  3403. static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
  3404. {
  3405. atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
  3406. }
  3407. /*
  3408. * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
  3409. * If the CPU is not running (e.g. waiting as idle) the function will
  3410. * return immediately. */
  3411. void exit_sie(struct kvm_vcpu *vcpu)
  3412. {
  3413. kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
  3414. kvm_s390_vsie_kick(vcpu);
  3415. while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
  3416. cpu_relax();
  3417. }
  3418. /* Kick a guest cpu out of SIE to process a request synchronously */
  3419. void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
  3420. {
  3421. __kvm_make_request(req, vcpu);
  3422. kvm_s390_vcpu_request(vcpu);
  3423. }
  3424. static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start,
  3425. unsigned long end)
  3426. {
  3427. struct kvm *kvm = gmap->private;
  3428. struct kvm_vcpu *vcpu;
  3429. unsigned long prefix;
  3430. unsigned long i;
  3431. if (gmap_is_shadow(gmap))
  3432. return;
  3433. if (start >= 1UL << 31)
  3434. /* We are only interested in prefix pages */
  3435. return;
  3436. kvm_for_each_vcpu(i, vcpu, kvm) {
  3437. /* match against both prefix pages */
  3438. prefix = kvm_s390_get_prefix(vcpu);
  3439. if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) {
  3440. VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx",
  3441. start, end);
  3442. kvm_s390_sync_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
  3443. }
  3444. }
  3445. }
  3446. bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
  3447. {
  3448. /* do not poll with more than halt_poll_max_steal percent of steal time */
  3449. if (S390_lowcore.avg_steal_timer * 100 / (TICK_USEC << 12) >=
  3450. READ_ONCE(halt_poll_max_steal)) {
  3451. vcpu->stat.halt_no_poll_steal++;
  3452. return true;
  3453. }
  3454. return false;
  3455. }
  3456. int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
  3457. {
  3458. /* kvm common code refers to this, but never calls it */
  3459. BUG();
  3460. return 0;
  3461. }
  3462. static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
  3463. struct kvm_one_reg *reg)
  3464. {
  3465. int r = -EINVAL;
  3466. switch (reg->id) {
  3467. case KVM_REG_S390_TODPR:
  3468. r = put_user(vcpu->arch.sie_block->todpr,
  3469. (u32 __user *)reg->addr);
  3470. break;
  3471. case KVM_REG_S390_EPOCHDIFF:
  3472. r = put_user(vcpu->arch.sie_block->epoch,
  3473. (u64 __user *)reg->addr);
  3474. break;
  3475. case KVM_REG_S390_CPU_TIMER:
  3476. r = put_user(kvm_s390_get_cpu_timer(vcpu),
  3477. (u64 __user *)reg->addr);
  3478. break;
  3479. case KVM_REG_S390_CLOCK_COMP:
  3480. r = put_user(vcpu->arch.sie_block->ckc,
  3481. (u64 __user *)reg->addr);
  3482. break;
  3483. case KVM_REG_S390_PFTOKEN:
  3484. r = put_user(vcpu->arch.pfault_token,
  3485. (u64 __user *)reg->addr);
  3486. break;
  3487. case KVM_REG_S390_PFCOMPARE:
  3488. r = put_user(vcpu->arch.pfault_compare,
  3489. (u64 __user *)reg->addr);
  3490. break;
  3491. case KVM_REG_S390_PFSELECT:
  3492. r = put_user(vcpu->arch.pfault_select,
  3493. (u64 __user *)reg->addr);
  3494. break;
  3495. case KVM_REG_S390_PP:
  3496. r = put_user(vcpu->arch.sie_block->pp,
  3497. (u64 __user *)reg->addr);
  3498. break;
  3499. case KVM_REG_S390_GBEA:
  3500. r = put_user(vcpu->arch.sie_block->gbea,
  3501. (u64 __user *)reg->addr);
  3502. break;
  3503. default:
  3504. break;
  3505. }
  3506. return r;
  3507. }
  3508. static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
  3509. struct kvm_one_reg *reg)
  3510. {
  3511. int r = -EINVAL;
  3512. __u64 val;
  3513. switch (reg->id) {
  3514. case KVM_REG_S390_TODPR:
  3515. r = get_user(vcpu->arch.sie_block->todpr,
  3516. (u32 __user *)reg->addr);
  3517. break;
  3518. case KVM_REG_S390_EPOCHDIFF:
  3519. r = get_user(vcpu->arch.sie_block->epoch,
  3520. (u64 __user *)reg->addr);
  3521. break;
  3522. case KVM_REG_S390_CPU_TIMER:
  3523. r = get_user(val, (u64 __user *)reg->addr);
  3524. if (!r)
  3525. kvm_s390_set_cpu_timer(vcpu, val);
  3526. break;
  3527. case KVM_REG_S390_CLOCK_COMP:
  3528. r = get_user(vcpu->arch.sie_block->ckc,
  3529. (u64 __user *)reg->addr);
  3530. break;
  3531. case KVM_REG_S390_PFTOKEN:
  3532. r = get_user(vcpu->arch.pfault_token,
  3533. (u64 __user *)reg->addr);
  3534. if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
  3535. kvm_clear_async_pf_completion_queue(vcpu);
  3536. break;
  3537. case KVM_REG_S390_PFCOMPARE:
  3538. r = get_user(vcpu->arch.pfault_compare,
  3539. (u64 __user *)reg->addr);
  3540. break;
  3541. case KVM_REG_S390_PFSELECT:
  3542. r = get_user(vcpu->arch.pfault_select,
  3543. (u64 __user *)reg->addr);
  3544. break;
  3545. case KVM_REG_S390_PP:
  3546. r = get_user(vcpu->arch.sie_block->pp,
  3547. (u64 __user *)reg->addr);
  3548. break;
  3549. case KVM_REG_S390_GBEA:
  3550. r = get_user(vcpu->arch.sie_block->gbea,
  3551. (u64 __user *)reg->addr);
  3552. break;
  3553. default:
  3554. break;
  3555. }
  3556. return r;
  3557. }
  3558. static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
  3559. {
  3560. vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
  3561. vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
  3562. memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
  3563. kvm_clear_async_pf_completion_queue(vcpu);
  3564. if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
  3565. kvm_s390_vcpu_stop(vcpu);
  3566. kvm_s390_clear_local_irqs(vcpu);
  3567. }
  3568. static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
  3569. {
  3570. /* Initial reset is a superset of the normal reset */
  3571. kvm_arch_vcpu_ioctl_normal_reset(vcpu);
  3572. /*
  3573. * This equals initial cpu reset in pop, but we don't switch to ESA.
  3574. * We do not only reset the internal data, but also ...
  3575. */
  3576. vcpu->arch.sie_block->gpsw.mask = 0;
  3577. vcpu->arch.sie_block->gpsw.addr = 0;
  3578. kvm_s390_set_prefix(vcpu, 0);
  3579. kvm_s390_set_cpu_timer(vcpu, 0);
  3580. vcpu->arch.sie_block->ckc = 0;
  3581. memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
  3582. vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
  3583. vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
  3584. /* ... the data in sync regs */
  3585. memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
  3586. vcpu->run->s.regs.ckc = 0;
  3587. vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
  3588. vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
  3589. vcpu->run->psw_addr = 0;
  3590. vcpu->run->psw_mask = 0;
  3591. vcpu->run->s.regs.todpr = 0;
  3592. vcpu->run->s.regs.cputm = 0;
  3593. vcpu->run->s.regs.ckc = 0;
  3594. vcpu->run->s.regs.pp = 0;
  3595. vcpu->run->s.regs.gbea = 1;
  3596. vcpu->run->s.regs.fpc = 0;
  3597. /*
  3598. * Do not reset these registers in the protected case, as some of
  3599. * them are overlayed and they are not accessible in this case
  3600. * anyway.
  3601. */
  3602. if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
  3603. vcpu->arch.sie_block->gbea = 1;
  3604. vcpu->arch.sie_block->pp = 0;
  3605. vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
  3606. vcpu->arch.sie_block->todpr = 0;
  3607. }
  3608. }
  3609. static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
  3610. {
  3611. struct kvm_sync_regs *regs = &vcpu->run->s.regs;
  3612. /* Clear reset is a superset of the initial reset */
  3613. kvm_arch_vcpu_ioctl_initial_reset(vcpu);
  3614. memset(&regs->gprs, 0, sizeof(regs->gprs));
  3615. memset(&regs->vrs, 0, sizeof(regs->vrs));
  3616. memset(&regs->acrs, 0, sizeof(regs->acrs));
  3617. memset(&regs->gscb, 0, sizeof(regs->gscb));
  3618. regs->etoken = 0;
  3619. regs->etoken_extension = 0;
  3620. }
  3621. int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3622. {
  3623. vcpu_load(vcpu);
  3624. memcpy(&vcpu->run->s.regs.gprs, &regs->gprs, sizeof(regs->gprs));
  3625. vcpu_put(vcpu);
  3626. return 0;
  3627. }
  3628. int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
  3629. {
  3630. vcpu_load(vcpu);
  3631. memcpy(&regs->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
  3632. vcpu_put(vcpu);
  3633. return 0;
  3634. }
  3635. int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
  3636. struct kvm_sregs *sregs)
  3637. {
  3638. vcpu_load(vcpu);
  3639. memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
  3640. memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
  3641. vcpu_put(vcpu);
  3642. return 0;
  3643. }
  3644. int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
  3645. struct kvm_sregs *sregs)
  3646. {
  3647. vcpu_load(vcpu);
  3648. memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
  3649. memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
  3650. vcpu_put(vcpu);
  3651. return 0;
  3652. }
  3653. int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  3654. {
  3655. int ret = 0;
  3656. vcpu_load(vcpu);
  3657. if (test_fp_ctl(fpu->fpc)) {
  3658. ret = -EINVAL;
  3659. goto out;
  3660. }
  3661. vcpu->run->s.regs.fpc = fpu->fpc;
  3662. if (MACHINE_HAS_VX)
  3663. convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
  3664. (freg_t *) fpu->fprs);
  3665. else
  3666. memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
  3667. out:
  3668. vcpu_put(vcpu);
  3669. return ret;
  3670. }
  3671. int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
  3672. {
  3673. vcpu_load(vcpu);
  3674. /* make sure we have the latest values */
  3675. save_fpu_regs();
  3676. if (MACHINE_HAS_VX)
  3677. convert_vx_to_fp((freg_t *) fpu->fprs,
  3678. (__vector128 *) vcpu->run->s.regs.vrs);
  3679. else
  3680. memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
  3681. fpu->fpc = vcpu->run->s.regs.fpc;
  3682. vcpu_put(vcpu);
  3683. return 0;
  3684. }
  3685. static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
  3686. {
  3687. int rc = 0;
  3688. if (!is_vcpu_stopped(vcpu))
  3689. rc = -EBUSY;
  3690. else {
  3691. vcpu->run->psw_mask = psw.mask;
  3692. vcpu->run->psw_addr = psw.addr;
  3693. }
  3694. return rc;
  3695. }
  3696. int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
  3697. struct kvm_translation *tr)
  3698. {
  3699. return -EINVAL; /* not implemented yet */
  3700. }
  3701. #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
  3702. KVM_GUESTDBG_USE_HW_BP | \
  3703. KVM_GUESTDBG_ENABLE)
  3704. int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
  3705. struct kvm_guest_debug *dbg)
  3706. {
  3707. int rc = 0;
  3708. vcpu_load(vcpu);
  3709. vcpu->guest_debug = 0;
  3710. kvm_s390_clear_bp_data(vcpu);
  3711. if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
  3712. rc = -EINVAL;
  3713. goto out;
  3714. }
  3715. if (!sclp.has_gpere) {
  3716. rc = -EINVAL;
  3717. goto out;
  3718. }
  3719. if (dbg->control & KVM_GUESTDBG_ENABLE) {
  3720. vcpu->guest_debug = dbg->control;
  3721. /* enforce guest PER */
  3722. kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
  3723. if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
  3724. rc = kvm_s390_import_bp_data(vcpu, dbg);
  3725. } else {
  3726. kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
  3727. vcpu->arch.guestdbg.last_bp = 0;
  3728. }
  3729. if (rc) {
  3730. vcpu->guest_debug = 0;
  3731. kvm_s390_clear_bp_data(vcpu);
  3732. kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
  3733. }
  3734. out:
  3735. vcpu_put(vcpu);
  3736. return rc;
  3737. }
  3738. int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
  3739. struct kvm_mp_state *mp_state)
  3740. {
  3741. int ret;
  3742. vcpu_load(vcpu);
  3743. /* CHECK_STOP and LOAD are not supported yet */
  3744. ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
  3745. KVM_MP_STATE_OPERATING;
  3746. vcpu_put(vcpu);
  3747. return ret;
  3748. }
  3749. int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
  3750. struct kvm_mp_state *mp_state)
  3751. {
  3752. int rc = 0;
  3753. vcpu_load(vcpu);
  3754. /* user space knows about this interface - let it control the state */
  3755. kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
  3756. switch (mp_state->mp_state) {
  3757. case KVM_MP_STATE_STOPPED:
  3758. rc = kvm_s390_vcpu_stop(vcpu);
  3759. break;
  3760. case KVM_MP_STATE_OPERATING:
  3761. rc = kvm_s390_vcpu_start(vcpu);
  3762. break;
  3763. case KVM_MP_STATE_LOAD:
  3764. if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
  3765. rc = -ENXIO;
  3766. break;
  3767. }
  3768. rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
  3769. break;
  3770. case KVM_MP_STATE_CHECK_STOP:
  3771. fallthrough; /* CHECK_STOP and LOAD are not supported yet */
  3772. default:
  3773. rc = -ENXIO;
  3774. }
  3775. vcpu_put(vcpu);
  3776. return rc;
  3777. }
  3778. static bool ibs_enabled(struct kvm_vcpu *vcpu)
  3779. {
  3780. return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
  3781. }
  3782. static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
  3783. {
  3784. retry:
  3785. kvm_s390_vcpu_request_handled(vcpu);
  3786. if (!kvm_request_pending(vcpu))
  3787. return 0;
  3788. /*
  3789. * If the guest prefix changed, re-arm the ipte notifier for the
  3790. * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
  3791. * This ensures that the ipte instruction for this request has
  3792. * already finished. We might race against a second unmapper that
  3793. * wants to set the blocking bit. Lets just retry the request loop.
  3794. */
  3795. if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) {
  3796. int rc;
  3797. rc = gmap_mprotect_notify(vcpu->arch.gmap,
  3798. kvm_s390_get_prefix(vcpu),
  3799. PAGE_SIZE * 2, PROT_WRITE);
  3800. if (rc) {
  3801. kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
  3802. return rc;
  3803. }
  3804. goto retry;
  3805. }
  3806. if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
  3807. vcpu->arch.sie_block->ihcpu = 0xffff;
  3808. goto retry;
  3809. }
  3810. if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
  3811. if (!ibs_enabled(vcpu)) {
  3812. trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
  3813. kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
  3814. }
  3815. goto retry;
  3816. }
  3817. if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
  3818. if (ibs_enabled(vcpu)) {
  3819. trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
  3820. kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
  3821. }
  3822. goto retry;
  3823. }
  3824. if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
  3825. vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
  3826. goto retry;
  3827. }
  3828. if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
  3829. /*
  3830. * Disable CMM virtualization; we will emulate the ESSA
  3831. * instruction manually, in order to provide additional
  3832. * functionalities needed for live migration.
  3833. */
  3834. vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
  3835. goto retry;
  3836. }
  3837. if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
  3838. /*
  3839. * Re-enable CMM virtualization if CMMA is available and
  3840. * CMM has been used.
  3841. */
  3842. if ((vcpu->kvm->arch.use_cmma) &&
  3843. (vcpu->kvm->mm->context.uses_cmm))
  3844. vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
  3845. goto retry;
  3846. }
  3847. /* we left the vsie handler, nothing to do, just clear the request */
  3848. kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
  3849. return 0;
  3850. }
  3851. static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
  3852. {
  3853. struct kvm_vcpu *vcpu;
  3854. union tod_clock clk;
  3855. unsigned long i;
  3856. preempt_disable();
  3857. store_tod_clock_ext(&clk);
  3858. kvm->arch.epoch = gtod->tod - clk.tod;
  3859. kvm->arch.epdx = 0;
  3860. if (test_kvm_facility(kvm, 139)) {
  3861. kvm->arch.epdx = gtod->epoch_idx - clk.ei;
  3862. if (kvm->arch.epoch > gtod->tod)
  3863. kvm->arch.epdx -= 1;
  3864. }
  3865. kvm_s390_vcpu_block_all(kvm);
  3866. kvm_for_each_vcpu(i, vcpu, kvm) {
  3867. vcpu->arch.sie_block->epoch = kvm->arch.epoch;
  3868. vcpu->arch.sie_block->epdx = kvm->arch.epdx;
  3869. }
  3870. kvm_s390_vcpu_unblock_all(kvm);
  3871. preempt_enable();
  3872. }
  3873. int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
  3874. {
  3875. if (!mutex_trylock(&kvm->lock))
  3876. return 0;
  3877. __kvm_s390_set_tod_clock(kvm, gtod);
  3878. mutex_unlock(&kvm->lock);
  3879. return 1;
  3880. }
  3881. /**
  3882. * kvm_arch_fault_in_page - fault-in guest page if necessary
  3883. * @vcpu: The corresponding virtual cpu
  3884. * @gpa: Guest physical address
  3885. * @writable: Whether the page should be writable or not
  3886. *
  3887. * Make sure that a guest page has been faulted-in on the host.
  3888. *
  3889. * Return: Zero on success, negative error code otherwise.
  3890. */
  3891. long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable)
  3892. {
  3893. return gmap_fault(vcpu->arch.gmap, gpa,
  3894. writable ? FAULT_FLAG_WRITE : 0);
  3895. }
  3896. static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
  3897. unsigned long token)
  3898. {
  3899. struct kvm_s390_interrupt inti;
  3900. struct kvm_s390_irq irq;
  3901. if (start_token) {
  3902. irq.u.ext.ext_params2 = token;
  3903. irq.type = KVM_S390_INT_PFAULT_INIT;
  3904. WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
  3905. } else {
  3906. inti.type = KVM_S390_INT_PFAULT_DONE;
  3907. inti.parm64 = token;
  3908. WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti));
  3909. }
  3910. }
  3911. bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
  3912. struct kvm_async_pf *work)
  3913. {
  3914. trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
  3915. __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
  3916. return true;
  3917. }
  3918. void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
  3919. struct kvm_async_pf *work)
  3920. {
  3921. trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
  3922. __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
  3923. }
  3924. void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
  3925. struct kvm_async_pf *work)
  3926. {
  3927. /* s390 will always inject the page directly */
  3928. }
  3929. bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
  3930. {
  3931. /*
  3932. * s390 will always inject the page directly,
  3933. * but we still want check_async_completion to cleanup
  3934. */
  3935. return true;
  3936. }
  3937. static bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
  3938. {
  3939. hva_t hva;
  3940. struct kvm_arch_async_pf arch;
  3941. if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
  3942. return false;
  3943. if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
  3944. vcpu->arch.pfault_compare)
  3945. return false;
  3946. if (psw_extint_disabled(vcpu))
  3947. return false;
  3948. if (kvm_s390_vcpu_has_irq(vcpu, 0))
  3949. return false;
  3950. if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
  3951. return false;
  3952. if (!vcpu->arch.gmap->pfault_enabled)
  3953. return false;
  3954. hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr));
  3955. hva += current->thread.gmap_addr & ~PAGE_MASK;
  3956. if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
  3957. return false;
  3958. return kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch);
  3959. }
  3960. static int vcpu_pre_run(struct kvm_vcpu *vcpu)
  3961. {
  3962. int rc, cpuflags;
  3963. /*
  3964. * On s390 notifications for arriving pages will be delivered directly
  3965. * to the guest but the house keeping for completed pfaults is
  3966. * handled outside the worker.
  3967. */
  3968. kvm_check_async_pf_completion(vcpu);
  3969. vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
  3970. vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
  3971. if (need_resched())
  3972. schedule();
  3973. if (!kvm_is_ucontrol(vcpu->kvm)) {
  3974. rc = kvm_s390_deliver_pending_interrupts(vcpu);
  3975. if (rc)
  3976. return rc;
  3977. }
  3978. rc = kvm_s390_handle_requests(vcpu);
  3979. if (rc)
  3980. return rc;
  3981. if (guestdbg_enabled(vcpu)) {
  3982. kvm_s390_backup_guest_per_regs(vcpu);
  3983. kvm_s390_patch_guest_per_regs(vcpu);
  3984. }
  3985. clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
  3986. vcpu->arch.sie_block->icptcode = 0;
  3987. cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
  3988. VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
  3989. trace_kvm_s390_sie_enter(vcpu, cpuflags);
  3990. return 0;
  3991. }
  3992. static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu)
  3993. {
  3994. struct kvm_s390_pgm_info pgm_info = {
  3995. .code = PGM_ADDRESSING,
  3996. };
  3997. u8 opcode, ilen;
  3998. int rc;
  3999. VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
  4000. trace_kvm_s390_sie_fault(vcpu);
  4001. /*
  4002. * We want to inject an addressing exception, which is defined as a
  4003. * suppressing or terminating exception. However, since we came here
  4004. * by a DAT access exception, the PSW still points to the faulting
  4005. * instruction since DAT exceptions are nullifying. So we've got
  4006. * to look up the current opcode to get the length of the instruction
  4007. * to be able to forward the PSW.
  4008. */
  4009. rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
  4010. ilen = insn_length(opcode);
  4011. if (rc < 0) {
  4012. return rc;
  4013. } else if (rc) {
  4014. /* Instruction-Fetching Exceptions - we can't detect the ilen.
  4015. * Forward by arbitrary ilc, injection will take care of
  4016. * nullification if necessary.
  4017. */
  4018. pgm_info = vcpu->arch.pgm;
  4019. ilen = 4;
  4020. }
  4021. pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
  4022. kvm_s390_forward_psw(vcpu, ilen);
  4023. return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
  4024. }
  4025. static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason)
  4026. {
  4027. struct mcck_volatile_info *mcck_info;
  4028. struct sie_page *sie_page;
  4029. VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
  4030. vcpu->arch.sie_block->icptcode);
  4031. trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
  4032. if (guestdbg_enabled(vcpu))
  4033. kvm_s390_restore_guest_per_regs(vcpu);
  4034. vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
  4035. vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
  4036. if (exit_reason == -EINTR) {
  4037. VCPU_EVENT(vcpu, 3, "%s", "machine check");
  4038. sie_page = container_of(vcpu->arch.sie_block,
  4039. struct sie_page, sie_block);
  4040. mcck_info = &sie_page->mcck_info;
  4041. kvm_s390_reinject_machine_check(vcpu, mcck_info);
  4042. return 0;
  4043. }
  4044. if (vcpu->arch.sie_block->icptcode > 0) {
  4045. int rc = kvm_handle_sie_intercept(vcpu);
  4046. if (rc != -EOPNOTSUPP)
  4047. return rc;
  4048. vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
  4049. vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
  4050. vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
  4051. vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
  4052. return -EREMOTE;
  4053. } else if (exit_reason != -EFAULT) {
  4054. vcpu->stat.exit_null++;
  4055. return 0;
  4056. } else if (kvm_is_ucontrol(vcpu->kvm)) {
  4057. vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
  4058. vcpu->run->s390_ucontrol.trans_exc_code =
  4059. current->thread.gmap_addr;
  4060. vcpu->run->s390_ucontrol.pgm_code = 0x10;
  4061. return -EREMOTE;
  4062. } else if (current->thread.gmap_pfault) {
  4063. trace_kvm_s390_major_guest_pfault(vcpu);
  4064. current->thread.gmap_pfault = 0;
  4065. if (kvm_arch_setup_async_pf(vcpu))
  4066. return 0;
  4067. vcpu->stat.pfault_sync++;
  4068. return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1);
  4069. }
  4070. return vcpu_post_run_fault_in_sie(vcpu);
  4071. }
  4072. #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
  4073. static int __vcpu_run(struct kvm_vcpu *vcpu)
  4074. {
  4075. int rc, exit_reason;
  4076. struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
  4077. /*
  4078. * We try to hold kvm->srcu during most of vcpu_run (except when run-
  4079. * ning the guest), so that memslots (and other stuff) are protected
  4080. */
  4081. kvm_vcpu_srcu_read_lock(vcpu);
  4082. do {
  4083. rc = vcpu_pre_run(vcpu);
  4084. if (rc)
  4085. break;
  4086. kvm_vcpu_srcu_read_unlock(vcpu);
  4087. /*
  4088. * As PF_VCPU will be used in fault handler, between
  4089. * guest_enter and guest_exit should be no uaccess.
  4090. */
  4091. local_irq_disable();
  4092. guest_enter_irqoff();
  4093. __disable_cpu_timer_accounting(vcpu);
  4094. local_irq_enable();
  4095. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4096. memcpy(sie_page->pv_grregs,
  4097. vcpu->run->s.regs.gprs,
  4098. sizeof(sie_page->pv_grregs));
  4099. }
  4100. if (test_cpu_flag(CIF_FPU))
  4101. load_fpu_regs();
  4102. exit_reason = sie64a(vcpu->arch.sie_block,
  4103. vcpu->run->s.regs.gprs);
  4104. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4105. memcpy(vcpu->run->s.regs.gprs,
  4106. sie_page->pv_grregs,
  4107. sizeof(sie_page->pv_grregs));
  4108. /*
  4109. * We're not allowed to inject interrupts on intercepts
  4110. * that leave the guest state in an "in-between" state
  4111. * where the next SIE entry will do a continuation.
  4112. * Fence interrupts in our "internal" PSW.
  4113. */
  4114. if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
  4115. vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
  4116. vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
  4117. }
  4118. }
  4119. local_irq_disable();
  4120. __enable_cpu_timer_accounting(vcpu);
  4121. guest_exit_irqoff();
  4122. local_irq_enable();
  4123. kvm_vcpu_srcu_read_lock(vcpu);
  4124. rc = vcpu_post_run(vcpu, exit_reason);
  4125. } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc);
  4126. kvm_vcpu_srcu_read_unlock(vcpu);
  4127. return rc;
  4128. }
  4129. static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
  4130. {
  4131. struct kvm_run *kvm_run = vcpu->run;
  4132. struct runtime_instr_cb *riccb;
  4133. struct gs_cb *gscb;
  4134. riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
  4135. gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
  4136. vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
  4137. vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
  4138. if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
  4139. vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
  4140. vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
  4141. vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
  4142. }
  4143. if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
  4144. vcpu->arch.pfault_token = kvm_run->s.regs.pft;
  4145. vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
  4146. vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
  4147. if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
  4148. kvm_clear_async_pf_completion_queue(vcpu);
  4149. }
  4150. if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
  4151. vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
  4152. vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
  4153. VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
  4154. }
  4155. /*
  4156. * If userspace sets the riccb (e.g. after migration) to a valid state,
  4157. * we should enable RI here instead of doing the lazy enablement.
  4158. */
  4159. if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
  4160. test_kvm_facility(vcpu->kvm, 64) &&
  4161. riccb->v &&
  4162. !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
  4163. VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
  4164. vcpu->arch.sie_block->ecb3 |= ECB3_RI;
  4165. }
  4166. /*
  4167. * If userspace sets the gscb (e.g. after migration) to non-zero,
  4168. * we should enable GS here instead of doing the lazy enablement.
  4169. */
  4170. if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
  4171. test_kvm_facility(vcpu->kvm, 133) &&
  4172. gscb->gssm &&
  4173. !vcpu->arch.gs_enabled) {
  4174. VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
  4175. vcpu->arch.sie_block->ecb |= ECB_GS;
  4176. vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
  4177. vcpu->arch.gs_enabled = 1;
  4178. }
  4179. if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
  4180. test_kvm_facility(vcpu->kvm, 82)) {
  4181. vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
  4182. vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
  4183. }
  4184. if (MACHINE_HAS_GS) {
  4185. preempt_disable();
  4186. __ctl_set_bit(2, 4);
  4187. if (current->thread.gs_cb) {
  4188. vcpu->arch.host_gscb = current->thread.gs_cb;
  4189. save_gs_cb(vcpu->arch.host_gscb);
  4190. }
  4191. if (vcpu->arch.gs_enabled) {
  4192. current->thread.gs_cb = (struct gs_cb *)
  4193. &vcpu->run->s.regs.gscb;
  4194. restore_gs_cb(current->thread.gs_cb);
  4195. }
  4196. preempt_enable();
  4197. }
  4198. /* SIE will load etoken directly from SDNX and therefore kvm_run */
  4199. }
  4200. static void sync_regs(struct kvm_vcpu *vcpu)
  4201. {
  4202. struct kvm_run *kvm_run = vcpu->run;
  4203. if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
  4204. kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
  4205. if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
  4206. memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
  4207. /* some control register changes require a tlb flush */
  4208. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  4209. }
  4210. if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
  4211. kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
  4212. vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
  4213. }
  4214. save_access_regs(vcpu->arch.host_acrs);
  4215. restore_access_regs(vcpu->run->s.regs.acrs);
  4216. /* save host (userspace) fprs/vrs */
  4217. save_fpu_regs();
  4218. vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc;
  4219. vcpu->arch.host_fpregs.regs = current->thread.fpu.regs;
  4220. if (MACHINE_HAS_VX)
  4221. current->thread.fpu.regs = vcpu->run->s.regs.vrs;
  4222. else
  4223. current->thread.fpu.regs = vcpu->run->s.regs.fprs;
  4224. current->thread.fpu.fpc = vcpu->run->s.regs.fpc;
  4225. if (test_fp_ctl(current->thread.fpu.fpc))
  4226. /* User space provided an invalid FPC, let's clear it */
  4227. current->thread.fpu.fpc = 0;
  4228. /* Sync fmt2 only data */
  4229. if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
  4230. sync_regs_fmt2(vcpu);
  4231. } else {
  4232. /*
  4233. * In several places we have to modify our internal view to
  4234. * not do things that are disallowed by the ultravisor. For
  4235. * example we must not inject interrupts after specific exits
  4236. * (e.g. 112 prefix page not secure). We do this by turning
  4237. * off the machine check, external and I/O interrupt bits
  4238. * of our PSW copy. To avoid getting validity intercepts, we
  4239. * do only accept the condition code from userspace.
  4240. */
  4241. vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
  4242. vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
  4243. PSW_MASK_CC;
  4244. }
  4245. kvm_run->kvm_dirty_regs = 0;
  4246. }
  4247. static void store_regs_fmt2(struct kvm_vcpu *vcpu)
  4248. {
  4249. struct kvm_run *kvm_run = vcpu->run;
  4250. kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
  4251. kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
  4252. kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
  4253. kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
  4254. kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
  4255. if (MACHINE_HAS_GS) {
  4256. preempt_disable();
  4257. __ctl_set_bit(2, 4);
  4258. if (vcpu->arch.gs_enabled)
  4259. save_gs_cb(current->thread.gs_cb);
  4260. current->thread.gs_cb = vcpu->arch.host_gscb;
  4261. restore_gs_cb(vcpu->arch.host_gscb);
  4262. if (!vcpu->arch.host_gscb)
  4263. __ctl_clear_bit(2, 4);
  4264. vcpu->arch.host_gscb = NULL;
  4265. preempt_enable();
  4266. }
  4267. /* SIE will save etoken directly into SDNX and therefore kvm_run */
  4268. }
  4269. static void store_regs(struct kvm_vcpu *vcpu)
  4270. {
  4271. struct kvm_run *kvm_run = vcpu->run;
  4272. kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
  4273. kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
  4274. kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
  4275. memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
  4276. kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
  4277. kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
  4278. kvm_run->s.regs.pft = vcpu->arch.pfault_token;
  4279. kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
  4280. kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
  4281. save_access_regs(vcpu->run->s.regs.acrs);
  4282. restore_access_regs(vcpu->arch.host_acrs);
  4283. /* Save guest register state */
  4284. save_fpu_regs();
  4285. vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
  4286. /* Restore will be done lazily at return */
  4287. current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc;
  4288. current->thread.fpu.regs = vcpu->arch.host_fpregs.regs;
  4289. if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
  4290. store_regs_fmt2(vcpu);
  4291. }
  4292. int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
  4293. {
  4294. struct kvm_run *kvm_run = vcpu->run;
  4295. int rc;
  4296. /*
  4297. * Running a VM while dumping always has the potential to
  4298. * produce inconsistent dump data. But for PV vcpus a SIE
  4299. * entry while dumping could also lead to a fatal validity
  4300. * intercept which we absolutely want to avoid.
  4301. */
  4302. if (vcpu->kvm->arch.pv.dumping)
  4303. return -EINVAL;
  4304. if (kvm_run->immediate_exit)
  4305. return -EINTR;
  4306. if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
  4307. kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
  4308. return -EINVAL;
  4309. vcpu_load(vcpu);
  4310. if (guestdbg_exit_pending(vcpu)) {
  4311. kvm_s390_prepare_debug_exit(vcpu);
  4312. rc = 0;
  4313. goto out;
  4314. }
  4315. kvm_sigset_activate(vcpu);
  4316. /*
  4317. * no need to check the return value of vcpu_start as it can only have
  4318. * an error for protvirt, but protvirt means user cpu state
  4319. */
  4320. if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
  4321. kvm_s390_vcpu_start(vcpu);
  4322. } else if (is_vcpu_stopped(vcpu)) {
  4323. pr_err_ratelimited("can't run stopped vcpu %d\n",
  4324. vcpu->vcpu_id);
  4325. rc = -EINVAL;
  4326. goto out;
  4327. }
  4328. sync_regs(vcpu);
  4329. enable_cpu_timer_accounting(vcpu);
  4330. might_fault();
  4331. rc = __vcpu_run(vcpu);
  4332. if (signal_pending(current) && !rc) {
  4333. kvm_run->exit_reason = KVM_EXIT_INTR;
  4334. rc = -EINTR;
  4335. }
  4336. if (guestdbg_exit_pending(vcpu) && !rc) {
  4337. kvm_s390_prepare_debug_exit(vcpu);
  4338. rc = 0;
  4339. }
  4340. if (rc == -EREMOTE) {
  4341. /* userspace support is needed, kvm_run has been prepared */
  4342. rc = 0;
  4343. }
  4344. disable_cpu_timer_accounting(vcpu);
  4345. store_regs(vcpu);
  4346. kvm_sigset_deactivate(vcpu);
  4347. vcpu->stat.exit_userspace++;
  4348. out:
  4349. vcpu_put(vcpu);
  4350. return rc;
  4351. }
  4352. /*
  4353. * store status at address
  4354. * we use have two special cases:
  4355. * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
  4356. * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
  4357. */
  4358. int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
  4359. {
  4360. unsigned char archmode = 1;
  4361. freg_t fprs[NUM_FPRS];
  4362. unsigned int px;
  4363. u64 clkcomp, cputm;
  4364. int rc;
  4365. px = kvm_s390_get_prefix(vcpu);
  4366. if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
  4367. if (write_guest_abs(vcpu, 163, &archmode, 1))
  4368. return -EFAULT;
  4369. gpa = 0;
  4370. } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
  4371. if (write_guest_real(vcpu, 163, &archmode, 1))
  4372. return -EFAULT;
  4373. gpa = px;
  4374. } else
  4375. gpa -= __LC_FPREGS_SAVE_AREA;
  4376. /* manually convert vector registers if necessary */
  4377. if (MACHINE_HAS_VX) {
  4378. convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
  4379. rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
  4380. fprs, 128);
  4381. } else {
  4382. rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
  4383. vcpu->run->s.regs.fprs, 128);
  4384. }
  4385. rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
  4386. vcpu->run->s.regs.gprs, 128);
  4387. rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
  4388. &vcpu->arch.sie_block->gpsw, 16);
  4389. rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
  4390. &px, 4);
  4391. rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
  4392. &vcpu->run->s.regs.fpc, 4);
  4393. rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
  4394. &vcpu->arch.sie_block->todpr, 4);
  4395. cputm = kvm_s390_get_cpu_timer(vcpu);
  4396. rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
  4397. &cputm, 8);
  4398. clkcomp = vcpu->arch.sie_block->ckc >> 8;
  4399. rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
  4400. &clkcomp, 8);
  4401. rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
  4402. &vcpu->run->s.regs.acrs, 64);
  4403. rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
  4404. &vcpu->arch.sie_block->gcr, 128);
  4405. return rc ? -EFAULT : 0;
  4406. }
  4407. int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
  4408. {
  4409. /*
  4410. * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
  4411. * switch in the run ioctl. Let's update our copies before we save
  4412. * it into the save area
  4413. */
  4414. save_fpu_regs();
  4415. vcpu->run->s.regs.fpc = current->thread.fpu.fpc;
  4416. save_access_regs(vcpu->run->s.regs.acrs);
  4417. return kvm_s390_store_status_unloaded(vcpu, addr);
  4418. }
  4419. static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
  4420. {
  4421. kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
  4422. kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
  4423. }
  4424. static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
  4425. {
  4426. unsigned long i;
  4427. struct kvm_vcpu *vcpu;
  4428. kvm_for_each_vcpu(i, vcpu, kvm) {
  4429. __disable_ibs_on_vcpu(vcpu);
  4430. }
  4431. }
  4432. static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
  4433. {
  4434. if (!sclp.has_ibs)
  4435. return;
  4436. kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
  4437. kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
  4438. }
  4439. int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
  4440. {
  4441. int i, online_vcpus, r = 0, started_vcpus = 0;
  4442. if (!is_vcpu_stopped(vcpu))
  4443. return 0;
  4444. trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
  4445. /* Only one cpu at a time may enter/leave the STOPPED state. */
  4446. spin_lock(&vcpu->kvm->arch.start_stop_lock);
  4447. online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
  4448. /* Let's tell the UV that we want to change into the operating state */
  4449. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4450. r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
  4451. if (r) {
  4452. spin_unlock(&vcpu->kvm->arch.start_stop_lock);
  4453. return r;
  4454. }
  4455. }
  4456. for (i = 0; i < online_vcpus; i++) {
  4457. if (!is_vcpu_stopped(kvm_get_vcpu(vcpu->kvm, i)))
  4458. started_vcpus++;
  4459. }
  4460. if (started_vcpus == 0) {
  4461. /* we're the only active VCPU -> speed it up */
  4462. __enable_ibs_on_vcpu(vcpu);
  4463. } else if (started_vcpus == 1) {
  4464. /*
  4465. * As we are starting a second VCPU, we have to disable
  4466. * the IBS facility on all VCPUs to remove potentially
  4467. * outstanding ENABLE requests.
  4468. */
  4469. __disable_ibs_on_all_vcpus(vcpu->kvm);
  4470. }
  4471. kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
  4472. /*
  4473. * The real PSW might have changed due to a RESTART interpreted by the
  4474. * ultravisor. We block all interrupts and let the next sie exit
  4475. * refresh our view.
  4476. */
  4477. if (kvm_s390_pv_cpu_is_protected(vcpu))
  4478. vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
  4479. /*
  4480. * Another VCPU might have used IBS while we were offline.
  4481. * Let's play safe and flush the VCPU at startup.
  4482. */
  4483. kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
  4484. spin_unlock(&vcpu->kvm->arch.start_stop_lock);
  4485. return 0;
  4486. }
  4487. int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
  4488. {
  4489. int i, online_vcpus, r = 0, started_vcpus = 0;
  4490. struct kvm_vcpu *started_vcpu = NULL;
  4491. if (is_vcpu_stopped(vcpu))
  4492. return 0;
  4493. trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
  4494. /* Only one cpu at a time may enter/leave the STOPPED state. */
  4495. spin_lock(&vcpu->kvm->arch.start_stop_lock);
  4496. online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
  4497. /* Let's tell the UV that we want to change into the stopped state */
  4498. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4499. r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
  4500. if (r) {
  4501. spin_unlock(&vcpu->kvm->arch.start_stop_lock);
  4502. return r;
  4503. }
  4504. }
  4505. /*
  4506. * Set the VCPU to STOPPED and THEN clear the interrupt flag,
  4507. * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
  4508. * have been fully processed. This will ensure that the VCPU
  4509. * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
  4510. */
  4511. kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
  4512. kvm_s390_clear_stop_irq(vcpu);
  4513. __disable_ibs_on_vcpu(vcpu);
  4514. for (i = 0; i < online_vcpus; i++) {
  4515. struct kvm_vcpu *tmp = kvm_get_vcpu(vcpu->kvm, i);
  4516. if (!is_vcpu_stopped(tmp)) {
  4517. started_vcpus++;
  4518. started_vcpu = tmp;
  4519. }
  4520. }
  4521. if (started_vcpus == 1) {
  4522. /*
  4523. * As we only have one VCPU left, we want to enable the
  4524. * IBS facility for that VCPU to speed it up.
  4525. */
  4526. __enable_ibs_on_vcpu(started_vcpu);
  4527. }
  4528. spin_unlock(&vcpu->kvm->arch.start_stop_lock);
  4529. return 0;
  4530. }
  4531. static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
  4532. struct kvm_enable_cap *cap)
  4533. {
  4534. int r;
  4535. if (cap->flags)
  4536. return -EINVAL;
  4537. switch (cap->cap) {
  4538. case KVM_CAP_S390_CSS_SUPPORT:
  4539. if (!vcpu->kvm->arch.css_support) {
  4540. vcpu->kvm->arch.css_support = 1;
  4541. VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
  4542. trace_kvm_s390_enable_css(vcpu->kvm);
  4543. }
  4544. r = 0;
  4545. break;
  4546. default:
  4547. r = -EINVAL;
  4548. break;
  4549. }
  4550. return r;
  4551. }
  4552. static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
  4553. struct kvm_s390_mem_op *mop)
  4554. {
  4555. void __user *uaddr = (void __user *)mop->buf;
  4556. int r = 0;
  4557. if (mop->flags || !mop->size)
  4558. return -EINVAL;
  4559. if (mop->size + mop->sida_offset < mop->size)
  4560. return -EINVAL;
  4561. if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
  4562. return -E2BIG;
  4563. if (!kvm_s390_pv_cpu_is_protected(vcpu))
  4564. return -EINVAL;
  4565. switch (mop->op) {
  4566. case KVM_S390_MEMOP_SIDA_READ:
  4567. if (copy_to_user(uaddr, (void *)(sida_origin(vcpu->arch.sie_block) +
  4568. mop->sida_offset), mop->size))
  4569. r = -EFAULT;
  4570. break;
  4571. case KVM_S390_MEMOP_SIDA_WRITE:
  4572. if (copy_from_user((void *)(sida_origin(vcpu->arch.sie_block) +
  4573. mop->sida_offset), uaddr, mop->size))
  4574. r = -EFAULT;
  4575. break;
  4576. }
  4577. return r;
  4578. }
  4579. static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
  4580. struct kvm_s390_mem_op *mop)
  4581. {
  4582. void __user *uaddr = (void __user *)mop->buf;
  4583. void *tmpbuf = NULL;
  4584. int r = 0;
  4585. const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION
  4586. | KVM_S390_MEMOP_F_CHECK_ONLY
  4587. | KVM_S390_MEMOP_F_SKEY_PROTECTION;
  4588. if (mop->flags & ~supported_flags || mop->ar >= NUM_ACRS || !mop->size)
  4589. return -EINVAL;
  4590. if (mop->size > MEM_OP_MAX_SIZE)
  4591. return -E2BIG;
  4592. if (kvm_s390_pv_cpu_is_protected(vcpu))
  4593. return -EINVAL;
  4594. if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
  4595. if (access_key_invalid(mop->key))
  4596. return -EINVAL;
  4597. } else {
  4598. mop->key = 0;
  4599. }
  4600. if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
  4601. tmpbuf = vmalloc(mop->size);
  4602. if (!tmpbuf)
  4603. return -ENOMEM;
  4604. }
  4605. switch (mop->op) {
  4606. case KVM_S390_MEMOP_LOGICAL_READ:
  4607. if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
  4608. r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
  4609. GACC_FETCH, mop->key);
  4610. break;
  4611. }
  4612. r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
  4613. mop->size, mop->key);
  4614. if (r == 0) {
  4615. if (copy_to_user(uaddr, tmpbuf, mop->size))
  4616. r = -EFAULT;
  4617. }
  4618. break;
  4619. case KVM_S390_MEMOP_LOGICAL_WRITE:
  4620. if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
  4621. r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
  4622. GACC_STORE, mop->key);
  4623. break;
  4624. }
  4625. if (copy_from_user(tmpbuf, uaddr, mop->size)) {
  4626. r = -EFAULT;
  4627. break;
  4628. }
  4629. r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
  4630. mop->size, mop->key);
  4631. break;
  4632. }
  4633. if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
  4634. kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
  4635. vfree(tmpbuf);
  4636. return r;
  4637. }
  4638. static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
  4639. struct kvm_s390_mem_op *mop)
  4640. {
  4641. int r, srcu_idx;
  4642. srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
  4643. switch (mop->op) {
  4644. case KVM_S390_MEMOP_LOGICAL_READ:
  4645. case KVM_S390_MEMOP_LOGICAL_WRITE:
  4646. r = kvm_s390_vcpu_mem_op(vcpu, mop);
  4647. break;
  4648. case KVM_S390_MEMOP_SIDA_READ:
  4649. case KVM_S390_MEMOP_SIDA_WRITE:
  4650. /* we are locked against sida going away by the vcpu->mutex */
  4651. r = kvm_s390_vcpu_sida_op(vcpu, mop);
  4652. break;
  4653. default:
  4654. r = -EINVAL;
  4655. }
  4656. srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
  4657. return r;
  4658. }
  4659. long kvm_arch_vcpu_async_ioctl(struct file *filp,
  4660. unsigned int ioctl, unsigned long arg)
  4661. {
  4662. struct kvm_vcpu *vcpu = filp->private_data;
  4663. void __user *argp = (void __user *)arg;
  4664. switch (ioctl) {
  4665. case KVM_S390_IRQ: {
  4666. struct kvm_s390_irq s390irq;
  4667. if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
  4668. return -EFAULT;
  4669. return kvm_s390_inject_vcpu(vcpu, &s390irq);
  4670. }
  4671. case KVM_S390_INTERRUPT: {
  4672. struct kvm_s390_interrupt s390int;
  4673. struct kvm_s390_irq s390irq = {};
  4674. if (copy_from_user(&s390int, argp, sizeof(s390int)))
  4675. return -EFAULT;
  4676. if (s390int_to_s390irq(&s390int, &s390irq))
  4677. return -EINVAL;
  4678. return kvm_s390_inject_vcpu(vcpu, &s390irq);
  4679. }
  4680. }
  4681. return -ENOIOCTLCMD;
  4682. }
  4683. static int kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu *vcpu,
  4684. struct kvm_pv_cmd *cmd)
  4685. {
  4686. struct kvm_s390_pv_dmp dmp;
  4687. void *data;
  4688. int ret;
  4689. /* Dump initialization is a prerequisite */
  4690. if (!vcpu->kvm->arch.pv.dumping)
  4691. return -EINVAL;
  4692. if (copy_from_user(&dmp, (__u8 __user *)cmd->data, sizeof(dmp)))
  4693. return -EFAULT;
  4694. /* We only handle this subcmd right now */
  4695. if (dmp.subcmd != KVM_PV_DUMP_CPU)
  4696. return -EINVAL;
  4697. /* CPU dump length is the same as create cpu storage donation. */
  4698. if (dmp.buff_len != uv_info.guest_cpu_stor_len)
  4699. return -EINVAL;
  4700. data = kvzalloc(uv_info.guest_cpu_stor_len, GFP_KERNEL);
  4701. if (!data)
  4702. return -ENOMEM;
  4703. ret = kvm_s390_pv_dump_cpu(vcpu, data, &cmd->rc, &cmd->rrc);
  4704. VCPU_EVENT(vcpu, 3, "PROTVIRT DUMP CPU %d rc %x rrc %x",
  4705. vcpu->vcpu_id, cmd->rc, cmd->rrc);
  4706. if (ret)
  4707. ret = -EINVAL;
  4708. /* On success copy over the dump data */
  4709. if (!ret && copy_to_user((__u8 __user *)dmp.buff_addr, data, uv_info.guest_cpu_stor_len))
  4710. ret = -EFAULT;
  4711. kvfree(data);
  4712. return ret;
  4713. }
  4714. long kvm_arch_vcpu_ioctl(struct file *filp,
  4715. unsigned int ioctl, unsigned long arg)
  4716. {
  4717. struct kvm_vcpu *vcpu = filp->private_data;
  4718. void __user *argp = (void __user *)arg;
  4719. int idx;
  4720. long r;
  4721. u16 rc, rrc;
  4722. vcpu_load(vcpu);
  4723. switch (ioctl) {
  4724. case KVM_S390_STORE_STATUS:
  4725. idx = srcu_read_lock(&vcpu->kvm->srcu);
  4726. r = kvm_s390_store_status_unloaded(vcpu, arg);
  4727. srcu_read_unlock(&vcpu->kvm->srcu, idx);
  4728. break;
  4729. case KVM_S390_SET_INITIAL_PSW: {
  4730. psw_t psw;
  4731. r = -EFAULT;
  4732. if (copy_from_user(&psw, argp, sizeof(psw)))
  4733. break;
  4734. r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
  4735. break;
  4736. }
  4737. case KVM_S390_CLEAR_RESET:
  4738. r = 0;
  4739. kvm_arch_vcpu_ioctl_clear_reset(vcpu);
  4740. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4741. r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
  4742. UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
  4743. VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
  4744. rc, rrc);
  4745. }
  4746. break;
  4747. case KVM_S390_INITIAL_RESET:
  4748. r = 0;
  4749. kvm_arch_vcpu_ioctl_initial_reset(vcpu);
  4750. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4751. r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
  4752. UVC_CMD_CPU_RESET_INITIAL,
  4753. &rc, &rrc);
  4754. VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
  4755. rc, rrc);
  4756. }
  4757. break;
  4758. case KVM_S390_NORMAL_RESET:
  4759. r = 0;
  4760. kvm_arch_vcpu_ioctl_normal_reset(vcpu);
  4761. if (kvm_s390_pv_cpu_is_protected(vcpu)) {
  4762. r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
  4763. UVC_CMD_CPU_RESET, &rc, &rrc);
  4764. VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
  4765. rc, rrc);
  4766. }
  4767. break;
  4768. case KVM_SET_ONE_REG:
  4769. case KVM_GET_ONE_REG: {
  4770. struct kvm_one_reg reg;
  4771. r = -EINVAL;
  4772. if (kvm_s390_pv_cpu_is_protected(vcpu))
  4773. break;
  4774. r = -EFAULT;
  4775. if (copy_from_user(&reg, argp, sizeof(reg)))
  4776. break;
  4777. if (ioctl == KVM_SET_ONE_REG)
  4778. r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, &reg);
  4779. else
  4780. r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, &reg);
  4781. break;
  4782. }
  4783. #ifdef CONFIG_KVM_S390_UCONTROL
  4784. case KVM_S390_UCAS_MAP: {
  4785. struct kvm_s390_ucas_mapping ucasmap;
  4786. if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
  4787. r = -EFAULT;
  4788. break;
  4789. }
  4790. if (!kvm_is_ucontrol(vcpu->kvm)) {
  4791. r = -EINVAL;
  4792. break;
  4793. }
  4794. r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr,
  4795. ucasmap.vcpu_addr, ucasmap.length);
  4796. break;
  4797. }
  4798. case KVM_S390_UCAS_UNMAP: {
  4799. struct kvm_s390_ucas_mapping ucasmap;
  4800. if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) {
  4801. r = -EFAULT;
  4802. break;
  4803. }
  4804. if (!kvm_is_ucontrol(vcpu->kvm)) {
  4805. r = -EINVAL;
  4806. break;
  4807. }
  4808. r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr,
  4809. ucasmap.length);
  4810. break;
  4811. }
  4812. #endif
  4813. case KVM_S390_VCPU_FAULT: {
  4814. r = gmap_fault(vcpu->arch.gmap, arg, 0);
  4815. break;
  4816. }
  4817. case KVM_ENABLE_CAP:
  4818. {
  4819. struct kvm_enable_cap cap;
  4820. r = -EFAULT;
  4821. if (copy_from_user(&cap, argp, sizeof(cap)))
  4822. break;
  4823. r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
  4824. break;
  4825. }
  4826. case KVM_S390_MEM_OP: {
  4827. struct kvm_s390_mem_op mem_op;
  4828. if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
  4829. r = kvm_s390_vcpu_memsida_op(vcpu, &mem_op);
  4830. else
  4831. r = -EFAULT;
  4832. break;
  4833. }
  4834. case KVM_S390_SET_IRQ_STATE: {
  4835. struct kvm_s390_irq_state irq_state;
  4836. r = -EFAULT;
  4837. if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
  4838. break;
  4839. if (irq_state.len > VCPU_IRQS_MAX_BUF ||
  4840. irq_state.len == 0 ||
  4841. irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
  4842. r = -EINVAL;
  4843. break;
  4844. }
  4845. /* do not use irq_state.flags, it will break old QEMUs */
  4846. r = kvm_s390_set_irq_state(vcpu,
  4847. (void __user *) irq_state.buf,
  4848. irq_state.len);
  4849. break;
  4850. }
  4851. case KVM_S390_GET_IRQ_STATE: {
  4852. struct kvm_s390_irq_state irq_state;
  4853. r = -EFAULT;
  4854. if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
  4855. break;
  4856. if (irq_state.len == 0) {
  4857. r = -EINVAL;
  4858. break;
  4859. }
  4860. /* do not use irq_state.flags, it will break old QEMUs */
  4861. r = kvm_s390_get_irq_state(vcpu,
  4862. (__u8 __user *) irq_state.buf,
  4863. irq_state.len);
  4864. break;
  4865. }
  4866. case KVM_S390_PV_CPU_COMMAND: {
  4867. struct kvm_pv_cmd cmd;
  4868. r = -EINVAL;
  4869. if (!is_prot_virt_host())
  4870. break;
  4871. r = -EFAULT;
  4872. if (copy_from_user(&cmd, argp, sizeof(cmd)))
  4873. break;
  4874. r = -EINVAL;
  4875. if (cmd.flags)
  4876. break;
  4877. /* We only handle this cmd right now */
  4878. if (cmd.cmd != KVM_PV_DUMP)
  4879. break;
  4880. r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd);
  4881. /* Always copy over UV rc / rrc data */
  4882. if (copy_to_user((__u8 __user *)argp, &cmd.rc,
  4883. sizeof(cmd.rc) + sizeof(cmd.rrc)))
  4884. r = -EFAULT;
  4885. break;
  4886. }
  4887. default:
  4888. r = -ENOTTY;
  4889. }
  4890. vcpu_put(vcpu);
  4891. return r;
  4892. }
  4893. vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
  4894. {
  4895. #ifdef CONFIG_KVM_S390_UCONTROL
  4896. if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
  4897. && (kvm_is_ucontrol(vcpu->kvm))) {
  4898. vmf->page = virt_to_page(vcpu->arch.sie_block);
  4899. get_page(vmf->page);
  4900. return 0;
  4901. }
  4902. #endif
  4903. return VM_FAULT_SIGBUS;
  4904. }
  4905. /* Section: memory related */
  4906. int kvm_arch_prepare_memory_region(struct kvm *kvm,
  4907. const struct kvm_memory_slot *old,
  4908. struct kvm_memory_slot *new,
  4909. enum kvm_mr_change change)
  4910. {
  4911. gpa_t size;
  4912. /* When we are protected, we should not change the memory slots */
  4913. if (kvm_s390_pv_get_handle(kvm))
  4914. return -EINVAL;
  4915. if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) {
  4916. /*
  4917. * A few sanity checks. We can have memory slots which have to be
  4918. * located/ended at a segment boundary (1MB). The memory in userland is
  4919. * ok to be fragmented into various different vmas. It is okay to mmap()
  4920. * and munmap() stuff in this slot after doing this call at any time
  4921. */
  4922. if (new->userspace_addr & 0xffffful)
  4923. return -EINVAL;
  4924. size = new->npages * PAGE_SIZE;
  4925. if (size & 0xffffful)
  4926. return -EINVAL;
  4927. if ((new->base_gfn * PAGE_SIZE) + size > kvm->arch.mem_limit)
  4928. return -EINVAL;
  4929. }
  4930. if (!kvm->arch.migration_mode)
  4931. return 0;
  4932. /*
  4933. * Turn off migration mode when:
  4934. * - userspace creates a new memslot with dirty logging off,
  4935. * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and
  4936. * dirty logging is turned off.
  4937. * Migration mode expects dirty page logging being enabled to store
  4938. * its dirty bitmap.
  4939. */
  4940. if (change != KVM_MR_DELETE &&
  4941. !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
  4942. WARN(kvm_s390_vm_stop_migration(kvm),
  4943. "Failed to stop migration mode");
  4944. return 0;
  4945. }
  4946. void kvm_arch_commit_memory_region(struct kvm *kvm,
  4947. struct kvm_memory_slot *old,
  4948. const struct kvm_memory_slot *new,
  4949. enum kvm_mr_change change)
  4950. {
  4951. int rc = 0;
  4952. switch (change) {
  4953. case KVM_MR_DELETE:
  4954. rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
  4955. old->npages * PAGE_SIZE);
  4956. break;
  4957. case KVM_MR_MOVE:
  4958. rc = gmap_unmap_segment(kvm->arch.gmap, old->base_gfn * PAGE_SIZE,
  4959. old->npages * PAGE_SIZE);
  4960. if (rc)
  4961. break;
  4962. fallthrough;
  4963. case KVM_MR_CREATE:
  4964. rc = gmap_map_segment(kvm->arch.gmap, new->userspace_addr,
  4965. new->base_gfn * PAGE_SIZE,
  4966. new->npages * PAGE_SIZE);
  4967. break;
  4968. case KVM_MR_FLAGS_ONLY:
  4969. break;
  4970. default:
  4971. WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
  4972. }
  4973. if (rc)
  4974. pr_warn("failed to commit memory region\n");
  4975. return;
  4976. }
  4977. static inline unsigned long nonhyp_mask(int i)
  4978. {
  4979. unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
  4980. return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
  4981. }
  4982. static int __init kvm_s390_init(void)
  4983. {
  4984. int i;
  4985. if (!sclp.has_sief2) {
  4986. pr_info("SIE is not available\n");
  4987. return -ENODEV;
  4988. }
  4989. if (nested && hpage) {
  4990. pr_info("A KVM host that supports nesting cannot back its KVM guests with huge pages\n");
  4991. return -EINVAL;
  4992. }
  4993. for (i = 0; i < 16; i++)
  4994. kvm_s390_fac_base[i] |=
  4995. stfle_fac_list[i] & nonhyp_mask(i);
  4996. return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
  4997. }
  4998. static void __exit kvm_s390_exit(void)
  4999. {
  5000. kvm_exit();
  5001. }
  5002. module_init(kvm_s390_init);
  5003. module_exit(kvm_s390_exit);
  5004. /*
  5005. * Enable autoloading of the kvm module.
  5006. * Note that we add the module alias here instead of virt/kvm/kvm_main.c
  5007. * since x86 takes a different approach.
  5008. */
  5009. #include <linux/miscdevice.h>
  5010. MODULE_ALIAS_MISCDEV(KVM_MINOR);
  5011. MODULE_ALIAS("devname:kvm");