adsprpc.c 247 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484548554865487548854895490549154925493549454955496549754985499550055015502550355045505550655075508550955105511551255135514551555165517551855195520552155225523552455255526552755285529553055315532553355345535553655375538553955405541554255435544554555465547554855495550555155525553555455555556555755585559556055615562556355645565556655675568556955705571557255735574557555765577557855795580558155825583558455855586558755885589559055915592559355945595559655975598559956005601560256035604560556065607560856095610561156125613561456155616561756185619562056215622562356245625562656275628562956305631563256335634563556365637563856395640564156425643564456455646564756485649565056515652565356545655565656575658565956605661566256635664566556665667566856695670567156725673567456755676567756785679568056815682568356845685568656875688568956905691569256935694569556965697569856995700570157025703570457055706570757085709571057115712571357145715571657175718571957205721572257235724572557265727572857295730573157325733573457355736573757385739574057415742574357445745574657475748574957505751575257535754575557565757575857595760576157625763576457655766576757685769577057715772577357745775577657775778577957805781578257835784578557865787578857895790579157925793579457955796579757985799580058015802580358045805580658075808580958105811581258135814581558165817581858195820582158225823582458255826582758285829583058315832583358345835583658375838583958405841584258435844584558465847584858495850585158525853585458555856585758585859586058615862586358645865586658675868586958705871587258735874587558765877587858795880588158825883588458855886588758885889589058915892589358945895589658975898589959005901590259035904590559065907590859095910591159125913591459155916591759185919592059215922592359245925592659275928592959305931593259335934593559365937593859395940594159425943594459455946594759485949595059515952595359545955595659575958595959605961596259635964596559665967596859695970597159725973597459755976597759785979598059815982598359845985598659875988598959905991599259935994599559965997599859996000600160026003600460056006600760086009601060116012601360146015601660176018601960206021602260236024602560266027602860296030603160326033603460356036603760386039604060416042604360446045604660476048604960506051605260536054605560566057605860596060606160626063606460656066606760686069607060716072607360746075607660776078607960806081608260836084608560866087608860896090609160926093609460956096609760986099610061016102610361046105610661076108610961106111611261136114611561166117611861196120612161226123612461256126612761286129613061316132613361346135613661376138613961406141614261436144614561466147614861496150615161526153615461556156615761586159616061616162616361646165616661676168616961706171617261736174617561766177617861796180618161826183618461856186618761886189619061916192619361946195619661976198619962006201620262036204620562066207620862096210621162126213621462156216621762186219622062216222622362246225622662276228622962306231623262336234623562366237623862396240624162426243624462456246624762486249625062516252625362546255625662576258625962606261626262636264626562666267626862696270627162726273627462756276627762786279628062816282628362846285628662876288628962906291629262936294629562966297629862996300630163026303630463056306630763086309631063116312631363146315631663176318631963206321632263236324632563266327632863296330633163326333633463356336633763386339634063416342634363446345634663476348634963506351635263536354635563566357635863596360636163626363636463656366636763686369637063716372637363746375637663776378637963806381638263836384638563866387638863896390639163926393639463956396639763986399640064016402640364046405640664076408640964106411641264136414641564166417641864196420642164226423642464256426642764286429643064316432643364346435643664376438643964406441644264436444644564466447644864496450645164526453645464556456645764586459646064616462646364646465646664676468646964706471647264736474647564766477647864796480648164826483648464856486648764886489649064916492649364946495649664976498649965006501650265036504650565066507650865096510651165126513651465156516651765186519652065216522652365246525652665276528652965306531653265336534653565366537653865396540654165426543654465456546654765486549655065516552655365546555655665576558655965606561656265636564656565666567656865696570657165726573657465756576657765786579658065816582658365846585658665876588658965906591659265936594659565966597659865996600660166026603660466056606660766086609661066116612661366146615661666176618661966206621662266236624662566266627662866296630663166326633663466356636663766386639664066416642664366446645664666476648664966506651665266536654665566566657665866596660666166626663666466656666666766686669667066716672667366746675667666776678667966806681668266836684668566866687668866896690669166926693669466956696669766986699670067016702670367046705670667076708670967106711671267136714671567166717671867196720672167226723672467256726672767286729673067316732673367346735673667376738673967406741674267436744674567466747674867496750675167526753675467556756675767586759676067616762676367646765676667676768676967706771677267736774677567766777677867796780678167826783678467856786678767886789679067916792679367946795679667976798679968006801680268036804680568066807680868096810681168126813681468156816681768186819682068216822682368246825682668276828682968306831683268336834683568366837683868396840684168426843684468456846684768486849685068516852685368546855685668576858685968606861686268636864686568666867686868696870687168726873687468756876687768786879688068816882688368846885688668876888688968906891689268936894689568966897689868996900690169026903690469056906690769086909691069116912691369146915691669176918691969206921692269236924692569266927692869296930693169326933693469356936693769386939694069416942694369446945694669476948694969506951695269536954695569566957695869596960696169626963696469656966696769686969697069716972697369746975697669776978697969806981698269836984698569866987698869896990699169926993699469956996699769986999700070017002700370047005700670077008700970107011701270137014701570167017701870197020702170227023702470257026702770287029703070317032703370347035703670377038703970407041704270437044704570467047704870497050705170527053705470557056705770587059706070617062706370647065706670677068706970707071707270737074707570767077707870797080708170827083708470857086708770887089709070917092709370947095709670977098709971007101710271037104710571067107710871097110711171127113711471157116711771187119712071217122712371247125712671277128712971307131713271337134713571367137713871397140714171427143714471457146714771487149715071517152715371547155715671577158715971607161716271637164716571667167716871697170717171727173717471757176717771787179718071817182718371847185718671877188718971907191719271937194719571967197719871997200720172027203720472057206720772087209721072117212721372147215721672177218721972207221722272237224722572267227722872297230723172327233723472357236723772387239724072417242724372447245724672477248724972507251725272537254725572567257725872597260726172627263726472657266726772687269727072717272727372747275727672777278727972807281728272837284728572867287728872897290729172927293729472957296729772987299730073017302730373047305730673077308730973107311731273137314731573167317731873197320732173227323732473257326732773287329733073317332733373347335733673377338733973407341734273437344734573467347734873497350735173527353735473557356735773587359736073617362736373647365736673677368736973707371737273737374737573767377737873797380738173827383738473857386738773887389739073917392739373947395739673977398739974007401740274037404740574067407740874097410741174127413741474157416741774187419742074217422742374247425742674277428742974307431743274337434743574367437743874397440744174427443744474457446744774487449745074517452745374547455745674577458745974607461746274637464746574667467746874697470747174727473747474757476747774787479748074817482748374847485748674877488748974907491749274937494749574967497749874997500750175027503750475057506750775087509751075117512751375147515751675177518751975207521752275237524752575267527752875297530753175327533753475357536753775387539754075417542754375447545754675477548754975507551755275537554755575567557755875597560756175627563756475657566756775687569757075717572757375747575757675777578757975807581758275837584758575867587758875897590759175927593759475957596759775987599760076017602760376047605760676077608760976107611761276137614761576167617761876197620762176227623762476257626762776287629763076317632763376347635763676377638763976407641764276437644764576467647764876497650765176527653765476557656765776587659766076617662766376647665766676677668766976707671767276737674767576767677767876797680768176827683768476857686768776887689769076917692769376947695769676977698769977007701770277037704770577067707770877097710771177127713771477157716771777187719772077217722772377247725772677277728772977307731773277337734773577367737773877397740774177427743774477457746774777487749775077517752775377547755775677577758775977607761776277637764776577667767776877697770777177727773777477757776777777787779778077817782778377847785778677877788778977907791779277937794779577967797779877997800780178027803780478057806780778087809781078117812781378147815781678177818781978207821782278237824782578267827782878297830783178327833783478357836783778387839784078417842784378447845784678477848784978507851785278537854785578567857785878597860786178627863786478657866786778687869787078717872787378747875787678777878787978807881788278837884788578867887788878897890789178927893789478957896789778987899790079017902790379047905790679077908790979107911791279137914791579167917791879197920792179227923792479257926792779287929793079317932793379347935793679377938793979407941794279437944794579467947794879497950795179527953795479557956795779587959796079617962796379647965796679677968796979707971797279737974797579767977797879797980798179827983798479857986798779887989799079917992799379947995799679977998799980008001800280038004800580068007800880098010801180128013801480158016801780188019802080218022802380248025802680278028802980308031803280338034803580368037803880398040804180428043804480458046804780488049805080518052805380548055805680578058805980608061806280638064806580668067806880698070807180728073807480758076807780788079808080818082808380848085808680878088808980908091809280938094809580968097809880998100810181028103810481058106810781088109811081118112811381148115811681178118811981208121812281238124812581268127812881298130813181328133813481358136813781388139814081418142814381448145814681478148814981508151815281538154815581568157815881598160816181628163816481658166816781688169817081718172817381748175817681778178817981808181818281838184818581868187818881898190819181928193819481958196819781988199820082018202820382048205820682078208820982108211821282138214821582168217821882198220822182228223822482258226822782288229823082318232823382348235823682378238823982408241824282438244824582468247824882498250825182528253825482558256825782588259826082618262826382648265826682678268826982708271827282738274827582768277827882798280828182828283828482858286828782888289829082918292829382948295829682978298829983008301830283038304830583068307830883098310831183128313831483158316831783188319832083218322832383248325832683278328832983308331833283338334833583368337833883398340834183428343834483458346834783488349835083518352835383548355835683578358835983608361836283638364836583668367836883698370837183728373837483758376837783788379838083818382838383848385838683878388838983908391839283938394839583968397839883998400840184028403840484058406840784088409841084118412841384148415841684178418841984208421842284238424842584268427842884298430843184328433843484358436843784388439844084418442844384448445844684478448844984508451845284538454845584568457845884598460846184628463846484658466846784688469847084718472847384748475847684778478847984808481848284838484848584868487848884898490849184928493849484958496849784988499850085018502850385048505850685078508850985108511851285138514851585168517851885198520852185228523852485258526852785288529853085318532853385348535853685378538853985408541854285438544854585468547854885498550855185528553855485558556855785588559856085618562856385648565856685678568856985708571857285738574857585768577857885798580858185828583858485858586858785888589859085918592859385948595859685978598859986008601860286038604860586068607860886098610861186128613861486158616861786188619862086218622862386248625862686278628862986308631863286338634863586368637863886398640864186428643864486458646864786488649865086518652865386548655865686578658865986608661866286638664866586668667866886698670867186728673867486758676867786788679868086818682868386848685868686878688868986908691869286938694869586968697869886998700870187028703870487058706870787088709871087118712871387148715871687178718871987208721872287238724872587268727872887298730873187328733873487358736873787388739874087418742874387448745874687478748874987508751875287538754875587568757875887598760876187628763876487658766876787688769877087718772877387748775877687778778877987808781878287838784878587868787878887898790879187928793879487958796879787988799880088018802880388048805880688078808880988108811881288138814881588168817881888198820882188228823882488258826882788288829883088318832883388348835883688378838883988408841884288438844884588468847884888498850885188528853885488558856885788588859886088618862886388648865886688678868886988708871887288738874887588768877887888798880888188828883888488858886888788888889889088918892889388948895889688978898889989008901890289038904890589068907890889098910891189128913891489158916891789188919892089218922892389248925892689278928892989308931893289338934893589368937893889398940894189428943894489458946894789488949895089518952895389548955895689578958895989608961896289638964896589668967896889698970897189728973897489758976897789788979898089818982898389848985898689878988898989908991899289938994899589968997899889999000900190029003900490059006900790089009901090119012901390149015901690179018901990209021902290239024902590269027902890299030903190329033903490359036903790389039904090419042904390449045904690479048904990509051905290539054905590569057905890599060906190629063906490659066906790689069907090719072907390749075907690779078907990809081908290839084908590869087908890899090909190929093909490959096909790989099910091019102910391049105910691079108910991109111911291139114911591169117911891199120912191229123912491259126912791289129913091319132913391349135913691379138913991409141914291439144914591469147914891499150915191529153915491559156915791589159916091619162916391649165916691679168916991709171917291739174917591769177917891799180918191829183918491859186918791889189919091919192919391949195919691979198919992009201920292039204920592069207920892099210921192129213921492159216921792189219922092219222922392249225922692279228922992309231923292339234923592369237923892399240924192429243924492459246
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. /* Uncomment this block to log an error on every VERIFY failure */
  7. /*
  8. * #ifndef VERIFY_PRINT_ERROR
  9. * #define VERIFY_PRINT_ERROR
  10. * #endif
  11. */
  12. #include <linux/dma-buf.h>
  13. #include <linux/dma-mapping.h>
  14. #include <linux/qcom-dma-mapping.h>
  15. #include <linux/slab.h>
  16. #include <linux/completion.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/mm.h>
  19. #include <linux/wait.h>
  20. #include <linux/sched.h>
  21. #include <linux/module.h>
  22. #include <linux/list.h>
  23. #include <linux/arch_topology.h>
  24. #include <linux/hash.h>
  25. #include <linux/msm_ion.h>
  26. #include <linux/qcom_scm.h>
  27. #include <linux/ipc_logging.h>
  28. #include <linux/remoteproc/qcom_rproc.h>
  29. #include <linux/scatterlist.h>
  30. #include <linux/uaccess.h>
  31. #include <linux/device.h>
  32. #include <linux/of.h>
  33. #include <linux/of_address.h>
  34. #include <linux/of_platform.h>
  35. #include <linux/dma-map-ops.h>
  36. #include <linux/cma.h>
  37. #include <linux/sort.h>
  38. #include <linux/cred.h>
  39. #include <linux/msm_dma_iommu_mapping.h>
  40. #include "adsprpc_compat.h"
  41. #include "adsprpc_shared.h"
  42. #include <soc/qcom/qcom_ramdump.h>
  43. #include <soc/qcom/minidump.h>
  44. #include <soc/qcom/secure_buffer.h>
  45. #include <linux/delay.h>
  46. #include <linux/debugfs.h>
  47. #include <linux/pm_qos.h>
  48. #include <linux/stat.h>
  49. #include <linux/preempt.h>
  50. #include <linux/of_reserved_mem.h>
  51. #include <linux/soc/qcom/pdr.h>
  52. #include <linux/soc/qcom/qmi.h>
  53. #include <linux/mem-buf.h>
  54. #include <linux/iommu.h>
  55. #include <asm/arch_timer.h>
  56. #include <linux/genalloc.h>
  57. #include <soc/qcom/socinfo.h>
  58. #ifdef CONFIG_HIBERNATION
  59. #include <linux/suspend.h>
  60. #include <linux/notifier.h>
  61. #endif
  62. #define CREATE_TRACE_POINTS
  63. #include "fastrpc_trace.h"
  64. #ifdef CONFIG_MSM_ADSPRPC_TRUSTED
  65. #include "../include/linux/fastrpc.h"
  66. #else
  67. #include "fastrpc.h"
  68. #endif
  69. #define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C
  70. #define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D
  71. #define TZ_PIL_AUTH_QDSP6_PROC 1
  72. #define FASTRPC_ENOSUCH 39
  73. #define VMID_SSC_Q6 5
  74. #define VMID_ADSP_Q6 6
  75. #define DEBUGFS_SIZE 3072
  76. #define PID_SIZE 10
  77. #define AUDIO_PDR_ADSP_DTSI_PROPERTY_NAME "qcom,fastrpc-adsp-audio-pdr"
  78. #define AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME "audio_pdr_adsprpc"
  79. #define AUDIO_PDR_ADSP_SERVICE_NAME "avs/audio"
  80. #define ADSP_AUDIOPD_NAME "msm/adsp/audio_pd"
  81. #define SENSORS_PDR_ADSP_DTSI_PROPERTY_NAME "qcom,fastrpc-adsp-sensors-pdr"
  82. #define SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME "sensors_pdr_adsprpc"
  83. #define SENSORS_PDR_ADSP_SERVICE_NAME "tms/servreg"
  84. #define ADSP_SENSORPD_NAME "msm/adsp/sensor_pd"
  85. #define SENSORS_PDR_SLPI_DTSI_PROPERTY_NAME "qcom,fastrpc-slpi-sensors-pdr"
  86. #define SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME "sensors_pdr_sdsprpc"
  87. #define SENSORS_PDR_SLPI_SERVICE_NAME SENSORS_PDR_ADSP_SERVICE_NAME
  88. #define SLPI_SENSORPD_NAME "msm/slpi/sensor_pd"
  89. #define FASTRPC_SECURE_WAKE_SOURCE_CLIENT_NAME "adsprpc-secure"
  90. #define FASTRPC_NON_SECURE_WAKE_SOURCE_CLIENT_NAME "adsprpc-non_secure"
  91. #define RPC_TIMEOUT (5 * HZ)
  92. #define BALIGN 128
  93. #define M_FDLIST (16)
  94. #define M_CRCLIST (64)
  95. #define M_KERNEL_PERF_LIST (PERF_KEY_MAX)
  96. #define M_DSP_PERF_LIST (12)
  97. #define FASTRPC_CTX_MAGIC (0xbeeddeed)
  98. /* Process status notifications from DSP will be sent with this unique context */
  99. #define FASTRPC_NOTIF_CTX_RESERVED 0xABCDABCD
  100. #define FASTRPC_CTX_JOB_TYPE_POS (4)
  101. #define FASTRPC_CTX_TABLE_IDX_POS (6)
  102. #define FASTRPC_CTX_JOBID_POS (16)
  103. #define FASTRPC_CTX_TABLE_IDX_MASK \
  104. ((FASTRPC_CTX_MAX - 1) << FASTRPC_CTX_TABLE_IDX_POS)
  105. #define FASTRPC_ASYNC_JOB_MASK (1)
  106. #define GET_TABLE_IDX_FROM_CTXID(ctxid) \
  107. ((ctxid & FASTRPC_CTX_TABLE_IDX_MASK) >> FASTRPC_CTX_TABLE_IDX_POS)
  108. /* Reserve few entries in context table for critical kernel and static RPC
  109. * calls to avoid user invocations from exhausting all entries.
  110. */
  111. #define NUM_KERNEL_AND_STATIC_ONLY_CONTEXTS (70)
  112. /* Maximum number of pending contexts per remote session */
  113. #define MAX_PENDING_CTX_PER_SESSION (64)
  114. #define NUM_DEVICES 2 /* adsprpc-smd, adsprpc-smd-secure */
  115. #define MINOR_NUM_DEV 0
  116. #define MINOR_NUM_SECURE_DEV 1
  117. #define NON_SECURE_CHANNEL 0
  118. #define SECURE_CHANNEL 1
  119. #define IS_CACHE_ALIGNED(x) (((x) & ((L1_CACHE_BYTES)-1)) == 0)
  120. #ifndef ION_FLAG_CACHED
  121. #define ION_FLAG_CACHED (1)
  122. #endif
  123. #ifndef topology_cluster_id
  124. #define topology_cluster_id(cpu) topology_physical_package_id(cpu)
  125. #endif
  126. /*
  127. * ctxid of every message is OR-ed with fastrpc_remote_pd_type before
  128. * it is sent to DSP. So mask 2 LSBs to retrieve actual context
  129. */
  130. #define CONTEXT_PD_CHECK (3)
  131. #define GET_CTXID_FROM_RSP_CTX(rsp_ctx) (rsp_ctx & ~CONTEXT_PD_CHECK)
  132. #define RH_CID ADSP_DOMAIN_ID
  133. #define FASTRPC_STATIC_HANDLE_PROCESS_GROUP (1)
  134. #define FASTRPC_STATIC_HANDLE_DSP_UTILITIES (2)
  135. #define FASTRPC_STATIC_HANDLE_LISTENER (3)
  136. #define FASTRPC_STATIC_HANDLE_MAX (20)
  137. #define FASTRPC_LATENCY_CTRL_ENB (1)
  138. /* Maximum PM timeout that can be voted through fastrpc */
  139. #define MAX_PM_TIMEOUT_MS 50
  140. /* timeout in us for busy polling after early response from remote processor */
  141. #define FASTRPC_POLL_TIME (4000)
  142. /* timeout in us for polling until memory barrier */
  143. #define FASTRPC_POLL_TIME_MEM_UPDATE (500)
  144. /* timeout in us for polling completion signal after user early hint */
  145. #define FASTRPC_USER_EARLY_HINT_TIMEOUT (500)
  146. /* Early wake up poll completion number received from remote processor */
  147. #define FASTRPC_EARLY_WAKEUP_POLL (0xabbccdde)
  148. /* Poll response number from remote processor for call completion */
  149. #define FASTRPC_POLL_RESPONSE (0xdecaf)
  150. /* latency in us, early wake up signal used below this value */
  151. #define FASTRPC_EARLY_WAKEUP_LATENCY (200)
  152. /* response version number */
  153. #define FASTRPC_RSP_VERSION2 (2)
  154. /* CPU feature information to DSP */
  155. #define FASTRPC_CPUINFO_DEFAULT (0)
  156. #define FASTRPC_CPUINFO_EARLY_WAKEUP (1)
  157. #define INIT_FILELEN_MAX (2*1024*1024)
  158. #define INIT_MEMLEN_MAX_STATIC (8*1024*1024)
  159. #define INIT_MEMLEN_MAX_DYNAMIC (200*1024*1024)
  160. #define INIT_MEMLEN_MIN_DYNAMIC (3*1024*1024)
  161. #define MAX_CACHE_BUF_SIZE (8*1024*1024)
  162. /* Maximum buffers cached in cached buffer list */
  163. #define MAX_CACHED_BUFS (32)
  164. /* Max no. of persistent headers pre-allocated per process */
  165. #define MAX_PERSISTENT_HEADERS (25)
  166. /* Max value of unique fastrpc tgid */
  167. #define MAX_FRPC_TGID 256
  168. #define PERF_CAPABILITY_SUPPORT (1 << 1)
  169. #define KERNEL_ERROR_CODE_V1_SUPPORT 1
  170. #define USERSPACE_ALLOCATION_SUPPORT 1
  171. #define DSPSIGNAL_SUPPORT 1
  172. #define MD_GMSG_BUFFER (1000)
  173. #define MINI_DUMP_DBG_SIZE (200*1024)
  174. /* Max number of region supported */
  175. #define MAX_UNIQUE_ID 5
  176. /* Convert the 19.2MHz clock count to micro-seconds */
  177. #define CONVERT_CNT_TO_US(CNT) (CNT * 10ull / 192ull)
  178. #define FASTRPC_USER_PD_FORCE_KILL 2
  179. /*
  180. * No. of pages shared with DSP during process init
  181. * First page for init-mem and second page for proc-attrs
  182. */
  183. #define PAGESLEN_WITH_SHAREDBUF 2
  184. /* Unique index flag used for mini dump */
  185. static int md_unique_index_flag[MAX_UNIQUE_ID] = { 0, 0, 0, 0, 0 };
  186. /* Array to keep track unique tgid_frpc usage */
  187. static bool frpc_tgid_usage_array[MAX_FRPC_TGID] = {0};
  188. /* Fastrpc remote process attributes */
  189. enum fastrpc_proc_attr {
  190. /* Macro for Debug attr */
  191. FASTRPC_MODE_DEBUG = 1 << 0,
  192. /* Macro for Ptrace */
  193. FASTRPC_MODE_PTRACE = 1 << 1,
  194. /* Macro for CRC Check */
  195. FASTRPC_MODE_CRC = 1 << 2,
  196. /* Macro for Unsigned PD */
  197. FASTRPC_MODE_UNSIGNED_MODULE = 1 << 3,
  198. /* Macro for Adaptive QoS */
  199. FASTRPC_MODE_ADAPTIVE_QOS = 1 << 4,
  200. /* Macro for System Process */
  201. FASTRPC_MODE_SYSTEM_PROCESS = 1 << 5,
  202. /* Macro for Prvileged Process */
  203. FASTRPC_MODE_PRIVILEGED = (1 << 6),
  204. /* Macro for system unsigned PD */
  205. FASTRPC_MODE_SYSTEM_UNSIGNED_PD = 1 << 17,
  206. };
  207. /* FastRPC remote subsystem state*/
  208. enum fastrpc_remote_subsys_state {
  209. SUBSYSTEM_RESTARTING = 0,
  210. SUBSYSTEM_DOWN,
  211. SUBSYSTEM_UP,
  212. };
  213. #define PERF_END ((void)0)
  214. #define PERF(enb, cnt, ff) \
  215. {\
  216. struct timespec64 startT = {0};\
  217. uint64_t *counter = cnt;\
  218. if (enb && counter) {\
  219. ktime_get_real_ts64(&startT);\
  220. } \
  221. ff ;\
  222. if (enb && counter) {\
  223. *counter += getnstimediff(&startT);\
  224. } \
  225. }
  226. #define GET_COUNTER(perf_ptr, offset) \
  227. (perf_ptr != NULL ?\
  228. (((offset >= 0) && (offset < PERF_KEY_MAX)) ?\
  229. (uint64_t *)(perf_ptr + offset)\
  230. : (uint64_t *)NULL) : (uint64_t *)NULL)
  231. /* Macro for comparing local client and PD names with those from callback */
  232. #define COMPARE_SERVICE_LOCATOR_NAMES(cb_client, local_client, \
  233. cb_pdname, local_pdname) \
  234. ((!strcmp(cb_client, local_client)) \
  235. && (!strcmp(cb_pdname, local_pdname)))
  236. #define IS_ASYNC_FASTRPC_AVAILABLE (1)
  237. /* Use the second definition to enable additional dspsignal debug logging */
  238. #define DSPSIGNAL_VERBOSE(x, ...)
  239. /*#define DSPSIGNAL_VERBOSE ADSPRPC_INFO*/
  240. MODULE_IMPORT_NS(DMA_BUF);
  241. static struct dentry *debugfs_root;
  242. static struct dentry *debugfs_global_file;
  243. static inline uint64_t buf_page_start(uint64_t buf)
  244. {
  245. uint64_t start = (uint64_t) buf & PAGE_MASK;
  246. return start;
  247. }
  248. static inline uint64_t buf_page_offset(uint64_t buf)
  249. {
  250. uint64_t offset = (uint64_t) buf & (PAGE_SIZE - 1);
  251. return offset;
  252. }
  253. static inline uint64_t buf_num_pages(uint64_t buf, size_t len)
  254. {
  255. uint64_t start = buf_page_start(buf) >> PAGE_SHIFT;
  256. uint64_t end = (((uint64_t) buf + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
  257. uint64_t nPages = end - start + 1;
  258. return nPages;
  259. }
  260. static inline uint64_t buf_page_size(uint32_t size)
  261. {
  262. uint64_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;
  263. return sz > PAGE_SIZE ? sz : PAGE_SIZE;
  264. }
  265. static inline void *uint64_to_ptr(uint64_t addr)
  266. {
  267. void *ptr = (void *)((uintptr_t)addr);
  268. return ptr;
  269. }
  270. static inline uint64_t ptr_to_uint64(void *ptr)
  271. {
  272. uint64_t addr = (uint64_t)((uintptr_t)ptr);
  273. return addr;
  274. }
  275. static struct fastrpc_apps gfa;
  276. static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
  277. {
  278. .name = "adsprpc-smd",
  279. .subsys = "lpass",
  280. .spd = {
  281. {
  282. .servloc_name =
  283. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME,
  284. .spdname = ADSP_AUDIOPD_NAME,
  285. .cid = ADSP_DOMAIN_ID,
  286. },
  287. {
  288. .servloc_name =
  289. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME,
  290. .spdname = ADSP_SENSORPD_NAME,
  291. .cid = ADSP_DOMAIN_ID,
  292. }
  293. },
  294. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  295. .cpuinfo_status = false,
  296. },
  297. {
  298. .name = "mdsprpc-smd",
  299. .subsys = "mpss",
  300. .spd = {
  301. {
  302. .cid = MDSP_DOMAIN_ID,
  303. }
  304. },
  305. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  306. .cpuinfo_status = false,
  307. },
  308. {
  309. .name = "sdsprpc-smd",
  310. .subsys = "dsps",
  311. .spd = {
  312. {
  313. .servloc_name =
  314. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME,
  315. .spdname = SLPI_SENSORPD_NAME,
  316. .cid = SDSP_DOMAIN_ID,
  317. }
  318. },
  319. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  320. .cpuinfo_status = false,
  321. },
  322. {
  323. .name = "cdsprpc-smd",
  324. .subsys = "cdsp",
  325. .spd = {
  326. {
  327. .cid = CDSP_DOMAIN_ID,
  328. }
  329. },
  330. .cpuinfo_todsp = FASTRPC_CPUINFO_EARLY_WAKEUP,
  331. .cpuinfo_status = false,
  332. },
  333. };
  334. static uint32_t kernel_capabilities[FASTRPC_MAX_ATTRIBUTES -
  335. FASTRPC_MAX_DSP_ATTRIBUTES] = {
  336. PERF_CAPABILITY_SUPPORT,
  337. /* PERF_LOGGING_V2_SUPPORT feature is supported, unsupported = 0 */
  338. KERNEL_ERROR_CODE_V1_SUPPORT,
  339. /* Fastrpc Driver error code changes present */
  340. USERSPACE_ALLOCATION_SUPPORT,
  341. /* Userspace allocation allowed for DSP memory request*/
  342. DSPSIGNAL_SUPPORT
  343. /* Lightweight driver-based signaling */
  344. };
  345. static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type);
  346. static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
  347. uint32_t flags, uintptr_t va, uint64_t phys,
  348. size_t size, uintptr_t *raddr);
  349. static inline void fastrpc_update_rxmsg_buf(struct fastrpc_channel_ctx *chan,
  350. uint64_t ctx, int retval, uint32_t rsp_flags,
  351. uint32_t early_wake_time, uint32_t ver, int64_t ns, uint64_t xo_time_in_us);
  352. static int fastrpc_file_get(struct fastrpc_file *fl);
  353. static void fastrpc_file_put(struct fastrpc_file *fl);
  354. /**
  355. * fastrpc_device_create - Create device for the fastrpc process file
  356. * @fl : Fastrpc process file
  357. * Returns: 0 on Success
  358. */
  359. static int fastrpc_device_create(struct fastrpc_file *fl);
  360. static inline int64_t getnstimediff(struct timespec64 *start)
  361. {
  362. int64_t ns;
  363. struct timespec64 ts, b;
  364. ktime_get_real_ts64(&ts);
  365. b = timespec64_sub(ts, *start);
  366. ns = timespec64_to_ns(&b);
  367. return ns;
  368. }
  369. /**
  370. * get_timestamp_in_ns - Gets time of day in nanoseconds
  371. *
  372. * Returns: Timestamp in nanoseconds
  373. */
  374. static inline int64_t get_timestamp_in_ns(void)
  375. {
  376. int64_t ns = 0;
  377. struct timespec64 ts;
  378. ktime_get_real_ts64(&ts);
  379. ns = timespec64_to_ns(&ts);
  380. return ns;
  381. }
  382. static inline int poll_for_remote_response(struct smq_invoke_ctx *ctx, uint32_t timeout)
  383. {
  384. int err = -EIO;
  385. uint32_t sc = ctx->sc, ii = 0, jj = 0;
  386. struct smq_invoke_buf *list;
  387. struct smq_phy_page *pages;
  388. uint64_t *fdlist = NULL;
  389. uint32_t *crclist = NULL, *poll = NULL;
  390. unsigned int inbufs, outbufs, handles;
  391. /* calculate poll memory location */
  392. inbufs = REMOTE_SCALARS_INBUFS(sc);
  393. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  394. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  395. list = smq_invoke_buf_start(ctx->rpra, sc);
  396. pages = smq_phy_page_start(sc, list);
  397. fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
  398. crclist = (uint32_t *)(fdlist + M_FDLIST);
  399. poll = (uint32_t *)(crclist + M_CRCLIST);
  400. /* poll on memory for DSP response. Return failure on timeout */
  401. for (ii = 0, jj = 0; ii < timeout; ii++, jj++) {
  402. if (*poll == FASTRPC_EARLY_WAKEUP_POLL) {
  403. /* Remote processor sent early response */
  404. err = 0;
  405. break;
  406. } else if (*poll == FASTRPC_POLL_RESPONSE) {
  407. /* Remote processor sent poll response to complete the call */
  408. err = 0;
  409. ctx->is_work_done = true;
  410. ctx->retval = 0;
  411. /* Update DSP response history */
  412. fastrpc_update_rxmsg_buf(&gfa.channel[ctx->fl->cid],
  413. ctx->msg.invoke.header.ctx, 0, POLL_MODE, 0,
  414. FASTRPC_RSP_VERSION2, get_timestamp_in_ns(),
  415. CONVERT_CNT_TO_US(__arch_counter_get_cntvct()));
  416. break;
  417. }
  418. if (jj == FASTRPC_POLL_TIME_MEM_UPDATE) {
  419. /* Wait for DSP to finish updating poll memory */
  420. rmb();
  421. jj = 0;
  422. }
  423. udelay(1);
  424. }
  425. return err;
  426. }
  427. enum interrupted_state {
  428. DEFAULT_STATE = 0,
  429. INTERRUPTED_STATE = 1,
  430. RESTORED_STATE = 2,
  431. };
  432. /**
  433. * fastrpc_update_txmsg_buf - Update history of sent glink messages
  434. * @msg : Pointer to RPC message to remote subsystem
  435. * @transport_send_err : Error from transport
  436. * @ns : Timestamp (in ns) of sent message
  437. * @xo_time_in_us : XO Timestamp (in us) of sent message
  438. * @ctx : invoke ctx
  439. * @interrupted : 0/1/2 (default/interrupted/restored)
  440. *
  441. * Returns none
  442. */
  443. static inline void fastrpc_update_txmsg_buf(struct smq_msg *msg,
  444. int transport_send_err, int64_t ns, uint64_t xo_time_in_us,
  445. struct smq_invoke_ctx *ctx, enum interrupted_state interrupted)
  446. {
  447. unsigned long flags = 0;
  448. unsigned int tx_index = 0;
  449. struct fastrpc_tx_msg *tx_msg = NULL;
  450. struct fastrpc_channel_ctx *chan = NULL;
  451. struct fastrpc_file *fl = ctx->fl;
  452. int err = 0, cid = -1;
  453. if (!fl) {
  454. err = -EBADF;
  455. goto bail;
  456. }
  457. cid = fl->cid;
  458. VERIFY(err, VALID_FASTRPC_CID(cid));
  459. if (err) {
  460. err = -ECHRNG;
  461. goto bail;
  462. }
  463. chan = &fl->apps->channel[cid];
  464. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  465. if (interrupted){
  466. if (ctx->tx_index >= 0 && ctx->tx_index < GLINK_MSG_HISTORY_LEN) {
  467. tx_msg = &chan->gmsg_log.tx_msgs[ctx->tx_index];
  468. if (tx_msg->msg.invoke.header.ctx == ctx->msg.invoke.header.ctx) {
  469. tx_msg->xo_time_in_us_interrupted = ctx->xo_time_in_us_interrupted;
  470. tx_msg->xo_time_in_us_restored = ctx->xo_time_in_us_restored;
  471. }
  472. }
  473. } else {
  474. tx_index = chan->gmsg_log.tx_index;
  475. ctx->tx_index = tx_index;
  476. tx_msg = &chan->gmsg_log.tx_msgs[tx_index];
  477. memcpy(&tx_msg->msg, msg, sizeof(struct smq_msg));
  478. tx_msg->transport_send_err = transport_send_err;
  479. tx_msg->ns = ns;
  480. tx_msg->xo_time_in_us = xo_time_in_us;
  481. tx_index++;
  482. chan->gmsg_log.tx_index =
  483. (tx_index > (GLINK_MSG_HISTORY_LEN - 1)) ? 0 : tx_index;
  484. }
  485. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  486. bail:
  487. if (err)
  488. ADSPRPC_ERR("adsprpc: %s: unable to update txmsg buf (err %d) for ctx: 0x%x\n",
  489. __func__, err, ctx->msg.invoke.header.ctx);
  490. }
  491. /**
  492. * fastrpc_update_rxmsg_buf - Update history of received glink responses
  493. * @chan : Channel context
  494. * @ctx : Context of received response from DSP
  495. * @retval : Return value for RPC call
  496. * @rsp_flags : Response type
  497. * @early_wake_time : Poll time for early wakeup
  498. * @ver : Version of response
  499. * @ns : Timestamp (in ns) of response
  500. * @xo_time_in_us : XO Timestamp (in us) of response
  501. *
  502. * Returns none
  503. */
  504. static inline void fastrpc_update_rxmsg_buf(struct fastrpc_channel_ctx *chan,
  505. uint64_t ctx, int retval, uint32_t rsp_flags,
  506. uint32_t early_wake_time, uint32_t ver, int64_t ns, uint64_t xo_time_in_us)
  507. {
  508. unsigned long flags = 0;
  509. unsigned int rx_index = 0;
  510. struct fastrpc_rx_msg *rx_msg = NULL;
  511. struct smq_invoke_rspv2 *rsp = NULL;
  512. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  513. rx_index = chan->gmsg_log.rx_index;
  514. rx_msg = &chan->gmsg_log.rx_msgs[rx_index];
  515. rsp = &rx_msg->rsp;
  516. rsp->ctx = ctx;
  517. rsp->retval = retval;
  518. rsp->flags = rsp_flags;
  519. rsp->early_wake_time = early_wake_time;
  520. rsp->version = ver;
  521. rx_msg->ns = ns;
  522. rx_msg->xo_time_in_us = xo_time_in_us;
  523. rx_index++;
  524. chan->gmsg_log.rx_index =
  525. (rx_index > (GLINK_MSG_HISTORY_LEN - 1)) ? 0 : rx_index;
  526. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  527. }
  528. static inline int get_unique_index(void)
  529. {
  530. int index = -1;
  531. mutex_lock(&gfa.mut_uid);
  532. for (index = 0; index < MAX_UNIQUE_ID; index++) {
  533. if (md_unique_index_flag[index] == 0) {
  534. md_unique_index_flag[index] = 1;
  535. mutex_unlock(&gfa.mut_uid);
  536. return index;
  537. }
  538. }
  539. mutex_unlock(&gfa.mut_uid);
  540. return index;
  541. }
  542. static inline void reset_unique_index(int index)
  543. {
  544. mutex_lock(&gfa.mut_uid);
  545. if (index > -1 && index < MAX_UNIQUE_ID)
  546. md_unique_index_flag[index] = 0;
  547. mutex_unlock(&gfa.mut_uid);
  548. }
  549. /**
  550. * fastrpc_minidump_add_region - Add mini dump region
  551. * @fastrpc_mmap : Input structure mmap
  552. *
  553. * Returns int
  554. */
  555. static int fastrpc_minidump_add_region(struct fastrpc_mmap *map)
  556. {
  557. int err = 0, ret_val = 0, md_index = 0;
  558. struct md_region md_entry;
  559. md_index = get_unique_index();
  560. if (md_index > -1 && md_index < MAX_UNIQUE_ID) {
  561. scnprintf(md_entry.name, MAX_NAME_LENGTH, "FRPC_%d", md_index);
  562. md_entry.virt_addr = map->va;
  563. md_entry.phys_addr = map->phys;
  564. md_entry.size = map->size;
  565. ret_val = msm_minidump_add_region(&md_entry);
  566. if (ret_val < 0) {
  567. ADSPRPC_ERR(
  568. "Failed to add/update CMA to Minidump for phys: 0x%llx, size: %zu, md_index %d, md_entry.name %s\n",
  569. map->phys,
  570. map->size, md_index,
  571. md_entry.name);
  572. reset_unique_index(md_index);
  573. err = ret_val;
  574. } else {
  575. map->frpc_md_index = md_index;
  576. }
  577. } else {
  578. pr_warn("failed to generate valid unique id for mini dump : %d\n", md_index);
  579. }
  580. return err;
  581. }
  582. /**
  583. * fastrpc_minidump_remove_region - Remove mini dump region if added
  584. * @fastrpc_mmap : Input structure mmap
  585. *
  586. * Returns int
  587. */
  588. static int fastrpc_minidump_remove_region(struct fastrpc_mmap *map)
  589. {
  590. int err = -EINVAL;
  591. struct md_region md_entry;
  592. if (map->frpc_md_index > -1 && map->frpc_md_index < MAX_UNIQUE_ID) {
  593. scnprintf(md_entry.name, MAX_NAME_LENGTH, "FRPC_%d",
  594. map->frpc_md_index);
  595. md_entry.virt_addr = map->va;
  596. md_entry.phys_addr = map->phys;
  597. md_entry.size = map->size;
  598. err = msm_minidump_remove_region(&md_entry);
  599. if (err < 0) {
  600. ADSPRPC_ERR(
  601. "Failed to remove CMA from Minidump for phys: 0x%llx, size: %zu index = %d\n",
  602. map->phys, map->size, map->frpc_md_index);
  603. } else {
  604. reset_unique_index(map->frpc_md_index);
  605. map->frpc_md_index = -1;
  606. }
  607. } else {
  608. ADSPRPC_WARN("mini-dump enabled with invalid unique id: %d\n", map->frpc_md_index);
  609. }
  610. return err;
  611. }
  612. static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache)
  613. {
  614. struct fastrpc_file *fl = buf == NULL ? NULL : buf->fl;
  615. int vmid, err = 0, cid = -1;
  616. if (!fl)
  617. return;
  618. if (buf->in_use) {
  619. /* Don't free persistent header buf. Just mark as available */
  620. spin_lock(&fl->hlock);
  621. buf->in_use = false;
  622. spin_unlock(&fl->hlock);
  623. return;
  624. }
  625. if (cache && buf->size < MAX_CACHE_BUF_SIZE) {
  626. spin_lock(&fl->hlock);
  627. if (fl->num_cached_buf > MAX_CACHED_BUFS) {
  628. spin_unlock(&fl->hlock);
  629. goto skip_buf_cache;
  630. }
  631. hlist_add_head(&buf->hn, &fl->cached_bufs);
  632. fl->num_cached_buf++;
  633. buf->type = -1;
  634. spin_unlock(&fl->hlock);
  635. return;
  636. }
  637. skip_buf_cache:
  638. if (buf->type == USERHEAP_BUF) {
  639. spin_lock(&fl->hlock);
  640. hlist_del_init(&buf->hn_rem);
  641. spin_unlock(&fl->hlock);
  642. buf->raddr = 0;
  643. }
  644. if (!IS_ERR_OR_NULL(buf->virt)) {
  645. VERIFY(err, fl->sctx != NULL);
  646. if (err)
  647. goto bail;
  648. if (fl->sctx->smmu.cb)
  649. buf->phys &= ~((uint64_t)fl->sctx->smmu.cb << 32);
  650. cid = fl->cid;
  651. VERIFY(err, VALID_FASTRPC_CID(cid));
  652. if (err) {
  653. err = -ECHRNG;
  654. ADSPRPC_ERR(
  655. "invalid channel 0x%zx set for session\n",
  656. cid);
  657. goto bail;
  658. }
  659. vmid = fl->apps->channel[cid].vmid;
  660. if ((vmid) && (fl->apps->channel[cid].in_hib == 0)) {
  661. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS)| BIT(vmid);
  662. struct qcom_scm_vmperm dest_perms = {0};
  663. int hyp_err = 0;
  664. dest_perms.vmid = QCOM_SCM_VMID_HLOS;
  665. dest_perms.perm = QCOM_SCM_PERM_RWX;
  666. hyp_err = qcom_scm_assign_mem(buf->phys,
  667. buf_page_size(buf->size),
  668. &src_perms, &dest_perms, 1);
  669. if (hyp_err) {
  670. ADSPRPC_ERR(
  671. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  672. hyp_err, buf->phys, buf->size);
  673. }
  674. }
  675. trace_fastrpc_dma_free(cid, buf->phys, buf->size);
  676. dma_free_attrs(fl->sctx->smmu.dev, buf->size, buf->virt,
  677. buf->phys, buf->dma_attr);
  678. }
  679. bail:
  680. kfree(buf);
  681. }
  682. static void fastrpc_cached_buf_list_free(struct fastrpc_file *fl)
  683. {
  684. struct fastrpc_buf *buf, *free;
  685. do {
  686. struct hlist_node *n;
  687. free = NULL;
  688. spin_lock(&fl->hlock);
  689. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  690. hlist_del_init(&buf->hn);
  691. fl->num_cached_buf--;
  692. free = buf;
  693. break;
  694. }
  695. spin_unlock(&fl->hlock);
  696. if (free)
  697. fastrpc_buf_free(free, 0);
  698. } while (free);
  699. }
  700. static void fastrpc_remote_buf_list_free(struct fastrpc_file *fl)
  701. {
  702. struct fastrpc_buf *buf, *free;
  703. do {
  704. struct hlist_node *n;
  705. free = NULL;
  706. spin_lock(&fl->hlock);
  707. hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {
  708. free = buf;
  709. break;
  710. }
  711. spin_unlock(&fl->hlock);
  712. if (free)
  713. fastrpc_buf_free(free, 0);
  714. } while (free);
  715. }
  716. static void fastrpc_mmap_add(struct fastrpc_mmap *map)
  717. {
  718. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  719. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  720. struct fastrpc_apps *me = &gfa;
  721. unsigned long irq_flags = 0;
  722. spin_lock_irqsave(&me->hlock, irq_flags);
  723. hlist_add_head(&map->hn, &me->maps);
  724. spin_unlock_irqrestore(&me->hlock, irq_flags);
  725. } else {
  726. struct fastrpc_file *fl = map->fl;
  727. hlist_add_head(&map->hn, &fl->maps);
  728. }
  729. }
  730. static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd,
  731. struct dma_buf *buf, uintptr_t va, size_t len, int mflags, int refs,
  732. struct fastrpc_mmap **ppmap)
  733. {
  734. struct fastrpc_apps *me = &gfa;
  735. struct fastrpc_mmap *match = NULL, *map = NULL;
  736. struct hlist_node *n;
  737. unsigned long irq_flags = 0;
  738. if ((va + len) < va)
  739. return -EFAULT;
  740. if (mflags == ADSP_MMAP_HEAP_ADDR ||
  741. mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  742. spin_lock_irqsave(&me->hlock, irq_flags);
  743. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  744. if (va >= map->va &&
  745. va + len <= map->va + map->len &&
  746. map->fd == fd) {
  747. if (refs) {
  748. if (map->refs + 1 == INT_MAX) {
  749. spin_unlock_irqrestore(&me->hlock, irq_flags);
  750. return -ETOOMANYREFS;
  751. }
  752. map->refs++;
  753. }
  754. match = map;
  755. break;
  756. }
  757. }
  758. spin_unlock_irqrestore(&me->hlock, irq_flags);
  759. } else if (mflags == ADSP_MMAP_DMA_BUFFER) {
  760. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  761. if (map->buf == buf) {
  762. if (refs) {
  763. if (map->refs + 1 == INT_MAX)
  764. return -ETOOMANYREFS;
  765. map->refs++;
  766. }
  767. match = map;
  768. break;
  769. }
  770. }
  771. } else {
  772. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  773. if (va >= map->va &&
  774. va + len <= map->va + map->len &&
  775. map->fd == fd) {
  776. if (refs) {
  777. if (map->refs + 1 == INT_MAX)
  778. return -ETOOMANYREFS;
  779. map->refs++;
  780. }
  781. match = map;
  782. break;
  783. }
  784. }
  785. }
  786. if (match) {
  787. *ppmap = match;
  788. return 0;
  789. }
  790. return -ENXIO;
  791. }
  792. static int fastrpc_alloc_cma_memory(dma_addr_t *region_phys, void **vaddr,
  793. size_t size, unsigned long dma_attr)
  794. {
  795. int err = 0;
  796. struct fastrpc_apps *me = &gfa;
  797. if (me->dev == NULL) {
  798. ADSPRPC_ERR(
  799. "failed to allocate CMA memory, device adsprpc-mem is not initialized\n");
  800. return -ENODEV;
  801. }
  802. VERIFY(err, size > 0 && size < me->max_size_limit);
  803. if (err) {
  804. err = -EFAULT;
  805. pr_err("adsprpc: %s: invalid allocation size 0x%zx\n",
  806. __func__, size);
  807. return err;
  808. }
  809. *vaddr = dma_alloc_attrs(me->dev, size, region_phys,
  810. GFP_KERNEL, dma_attr);
  811. if (IS_ERR_OR_NULL(*vaddr)) {
  812. ADSPRPC_ERR(
  813. "dma_alloc_attrs failed for device %s size 0x%zx dma_attr %lu, returned %ld\n",
  814. dev_name(me->dev), size, dma_attr, PTR_ERR(*vaddr));
  815. return -ENOBUFS;
  816. }
  817. return 0;
  818. }
  819. static int fastrpc_mmap_remove(struct fastrpc_file *fl, int fd, uintptr_t va,
  820. size_t len, struct fastrpc_mmap **ppmap)
  821. {
  822. struct fastrpc_mmap *match = NULL, *map;
  823. struct hlist_node *n;
  824. struct fastrpc_apps *me = &gfa;
  825. unsigned long irq_flags = 0;
  826. /*
  827. * Search for a mapping by matching fd, remote address and length.
  828. * For backward compatibility, search for a mapping by matching is
  829. * limited to remote address and length when passed fd < 0.
  830. */
  831. spin_lock_irqsave(&me->hlock, irq_flags);
  832. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  833. if ((fd < 0 || map->fd == fd) && map->raddr == va &&
  834. map->raddr + map->len == va + len &&
  835. map->refs == 1 && !map->is_persistent &&
  836. /* Skip unmap if it is fastrpc shell memory */
  837. !map->is_filemap) {
  838. match = map;
  839. hlist_del_init(&map->hn);
  840. break;
  841. }
  842. }
  843. spin_unlock_irqrestore(&me->hlock, irq_flags);
  844. if (match) {
  845. *ppmap = match;
  846. return 0;
  847. }
  848. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  849. if ((fd < 0 || map->fd == fd) && map->raddr == va &&
  850. map->raddr + map->len == va + len &&
  851. map->refs == 1 &&
  852. /* Remove map only if it isn't being used in any pending RPC calls */
  853. !map->ctx_refs &&
  854. /* Skip unmap if it is fastrpc shell memory */
  855. !map->is_filemap) {
  856. match = map;
  857. hlist_del_init(&map->hn);
  858. break;
  859. }
  860. }
  861. if (match) {
  862. *ppmap = match;
  863. return 0;
  864. }
  865. return -ETOOMANYREFS;
  866. }
  867. static void fastrpc_mmap_free(struct fastrpc_mmap *map, uint32_t flags)
  868. {
  869. struct fastrpc_apps *me = &gfa;
  870. struct fastrpc_file *fl;
  871. int vmid, cid = -1, err = 0;
  872. struct fastrpc_session_ctx *sess;
  873. unsigned long irq_flags = 0;
  874. if (!map)
  875. return;
  876. fl = map->fl;
  877. if (fl && !(map->flags == ADSP_MMAP_HEAP_ADDR ||
  878. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)) {
  879. cid = fl->cid;
  880. VERIFY(err, VALID_FASTRPC_CID(cid));
  881. if (err) {
  882. err = -ECHRNG;
  883. pr_err("adsprpc: ERROR:%s, Invalid channel id: %d, err:%d\n",
  884. __func__, cid, err);
  885. return;
  886. }
  887. }
  888. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  889. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  890. spin_lock_irqsave(&me->hlock, irq_flags);
  891. map->refs--;
  892. if (!map->refs && !map->is_persistent && !map->ctx_refs)
  893. hlist_del_init(&map->hn);
  894. if (map->refs > 0) {
  895. ADSPRPC_WARN(
  896. "multiple references for remote heap size %zu va 0x%lx ref count is %d\n",
  897. map->size, map->va, map->refs);
  898. spin_unlock_irqrestore(&me->hlock, irq_flags);
  899. return;
  900. }
  901. if (map->is_persistent && map->in_use)
  902. map->in_use = false;
  903. spin_unlock_irqrestore(&me->hlock, irq_flags);
  904. } else {
  905. map->refs--;
  906. if (!map->refs && !map->ctx_refs)
  907. hlist_del_init(&map->hn);
  908. if (map->refs > 0 && !flags)
  909. return;
  910. }
  911. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  912. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  913. if (me->dev == NULL) {
  914. ADSPRPC_ERR(
  915. "failed to free remote heap allocation, device is not initialized\n");
  916. return;
  917. }
  918. if (msm_minidump_enabled() && !map->is_persistent)
  919. err = fastrpc_minidump_remove_region(map);
  920. if (map->phys && !map->is_persistent) {
  921. trace_fastrpc_dma_free(-1, map->phys, map->size);
  922. dma_free_attrs(me->dev, map->size, (void *)map->va,
  923. (dma_addr_t)map->phys, (unsigned long)map->attr);
  924. }
  925. } else if (map->flags == FASTRPC_MAP_FD_NOMAP) {
  926. trace_fastrpc_dma_unmap(cid, map->phys, map->size);
  927. if (!IS_ERR_OR_NULL(map->table))
  928. dma_buf_unmap_attachment(map->attach, map->table,
  929. DMA_BIDIRECTIONAL);
  930. if (!IS_ERR_OR_NULL(map->attach))
  931. dma_buf_detach(map->buf, map->attach);
  932. if (!IS_ERR_OR_NULL(map->buf))
  933. dma_buf_put(map->buf);
  934. } else {
  935. if (!fl)
  936. goto bail;
  937. if (map->secure)
  938. sess = fl->secsctx;
  939. else
  940. sess = fl->sctx;
  941. vmid = fl->apps->channel[cid].vmid;
  942. if (vmid && map->phys && (me->channel[cid].in_hib == 0)) {
  943. int hyp_err = 0;
  944. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
  945. struct qcom_scm_vmperm dst_perms = {0};
  946. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  947. dst_perms.perm = QCOM_SCM_PERM_RWX;
  948. hyp_err = qcom_scm_assign_mem(map->phys,
  949. buf_page_size(map->size),
  950. &src_perms, &dst_perms, 1);
  951. if (hyp_err) {
  952. ADSPRPC_ERR(
  953. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  954. hyp_err, map->phys, map->size);
  955. }
  956. }
  957. trace_fastrpc_dma_unmap(cid, map->phys, map->size);
  958. if (!IS_ERR_OR_NULL(map->table))
  959. dma_buf_unmap_attachment(map->attach, map->table,
  960. DMA_BIDIRECTIONAL);
  961. if (!IS_ERR_OR_NULL(map->attach))
  962. dma_buf_detach(map->buf, map->attach);
  963. if (!IS_ERR_OR_NULL(map->buf))
  964. dma_buf_put(map->buf);
  965. }
  966. if (fl) {
  967. spin_lock(&fl->hlock);
  968. if ((map->flags == ADSP_MMAP_ADD_PAGES) || (map->flags == ADSP_MMAP_ADD_PAGES_LLC))
  969. fl->mem_snap.heap_bufs_size -= map->size;
  970. else
  971. fl->mem_snap.nonheap_bufs_size -= map->size;
  972. spin_unlock(&fl->hlock);
  973. }
  974. bail:
  975. if (!map->is_persistent)
  976. kfree(map);
  977. }
  978. static int fastrpc_session_alloc_secure_memory(
  979. struct fastrpc_channel_ctx *chan, int secure,
  980. int sharedcb, int pd_type, struct fastrpc_session_ctx **session);
  981. static inline bool fastrpc_get_persistent_map(size_t len, struct fastrpc_mmap **pers_map)
  982. {
  983. struct fastrpc_apps *me = &gfa;
  984. struct fastrpc_mmap *map = NULL;
  985. struct hlist_node *n = NULL;
  986. bool found = false;
  987. unsigned long irq_flags = 0;
  988. spin_lock_irqsave(&me->hlock, irq_flags);
  989. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  990. if (len == map->len &&
  991. map->is_persistent && !map->in_use) {
  992. *pers_map = map;
  993. map->in_use = true;
  994. /*
  995. * Incrementing map reference count when getting
  996. * the map to avoid negative reference count when
  997. * freeing the map.
  998. */
  999. map->refs++;
  1000. found = true;
  1001. break;
  1002. }
  1003. }
  1004. spin_unlock_irqrestore(&me->hlock, irq_flags);
  1005. return found;
  1006. }
  1007. static int fastrpc_mmap_create_remote_heap(struct fastrpc_file *fl,
  1008. struct fastrpc_mmap *map, size_t len, int mflags)
  1009. {
  1010. int err = 0;
  1011. struct fastrpc_apps *me = &gfa;
  1012. dma_addr_t region_phys = 0;
  1013. void *region_vaddr = NULL;
  1014. map->apps = me;
  1015. map->fl = NULL;
  1016. map->attr |= DMA_ATTR_NO_KERNEL_MAPPING;
  1017. err = fastrpc_alloc_cma_memory(&region_phys, &region_vaddr,
  1018. len, (unsigned long) map->attr);
  1019. if (err)
  1020. goto bail;
  1021. trace_fastrpc_dma_alloc(fl->cid, (uint64_t)region_phys, len,
  1022. (unsigned long)map->attr, mflags);
  1023. map->phys = (uintptr_t)region_phys;
  1024. map->size = len;
  1025. map->va = (uintptr_t)region_vaddr;
  1026. map->servloc_name = fl->servloc_name;
  1027. bail:
  1028. return err;
  1029. }
  1030. static int get_buffer_attr(struct dma_buf *buf, bool *exclusive_access, bool *hlos_access)
  1031. {
  1032. const int *vmids_list = NULL, *perms = NULL;
  1033. int err = 0, vmids_list_len = 0;
  1034. *exclusive_access = false;
  1035. *hlos_access = false;
  1036. err = mem_buf_dma_buf_get_vmperm(buf, &vmids_list, &perms, &vmids_list_len);
  1037. if (err)
  1038. goto bail;
  1039. /*
  1040. * If one VM has access to buffer and is the current VM,
  1041. * then VM has exclusive access to buffer
  1042. */
  1043. if (vmids_list_len == 1 && vmids_list[0] == mem_buf_current_vmid())
  1044. *exclusive_access = true;
  1045. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1046. for (int ii = 0; ii < vmids_list_len; ii++) {
  1047. if (vmids_list[ii] == VMID_HLOS) {
  1048. *hlos_access = true;
  1049. break;
  1050. }
  1051. }
  1052. #endif
  1053. bail:
  1054. return err;
  1055. }
  1056. static int set_buffer_secure_type(struct fastrpc_mmap *map)
  1057. {
  1058. int err = 0;
  1059. bool hlos_access = false, exclusive_access = false;
  1060. VERIFY(err, 0 == (err = get_buffer_attr(map->buf, &exclusive_access, &hlos_access)));
  1061. if (err) {
  1062. ADSPRPC_ERR("failed to obtain buffer attributes for fd %d ret %d\n", map->fd, err);
  1063. err = -EBADFD;
  1064. goto bail;
  1065. }
  1066. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1067. /*
  1068. * PVM (HLOS) can share buffers with TVM, in case buffers are to be shared to secure PD,
  1069. * PVM is expected to relinquish its ownership to those buffers before sharing.
  1070. * If PVM still retains access, then those buffers cannot be shared to secure PD.
  1071. */
  1072. if (hlos_access) {
  1073. ADSPRPC_ERR("Buffers with HLOS access (fd %d) are not allowed on TVM\n", map->fd);
  1074. err = -EACCES;
  1075. goto bail;
  1076. }
  1077. #endif
  1078. /*
  1079. * Secure buffers would always be owned by multiple VMs.
  1080. * If current VM is the exclusive owner of a buffer, it is considered non-secure.
  1081. * In PVM:
  1082. * - CPZ buffers are secure
  1083. * - All other buffers are non-secure
  1084. * In TVM:
  1085. * - Since it is a secure environment by default, there are no explicit "secure" buffers
  1086. * - All buffers are marked "non-secure"
  1087. */
  1088. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1089. map->secure = 0;
  1090. #else
  1091. map->secure = (exclusive_access) ? 0 : 1;
  1092. #endif
  1093. bail:
  1094. return err;
  1095. }
  1096. static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, struct dma_buf *buf,
  1097. unsigned int attr, uintptr_t va, size_t len, int mflags,
  1098. struct fastrpc_mmap **ppmap)
  1099. {
  1100. struct fastrpc_apps *me = &gfa;
  1101. struct fastrpc_session_ctx *sess;
  1102. struct fastrpc_apps *apps = NULL;
  1103. int cid = -1;
  1104. struct fastrpc_channel_ctx *chan = NULL;
  1105. struct fastrpc_mmap *map = NULL;
  1106. int err = 0, vmid, sgl_index = 0;
  1107. struct scatterlist *sgl = NULL;
  1108. bool dma_attach_fail = false;
  1109. size_t tot_bufs_size = 0;
  1110. if (!fl) {
  1111. err = -EBADF;
  1112. goto bail;
  1113. }
  1114. apps = fl->apps;
  1115. cid = fl->cid;
  1116. VERIFY(err, VALID_FASTRPC_CID(cid));
  1117. if (err) {
  1118. err = -ECHRNG;
  1119. goto bail;
  1120. }
  1121. chan = &apps->channel[cid];
  1122. if (!fastrpc_mmap_find(fl, fd, NULL, va, len, mflags, 1, ppmap))
  1123. return 0;
  1124. map = kzalloc(sizeof(*map), GFP_KERNEL);
  1125. VERIFY(err, !IS_ERR_OR_NULL(map));
  1126. if (err) {
  1127. err = -ENOMEM;
  1128. goto bail;
  1129. }
  1130. INIT_HLIST_NODE(&map->hn);
  1131. map->flags = mflags;
  1132. map->refs = 1;
  1133. map->fl = fl;
  1134. map->fd = fd;
  1135. map->attr = attr;
  1136. map->buf = buf;
  1137. map->frpc_md_index = -1;
  1138. map->is_filemap = false;
  1139. ktime_get_real_ts64(&map->map_start_time);
  1140. if (mflags == ADSP_MMAP_HEAP_ADDR ||
  1141. mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  1142. VERIFY(err, 0 == (err = fastrpc_mmap_create_remote_heap(fl, map,
  1143. len, mflags)));
  1144. if (err)
  1145. goto bail;
  1146. if (msm_minidump_enabled()) {
  1147. err = fastrpc_minidump_add_region(map);
  1148. if (err)
  1149. goto bail;
  1150. }
  1151. } else if (mflags == FASTRPC_MAP_FD_NOMAP) {
  1152. VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));
  1153. if (err) {
  1154. ADSPRPC_ERR("dma_buf_get failed for fd %d ret %ld\n",
  1155. fd, PTR_ERR(map->buf));
  1156. err = -EBADFD;
  1157. goto bail;
  1158. }
  1159. err = set_buffer_secure_type(map);
  1160. if (err)
  1161. goto bail;
  1162. map->va = 0;
  1163. map->phys = 0;
  1164. VERIFY(err, !IS_ERR_OR_NULL(map->attach =
  1165. dma_buf_attach(map->buf, me->dev)));
  1166. if (err) {
  1167. ADSPRPC_ERR(
  1168. "dma_buf_attach for fd %d for len 0x%zx failed to map buffer on SMMU device %s ret %ld\n",
  1169. fd, len, dev_name(me->dev), PTR_ERR(map->attach));
  1170. dma_attach_fail = true;
  1171. err = -EFAULT;
  1172. goto bail;
  1173. }
  1174. map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  1175. VERIFY(err, !IS_ERR_OR_NULL(map->table =
  1176. dma_buf_map_attachment(map->attach,
  1177. DMA_BIDIRECTIONAL)));
  1178. if (err) {
  1179. ADSPRPC_ERR(
  1180. "dma_buf_map_attachment for fd %d for len 0x%zx failed on device %s ret %ld\n",
  1181. fd, len, dev_name(me->dev), PTR_ERR(map->table));
  1182. err = -EFAULT;
  1183. goto bail;
  1184. }
  1185. VERIFY(err, map->table->nents == 1);
  1186. if (err) {
  1187. ADSPRPC_ERR(
  1188. "multiple scatter-gather entries (%u) present for NOMAP fd %d\n",
  1189. map->table->nents, fd);
  1190. err = -EFAULT;
  1191. goto bail;
  1192. }
  1193. map->phys = sg_dma_address(map->table->sgl);
  1194. map->size = len;
  1195. map->flags = FASTRPC_MAP_FD_DELAYED;
  1196. trace_fastrpc_dma_map(cid, fd, map->phys, map->size,
  1197. len, map->attach->dma_map_attrs, mflags);
  1198. } else {
  1199. if (map->attr && (map->attr & FASTRPC_ATTR_KEEP_MAP)) {
  1200. ADSPRPC_INFO("buffer mapped with persist attr 0x%x\n",
  1201. (unsigned int)map->attr);
  1202. map->refs = 2;
  1203. }
  1204. if (mflags == ADSP_MMAP_DMA_BUFFER) {
  1205. VERIFY(err, !IS_ERR_OR_NULL(map->buf));
  1206. if (err) {
  1207. ADSPRPC_ERR("Invalid DMA buffer address %pK\n",
  1208. map->buf);
  1209. err = -EFAULT;
  1210. goto bail;
  1211. }
  1212. /* Increment DMA buffer ref count,
  1213. * so that client cannot unmap DMA buffer, before freeing buffer
  1214. */
  1215. get_dma_buf(map->buf);
  1216. } else {
  1217. VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));
  1218. if (err) {
  1219. ADSPRPC_ERR("dma_buf_get failed for fd %d ret %ld\n",
  1220. fd, PTR_ERR(map->buf));
  1221. err = -EBADFD;
  1222. goto bail;
  1223. }
  1224. }
  1225. err = set_buffer_secure_type(map);
  1226. if (err)
  1227. goto bail;
  1228. if (map->secure) {
  1229. if (!fl->secsctx)
  1230. err = fastrpc_session_alloc_secure_memory(chan, 1,
  1231. me->share_securecb, fl->pd_type, &fl->secsctx);
  1232. if (err) {
  1233. ADSPRPC_ERR(
  1234. "fastrpc_session_alloc_secure_memory failed for fd %d ret %d\n",
  1235. fd, err);
  1236. err = -ENOSR;
  1237. goto bail;
  1238. }
  1239. }
  1240. if (map->secure)
  1241. sess = fl->secsctx;
  1242. else
  1243. sess = fl->sctx;
  1244. VERIFY(err, !IS_ERR_OR_NULL(sess));
  1245. if (err) {
  1246. ADSPRPC_ERR(
  1247. "session is invalid for fd %d, secure flag %d\n",
  1248. fd, map->secure);
  1249. err = -EBADR;
  1250. goto bail;
  1251. }
  1252. VERIFY(err, !IS_ERR_OR_NULL(map->attach =
  1253. dma_buf_attach(map->buf, sess->smmu.dev)));
  1254. if (err) {
  1255. ADSPRPC_ERR(
  1256. "dma_buf_attach for fd %d failed for len 0x%zx to map buffer on SMMU device %s ret %ld\n",
  1257. fd, len, dev_name(sess->smmu.dev),
  1258. PTR_ERR(map->attach));
  1259. dma_attach_fail = true;
  1260. err = -EFAULT;
  1261. goto bail;
  1262. }
  1263. map->attach->dma_map_attrs |= DMA_ATTR_DELAYED_UNMAP;
  1264. /*
  1265. * Skip CPU sync if IO Cohernecy is not supported
  1266. */
  1267. if (!sess->smmu.coherent)
  1268. map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  1269. VERIFY(err, !IS_ERR_OR_NULL(map->table =
  1270. dma_buf_map_attachment(map->attach,
  1271. DMA_BIDIRECTIONAL)));
  1272. if (err) {
  1273. ADSPRPC_ERR(
  1274. "dma_buf_map_attachment for fd %d failed for len 0x%zx on device %s ret %ld\n",
  1275. fd, len, dev_name(sess->smmu.dev),
  1276. PTR_ERR(map->table));
  1277. err = -EFAULT;
  1278. goto bail;
  1279. }
  1280. if (!sess->smmu.enabled) {
  1281. VERIFY(err, map->table->nents == 1);
  1282. if (err) {
  1283. ADSPRPC_ERR(
  1284. "multiple scatter-gather entries (%u) present for fd %d mapped on SMMU disabled device\n",
  1285. map->table->nents, fd);
  1286. err = -EFAULT;
  1287. goto bail;
  1288. }
  1289. }
  1290. map->phys = sg_dma_address(map->table->sgl);
  1291. if (sess->smmu.cb) {
  1292. map->phys += ((uint64_t)sess->smmu.cb << 32);
  1293. for_each_sg(map->table->sgl, sgl, map->table->nents,
  1294. sgl_index)
  1295. map->size += sg_dma_len(sgl);
  1296. } else {
  1297. map->size = buf_page_size(len);
  1298. }
  1299. trace_fastrpc_dma_map(cid, fd, map->phys, map->size,
  1300. len, map->attach->dma_map_attrs, mflags);
  1301. VERIFY(err, map->size >= len && map->size < me->max_size_limit);
  1302. if (err) {
  1303. err = -EFAULT;
  1304. pr_err("adsprpc: %s: invalid map size 0x%zx len 0x%zx\n",
  1305. __func__, map->size, len);
  1306. goto bail;
  1307. }
  1308. vmid = fl->apps->channel[cid].vmid;
  1309. if (vmid) {
  1310. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  1311. struct qcom_scm_vmperm dst_perms[2] = {0};
  1312. dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
  1313. dst_perms[0].perm = QCOM_SCM_PERM_RW;
  1314. dst_perms[1].vmid = vmid;
  1315. dst_perms[1].perm = QCOM_SCM_PERM_RWX;
  1316. err = qcom_scm_assign_mem(map->phys,
  1317. buf_page_size(map->size),
  1318. &src_perms, dst_perms, 2);
  1319. if (err) {
  1320. ADSPRPC_ERR(
  1321. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  1322. err, map->phys, map->size);
  1323. err = -EADDRNOTAVAIL;
  1324. goto bail;
  1325. }
  1326. }
  1327. map->va = va;
  1328. }
  1329. map->len = len;
  1330. spin_lock(&fl->hlock);
  1331. if ((mflags == ADSP_MMAP_ADD_PAGES) || (mflags == ADSP_MMAP_ADD_PAGES_LLC))
  1332. fl->mem_snap.heap_bufs_size += map->size;
  1333. else
  1334. fl->mem_snap.nonheap_bufs_size += map->size;
  1335. spin_unlock(&fl->hlock);
  1336. fastrpc_mmap_add(map);
  1337. *ppmap = map;
  1338. bail:
  1339. if (dma_attach_fail && fl) {
  1340. tot_bufs_size = fl->mem_snap.heap_bufs_size
  1341. + fl->mem_snap.nonheap_bufs_size;
  1342. ADSPRPC_INFO("Heapbufs size: %zu, non-heapbufs size: %zu, total size: %zu\n",
  1343. fl->mem_snap.heap_bufs_size, fl->mem_snap.nonheap_bufs_size,
  1344. tot_bufs_size);
  1345. }
  1346. if (map)
  1347. ktime_get_real_ts64(&map->map_end_time);
  1348. if (err && map)
  1349. fastrpc_mmap_free(map, 0);
  1350. return err;
  1351. }
  1352. static inline bool fastrpc_get_cached_buf(struct fastrpc_file *fl,
  1353. size_t size, int buf_type, struct fastrpc_buf **obuf)
  1354. {
  1355. bool found = false;
  1356. struct fastrpc_buf *buf = NULL, *fr = NULL;
  1357. struct hlist_node *n = NULL;
  1358. if (buf_type == USERHEAP_BUF)
  1359. goto bail;
  1360. /* find the smallest buffer that fits in the cache */
  1361. spin_lock(&fl->hlock);
  1362. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  1363. if (buf->size >= size && (!fr || fr->size > buf->size))
  1364. fr = buf;
  1365. }
  1366. if (fr) {
  1367. hlist_del_init(&fr->hn);
  1368. fl->num_cached_buf--;
  1369. }
  1370. spin_unlock(&fl->hlock);
  1371. if (fr) {
  1372. fr->type = buf_type;
  1373. *obuf = fr;
  1374. found = true;
  1375. }
  1376. bail:
  1377. return found;
  1378. }
  1379. static inline bool fastrpc_get_persistent_buf(struct fastrpc_file *fl,
  1380. size_t size, int buf_type, struct fastrpc_buf **obuf)
  1381. {
  1382. unsigned int i = 0;
  1383. bool found = false;
  1384. struct fastrpc_buf *buf = NULL;
  1385. spin_lock(&fl->hlock);
  1386. if (!fl->num_pers_hdrs)
  1387. goto bail;
  1388. /*
  1389. * Persistent header buffer can be used only if
  1390. * metadata length is less than 1 page size.
  1391. */
  1392. if (buf_type != METADATA_BUF || size > PAGE_SIZE)
  1393. goto bail;
  1394. for (i = 0; i < fl->num_pers_hdrs; i++) {
  1395. buf = &fl->hdr_bufs[i];
  1396. /* If buffer not in use, then assign it for requested alloc */
  1397. if (!buf->in_use) {
  1398. buf->in_use = true;
  1399. *obuf = buf;
  1400. found = true;
  1401. break;
  1402. }
  1403. }
  1404. bail:
  1405. spin_unlock(&fl->hlock);
  1406. return found;
  1407. }
  1408. static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
  1409. unsigned long dma_attr, uint32_t rflags,
  1410. int buf_type, struct fastrpc_buf **obuf)
  1411. {
  1412. int err = 0, vmid;
  1413. struct fastrpc_apps *me = &gfa;
  1414. struct fastrpc_buf *buf = NULL;
  1415. int cid = -1;
  1416. VERIFY(err, fl && fl->sctx != NULL);
  1417. if (err) {
  1418. err = -EBADR;
  1419. goto bail;
  1420. }
  1421. cid = fl->cid;
  1422. VERIFY(err, VALID_FASTRPC_CID(cid));
  1423. if (err) {
  1424. err = -ECHRNG;
  1425. goto bail;
  1426. }
  1427. VERIFY(err, size > 0 && size < me->max_size_limit);
  1428. if (err) {
  1429. err = -EFAULT;
  1430. pr_err("adsprpc: %s: invalid allocation size 0x%zx\n",
  1431. __func__, size);
  1432. goto bail;
  1433. }
  1434. VERIFY(err, size > 0 && fl->sctx->smmu.dev);
  1435. if (err) {
  1436. err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;
  1437. goto bail;
  1438. }
  1439. if (fastrpc_get_persistent_buf(fl, size, buf_type, obuf))
  1440. return err;
  1441. if (fastrpc_get_cached_buf(fl, size, buf_type, obuf))
  1442. return err;
  1443. /* If unable to get persistent or cached buf, allocate new buffer */
  1444. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  1445. if (err) {
  1446. err = -ENOMEM;
  1447. goto bail;
  1448. }
  1449. INIT_HLIST_NODE(&buf->hn);
  1450. buf->fl = fl;
  1451. buf->virt = NULL;
  1452. buf->phys = 0;
  1453. buf->size = size;
  1454. buf->dma_attr = dma_attr;
  1455. buf->flags = rflags;
  1456. buf->raddr = 0;
  1457. buf->type = buf_type;
  1458. ktime_get_real_ts64(&buf->buf_start_time);
  1459. buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
  1460. (dma_addr_t *)&buf->phys,
  1461. GFP_KERNEL, buf->dma_attr);
  1462. if (IS_ERR_OR_NULL(buf->virt)) {
  1463. /* free cache and retry */
  1464. fastrpc_cached_buf_list_free(fl);
  1465. buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
  1466. (dma_addr_t *)&buf->phys, GFP_KERNEL,
  1467. buf->dma_attr);
  1468. VERIFY(err, !IS_ERR_OR_NULL(buf->virt));
  1469. }
  1470. if (err) {
  1471. ADSPRPC_ERR(
  1472. "dma_alloc_attrs failed for size 0x%zx, returned %pK\n",
  1473. size, buf->virt);
  1474. err = -ENOBUFS;
  1475. goto bail;
  1476. }
  1477. if (fl->sctx->smmu.cb)
  1478. buf->phys += ((uint64_t)fl->sctx->smmu.cb << 32);
  1479. trace_fastrpc_dma_alloc(cid, buf->phys, size,
  1480. dma_attr, (int)rflags);
  1481. vmid = fl->apps->channel[cid].vmid;
  1482. if (vmid) {
  1483. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  1484. struct qcom_scm_vmperm dst_perms[2] = {0};
  1485. dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
  1486. dst_perms[0].perm = QCOM_SCM_PERM_RW;
  1487. dst_perms[1].vmid = vmid;
  1488. dst_perms[1].perm = QCOM_SCM_PERM_RWX;
  1489. err = qcom_scm_assign_mem(buf->phys, buf_page_size(size),
  1490. &src_perms, dst_perms, 2);
  1491. if (err) {
  1492. ADSPRPC_DEBUG(
  1493. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  1494. err, buf->phys, size);
  1495. err = -EADDRNOTAVAIL;
  1496. goto bail;
  1497. }
  1498. }
  1499. if (buf_type == USERHEAP_BUF) {
  1500. INIT_HLIST_NODE(&buf->hn_rem);
  1501. spin_lock(&fl->hlock);
  1502. hlist_add_head(&buf->hn_rem, &fl->remote_bufs);
  1503. spin_unlock(&fl->hlock);
  1504. }
  1505. *obuf = buf;
  1506. bail:
  1507. if (buf)
  1508. ktime_get_real_ts64(&buf->buf_end_time);
  1509. if (err && buf)
  1510. fastrpc_buf_free(buf, 0);
  1511. return err;
  1512. }
  1513. static int context_restore_interrupted(struct fastrpc_file *fl,
  1514. struct fastrpc_ioctl_invoke_async *inv,
  1515. struct smq_invoke_ctx **po)
  1516. {
  1517. int err = 0;
  1518. struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;
  1519. struct hlist_node *n;
  1520. struct fastrpc_ioctl_invoke *invoke = &inv->inv;
  1521. spin_lock(&fl->hlock);
  1522. hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {
  1523. if (ictx->pid == current->pid) {
  1524. if (invoke->sc != ictx->sc || ictx->fl != fl) {
  1525. err = -EINVAL;
  1526. ictx->sc_interrupted = invoke->sc;
  1527. ictx->fl_interrupted = fl;
  1528. ictx->handle_interrupted = invoke->handle;
  1529. ADSPRPC_ERR(
  1530. "interrupted sc (0x%x) or fl (%pK) does not match with invoke sc (0x%x) or fl (%pK)\n",
  1531. ictx->sc, ictx->fl, invoke->sc, fl);
  1532. } else {
  1533. ictx->xo_time_in_us_restored = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1534. fastrpc_update_txmsg_buf(NULL, 0, 0, 0, ictx, RESTORED_STATE);
  1535. ADSPRPC_DEBUG(
  1536. "restored sc (0x%x) of fl (%pK), interrupt ts 0x%llx, restore ts 0x%llx \n",
  1537. ictx->sc, ictx->fl, ictx->xo_time_in_us_interrupted, ictx->xo_time_in_us_restored);
  1538. ctx = ictx;
  1539. hlist_del_init(&ctx->hn);
  1540. hlist_add_head(&ctx->hn, &fl->clst.pending);
  1541. }
  1542. break;
  1543. }
  1544. }
  1545. spin_unlock(&fl->hlock);
  1546. if (ctx)
  1547. *po = ctx;
  1548. return err;
  1549. }
  1550. static unsigned int sorted_lists_intersection(unsigned int *listA,
  1551. unsigned int lenA, unsigned int *listB, unsigned int lenB)
  1552. {
  1553. unsigned int i = 0, j = 0;
  1554. while (i < lenA && j < lenB) {
  1555. if (listA[i] < listB[j])
  1556. i++;
  1557. else if (listA[i] > listB[j])
  1558. j++;
  1559. else
  1560. return listA[i];
  1561. }
  1562. return 0;
  1563. }
  1564. #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
  1565. static int uint_cmp_func(const void *p1, const void *p2)
  1566. {
  1567. unsigned int a1 = *((unsigned int *)p1);
  1568. unsigned int a2 = *((unsigned int *)p2);
  1569. return CMP(a1, a2);
  1570. }
  1571. static int overlap_ptr_cmp(const void *a, const void *b)
  1572. {
  1573. struct overlap *pa = *((struct overlap **)a);
  1574. struct overlap *pb = *((struct overlap **)b);
  1575. /* sort with lowest starting buffer first */
  1576. int st = CMP(pa->start, pb->start);
  1577. /* sort with highest ending buffer first */
  1578. int ed = CMP(pb->end, pa->end);
  1579. return st == 0 ? ed : st;
  1580. }
  1581. static int context_build_overlap(struct smq_invoke_ctx *ctx)
  1582. {
  1583. int i, err = 0;
  1584. remote_arg_t *lpra = ctx->lpra;
  1585. int inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
  1586. int outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
  1587. int nbufs = inbufs + outbufs;
  1588. struct overlap max;
  1589. for (i = 0; i < nbufs; ++i) {
  1590. ctx->overs[i].start = (uintptr_t)lpra[i].buf.pv;
  1591. ctx->overs[i].end = ctx->overs[i].start + lpra[i].buf.len;
  1592. if (lpra[i].buf.len) {
  1593. VERIFY(err, ctx->overs[i].end > ctx->overs[i].start);
  1594. if (err) {
  1595. err = -EFAULT;
  1596. ADSPRPC_ERR(
  1597. "Invalid address 0x%llx and size %zu\n",
  1598. (uintptr_t)lpra[i].buf.pv,
  1599. lpra[i].buf.len);
  1600. goto bail;
  1601. }
  1602. }
  1603. ctx->overs[i].raix = i;
  1604. ctx->overps[i] = &ctx->overs[i];
  1605. }
  1606. sort(ctx->overps, nbufs, sizeof(*ctx->overps), overlap_ptr_cmp, NULL);
  1607. max.start = 0;
  1608. max.end = 0;
  1609. for (i = 0; i < nbufs; ++i) {
  1610. if (ctx->overps[i]->start < max.end) {
  1611. ctx->overps[i]->mstart = max.end;
  1612. ctx->overps[i]->mend = ctx->overps[i]->end;
  1613. ctx->overps[i]->offset = max.end -
  1614. ctx->overps[i]->start;
  1615. if (ctx->overps[i]->end > max.end) {
  1616. max.end = ctx->overps[i]->end;
  1617. } else {
  1618. if ((max.raix < inbufs &&
  1619. ctx->overps[i]->raix + 1 > inbufs) ||
  1620. (ctx->overps[i]->raix < inbufs &&
  1621. max.raix + 1 > inbufs))
  1622. ctx->overps[i]->do_cmo = 1;
  1623. ctx->overps[i]->mend = 0;
  1624. ctx->overps[i]->mstart = 0;
  1625. }
  1626. } else {
  1627. ctx->overps[i]->mend = ctx->overps[i]->end;
  1628. ctx->overps[i]->mstart = ctx->overps[i]->start;
  1629. ctx->overps[i]->offset = 0;
  1630. max = *ctx->overps[i];
  1631. }
  1632. }
  1633. bail:
  1634. return err;
  1635. }
  1636. #define K_COPY_FROM_USER(err, kernel, dst, src, size) \
  1637. do {\
  1638. if (!(kernel))\
  1639. err = copy_from_user((dst),\
  1640. (void const __user *)(src),\
  1641. (size));\
  1642. else\
  1643. memmove((dst), (src), (size));\
  1644. } while (0)
  1645. #define K_COPY_TO_USER(err, kernel, dst, src, size) \
  1646. do {\
  1647. if (!(kernel))\
  1648. err = copy_to_user((void __user *)(dst),\
  1649. (src), (size));\
  1650. else\
  1651. memmove((dst), (src), (size));\
  1652. } while (0)
  1653. static void context_free(struct smq_invoke_ctx *ctx);
  1654. static int context_alloc(struct fastrpc_file *fl, uint32_t kernel,
  1655. struct fastrpc_ioctl_invoke_async *invokefd,
  1656. struct smq_invoke_ctx **po)
  1657. {
  1658. struct fastrpc_apps *me = &gfa;
  1659. int err = 0, bufs, ii, size = 0, cid = fl->cid;
  1660. struct smq_invoke_ctx *ctx = NULL;
  1661. struct fastrpc_ctx_lst *clst = &fl->clst;
  1662. struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;
  1663. struct fastrpc_channel_ctx *chan = NULL;
  1664. unsigned long irq_flags = 0;
  1665. uint32_t is_kernel_memory = 0;
  1666. spin_lock(&fl->hlock);
  1667. if (fl->clst.num_active_ctxs > MAX_PENDING_CTX_PER_SESSION &&
  1668. !(kernel || invoke->handle < FASTRPC_STATIC_HANDLE_MAX)) {
  1669. err = -EDQUOT;
  1670. spin_unlock(&fl->hlock);
  1671. goto bail;
  1672. }
  1673. spin_unlock(&fl->hlock);
  1674. bufs = REMOTE_SCALARS_LENGTH(invoke->sc);
  1675. size = bufs * sizeof(*ctx->lpra) + bufs * sizeof(*ctx->maps) +
  1676. sizeof(*ctx->fds) * (bufs) +
  1677. sizeof(*ctx->attrs) * (bufs) +
  1678. sizeof(*ctx->overs) * (bufs) +
  1679. sizeof(*ctx->overps) * (bufs);
  1680. VERIFY(err, NULL != (ctx = kzalloc(sizeof(*ctx) + size, GFP_KERNEL)));
  1681. if (err) {
  1682. err = -ENOMEM;
  1683. goto bail;
  1684. }
  1685. INIT_HLIST_NODE(&ctx->hn);
  1686. INIT_HLIST_NODE(&ctx->asyncn);
  1687. hlist_add_fake(&ctx->hn);
  1688. hlist_add_fake(&ctx->asyncn);
  1689. ctx->fl = fl;
  1690. ctx->maps = (struct fastrpc_mmap **)(&ctx[1]);
  1691. ctx->lpra = (remote_arg_t *)(&ctx->maps[bufs]);
  1692. ctx->fds = (int *)(&ctx->lpra[bufs]);
  1693. ctx->attrs = (unsigned int *)(&ctx->fds[bufs]);
  1694. ctx->overs = (struct overlap *)(&ctx->attrs[bufs]);
  1695. ctx->overps = (struct overlap **)(&ctx->overs[bufs]);
  1696. /* If user message, do not use copy_from_user to copy buffers for
  1697. * compat driver,as memory is already copied to kernel memory
  1698. * for compat driver
  1699. */
  1700. is_kernel_memory = ((kernel == USER_MSG) ? (fl->is_compat) : kernel);
  1701. K_COPY_FROM_USER(err, is_kernel_memory, (void *)ctx->lpra, invoke->pra,
  1702. bufs * sizeof(*ctx->lpra));
  1703. if (err) {
  1704. ADSPRPC_ERR(
  1705. "copy from user failed with %d for remote arguments list\n",
  1706. err);
  1707. err = -EFAULT;
  1708. goto bail;
  1709. }
  1710. if (invokefd->fds) {
  1711. K_COPY_FROM_USER(err, kernel, ctx->fds, invokefd->fds,
  1712. bufs * sizeof(*ctx->fds));
  1713. if (err) {
  1714. ADSPRPC_ERR(
  1715. "copy from user failed with %d for fd list\n",
  1716. err);
  1717. err = -EFAULT;
  1718. goto bail;
  1719. }
  1720. } else {
  1721. ctx->fds = NULL;
  1722. }
  1723. if (invokefd->attrs) {
  1724. K_COPY_FROM_USER(err, kernel, ctx->attrs, invokefd->attrs,
  1725. bufs * sizeof(*ctx->attrs));
  1726. if (err) {
  1727. ADSPRPC_ERR(
  1728. "copy from user failed with %d for attribute list\n",
  1729. err);
  1730. err = -EFAULT;
  1731. goto bail;
  1732. }
  1733. }
  1734. ctx->crc = (uint32_t *)invokefd->crc;
  1735. ctx->perf_dsp = (uint64_t *)invokefd->perf_dsp;
  1736. ctx->perf_kernel = (uint64_t *)invokefd->perf_kernel;
  1737. ctx->handle = invoke->handle;
  1738. ctx->sc = invoke->sc;
  1739. if (bufs) {
  1740. VERIFY(err, 0 == (err = context_build_overlap(ctx)));
  1741. if (err)
  1742. goto bail;
  1743. }
  1744. ctx->retval = -1;
  1745. ctx->pid = current->pid;
  1746. /* Store HLOS PID in context, it is not being sent to DSP */
  1747. ctx->tgid = fl->tgid;
  1748. init_completion(&ctx->work);
  1749. ctx->magic = FASTRPC_CTX_MAGIC;
  1750. ctx->rsp_flags = NORMAL_RESPONSE;
  1751. ctx->is_work_done = false;
  1752. ctx->copybuf = NULL;
  1753. ctx->is_early_wakeup = false;
  1754. ctx->is_job_sent_to_remote_ss = false;
  1755. if (ctx->fl->profile) {
  1756. ctx->perf = kzalloc(sizeof(*(ctx->perf)), GFP_KERNEL);
  1757. VERIFY(err, !IS_ERR_OR_NULL(ctx->perf));
  1758. if (err) {
  1759. kfree(ctx->perf);
  1760. err = -ENOMEM;
  1761. goto bail;
  1762. }
  1763. memset(ctx->perf, 0, sizeof(*(ctx->perf)));
  1764. /* Use HLOS PID, as perf tid is not being sent to DSP and is used to log in traces */
  1765. ctx->perf->tid = fl->tgid;
  1766. }
  1767. if (invokefd->job) {
  1768. K_COPY_FROM_USER(err, kernel, &ctx->asyncjob, invokefd->job,
  1769. sizeof(ctx->asyncjob));
  1770. if (err)
  1771. goto bail;
  1772. }
  1773. VERIFY(err, VALID_FASTRPC_CID(cid));
  1774. if (err) {
  1775. err = -ECHRNG;
  1776. goto bail;
  1777. }
  1778. chan = &me->channel[cid];
  1779. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  1780. me->jobid[cid]++;
  1781. for (ii = ((kernel || ctx->handle < FASTRPC_STATIC_HANDLE_MAX)
  1782. ? 0 : NUM_KERNEL_AND_STATIC_ONLY_CONTEXTS);
  1783. ii < FASTRPC_CTX_MAX; ii++) {
  1784. if (!chan->ctxtable[ii]) {
  1785. chan->ctxtable[ii] = ctx;
  1786. ctx->ctxid = (me->jobid[cid] << FASTRPC_CTX_JOBID_POS)
  1787. | (ii << FASTRPC_CTX_TABLE_IDX_POS)
  1788. | ((ctx->asyncjob.isasyncjob &&
  1789. FASTRPC_ASYNC_JOB_MASK) << FASTRPC_CTX_JOB_TYPE_POS);
  1790. break;
  1791. }
  1792. }
  1793. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  1794. VERIFY(err, ii < FASTRPC_CTX_MAX);
  1795. if (err) {
  1796. ADSPRPC_ERR(
  1797. "adsprpc: out of context table entries for handle 0x%x, sc 0x%x\n",
  1798. ctx->handle, ctx->sc);
  1799. err = -ENOKEY;
  1800. goto bail;
  1801. }
  1802. ctx->xo_time_in_us_created = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1803. spin_lock(&fl->hlock);
  1804. hlist_add_head(&ctx->hn, &clst->pending);
  1805. clst->num_active_ctxs++;
  1806. spin_unlock(&fl->hlock);
  1807. trace_fastrpc_context_alloc((uint64_t)ctx,
  1808. ctx->ctxid | fl->pd, ctx->handle, ctx->sc);
  1809. *po = ctx;
  1810. bail:
  1811. if (ctx && err)
  1812. context_free(ctx);
  1813. return err;
  1814. }
  1815. static void context_save_interrupted(struct smq_invoke_ctx *ctx)
  1816. {
  1817. struct fastrpc_ctx_lst *clst = &ctx->fl->clst;
  1818. ctx->xo_time_in_us_interrupted = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1819. fastrpc_update_txmsg_buf(NULL, 0, 0, 0, ctx, INTERRUPTED_STATE);
  1820. spin_lock(&ctx->fl->hlock);
  1821. hlist_del_init(&ctx->hn);
  1822. hlist_add_head(&ctx->hn, &clst->interrupted);
  1823. spin_unlock(&ctx->fl->hlock);
  1824. }
  1825. static void context_free(struct smq_invoke_ctx *ctx)
  1826. {
  1827. uint32_t i = 0;
  1828. struct fastrpc_apps *me = &gfa;
  1829. int nbufs = REMOTE_SCALARS_INBUFS(ctx->sc) +
  1830. REMOTE_SCALARS_OUTBUFS(ctx->sc);
  1831. int cid = ctx->fl->cid;
  1832. struct fastrpc_channel_ctx *chan = NULL;
  1833. unsigned long irq_flags = 0;
  1834. int err = 0;
  1835. VERIFY(err, VALID_FASTRPC_CID(cid));
  1836. if (err) {
  1837. ADSPRPC_ERR(
  1838. "invalid channel 0x%zx set for session\n",
  1839. cid);
  1840. return;
  1841. }
  1842. chan = &me->channel[cid];
  1843. i = (uint32_t)GET_TABLE_IDX_FROM_CTXID(ctx->ctxid);
  1844. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  1845. if (i < FASTRPC_CTX_MAX && chan->ctxtable[i] == ctx) {
  1846. chan->ctxtable[i] = NULL;
  1847. } else {
  1848. for (i = 0; i < FASTRPC_CTX_MAX; i++) {
  1849. if (chan->ctxtable[i] == ctx) {
  1850. chan->ctxtable[i] = NULL;
  1851. break;
  1852. }
  1853. }
  1854. }
  1855. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  1856. spin_lock(&ctx->fl->hlock);
  1857. if (!hlist_unhashed(&ctx->hn)) {
  1858. hlist_del_init(&ctx->hn);
  1859. ctx->fl->clst.num_active_ctxs--;
  1860. }
  1861. spin_unlock(&ctx->fl->hlock);
  1862. mutex_lock(&ctx->fl->map_mutex);
  1863. for (i = 0; i < nbufs; ++i) {
  1864. /*
  1865. * Decrement ctx refs count before mmap free,
  1866. * indicate remote call no longer using it
  1867. */
  1868. if (ctx->maps[i] && ctx->maps[i]->ctx_refs)
  1869. ctx->maps[i]->ctx_refs--;
  1870. fastrpc_mmap_free(ctx->maps[i], 0);
  1871. }
  1872. mutex_unlock(&ctx->fl->map_mutex);
  1873. fastrpc_buf_free(ctx->buf, 1);
  1874. if (ctx->copybuf != ctx->buf)
  1875. fastrpc_buf_free(ctx->copybuf, 1);
  1876. kfree(ctx->lrpra);
  1877. ctx->lrpra = NULL;
  1878. ctx->magic = 0;
  1879. ctx->ctxid = 0;
  1880. if (ctx->fl->profile)
  1881. kfree(ctx->perf);
  1882. trace_fastrpc_context_free((uint64_t)ctx,
  1883. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  1884. kfree(ctx);
  1885. }
  1886. static void fastrpc_queue_completed_async_job(struct smq_invoke_ctx *ctx)
  1887. {
  1888. struct fastrpc_file *fl = ctx->fl;
  1889. unsigned long flags;
  1890. spin_lock_irqsave(&fl->aqlock, flags);
  1891. if (ctx->is_early_wakeup)
  1892. goto bail;
  1893. if (!hlist_unhashed(&ctx->asyncn)) {
  1894. hlist_add_head(&ctx->asyncn, &fl->clst.async_queue);
  1895. atomic_add(1, &fl->async_queue_job_count);
  1896. ctx->is_early_wakeup = true;
  1897. wake_up_interruptible(&fl->async_wait_queue);
  1898. }
  1899. bail:
  1900. spin_unlock_irqrestore(&fl->aqlock, flags);
  1901. }
  1902. static void fastrpc_queue_pd_status(struct fastrpc_file *fl, int domain, int status, int sessionid)
  1903. {
  1904. struct smq_notif_rsp *notif_rsp = NULL;
  1905. unsigned long flags;
  1906. int err = 0;
  1907. VERIFY(err, NULL != (notif_rsp = kzalloc(sizeof(*notif_rsp), GFP_ATOMIC)));
  1908. if (err) {
  1909. ADSPRPC_ERR(
  1910. "allocation failed for size 0x%zx\n",
  1911. sizeof(*notif_rsp));
  1912. return;
  1913. }
  1914. notif_rsp->status = status;
  1915. notif_rsp->domain = domain;
  1916. notif_rsp->session = sessionid;
  1917. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  1918. list_add_tail(&notif_rsp->notifn, &fl->clst.notif_queue);
  1919. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  1920. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  1921. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  1922. }
  1923. static void fastrpc_notif_find_process(int domain, struct smq_notif_rspv3 *notif)
  1924. {
  1925. struct fastrpc_apps *me = &gfa;
  1926. struct fastrpc_file *fl = NULL;
  1927. struct hlist_node *n;
  1928. bool is_process_found = false;
  1929. unsigned long irq_flags = 0;
  1930. int err = 0;
  1931. spin_lock_irqsave(&me->hlock, irq_flags);
  1932. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  1933. if (fl->tgid_frpc == notif->pid) {
  1934. is_process_found = true;
  1935. err = fastrpc_file_get(fl);
  1936. if (err) {
  1937. ADSPRPC_ERR("Failed to get user process reference\n");
  1938. is_process_found = false;
  1939. }
  1940. break;
  1941. }
  1942. }
  1943. spin_unlock_irqrestore(&me->hlock, irq_flags);
  1944. if (!is_process_found)
  1945. return;
  1946. fastrpc_queue_pd_status(fl, domain, notif->status, fl->sessionid);
  1947. fastrpc_file_put(fl);
  1948. }
  1949. static void context_notify_user(struct smq_invoke_ctx *ctx,
  1950. int retval, uint32_t rsp_flags, uint32_t early_wake_time)
  1951. {
  1952. fastrpc_pm_awake(ctx->fl, gcinfo[ctx->fl->cid].secure);
  1953. ctx->retval = retval;
  1954. ctx->rsp_flags = (enum fastrpc_response_flags)rsp_flags;
  1955. trace_fastrpc_context_complete(ctx->fl->cid, (uint64_t)ctx, retval,
  1956. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  1957. switch (rsp_flags) {
  1958. case NORMAL_RESPONSE:
  1959. fallthrough;
  1960. case COMPLETE_SIGNAL:
  1961. /* normal and complete response with return value */
  1962. ctx->is_work_done = true;
  1963. if (ctx->asyncjob.isasyncjob)
  1964. fastrpc_queue_completed_async_job(ctx);
  1965. trace_fastrpc_msg("wakeup_task: begin");
  1966. complete(&ctx->work);
  1967. trace_fastrpc_msg("wakeup_task: end");
  1968. break;
  1969. case USER_EARLY_SIGNAL:
  1970. /* user hint of approximate time of completion */
  1971. ctx->early_wake_time = early_wake_time;
  1972. if (ctx->asyncjob.isasyncjob)
  1973. break;
  1974. fallthrough;
  1975. case EARLY_RESPONSE:
  1976. /* rpc framework early response with return value */
  1977. if (ctx->asyncjob.isasyncjob)
  1978. fastrpc_queue_completed_async_job(ctx);
  1979. else {
  1980. trace_fastrpc_msg("wakeup_task: begin");
  1981. complete(&ctx->work);
  1982. trace_fastrpc_msg("wakeup_task: end");
  1983. }
  1984. break;
  1985. default:
  1986. break;
  1987. }
  1988. }
  1989. static void fastrpc_notify_users(struct fastrpc_file *me)
  1990. {
  1991. struct smq_invoke_ctx *ictx;
  1992. struct hlist_node *n;
  1993. unsigned long irq_flags = 0;
  1994. spin_lock_irqsave(&me->hlock, irq_flags);
  1995. hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
  1996. ictx->is_work_done = true;
  1997. ictx->retval = -ECONNRESET;
  1998. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  1999. ictx->retval, ictx->msg.invoke.header.ctx,
  2000. ictx->handle, ictx->sc);
  2001. if (ictx->asyncjob.isasyncjob && ictx->is_job_sent_to_remote_ss)
  2002. fastrpc_queue_completed_async_job(ictx);
  2003. else
  2004. complete(&ictx->work);
  2005. }
  2006. hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
  2007. ictx->is_work_done = true;
  2008. ictx->retval = -ECONNRESET;
  2009. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2010. ictx->retval, ictx->msg.invoke.header.ctx,
  2011. ictx->handle, ictx->sc);
  2012. complete(&ictx->work);
  2013. }
  2014. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2015. }
  2016. static void fastrpc_notify_users_staticpd_pdr(struct fastrpc_file *me)
  2017. {
  2018. struct smq_invoke_ctx *ictx;
  2019. struct hlist_node *n;
  2020. unsigned long irq_flags = 0;
  2021. spin_lock_irqsave(&me->hlock, irq_flags);
  2022. hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
  2023. if (ictx->msg.pid) {
  2024. ictx->is_work_done = true;
  2025. ictx->retval = -ECONNRESET;
  2026. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2027. ictx->retval, ictx->msg.invoke.header.ctx,
  2028. ictx->handle, ictx->sc);
  2029. if (ictx->asyncjob.isasyncjob && ictx->is_job_sent_to_remote_ss)
  2030. fastrpc_queue_completed_async_job(ictx);
  2031. else
  2032. complete(&ictx->work);
  2033. }
  2034. }
  2035. hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
  2036. if (ictx->msg.pid) {
  2037. ictx->is_work_done = true;
  2038. ictx->retval = -ECONNRESET;
  2039. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2040. ictx->retval, ictx->msg.invoke.header.ctx,
  2041. ictx->handle, ictx->sc);
  2042. complete(&ictx->work);
  2043. }
  2044. }
  2045. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2046. }
  2047. static void fastrpc_update_ramdump_status(int cid)
  2048. {
  2049. struct fastrpc_file *fl = NULL;
  2050. struct hlist_node *n = NULL;
  2051. struct fastrpc_apps *me = &gfa;
  2052. struct fastrpc_channel_ctx *chan = &me->channel[cid];
  2053. unsigned long irq_flags = 0;
  2054. spin_lock_irqsave(&me->hlock, irq_flags);
  2055. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2056. if (fl->cid == cid && fl->init_mem &&
  2057. fl->file_close < FASTRPC_PROCESS_DSP_EXIT_COMPLETE &&
  2058. fl->dsp_proc_init) {
  2059. hlist_add_head(&fl->init_mem->hn_init, &chan->initmems);
  2060. fl->is_ramdump_pend = true;
  2061. }
  2062. }
  2063. if (chan->buf)
  2064. hlist_add_head(&chan->buf->hn_init, &chan->initmems);
  2065. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2066. }
  2067. static void fastrpc_ramdump_collection(int cid)
  2068. {
  2069. struct fastrpc_file *fl = NULL;
  2070. struct hlist_node *n = NULL;
  2071. struct fastrpc_apps *me = &gfa;
  2072. struct fastrpc_channel_ctx *chan = &me->channel[cid];
  2073. struct qcom_dump_segment ramdump_entry;
  2074. struct fastrpc_buf *buf = NULL;
  2075. int ret = 0;
  2076. unsigned long irq_flags = 0;
  2077. struct list_head head;
  2078. hlist_for_each_entry_safe(buf, n, &chan->initmems, hn_init) {
  2079. fl = buf->fl;
  2080. memset(&ramdump_entry, 0, sizeof(ramdump_entry));
  2081. ramdump_entry.da = buf->phys;
  2082. ramdump_entry.va = (void *)buf->virt;
  2083. ramdump_entry.size = buf->size;
  2084. INIT_LIST_HEAD(&head);
  2085. list_add(&ramdump_entry.node, &head);
  2086. if (fl) {
  2087. ret = fastrpc_file_get(fl);
  2088. if (ret) {
  2089. ADSPRPC_ERR("Failed to get user process reference\n");
  2090. continue;
  2091. }
  2092. if (fl && fl->sctx && fl->sctx->smmu.dev) {
  2093. ret = qcom_elf_dump(&head, fl->sctx->smmu.dev, ELF_CLASS);
  2094. } else {
  2095. if (me->dev != NULL)
  2096. ret = qcom_elf_dump(&head, me->dev, ELF_CLASS);
  2097. }
  2098. if (ret < 0)
  2099. ADSPRPC_ERR("adsprpc: %s: unable to dump PD memory (err %d)\n",
  2100. __func__, ret);
  2101. hlist_del_init(&buf->hn_init);
  2102. if (fl) {
  2103. spin_lock_irqsave(&me->hlock, irq_flags);
  2104. if (fl->file_close)
  2105. complete(&fl->work);
  2106. fl->is_ramdump_pend = false;
  2107. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2108. fastrpc_file_put(fl);
  2109. }
  2110. }
  2111. }
  2112. }
  2113. static void fastrpc_notify_drivers(struct fastrpc_apps *me, int cid)
  2114. {
  2115. struct fastrpc_file *fl;
  2116. struct hlist_node *n;
  2117. unsigned long irq_flags = 0;
  2118. spin_lock_irqsave(&me->hlock, irq_flags);
  2119. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2120. if (fl->cid == cid) {
  2121. fastrpc_queue_pd_status(fl, cid, FASTRPC_DSP_SSR, fl->sessionid);
  2122. fastrpc_notify_users(fl);
  2123. }
  2124. }
  2125. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2126. }
  2127. static void fastrpc_notify_pdr_drivers(struct fastrpc_apps *me,
  2128. char *servloc_name)
  2129. {
  2130. struct fastrpc_file *fl;
  2131. struct hlist_node *n;
  2132. unsigned long irq_flags = 0;
  2133. spin_lock_irqsave(&me->hlock, irq_flags);
  2134. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2135. if (fl->servloc_name && !strcmp(servloc_name, fl->servloc_name))
  2136. fastrpc_notify_users_staticpd_pdr(fl);
  2137. }
  2138. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2139. }
  2140. static void context_list_ctor(struct fastrpc_ctx_lst *me)
  2141. {
  2142. INIT_HLIST_HEAD(&me->interrupted);
  2143. INIT_HLIST_HEAD(&me->pending);
  2144. me->num_active_ctxs = 0;
  2145. INIT_HLIST_HEAD(&me->async_queue);
  2146. INIT_LIST_HEAD(&me->notif_queue);
  2147. }
  2148. static void fastrpc_context_list_dtor(struct fastrpc_file *fl)
  2149. {
  2150. struct fastrpc_ctx_lst *clst = &fl->clst;
  2151. struct smq_invoke_ctx *ictx = NULL, *ctxfree;
  2152. struct hlist_node *n;
  2153. unsigned long irq_flags = 0;
  2154. struct smq_notif_rsp *inotif = NULL, *n1 = NULL;
  2155. do {
  2156. ctxfree = NULL;
  2157. spin_lock(&fl->hlock);
  2158. hlist_for_each_entry_safe(ictx, n, &clst->interrupted, hn) {
  2159. hlist_del_init(&ictx->hn);
  2160. clst->num_active_ctxs--;
  2161. ctxfree = ictx;
  2162. break;
  2163. }
  2164. spin_unlock(&fl->hlock);
  2165. if (ctxfree)
  2166. context_free(ctxfree);
  2167. } while (ctxfree);
  2168. do {
  2169. ctxfree = NULL;
  2170. spin_lock(&fl->hlock);
  2171. hlist_for_each_entry_safe(ictx, n, &clst->pending, hn) {
  2172. hlist_del_init(&ictx->hn);
  2173. clst->num_active_ctxs--;
  2174. ctxfree = ictx;
  2175. break;
  2176. }
  2177. spin_unlock(&fl->hlock);
  2178. if (ctxfree)
  2179. context_free(ctxfree);
  2180. } while (ctxfree);
  2181. spin_lock_irqsave(&fl->proc_state_notif.nqlock, irq_flags);
  2182. list_for_each_entry_safe(inotif, n1, &clst->notif_queue, notifn) {
  2183. list_del_init(&inotif->notifn);
  2184. atomic_sub(1, &fl->proc_state_notif.notif_queue_count);
  2185. kfree(inotif);
  2186. }
  2187. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, irq_flags);
  2188. }
  2189. static void fastrpc_file_free(struct kref *ref);
  2190. static void fastrpc_file_list_dtor(struct fastrpc_apps *me)
  2191. {
  2192. struct fastrpc_file *fl, *free;
  2193. struct hlist_node *n;
  2194. unsigned long irq_flags = 0;
  2195. do {
  2196. free = NULL;
  2197. spin_lock_irqsave(&me->hlock, irq_flags);
  2198. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2199. hlist_del_init(&fl->hn);
  2200. free = fl;
  2201. break;
  2202. }
  2203. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2204. if (free)
  2205. fastrpc_file_put(free);
  2206. } while (free);
  2207. }
  2208. static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
  2209. {
  2210. remote_arg64_t *rpra, *lrpra;
  2211. remote_arg_t *lpra = ctx->lpra;
  2212. struct smq_invoke_buf *list;
  2213. struct smq_phy_page *pages, *ipage;
  2214. uint32_t sc = ctx->sc;
  2215. int inbufs = REMOTE_SCALARS_INBUFS(sc);
  2216. int outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2217. int handles, bufs = inbufs + outbufs;
  2218. uintptr_t args = 0;
  2219. size_t rlen = 0, copylen = 0, metalen = 0, lrpralen = 0, templen = 0;
  2220. size_t totallen = 0; //header and non ion copy buf len
  2221. int i, oix;
  2222. int err = 0, j = 0;
  2223. int mflags = 0;
  2224. uint64_t *fdlist = NULL;
  2225. uint32_t *crclist = NULL;
  2226. uint32_t early_hint;
  2227. uint64_t *perf_counter = NULL;
  2228. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  2229. if (ctx->fl->profile)
  2230. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  2231. /* calculate size of the metadata */
  2232. rpra = NULL;
  2233. lrpra = NULL;
  2234. list = smq_invoke_buf_start(rpra, sc);
  2235. pages = smq_phy_page_start(sc, list);
  2236. ipage = pages;
  2237. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_MAP),
  2238. for (i = 0; i < bufs; ++i) {
  2239. uintptr_t buf = (uintptr_t)lpra[i].buf.pv;
  2240. size_t len = lpra[i].buf.len;
  2241. mutex_lock(&ctx->fl->map_mutex);
  2242. if (ctx->fds && (ctx->fds[i] != -1))
  2243. err = fastrpc_mmap_create(ctx->fl, ctx->fds[i], NULL,
  2244. ctx->attrs[i], buf, len,
  2245. mflags, &ctx->maps[i]);
  2246. /*
  2247. * Increment ctx refs count for in/out buffer if map created,
  2248. * indicate map under use in remote call
  2249. */
  2250. if (ctx->maps[i])
  2251. ctx->maps[i]->ctx_refs++;
  2252. mutex_unlock(&ctx->fl->map_mutex);
  2253. if (err)
  2254. goto bail;
  2255. ipage += 1;
  2256. }
  2257. PERF_END);
  2258. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  2259. mutex_lock(&ctx->fl->map_mutex);
  2260. for (i = bufs; i < bufs + handles; i++) {
  2261. int dmaflags = 0;
  2262. if (ctx->attrs && (ctx->attrs[i] & FASTRPC_ATTR_NOMAP))
  2263. dmaflags = FASTRPC_MAP_FD_NOMAP;
  2264. VERIFY(err, VALID_FASTRPC_CID(ctx->fl->cid));
  2265. if (err) {
  2266. err = -ECHRNG;
  2267. mutex_unlock(&ctx->fl->map_mutex);
  2268. goto bail;
  2269. }
  2270. dsp_cap_ptr = &gcinfo[ctx->fl->cid].dsp_cap_kernel;
  2271. // Skip cpu mapping if DMA_HANDLE_REVERSE_RPC_CAP is true.
  2272. if (!dsp_cap_ptr->dsp_attributes[DMA_HANDLE_REVERSE_RPC_CAP] &&
  2273. ctx->fds && (ctx->fds[i] != -1))
  2274. err = fastrpc_mmap_create(ctx->fl, ctx->fds[i], NULL,
  2275. FASTRPC_ATTR_NOVA, 0, 0, dmaflags,
  2276. &ctx->maps[i]);
  2277. if (err) {
  2278. for (j = bufs; j < i; j++) {
  2279. /*
  2280. * Due to error decrement ctx refs count before mmap free
  2281. * for each in/out handle, if map created
  2282. */
  2283. if (ctx->maps[j] && ctx->maps[j]->ctx_refs)
  2284. ctx->maps[j]->ctx_refs--;
  2285. fastrpc_mmap_free(ctx->maps[j], 0);
  2286. }
  2287. mutex_unlock(&ctx->fl->map_mutex);
  2288. goto bail;
  2289. } else if (ctx->maps[i]) {
  2290. /*
  2291. * Increment ctx refs count for in/out handle if map created
  2292. * and no error, indicate map under use in remote call
  2293. */
  2294. ctx->maps[i]->ctx_refs++;
  2295. }
  2296. ipage += 1;
  2297. }
  2298. mutex_unlock(&ctx->fl->map_mutex);
  2299. /* metalen includes meta data, fds, crc, dsp perf and early wakeup hint */
  2300. metalen = totallen = (size_t)&ipage[0] + (sizeof(uint64_t) * M_FDLIST) +
  2301. (sizeof(uint32_t) * M_CRCLIST) + (sizeof(uint64_t) * M_DSP_PERF_LIST) +
  2302. sizeof(early_hint);
  2303. if (metalen) {
  2304. err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0,
  2305. METADATA_BUF, &ctx->buf);
  2306. if (err)
  2307. goto bail;
  2308. VERIFY(err, !IS_ERR_OR_NULL(ctx->buf->virt));
  2309. if (err)
  2310. goto bail;
  2311. memset(ctx->buf->virt, 0, ctx->buf->size);
  2312. }
  2313. ctx->used = metalen;
  2314. /* allocate new local rpra buffer */
  2315. lrpralen = (size_t)&list[0];
  2316. if (lrpralen) {
  2317. lrpra = kzalloc(lrpralen, GFP_KERNEL);
  2318. VERIFY(err, !IS_ERR_OR_NULL(lrpra));
  2319. if (err) {
  2320. err = -ENOMEM;
  2321. goto bail;
  2322. }
  2323. }
  2324. ctx->lrpra = lrpra;
  2325. /* calculate len required for copying */
  2326. for (oix = 0; oix < inbufs + outbufs; ++oix) {
  2327. int i = ctx->overps[oix]->raix;
  2328. uintptr_t mstart, mend;
  2329. size_t len = lpra[i].buf.len;
  2330. if (!len)
  2331. continue;
  2332. if (ctx->maps[i])
  2333. continue;
  2334. if (ctx->overps[oix]->offset == 0)
  2335. copylen = ALIGN(copylen, BALIGN);
  2336. mstart = ctx->overps[oix]->mstart;
  2337. mend = ctx->overps[oix]->mend;
  2338. templen = mend - mstart;
  2339. VERIFY(err, ((templen <= LONG_MAX) && (copylen <= (LONG_MAX - templen))));
  2340. if (err) {
  2341. err = -EFAULT;
  2342. goto bail;
  2343. }
  2344. if (templen > DEBUG_PRINT_SIZE_LIMIT)
  2345. ADSPRPC_WARN(
  2346. "user passed non ion buffer size %zu, mend 0x%llx mstart 0x%llx, sc 0x%x handle 0x%x\n",
  2347. templen, mend, mstart, sc, ctx->handle);
  2348. copylen += templen;
  2349. }
  2350. totallen = ALIGN(totallen, BALIGN) + copylen;
  2351. /* allocate non -ion copy buffer */
  2352. /* Checking if copylen can be accomodated in metalen*/
  2353. /*if not allocating new buffer */
  2354. if (totallen <= (size_t)buf_page_size(metalen)) {
  2355. args = (uintptr_t)ctx->buf->virt + metalen;
  2356. ctx->copybuf = ctx->buf;
  2357. rlen = totallen - metalen;
  2358. } else if (copylen) {
  2359. err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, COPYDATA_BUF,
  2360. &ctx->copybuf);
  2361. if (err)
  2362. goto bail;
  2363. memset(ctx->copybuf->virt, 0, copylen);
  2364. args = (uintptr_t)ctx->copybuf->virt;
  2365. rlen = copylen;
  2366. totallen = copylen;
  2367. }
  2368. /* copy metadata */
  2369. rpra = ctx->buf->virt;
  2370. ctx->rpra = rpra;
  2371. list = smq_invoke_buf_start(rpra, sc);
  2372. pages = smq_phy_page_start(sc, list);
  2373. ipage = pages;
  2374. for (i = 0; i < bufs + handles; ++i) {
  2375. if (lpra[i].buf.len)
  2376. list[i].num = 1;
  2377. else
  2378. list[i].num = 0;
  2379. list[i].pgidx = ipage - pages;
  2380. ipage++;
  2381. }
  2382. /* map ion buffers */
  2383. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_MAP),
  2384. for (i = 0; rpra && i < inbufs + outbufs; ++i) {
  2385. struct fastrpc_mmap *map = ctx->maps[i];
  2386. uint64_t buf = ptr_to_uint64(lpra[i].buf.pv);
  2387. size_t len = lpra[i].buf.len;
  2388. uint64_t buf_start = 0;
  2389. rpra[i].buf.pv = 0;
  2390. rpra[i].buf.len = len;
  2391. if (!len)
  2392. continue;
  2393. if (map) {
  2394. struct vm_area_struct *vma;
  2395. uintptr_t offset;
  2396. uint64_t num = buf_num_pages(buf, len);
  2397. int idx = list[i].pgidx;
  2398. if (map->attr & FASTRPC_ATTR_NOVA) {
  2399. offset = 0;
  2400. } else {
  2401. down_read(&current->mm->mmap_lock);
  2402. VERIFY(err, NULL != (vma = find_vma(current->mm,
  2403. map->va)));
  2404. if (err) {
  2405. up_read(&current->mm->mmap_lock);
  2406. goto bail;
  2407. }
  2408. buf_start = buf_page_start(buf);
  2409. VERIFY(err, vma->vm_start <= buf_start);
  2410. if (err) {
  2411. up_read(&current->mm->mmap_lock);
  2412. ADSPRPC_ERR(
  2413. "buffer VA invalid for fd %d, IPA 0x%llx, VA 0x%llx, vma start 0x%llx\n",
  2414. map->fd, map->phys, map->va, vma->vm_start);
  2415. err = -EFAULT;
  2416. goto bail;
  2417. }
  2418. offset = buf_start - vma->vm_start;
  2419. up_read(&current->mm->mmap_lock);
  2420. VERIFY(err, offset + len <= (uintptr_t)map->size);
  2421. if (err) {
  2422. ADSPRPC_ERR(
  2423. "buffer address is invalid for the fd passed for %d address 0x%llx and size %zu\n",
  2424. i, (uintptr_t)lpra[i].buf.pv,
  2425. lpra[i].buf.len);
  2426. err = -EFAULT;
  2427. goto bail;
  2428. }
  2429. }
  2430. pages[idx].addr = map->phys + offset;
  2431. pages[idx].size = num << PAGE_SHIFT;
  2432. }
  2433. rpra[i].buf.pv = buf;
  2434. }
  2435. PERF_END);
  2436. for (i = bufs; i < bufs + handles; ++i) {
  2437. struct fastrpc_mmap *map = ctx->maps[i];
  2438. if (map) {
  2439. pages[i].addr = map->phys;
  2440. pages[i].size = map->size;
  2441. }
  2442. }
  2443. fdlist = (uint64_t *)&pages[bufs + handles];
  2444. crclist = (uint32_t *)&fdlist[M_FDLIST];
  2445. /* reset fds, crc and early wakeup hint memory */
  2446. /* remote process updates these values before responding */
  2447. memset(fdlist, 0, sizeof(uint64_t)*M_FDLIST + sizeof(uint32_t)*M_CRCLIST +
  2448. (sizeof(uint64_t) * M_DSP_PERF_LIST) + sizeof(early_hint));
  2449. /* copy non ion buffers */
  2450. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_COPY),
  2451. for (oix = 0; rpra && oix < inbufs + outbufs; ++oix) {
  2452. int i = ctx->overps[oix]->raix;
  2453. struct fastrpc_mmap *map = ctx->maps[i];
  2454. size_t mlen;
  2455. uint64_t buf;
  2456. size_t len = lpra[i].buf.len;
  2457. if (!len)
  2458. continue;
  2459. if (map)
  2460. continue;
  2461. if (ctx->overps[oix]->offset == 0) {
  2462. rlen -= ALIGN(args, BALIGN) - args;
  2463. args = ALIGN(args, BALIGN);
  2464. }
  2465. mlen = ctx->overps[oix]->mend - ctx->overps[oix]->mstart;
  2466. VERIFY(err, rlen >= mlen);
  2467. if (err) {
  2468. err = -EFAULT;
  2469. goto bail;
  2470. }
  2471. rpra[i].buf.pv =
  2472. (args - ctx->overps[oix]->offset);
  2473. pages[list[i].pgidx].addr = ctx->copybuf->phys -
  2474. ctx->overps[oix]->offset +
  2475. (totallen - rlen);
  2476. pages[list[i].pgidx].addr =
  2477. buf_page_start(pages[list[i].pgidx].addr);
  2478. buf = rpra[i].buf.pv;
  2479. pages[list[i].pgidx].size = buf_num_pages(buf, len) * PAGE_SIZE;
  2480. if (i < inbufs) {
  2481. K_COPY_FROM_USER(err, kernel, uint64_to_ptr(buf),
  2482. lpra[i].buf.pv, len);
  2483. if (err) {
  2484. ADSPRPC_ERR(
  2485. "copy from user failed with %d for dst 0x%llx, src %pK, size 0x%zx, arg %d\n",
  2486. err, buf, lpra[i].buf.pv, len, i+1);
  2487. err = -EFAULT;
  2488. goto bail;
  2489. }
  2490. }
  2491. if (len > DEBUG_PRINT_SIZE_LIMIT)
  2492. ADSPRPC_DEBUG(
  2493. "copied non ion buffer sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu\n",
  2494. sc, rpra[i].buf.pv,
  2495. ctx->overps[oix]->mend,
  2496. ctx->overps[oix]->mstart, len);
  2497. args = args + mlen;
  2498. rlen -= mlen;
  2499. }
  2500. PERF_END);
  2501. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_FLUSH),
  2502. for (oix = 0; oix < inbufs + outbufs; ++oix) {
  2503. int i = ctx->overps[oix]->raix;
  2504. struct fastrpc_mmap *map = ctx->maps[i];
  2505. if (i+1 > inbufs) // Avoiding flush for outbufs
  2506. continue;
  2507. if (ctx->fl->sctx && ctx->fl->sctx->smmu.coherent)
  2508. continue;
  2509. if (map && (map->attr & FASTRPC_ATTR_FORCE_NOFLUSH))
  2510. continue;
  2511. if (rpra && rpra[i].buf.len && (ctx->overps[oix]->mstart ||
  2512. ctx->overps[oix]->do_cmo == 1)) {
  2513. if (map && map->buf) {
  2514. if (((buf_page_size(ctx->overps[oix]->mend -
  2515. ctx->overps[oix]->mstart)) == map->size) ||
  2516. ctx->overps[oix]->do_cmo) {
  2517. dma_buf_begin_cpu_access(map->buf,
  2518. DMA_TO_DEVICE);
  2519. dma_buf_end_cpu_access(map->buf,
  2520. DMA_TO_DEVICE);
  2521. ADSPRPC_DEBUG(
  2522. "sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2523. sc, rpra[i].buf.pv,
  2524. ctx->overps[oix]->mend,
  2525. ctx->overps[oix]->mstart,
  2526. rpra[i].buf.len, map->size);
  2527. } else {
  2528. uintptr_t offset;
  2529. uint64_t flush_len;
  2530. struct vm_area_struct *vma;
  2531. down_read(&current->mm->mmap_lock);
  2532. VERIFY(err, NULL != (vma = find_vma(
  2533. current->mm, rpra[i].buf.pv)));
  2534. if (err) {
  2535. up_read(&current->mm->mmap_lock);
  2536. goto bail;
  2537. }
  2538. if (ctx->overps[oix]->do_cmo) {
  2539. offset = rpra[i].buf.pv -
  2540. vma->vm_start;
  2541. flush_len = rpra[i].buf.len;
  2542. } else {
  2543. offset =
  2544. ctx->overps[oix]->mstart
  2545. - vma->vm_start;
  2546. flush_len =
  2547. ctx->overps[oix]->mend -
  2548. ctx->overps[oix]->mstart;
  2549. }
  2550. up_read(&current->mm->mmap_lock);
  2551. dma_buf_begin_cpu_access_partial(
  2552. map->buf, DMA_TO_DEVICE, offset,
  2553. flush_len);
  2554. dma_buf_end_cpu_access_partial(
  2555. map->buf, DMA_TO_DEVICE, offset,
  2556. flush_len);
  2557. ADSPRPC_DEBUG(
  2558. "sc 0x%x vm_start 0x%llx pv 0x%llx, offset 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2559. sc, vma->vm_start,
  2560. rpra[i].buf.pv, offset,
  2561. ctx->overps[oix]->mend,
  2562. ctx->overps[oix]->mstart,
  2563. rpra[i].buf.len, map->size);
  2564. }
  2565. }
  2566. }
  2567. }
  2568. PERF_END);
  2569. for (i = bufs; ctx->fds && rpra && i < bufs + handles; i++) {
  2570. rpra[i].dma.fd = ctx->fds[i];
  2571. rpra[i].dma.len = (uint32_t)lpra[i].buf.len;
  2572. rpra[i].dma.offset =
  2573. (uint32_t)(uintptr_t)lpra[i].buf.pv;
  2574. }
  2575. /* Copy rpra to local buffer */
  2576. if (ctx->lrpra && rpra && lrpralen > 0)
  2577. memcpy(ctx->lrpra, rpra, lrpralen);
  2578. bail:
  2579. return err;
  2580. }
  2581. static int put_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
  2582. remote_arg_t *upra)
  2583. {
  2584. uint32_t sc = ctx->sc;
  2585. struct smq_invoke_buf *list;
  2586. struct smq_phy_page *pages;
  2587. struct fastrpc_mmap *mmap;
  2588. uint64_t *fdlist;
  2589. uint32_t *crclist = NULL, *poll = NULL;
  2590. uint64_t *perf_dsp_list = NULL;
  2591. remote_arg64_t *rpra = ctx->lrpra;
  2592. int i, inbufs, outbufs, handles;
  2593. int err = 0, perfErr = 0;
  2594. inbufs = REMOTE_SCALARS_INBUFS(sc);
  2595. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2596. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  2597. list = smq_invoke_buf_start(ctx->rpra, sc);
  2598. pages = smq_phy_page_start(sc, list);
  2599. fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
  2600. crclist = (uint32_t *)(fdlist + M_FDLIST);
  2601. poll = (uint32_t *)(crclist + M_CRCLIST);
  2602. perf_dsp_list = (uint64_t *)(poll + 1);
  2603. for (i = inbufs; i < inbufs + outbufs; ++i) {
  2604. if (!ctx->maps[i]) {
  2605. K_COPY_TO_USER(err, kernel,
  2606. ctx->lpra[i].buf.pv,
  2607. uint64_to_ptr(rpra[i].buf.pv),
  2608. rpra[i].buf.len);
  2609. if (err) {
  2610. ADSPRPC_ERR(
  2611. "Invalid size 0x%llx for output argument %d ret %ld\n",
  2612. rpra[i].buf.len, i+1, err);
  2613. err = -EFAULT;
  2614. goto bail;
  2615. }
  2616. } else {
  2617. mutex_lock(&ctx->fl->map_mutex);
  2618. /*
  2619. * Decrement ctx refs count before mmap free,
  2620. * indicate remote call no longer using it
  2621. */
  2622. if (ctx->maps[i]->ctx_refs)
  2623. ctx->maps[i]->ctx_refs--;
  2624. fastrpc_mmap_free(ctx->maps[i], 0);
  2625. mutex_unlock(&ctx->fl->map_mutex);
  2626. ctx->maps[i] = NULL;
  2627. }
  2628. }
  2629. mutex_lock(&ctx->fl->map_mutex);
  2630. for (i = 0; i < M_FDLIST; i++) {
  2631. if (!fdlist[i])
  2632. break;
  2633. if (!fastrpc_mmap_find(ctx->fl, (int)fdlist[i], NULL, 0, 0,
  2634. 0, 0, &mmap)) {
  2635. /*
  2636. * Decrement ctx refs count before mmap free,
  2637. * indicate remote call no longer using it
  2638. */
  2639. if (mmap && mmap->ctx_refs)
  2640. mmap->ctx_refs--;
  2641. fastrpc_mmap_free(mmap, 0);
  2642. }
  2643. }
  2644. mutex_unlock(&ctx->fl->map_mutex);
  2645. if (ctx->crc && crclist && rpra)
  2646. K_COPY_TO_USER(err, kernel, ctx->crc,
  2647. crclist, M_CRCLIST*sizeof(uint32_t));
  2648. if (ctx->perf_dsp && perf_dsp_list) {
  2649. K_COPY_TO_USER(perfErr, kernel, ctx->perf_dsp,
  2650. perf_dsp_list, M_DSP_PERF_LIST*sizeof(uint64_t));
  2651. if (perfErr)
  2652. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  2653. }
  2654. bail:
  2655. return err;
  2656. }
  2657. static void inv_args(struct smq_invoke_ctx *ctx)
  2658. {
  2659. int i, inbufs, outbufs;
  2660. uint32_t sc = ctx->sc;
  2661. remote_arg64_t *rpra = ctx->lrpra;
  2662. int err = 0;
  2663. inbufs = REMOTE_SCALARS_INBUFS(sc);
  2664. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2665. for (i = 0; i < inbufs + outbufs; ++i) {
  2666. int over = ctx->overps[i]->raix;
  2667. struct fastrpc_mmap *map = ctx->maps[over];
  2668. if ((over + 1 <= inbufs))
  2669. continue;
  2670. if (!rpra[over].buf.len)
  2671. continue;
  2672. if (ctx->fl && ctx->fl->sctx && ctx->fl->sctx->smmu.coherent)
  2673. continue;
  2674. if (map && (map->attr & FASTRPC_ATTR_FORCE_NOINVALIDATE))
  2675. continue;
  2676. if (buf_page_start(ptr_to_uint64((void *)rpra)) ==
  2677. buf_page_start(rpra[over].buf.pv)) {
  2678. continue;
  2679. }
  2680. if (ctx->overps[i]->mstart || ctx->overps[i]->do_cmo == 1) {
  2681. if (map && map->buf) {
  2682. if (((buf_page_size(ctx->overps[i]->mend -
  2683. ctx->overps[i]->mstart)) == map->size) ||
  2684. ctx->overps[i]->do_cmo) {
  2685. dma_buf_begin_cpu_access(map->buf,
  2686. DMA_FROM_DEVICE);
  2687. dma_buf_end_cpu_access(map->buf,
  2688. DMA_TO_DEVICE);
  2689. ADSPRPC_DEBUG(
  2690. "sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2691. sc, rpra[over].buf.pv,
  2692. ctx->overps[i]->mend,
  2693. ctx->overps[i]->mstart,
  2694. rpra[over].buf.len, map->size);
  2695. } else {
  2696. uintptr_t offset;
  2697. uint64_t inv_len;
  2698. struct vm_area_struct *vma;
  2699. down_read(&current->mm->mmap_lock);
  2700. VERIFY(err, NULL != (vma = find_vma(
  2701. current->mm,
  2702. rpra[over].buf.pv)));
  2703. if (err) {
  2704. up_read(&current->mm->mmap_lock);
  2705. goto bail;
  2706. }
  2707. if (ctx->overps[i]->do_cmo) {
  2708. offset = rpra[over].buf.pv -
  2709. vma->vm_start;
  2710. inv_len = rpra[over].buf.len;
  2711. } else {
  2712. offset =
  2713. ctx->overps[i]->mstart -
  2714. vma->vm_start;
  2715. inv_len =
  2716. ctx->overps[i]->mend -
  2717. ctx->overps[i]->mstart;
  2718. }
  2719. up_read(&current->mm->mmap_lock);
  2720. dma_buf_begin_cpu_access_partial(
  2721. map->buf, DMA_FROM_DEVICE, offset,
  2722. inv_len);
  2723. dma_buf_end_cpu_access_partial(map->buf,
  2724. DMA_TO_DEVICE, offset,
  2725. inv_len);
  2726. ADSPRPC_DEBUG(
  2727. "sc 0x%x vm_start 0x%llx pv 0x%llx, offset 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2728. sc, vma->vm_start,
  2729. rpra[over].buf.pv,
  2730. offset, ctx->overps[i]->mend,
  2731. ctx->overps[i]->mstart,
  2732. rpra[over].buf.len, map->size);
  2733. }
  2734. }
  2735. }
  2736. }
  2737. bail:
  2738. return;
  2739. }
  2740. static int fastrpc_invoke_send(struct smq_invoke_ctx *ctx,
  2741. uint32_t kernel, uint32_t handle)
  2742. {
  2743. struct smq_msg *msg = &ctx->msg;
  2744. struct smq_msg msg_temp;
  2745. struct smq_invoke_ctx ctx_temp;
  2746. struct fastrpc_file *fl = ctx->fl;
  2747. struct fastrpc_channel_ctx *channel_ctx = NULL;
  2748. int err = 0, cid = -1;
  2749. uint32_t sc = ctx->sc;
  2750. int64_t ns = 0;
  2751. uint64_t xo_time_in_us = 0;
  2752. int isasync = (ctx->asyncjob.isasyncjob ? true : false);
  2753. unsigned long irq_flags = 0;
  2754. uint32_t index = 0;
  2755. if (!fl) {
  2756. err = -EBADF;
  2757. goto bail;
  2758. }
  2759. cid = fl->cid;
  2760. VERIFY(err, VALID_FASTRPC_CID(cid));
  2761. if (err) {
  2762. err = -ECHRNG;
  2763. goto bail;
  2764. }
  2765. channel_ctx = &fl->apps->channel[cid];
  2766. mutex_lock(&channel_ctx->smd_mutex);
  2767. /* Send unique fastrpc process ID to dsp */
  2768. msg->pid = fl->tgid_frpc;
  2769. msg->tid = current->pid;
  2770. if (kernel == KERNEL_MSG_WITH_ZERO_PID)
  2771. msg->pid = 0;
  2772. msg->invoke.header.ctx = ctx->ctxid | fl->pd;
  2773. msg->invoke.header.handle = handle;
  2774. msg->invoke.header.sc = sc;
  2775. msg->invoke.page.addr = ctx->buf ? ctx->buf->phys : 0;
  2776. msg->invoke.page.size = buf_page_size(ctx->used);
  2777. if (fl->ssrcount != channel_ctx->ssrcount) {
  2778. err = -ECONNRESET;
  2779. mutex_unlock(&channel_ctx->smd_mutex);
  2780. goto bail;
  2781. }
  2782. mutex_unlock(&channel_ctx->smd_mutex);
  2783. xo_time_in_us = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  2784. if (isasync) {
  2785. /*
  2786. * After message is sent to DSP, async response thread could immediately
  2787. * get the response and free context, which will result in a use-after-free
  2788. * in this function. So use a local variable for message and context.
  2789. */
  2790. memcpy(&msg_temp, msg, sizeof(struct smq_msg));
  2791. msg = &msg_temp;
  2792. memcpy(&ctx_temp, ctx, sizeof(struct smq_invoke_ctx));
  2793. index = (uint32_t)GET_TABLE_IDX_FROM_CTXID(ctx->ctxid);
  2794. }
  2795. err = fastrpc_transport_send(cid, (void *)msg, sizeof(*msg), fl->tvm_remote_domain);
  2796. if (isasync) {
  2797. if (!err) {
  2798. /*
  2799. * Validate the ctx as this could have been already
  2800. * freed by async response.
  2801. */
  2802. spin_lock_irqsave(&channel_ctx->ctxlock, irq_flags);
  2803. if (index < FASTRPC_CTX_MAX && channel_ctx->ctxtable[index] == ctx)
  2804. ctx->is_job_sent_to_remote_ss = true;
  2805. spin_unlock_irqrestore(&channel_ctx->ctxlock, irq_flags);
  2806. }
  2807. ctx = &ctx_temp;
  2808. }
  2809. trace_fastrpc_transport_send(cid, (uint64_t)ctx, msg->invoke.header.ctx,
  2810. handle, sc, msg->invoke.page.addr, msg->invoke.page.size);
  2811. ns = get_timestamp_in_ns();
  2812. fastrpc_update_txmsg_buf(msg, err, ns, xo_time_in_us, ctx, DEFAULT_STATE);
  2813. bail:
  2814. return err;
  2815. }
  2816. /* fastrpc_get_nsp_status() - Reads the property string from soc_info
  2817. * denoted for nsp part, and updates the nsp device avialbility status
  2818. * if the nsp is not defective.
  2819. * @me : pointer to fastrpc_apps.
  2820. */
  2821. static void fastrpc_get_nsp_status(struct fastrpc_apps *me)
  2822. {
  2823. if (socinfo_get_part_info(PART_NSP)) {
  2824. me->fastrpc_nsp_status = 0;
  2825. ADSPRPC_ERR(
  2826. "nsp part defective with status:%x\n", me->fastrpc_nsp_status);
  2827. } else {
  2828. me->fastrpc_nsp_status = 1;
  2829. ADSPRPC_INFO("nsp available with status: %x\n", me->fastrpc_nsp_status);
  2830. }
  2831. }
  2832. /*
  2833. * Counts number of cores corresponding
  2834. * to cluster id 0. If a core is defective or unavailable, skip counting
  2835. * that core.
  2836. * @me : pointer to fastrpc_apps.
  2837. */
  2838. static void fastrpc_lowest_capacity_corecount(struct fastrpc_apps *me)
  2839. {
  2840. unsigned int cpu = 0;
  2841. cpu = cpumask_first(cpu_possible_mask);
  2842. for_each_cpu(cpu, cpu_possible_mask) {
  2843. if (topology_cluster_id(cpu) == 0)
  2844. me->lowest_capacity_core_count++;
  2845. }
  2846. ADSPRPC_INFO("lowest capacity core count: %u\n",
  2847. me->lowest_capacity_core_count);
  2848. }
  2849. static void fastrpc_init(struct fastrpc_apps *me)
  2850. {
  2851. int i, jj;
  2852. INIT_HLIST_HEAD(&me->drivers);
  2853. INIT_HLIST_HEAD(&me->maps);
  2854. spin_lock_init(&me->hlock);
  2855. me->channel = &gcinfo[0];
  2856. mutex_init(&me->mut_uid);
  2857. me->max_sess_per_proc = DEFAULT_MAX_SESS_PER_PROC;
  2858. for (i = 0; i < NUM_CHANNELS; i++) {
  2859. init_completion(&me->channel[i].work);
  2860. init_completion(&me->channel[i].workport);
  2861. me->channel[i].sesscount = 0;
  2862. /* All channels are secure by default except CDSP */
  2863. me->channel[i].secure = SECURE_CHANNEL;
  2864. me->channel[i].unsigned_support = false;
  2865. mutex_init(&me->channel[i].smd_mutex);
  2866. fastrpc_transport_session_init(i, me->channel[i].subsys);
  2867. spin_lock_init(&me->channel[i].ctxlock);
  2868. spin_lock_init(&me->channel[i].gmsg_log.lock);
  2869. INIT_HLIST_HEAD(&me->channel[i].initmems);
  2870. for (jj = 0; jj < NUM_SESSIONS; jj++)
  2871. init_waitqueue_head(&me->channel[i].spd[jj].wait_for_pdup);
  2872. }
  2873. /* Set CDSP channel to non secure */
  2874. me->channel[CDSP_DOMAIN_ID].secure = NON_SECURE_CHANNEL;
  2875. me->channel[CDSP_DOMAIN_ID].unsigned_support = true;
  2876. }
  2877. static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type)
  2878. {
  2879. struct fastrpc_apps *me = &gfa;
  2880. struct wakeup_source *wake_source = NULL;
  2881. if (!fl->wake_enable)
  2882. return;
  2883. /*
  2884. * Vote with PM to abort any suspend in progress and
  2885. * keep system awake for specified timeout
  2886. */
  2887. if (channel_type == SECURE_CHANNEL)
  2888. wake_source = me->wake_source_secure;
  2889. else if (channel_type == NON_SECURE_CHANNEL)
  2890. wake_source = me->wake_source;
  2891. if (wake_source)
  2892. pm_wakeup_ws_event(wake_source, fl->ws_timeout, true);
  2893. }
  2894. static inline void fastrpc_pm_relax(struct fastrpc_file *fl, int channel_type)
  2895. {
  2896. struct fastrpc_apps *me = &gfa;
  2897. struct wakeup_source *wake_source = NULL;
  2898. if (!fl->wake_enable)
  2899. return;
  2900. if (channel_type == SECURE_CHANNEL)
  2901. wake_source = me->wake_source_secure;
  2902. else if (channel_type == NON_SECURE_CHANNEL)
  2903. wake_source = me->wake_source;
  2904. ADSPRPC_INFO("done for tgid %d\n", fl->tgid);
  2905. if (wake_source)
  2906. __pm_relax(wake_source);
  2907. }
  2908. static inline int fastrpc_wait_for_response(struct smq_invoke_ctx *ctx,
  2909. uint32_t kernel)
  2910. {
  2911. int interrupted = 0;
  2912. if (kernel)
  2913. wait_for_completion(&ctx->work);
  2914. else
  2915. interrupted = wait_for_completion_interruptible(&ctx->work);
  2916. return interrupted;
  2917. }
  2918. static void fastrpc_wait_for_completion(struct smq_invoke_ctx *ctx,
  2919. int *ptr_interrupted, uint32_t kernel, uint32_t async,
  2920. bool *ptr_isworkdone)
  2921. {
  2922. int interrupted = 0, err = 0;
  2923. int jj;
  2924. bool wait_resp;
  2925. uint32_t wTimeout = FASTRPC_USER_EARLY_HINT_TIMEOUT;
  2926. uint32_t wakeTime = 0;
  2927. unsigned long flags;
  2928. if (!ctx) {
  2929. /* This failure is not expected */
  2930. err = *ptr_interrupted = EFAULT;
  2931. *ptr_isworkdone = false;
  2932. ADSPRPC_ERR("ctx is NULL, cannot wait for response err %d\n",
  2933. err);
  2934. return;
  2935. }
  2936. wakeTime = ctx->early_wake_time;
  2937. do {
  2938. switch (ctx->rsp_flags) {
  2939. /* try polling on completion with timeout */
  2940. case USER_EARLY_SIGNAL:
  2941. /* try wait if completion time is less than timeout */
  2942. /* disable preempt to avoid context switch latency */
  2943. preempt_disable();
  2944. jj = 0;
  2945. wait_resp = false;
  2946. for (; wakeTime < wTimeout && jj < wTimeout; jj++) {
  2947. wait_resp = try_wait_for_completion(&ctx->work);
  2948. if (wait_resp)
  2949. break;
  2950. udelay(1);
  2951. }
  2952. preempt_enable();
  2953. if (async) {
  2954. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2955. if (!ctx->is_work_done) {
  2956. ctx->is_early_wakeup = false;
  2957. *ptr_isworkdone = false;
  2958. } else
  2959. *ptr_isworkdone = true;
  2960. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2961. goto bail;
  2962. } else if (!wait_resp) {
  2963. interrupted = fastrpc_wait_for_response(ctx,
  2964. kernel);
  2965. *ptr_interrupted = interrupted;
  2966. if (interrupted || ctx->is_work_done)
  2967. goto bail;
  2968. }
  2969. break;
  2970. /* busy poll on memory for actual job done */
  2971. case EARLY_RESPONSE:
  2972. trace_fastrpc_msg("early_response: poll_begin");
  2973. err = poll_for_remote_response(ctx, FASTRPC_POLL_TIME);
  2974. /* Mark job done if poll on memory successful */
  2975. /* Wait for completion if poll on memory timoeut */
  2976. if (!err) {
  2977. ctx->is_work_done = true;
  2978. *ptr_isworkdone = true;
  2979. goto bail;
  2980. }
  2981. trace_fastrpc_msg("early_response: poll_timeout");
  2982. ADSPRPC_INFO("early rsp poll timeout (%u us) for handle 0x%x, sc 0x%x\n",
  2983. FASTRPC_POLL_TIME, ctx->handle, ctx->sc);
  2984. if (async) {
  2985. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2986. if (!ctx->is_work_done) {
  2987. ctx->is_early_wakeup = false;
  2988. *ptr_isworkdone = false;
  2989. } else
  2990. *ptr_isworkdone = true;
  2991. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2992. goto bail;
  2993. } else if (!ctx->is_work_done) {
  2994. interrupted = fastrpc_wait_for_response(ctx,
  2995. kernel);
  2996. *ptr_interrupted = interrupted;
  2997. if (interrupted || ctx->is_work_done)
  2998. goto bail;
  2999. }
  3000. break;
  3001. case COMPLETE_SIGNAL:
  3002. case NORMAL_RESPONSE:
  3003. if (!async) {
  3004. interrupted = fastrpc_wait_for_response(ctx,
  3005. kernel);
  3006. *ptr_interrupted = interrupted;
  3007. if (interrupted || ctx->is_work_done)
  3008. goto bail;
  3009. } else {
  3010. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  3011. if (!ctx->is_work_done) {
  3012. ctx->is_early_wakeup = false;
  3013. *ptr_isworkdone = false;
  3014. } else
  3015. *ptr_isworkdone = true;
  3016. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  3017. goto bail;
  3018. }
  3019. break;
  3020. case POLL_MODE:
  3021. trace_fastrpc_msg("poll_mode: begin");
  3022. err = poll_for_remote_response(ctx, ctx->fl->poll_timeout);
  3023. /* If polling timed out, move to normal response state */
  3024. if (err) {
  3025. trace_fastrpc_msg("poll_mode: timeout");
  3026. ADSPRPC_INFO("poll mode timeout (%u us) for handle 0x%x, sc 0x%x\n",
  3027. ctx->fl->poll_timeout, ctx->handle, ctx->sc);
  3028. ctx->rsp_flags = NORMAL_RESPONSE;
  3029. } else {
  3030. *ptr_interrupted = 0;
  3031. *ptr_isworkdone = true;
  3032. }
  3033. break;
  3034. default:
  3035. *ptr_interrupted = EBADR;
  3036. *ptr_isworkdone = false;
  3037. ADSPRPC_ERR(
  3038. "unsupported response flags 0x%x for handle 0x%x, sc 0x%x\n",
  3039. ctx->rsp_flags, ctx->handle, ctx->sc);
  3040. goto bail;
  3041. } /* end of switch */
  3042. } while (!ctx->is_work_done);
  3043. bail:
  3044. return;
  3045. }
  3046. static void fastrpc_update_invoke_count(uint32_t handle, uint64_t *perf_counter,
  3047. struct timespec64 *invoket)
  3048. {
  3049. /* update invoke count for dynamic handles */
  3050. if (handle != FASTRPC_STATIC_HANDLE_LISTENER) {
  3051. uint64_t *count = GET_COUNTER(perf_counter, PERF_INVOKE);
  3052. if (count)
  3053. *count += getnstimediff(invoket);
  3054. }
  3055. if (handle > FASTRPC_STATIC_HANDLE_MAX) {
  3056. uint64_t *count = GET_COUNTER(perf_counter, PERF_COUNT);
  3057. if (count)
  3058. *count += 1;
  3059. }
  3060. }
  3061. static int fastrpc_check_pd_status(struct fastrpc_file *fl, char *sloc_name);
  3062. int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
  3063. uint32_t kernel,
  3064. struct fastrpc_ioctl_invoke_async *inv)
  3065. {
  3066. struct smq_invoke_ctx *ctx = NULL;
  3067. struct fastrpc_ioctl_invoke *invoke = &inv->inv;
  3068. int err = 0, interrupted = 0, cid = -1, perfErr = 0;
  3069. struct timespec64 invoket = {0};
  3070. uint64_t *perf_counter = NULL;
  3071. bool isasyncinvoke = false, isworkdone = false;
  3072. cid = fl->cid;
  3073. VERIFY(err, VALID_FASTRPC_CID(cid) &&
  3074. fl->sctx != NULL);
  3075. if (err) {
  3076. ADSPRPC_ERR("kernel session not initialized yet for %s\n",
  3077. current->comm);
  3078. err = -EBADR;
  3079. goto bail;
  3080. }
  3081. if (fl->profile)
  3082. ktime_get_real_ts64(&invoket);
  3083. if (!kernel) {
  3084. VERIFY(err, invoke->handle !=
  3085. FASTRPC_STATIC_HANDLE_PROCESS_GROUP);
  3086. VERIFY(err, invoke->handle !=
  3087. FASTRPC_STATIC_HANDLE_DSP_UTILITIES);
  3088. if (err) {
  3089. err = -EINVAL;
  3090. ADSPRPC_ERR(
  3091. "user application trying to send a kernel RPC message to channel %d, handle 0x%x\n",
  3092. cid, invoke->handle);
  3093. goto bail;
  3094. }
  3095. }
  3096. if (!kernel) {
  3097. VERIFY(err, 0 == (err = context_restore_interrupted(fl,
  3098. inv, &ctx)));
  3099. if (err)
  3100. goto bail;
  3101. if (fl->sctx->smmu.faults)
  3102. err = -FASTRPC_ENOSUCH;
  3103. if (err)
  3104. goto bail;
  3105. if (ctx) {
  3106. trace_fastrpc_context_restore(cid, (uint64_t)ctx,
  3107. ctx->msg.invoke.header.ctx,
  3108. ctx->handle, ctx->sc);
  3109. goto wait;
  3110. }
  3111. }
  3112. trace_fastrpc_msg("context_alloc: begin");
  3113. VERIFY(err, 0 == (err = context_alloc(fl, kernel, inv, &ctx)));
  3114. trace_fastrpc_msg("context_alloc: end");
  3115. if (err)
  3116. goto bail;
  3117. if (fl->servloc_name) {
  3118. err = fastrpc_check_pd_status(fl,
  3119. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME);
  3120. err |= fastrpc_check_pd_status(fl,
  3121. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME);
  3122. err |= fastrpc_check_pd_status(fl,
  3123. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME);
  3124. if (err)
  3125. goto bail;
  3126. }
  3127. isasyncinvoke = (ctx->asyncjob.isasyncjob ? true : false);
  3128. if (fl->profile)
  3129. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  3130. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_GETARGS),
  3131. VERIFY(err, 0 == (err = get_args(kernel, ctx)));
  3132. PERF_END);
  3133. trace_fastrpc_msg("get_args: end");
  3134. if (err)
  3135. goto bail;
  3136. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3137. inv_args(ctx);
  3138. PERF_END);
  3139. trace_fastrpc_msg("inv_args_1: end");
  3140. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_LINK),
  3141. VERIFY(err, 0 == (err = fastrpc_invoke_send(ctx,
  3142. kernel, invoke->handle)));
  3143. PERF_END);
  3144. trace_fastrpc_msg("invoke_send: end");
  3145. if (err)
  3146. goto bail;
  3147. if (isasyncinvoke)
  3148. goto invoke_end;
  3149. wait:
  3150. /* Poll mode allowed only for non-static handle calls to dynamic CDSP process */
  3151. if (fl->poll_mode && (invoke->handle > FASTRPC_STATIC_HANDLE_MAX)
  3152. && (cid == CDSP_DOMAIN_ID)
  3153. && (fl->proc_flags == FASTRPC_INIT_CREATE))
  3154. ctx->rsp_flags = POLL_MODE;
  3155. fastrpc_wait_for_completion(ctx, &interrupted, kernel, 0, &isworkdone);
  3156. trace_fastrpc_msg("wait_for_completion: end");
  3157. VERIFY(err, 0 == (err = interrupted));
  3158. if (err)
  3159. goto bail;
  3160. if (!ctx->is_work_done) {
  3161. err = -ETIMEDOUT;
  3162. ADSPRPC_ERR(
  3163. "WorkDone state is invalid for handle 0x%x, sc 0x%x\n",
  3164. invoke->handle, ctx->sc);
  3165. goto bail;
  3166. }
  3167. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3168. inv_args(ctx);
  3169. PERF_END);
  3170. trace_fastrpc_msg("inv_args_2: end");
  3171. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_PUTARGS),
  3172. VERIFY(err, 0 == (err = put_args(kernel, ctx, invoke->pra)));
  3173. PERF_END);
  3174. trace_fastrpc_msg("put_args: end");
  3175. if (err)
  3176. goto bail;
  3177. VERIFY(err, 0 == (err = ctx->retval));
  3178. if (err)
  3179. goto bail;
  3180. bail:
  3181. if (ctx && interrupted == -ERESTARTSYS) {
  3182. trace_fastrpc_context_interrupt(cid, (uint64_t)ctx,
  3183. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  3184. context_save_interrupted(ctx);
  3185. } else if (ctx) {
  3186. if (fl->profile && !interrupted)
  3187. fastrpc_update_invoke_count(invoke->handle,
  3188. perf_counter, &invoket);
  3189. if (fl->profile && ctx->perf && ctx->handle > FASTRPC_STATIC_HANDLE_MAX) {
  3190. trace_fastrpc_perf_counters(ctx->handle, ctx->sc,
  3191. ctx->perf->count, ctx->perf->flush, ctx->perf->map,
  3192. ctx->perf->copy, ctx->perf->link, ctx->perf->getargs,
  3193. ctx->perf->putargs, ctx->perf->invargs,
  3194. ctx->perf->invoke, ctx->perf->tid);
  3195. if (ctx->perf_kernel) {
  3196. K_COPY_TO_USER(perfErr, kernel, ctx->perf_kernel,
  3197. ctx->perf, M_KERNEL_PERF_LIST*sizeof(uint64_t));
  3198. if (perfErr)
  3199. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  3200. }
  3201. }
  3202. context_free(ctx);
  3203. trace_fastrpc_msg("context_free: end");
  3204. }
  3205. if (!kernel) {
  3206. if (VALID_FASTRPC_CID(cid)
  3207. && (fl->ssrcount != fl->apps->channel[cid].ssrcount))
  3208. err = -ECONNRESET;
  3209. }
  3210. invoke_end:
  3211. if (fl->profile && !interrupted && isasyncinvoke)
  3212. fastrpc_update_invoke_count(invoke->handle, perf_counter,
  3213. &invoket);
  3214. return err;
  3215. }
  3216. static int fastrpc_wait_on_async_queue(
  3217. struct fastrpc_ioctl_async_response *async_res,
  3218. struct fastrpc_file *fl)
  3219. {
  3220. int err = 0, ierr = 0, interrupted = 0, perfErr = 0;
  3221. struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;
  3222. unsigned long flags;
  3223. uint64_t *perf_counter = NULL;
  3224. bool isworkdone = false;
  3225. struct hlist_node *n;
  3226. read_async_job:
  3227. if (!fl) {
  3228. err = -EBADF;
  3229. goto bail;
  3230. }
  3231. interrupted = wait_event_interruptible(fl->async_wait_queue,
  3232. atomic_read(&fl->async_queue_job_count));
  3233. if (fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  3234. err = -EBADF;
  3235. goto bail;
  3236. }
  3237. if (fl->exit_async) {
  3238. err = -EFAULT;
  3239. goto bail;
  3240. }
  3241. VERIFY(err, 0 == (err = interrupted));
  3242. if (err)
  3243. goto bail;
  3244. if (err) {
  3245. ADSPRPC_ERR("Failed to get user process reference\n");
  3246. goto bail;
  3247. }
  3248. spin_lock_irqsave(&fl->aqlock, flags);
  3249. hlist_for_each_entry_safe(ictx, n, &fl->clst.async_queue, asyncn) {
  3250. hlist_del_init(&ictx->asyncn);
  3251. atomic_sub(1, &fl->async_queue_job_count);
  3252. ctx = ictx;
  3253. break;
  3254. }
  3255. spin_unlock_irqrestore(&fl->aqlock, flags);
  3256. if (ctx) {
  3257. if (fl->profile)
  3258. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  3259. fastrpc_wait_for_completion(ctx, &interrupted, 0, 1,
  3260. &isworkdone);
  3261. if (!isworkdone) {//In valid workdone state
  3262. ADSPRPC_DEBUG(
  3263. "Async early wake response did not reach on time for thread %d handle 0x%x, sc 0x%x\n",
  3264. ctx->pid, ctx->handle, ctx->sc);
  3265. goto read_async_job;
  3266. }
  3267. async_res->jobid = ctx->asyncjob.jobid;
  3268. async_res->result = ctx->retval;
  3269. async_res->handle = ctx->handle;
  3270. async_res->sc = ctx->sc;
  3271. async_res->perf_dsp = (uint64_t *)ctx->perf_dsp;
  3272. async_res->perf_kernel = (uint64_t *)ctx->perf_kernel;
  3273. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3274. inv_args(ctx);
  3275. PERF_END);
  3276. if (ctx->retval != 0)
  3277. goto bail;
  3278. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_PUTARGS),
  3279. VERIFY(ierr, 0 == (ierr = put_args(0, ctx, NULL)));
  3280. PERF_END);
  3281. if (ierr)
  3282. goto bail;
  3283. } else { // Go back to wait if ctx is invalid
  3284. ADSPRPC_ERR("Invalid async job wake up\n");
  3285. goto read_async_job;
  3286. }
  3287. bail:
  3288. if (ierr)
  3289. async_res->result = ierr;
  3290. if (ctx) {
  3291. if (fl->profile && ctx->perf && ctx->handle > FASTRPC_STATIC_HANDLE_MAX) {
  3292. trace_fastrpc_perf_counters(ctx->handle, ctx->sc,
  3293. ctx->perf->count, ctx->perf->flush, ctx->perf->map,
  3294. ctx->perf->copy, ctx->perf->link, ctx->perf->getargs,
  3295. ctx->perf->putargs, ctx->perf->invargs,
  3296. ctx->perf->invoke, ctx->perf->tid);
  3297. if (ctx->perf_kernel) {
  3298. K_COPY_TO_USER(perfErr, 0, ctx->perf_kernel,
  3299. ctx->perf, M_KERNEL_PERF_LIST*sizeof(uint64_t));
  3300. if (perfErr)
  3301. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  3302. }
  3303. }
  3304. context_free(ctx);
  3305. }
  3306. return err;
  3307. }
  3308. static int fastrpc_wait_on_notif_queue(
  3309. struct fastrpc_ioctl_notif_rsp *notif_rsp,
  3310. struct fastrpc_file *fl)
  3311. {
  3312. int err = 0, interrupted = 0;
  3313. unsigned long flags;
  3314. struct smq_notif_rsp *notif = NULL, *inotif = NULL, *n = NULL;
  3315. read_notif_status:
  3316. if (!fl) {
  3317. err = -EBADF;
  3318. goto bail;
  3319. }
  3320. interrupted = wait_event_interruptible(fl->proc_state_notif.notif_wait_queue,
  3321. atomic_read(&fl->proc_state_notif.notif_queue_count));
  3322. if (fl->exit_notif) {
  3323. err = -EFAULT;
  3324. goto bail;
  3325. }
  3326. if (fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  3327. err = -EBADF;
  3328. goto bail;
  3329. }
  3330. VERIFY(err, 0 == (err = interrupted));
  3331. if (err)
  3332. goto bail;
  3333. if (err) {
  3334. ADSPRPC_ERR("Failed to get user process reference\n");
  3335. goto bail;
  3336. }
  3337. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  3338. list_for_each_entry_safe(inotif, n, &fl->clst.notif_queue, notifn) {
  3339. list_del_init(&inotif->notifn);
  3340. atomic_sub(1, &fl->proc_state_notif.notif_queue_count);
  3341. notif = inotif;
  3342. break;
  3343. }
  3344. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  3345. if (notif) {
  3346. notif_rsp->status = notif->status;
  3347. notif_rsp->domain = notif->domain;
  3348. notif_rsp->session = notif->session;
  3349. } else {// Go back to wait if ctx is invalid
  3350. ADSPRPC_ERR("Invalid status notification response\n");
  3351. goto read_notif_status;
  3352. }
  3353. bail:
  3354. kfree(notif);
  3355. return err;
  3356. }
  3357. static int fastrpc_get_async_response(
  3358. struct fastrpc_ioctl_async_response *async_res,
  3359. void *param, struct fastrpc_file *fl)
  3360. {
  3361. int err = 0;
  3362. err = fastrpc_wait_on_async_queue(async_res, fl);
  3363. if (err)
  3364. goto bail;
  3365. K_COPY_TO_USER(err, 0, param, async_res,
  3366. sizeof(struct fastrpc_ioctl_async_response));
  3367. bail:
  3368. return err;
  3369. }
  3370. static int fastrpc_get_notif_response(
  3371. struct fastrpc_ioctl_notif_rsp *notif,
  3372. void *param, struct fastrpc_file *fl)
  3373. {
  3374. int err = 0;
  3375. err = fastrpc_wait_on_notif_queue(notif, fl);
  3376. if (err)
  3377. goto bail;
  3378. K_COPY_TO_USER(err, 0, param, notif,
  3379. sizeof(struct fastrpc_ioctl_notif_rsp));
  3380. bail:
  3381. return err;
  3382. }
  3383. static int fastrpc_set_session_info(
  3384. struct fastrpc_proc_sess_info *sess_info,
  3385. void *param, struct fastrpc_file *fl)
  3386. {
  3387. int err = 0;
  3388. struct fastrpc_apps *me = &gfa;
  3389. if (fl->set_session_info) {
  3390. ADSPRPC_ERR("Set session info invoked multiple times\n");
  3391. err = -EBADR;
  3392. goto bail;
  3393. }
  3394. /*
  3395. * Third-party apps don't have permission to open the fastrpc device, so
  3396. * it is opened on their behalf by DSP HAL. This is detected by
  3397. * comparing current PID with the one stored during device open.
  3398. */
  3399. if (current->tgid != fl->tgid_open)
  3400. fl->untrusted_process = true;
  3401. VERIFY(err, sess_info->pd_type > DEFAULT_UNUSED &&
  3402. sess_info->pd_type < MAX_PD_TYPE);
  3403. if (err) {
  3404. ADSPRPC_ERR(
  3405. "Session PD type %u is invalid for the process\n",
  3406. sess_info->pd_type);
  3407. err = -EBADR;
  3408. goto bail;
  3409. }
  3410. if (fl->untrusted_process && sess_info->pd_type != USERPD) {
  3411. ADSPRPC_ERR(
  3412. "Session PD type %u not allowed for untrusted process\n",
  3413. sess_info->pd_type);
  3414. err = -EBADR;
  3415. goto bail;
  3416. }
  3417. /*
  3418. * If PD type is not configured for context banks,
  3419. * ignore PD type passed by the user, leave pd_type set to DEFAULT_UNUSED(0)
  3420. */
  3421. if (me->cb_pd_type)
  3422. fl->pd_type = sess_info->pd_type;
  3423. // Processes attaching to Sensor Static PD, share context bank.
  3424. if (sess_info->pd_type == SENSORS_STATICPD)
  3425. fl->sharedcb = 1;
  3426. if (sess_info->session_id >= me->max_sess_per_proc) {
  3427. ADSPRPC_ERR(
  3428. "Session ID %u cannot be beyond %u\n",
  3429. sess_info->session_id, me->max_sess_per_proc);
  3430. err = -EBADR;
  3431. goto bail;
  3432. }
  3433. fl->sessionid = sess_info->session_id;
  3434. // Set multi_session_support, to disable old way of setting session_id
  3435. fl->multi_session_support = true;
  3436. VERIFY(err, 0 == (err = fastrpc_get_info(fl, &(sess_info->domain_id))));
  3437. if (err)
  3438. goto bail;
  3439. K_COPY_TO_USER(err, 0, param, sess_info,
  3440. sizeof(struct fastrpc_proc_sess_info));
  3441. bail:
  3442. return err;
  3443. }
  3444. static int fastrpc_create_persistent_headers(struct fastrpc_file *fl,
  3445. uint32_t user_concurrency)
  3446. {
  3447. int err = 0, i = 0;
  3448. uint64_t virtb = 0;
  3449. struct fastrpc_buf *pers_hdr_buf = NULL, *hdr_bufs = NULL, *buf = NULL;
  3450. unsigned int num_pers_hdrs = 0;
  3451. size_t hdr_buf_alloc_len = 0;
  3452. if (fl->pers_hdr_buf || !user_concurrency)
  3453. goto bail;
  3454. /*
  3455. * Pre-allocate memory for persistent header buffers based
  3456. * on concurrency info passed by user. Upper limit enforced.
  3457. */
  3458. num_pers_hdrs = (user_concurrency > MAX_PERSISTENT_HEADERS) ?
  3459. MAX_PERSISTENT_HEADERS : user_concurrency;
  3460. hdr_buf_alloc_len = num_pers_hdrs*PAGE_SIZE;
  3461. err = fastrpc_buf_alloc(fl, hdr_buf_alloc_len, 0, 0,
  3462. METADATA_BUF, &pers_hdr_buf);
  3463. if (err)
  3464. goto bail;
  3465. virtb = ptr_to_uint64(pers_hdr_buf->virt);
  3466. /* Map entire buffer on remote subsystem in single RPC call */
  3467. err = fastrpc_mem_map_to_dsp(fl, -1, 0, ADSP_MMAP_PERSIST_HDR, 0,
  3468. pers_hdr_buf->phys, pers_hdr_buf->size,
  3469. &pers_hdr_buf->raddr);
  3470. if (err)
  3471. goto bail;
  3472. /* Divide and store as N chunks, each of 1 page size */
  3473. hdr_bufs = kcalloc(num_pers_hdrs, sizeof(struct fastrpc_buf),
  3474. GFP_KERNEL);
  3475. if (!hdr_bufs) {
  3476. err = -ENOMEM;
  3477. goto bail;
  3478. }
  3479. spin_lock(&fl->hlock);
  3480. fl->pers_hdr_buf = pers_hdr_buf;
  3481. fl->num_pers_hdrs = num_pers_hdrs;
  3482. fl->hdr_bufs = hdr_bufs;
  3483. for (i = 0; i < num_pers_hdrs; i++) {
  3484. buf = &fl->hdr_bufs[i];
  3485. buf->fl = fl;
  3486. buf->virt = uint64_to_ptr(virtb + (i*PAGE_SIZE));
  3487. buf->phys = pers_hdr_buf->phys + (i*PAGE_SIZE);
  3488. buf->size = PAGE_SIZE;
  3489. buf->dma_attr = pers_hdr_buf->dma_attr;
  3490. buf->flags = pers_hdr_buf->flags;
  3491. buf->type = pers_hdr_buf->type;
  3492. buf->in_use = false;
  3493. }
  3494. spin_unlock(&fl->hlock);
  3495. bail:
  3496. if (err) {
  3497. ADSPRPC_ERR(
  3498. "failed to map len %zu, flags %d, user concurrency %u, num headers %u with err %d\n",
  3499. hdr_buf_alloc_len, ADSP_MMAP_PERSIST_HDR,
  3500. user_concurrency, num_pers_hdrs, err);
  3501. fl->pers_hdr_buf = NULL;
  3502. fl->hdr_bufs = NULL;
  3503. fl->num_pers_hdrs = 0;
  3504. if (!IS_ERR_OR_NULL(pers_hdr_buf))
  3505. fastrpc_buf_free(pers_hdr_buf, 0);
  3506. if (!IS_ERR_OR_NULL(hdr_bufs))
  3507. kfree(hdr_bufs);
  3508. }
  3509. return err;
  3510. }
  3511. int fastrpc_internal_invoke2(struct fastrpc_file *fl,
  3512. struct fastrpc_ioctl_invoke2 *inv2)
  3513. {
  3514. union {
  3515. struct fastrpc_ioctl_invoke_async inv;
  3516. struct fastrpc_ioctl_invoke_async_no_perf inv3;
  3517. struct fastrpc_ioctl_async_response async_res;
  3518. uint32_t user_concurrency;
  3519. struct fastrpc_ioctl_notif_rsp notif;
  3520. struct fastrpc_proc_sharedbuf_info buff_info;
  3521. struct fastrpc_proc_sess_info sess_info;
  3522. } p;
  3523. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  3524. uint32_t size = 0;
  3525. int err = 0, domain = fl->cid;
  3526. if (inv2->req == FASTRPC_INVOKE2_ASYNC ||
  3527. inv2->req == FASTRPC_INVOKE2_ASYNC_RESPONSE) {
  3528. VERIFY(err, domain == CDSP_DOMAIN_ID && fl->sctx != NULL);
  3529. if (err) {
  3530. err = -EBADR;
  3531. goto bail;
  3532. }
  3533. dsp_cap_ptr = &gcinfo[domain].dsp_cap_kernel;
  3534. VERIFY(err,
  3535. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP] == 1);
  3536. if (err) {
  3537. err = -EPROTONOSUPPORT;
  3538. goto bail;
  3539. }
  3540. }
  3541. switch (inv2->req) {
  3542. case FASTRPC_INVOKE2_ASYNC:
  3543. size = sizeof(struct fastrpc_ioctl_invoke_async);
  3544. VERIFY(err, size >= inv2->size);
  3545. if (err) {
  3546. err = -EBADE;
  3547. goto bail;
  3548. }
  3549. if (size > inv2->size) {
  3550. K_COPY_FROM_USER(err, fl->is_compat, &p.inv3, (void *)inv2->invparam,
  3551. sizeof(struct fastrpc_ioctl_invoke_async_no_perf));
  3552. if (err)
  3553. goto bail;
  3554. memcpy(&p.inv, &p.inv3, sizeof(struct fastrpc_ioctl_invoke_crc));
  3555. memcpy(&p.inv.job, &p.inv3.job, sizeof(p.inv.job));
  3556. } else {
  3557. K_COPY_FROM_USER(err, fl->is_compat, &p.inv, (void *)inv2->invparam, size);
  3558. if (err)
  3559. goto bail;
  3560. }
  3561. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode,
  3562. USER_MSG, &p.inv)));
  3563. if (err)
  3564. goto bail;
  3565. break;
  3566. case FASTRPC_INVOKE2_ASYNC_RESPONSE:
  3567. VERIFY(err,
  3568. sizeof(struct fastrpc_ioctl_async_response) >= inv2->size);
  3569. if (err) {
  3570. err = -EBADE;
  3571. goto bail;
  3572. }
  3573. err = fastrpc_get_async_response(&p.async_res,
  3574. (void *)inv2->invparam, fl);
  3575. break;
  3576. case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:
  3577. size = sizeof(uint32_t);
  3578. if (inv2->size != size) {
  3579. err = -EBADE;
  3580. goto bail;
  3581. }
  3582. K_COPY_FROM_USER(err, 0, &p.user_concurrency,
  3583. (void *)inv2->invparam, size);
  3584. if (err)
  3585. goto bail;
  3586. err = fastrpc_create_persistent_headers(fl,
  3587. p.user_concurrency);
  3588. break;
  3589. case FASTRPC_INVOKE2_STATUS_NOTIF:
  3590. VERIFY(err,
  3591. sizeof(struct fastrpc_ioctl_notif_rsp) >= inv2->size);
  3592. if (err) {
  3593. err = -EBADE;
  3594. goto bail;
  3595. }
  3596. err = fastrpc_get_notif_response(&p.notif,
  3597. (void *)inv2->invparam, fl);
  3598. break;
  3599. case FASTRPC_INVOKE2_PROC_SHAREDBUF_INFO:
  3600. VERIFY(err,
  3601. sizeof(struct fastrpc_proc_sharedbuf_info) >= inv2->size);
  3602. if (err) {
  3603. err = -EBADE;
  3604. goto bail;
  3605. }
  3606. K_COPY_FROM_USER(err, fl->is_compat, &p.buff_info,
  3607. (void *)inv2->invparam, inv2->size);
  3608. if (err)
  3609. goto bail;
  3610. fl->sharedbuf_info.buf_fd = p.buff_info.buf_fd;
  3611. fl->sharedbuf_info.buf_size = p.buff_info.buf_size;
  3612. break;
  3613. case FASTRPC_INVOKE2_SESS_INFO:
  3614. VERIFY(err,
  3615. sizeof(struct fastrpc_proc_sess_info) >= inv2->size);
  3616. if (err) {
  3617. err = -EBADE;
  3618. goto bail;
  3619. }
  3620. K_COPY_FROM_USER(err, fl->is_compat, &p.sess_info,
  3621. (void *)inv2->invparam, inv2->size);
  3622. if (err)
  3623. goto bail;
  3624. err = fastrpc_set_session_info(&p.sess_info,
  3625. (void *)inv2->invparam, fl);
  3626. break;
  3627. default:
  3628. err = -ENOTTY;
  3629. break;
  3630. }
  3631. bail:
  3632. return err;
  3633. }
  3634. static int fastrpc_get_spd_session(char *name, int *session, int *cid)
  3635. {
  3636. struct fastrpc_apps *me = &gfa;
  3637. int err = 0, i, j, match = 0;
  3638. for (i = 0; i < NUM_CHANNELS; i++) {
  3639. for (j = 0; j < NUM_SESSIONS; j++) {
  3640. if (!me->channel[i].spd[j].servloc_name)
  3641. continue;
  3642. if (!strcmp(name, me->channel[i].spd[j].servloc_name)) {
  3643. match = 1;
  3644. break;
  3645. }
  3646. }
  3647. if (match)
  3648. break;
  3649. }
  3650. VERIFY(err, i < NUM_CHANNELS && j < NUM_SESSIONS);
  3651. if (err) {
  3652. err = -EUSERS;
  3653. goto bail;
  3654. }
  3655. *cid = i;
  3656. *session = j;
  3657. bail:
  3658. return err;
  3659. }
  3660. static int fastrpc_mmap_remove_pdr(struct fastrpc_file *fl);
  3661. static int fastrpc_channel_open(struct fastrpc_file *fl, uint32_t flags);
  3662. static int fastrpc_dsp_restart_handler(struct fastrpc_file *fl, int locked, bool dump_req);
  3663. /*
  3664. * This function makes a call to create a thread group in the root
  3665. * process or static process on the remote subsystem.
  3666. * Examples:
  3667. * - guestOS daemons on all DSPs
  3668. * - sensors daemon on sensorsPD on SLPI/ADSP
  3669. */
  3670. static int fastrpc_init_attach_process(struct fastrpc_file *fl,
  3671. struct fastrpc_ioctl_init *init)
  3672. {
  3673. int err = 0, tgid = fl->tgid_frpc;
  3674. remote_arg_t ra[1];
  3675. struct fastrpc_ioctl_invoke_async ioctl;
  3676. if (fl->dev_minor == MINOR_NUM_DEV) {
  3677. err = -ECONNREFUSED;
  3678. ADSPRPC_ERR(
  3679. "untrusted app trying to attach to privileged DSP PD\n");
  3680. return err;
  3681. }
  3682. /*
  3683. * Prepare remote arguments for creating thread group
  3684. * in guestOS/staticPD on the remote subsystem.
  3685. * Send unique fastrpc id to dsp
  3686. */
  3687. ra[0].buf.pv = (void *)&tgid;
  3688. ra[0].buf.len = sizeof(tgid);
  3689. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  3690. ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 0);
  3691. ioctl.inv.pra = ra;
  3692. ioctl.fds = NULL;
  3693. ioctl.attrs = NULL;
  3694. ioctl.crc = NULL;
  3695. ioctl.perf_kernel = NULL;
  3696. ioctl.perf_dsp = NULL;
  3697. ioctl.job = NULL;
  3698. if (init->flags == FASTRPC_INIT_ATTACH)
  3699. fl->pd = FASTRPC_ROOT_PD;
  3700. else if (init->flags == FASTRPC_INIT_ATTACH_SENSORS)
  3701. /* Setting to 2 will route the message to sensorsPD */
  3702. fl->pd = FASTRPC_SENSORS_PD;
  3703. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  3704. if (err)
  3705. goto bail;
  3706. bail:
  3707. return err;
  3708. }
  3709. /*
  3710. * This function makes a call to spawn a dynamic process
  3711. * on the remote subsystem.
  3712. * Example: all compute offloads to CDSP
  3713. */
  3714. static int fastrpc_init_create_dynamic_process(struct fastrpc_file *fl,
  3715. struct fastrpc_ioctl_init_attrs *uproc)
  3716. {
  3717. int err = 0, memlen = 0, mflags = 0, locked = 0, glocked = 0;
  3718. struct fastrpc_ioctl_invoke_async ioctl;
  3719. struct fastrpc_ioctl_init *init = &uproc->init;
  3720. /* First page for init-mem and second page for proc-attrs */
  3721. struct smq_phy_page pages[PAGESLEN_WITH_SHAREDBUF];
  3722. struct fastrpc_mmap *file = NULL;
  3723. struct fastrpc_buf *imem = NULL;
  3724. unsigned long imem_dma_attr = 0;
  3725. remote_arg_t ra[6];
  3726. int fds[6];
  3727. unsigned int gid = 0, one_mb = 1024*1024;
  3728. unsigned int dsp_userpd_memlen = 0;
  3729. struct fastrpc_buf *init_mem;
  3730. struct fastrpc_mmap *sharedbuf_map = NULL;
  3731. struct fastrpc_apps *me = &gfa;
  3732. unsigned long irq_flags = 0;
  3733. struct {
  3734. int pgid;
  3735. unsigned int namelen;
  3736. unsigned int filelen;
  3737. unsigned int pageslen;
  3738. int attrs;
  3739. int siglen;
  3740. } inbuf;
  3741. spin_lock(&fl->hlock);
  3742. if (fl->dsp_process_state) {
  3743. err = -EALREADY;
  3744. ADSPRPC_ERR("Already in create dynamic process\n");
  3745. spin_unlock(&fl->hlock);
  3746. return err;
  3747. }
  3748. fl->dsp_process_state = PROCESS_CREATE_IS_INPROGRESS;
  3749. if (init->memlen) {
  3750. if(init->memlen > INIT_MEMLEN_MAX_DYNAMIC || init->memlen < INIT_MEMLEN_MIN_DYNAMIC) {
  3751. ADSPRPC_ERR(
  3752. "init memory for process %d should be between %d and %d\n",
  3753. init->memlen, INIT_MEMLEN_MIN_DYNAMIC, INIT_MEMLEN_MAX_DYNAMIC);
  3754. err = -EINVAL;
  3755. spin_unlock(&fl->hlock);
  3756. goto bail;
  3757. }
  3758. dsp_userpd_memlen = init->memlen;
  3759. } else {
  3760. dsp_userpd_memlen = 3*one_mb;
  3761. }
  3762. spin_unlock(&fl->hlock);
  3763. inbuf.pgid = fl->tgid_frpc;
  3764. inbuf.namelen = strlen(current->comm) + 1;
  3765. inbuf.filelen = init->filelen;
  3766. fl->pd = FASTRPC_USER_PD;
  3767. if (uproc->attrs & FASTRPC_MODE_UNSIGNED_MODULE)
  3768. fl->is_unsigned_pd = true;
  3769. /* Check if file memory passed by userspace is valid */
  3770. VERIFY(err, access_ok((void __user *)init->file, init->filelen));
  3771. if (err)
  3772. goto bail;
  3773. if (init->filelen) {
  3774. /* Map the shell file buffer to remote subsystem */
  3775. mutex_lock(&fl->map_mutex);
  3776. err = fastrpc_mmap_create(fl, init->filefd, NULL, 0,
  3777. init->file, init->filelen, mflags, &file);
  3778. if (file)
  3779. file->is_filemap = true;
  3780. mutex_unlock(&fl->map_mutex);
  3781. if (err)
  3782. goto bail;
  3783. }
  3784. inbuf.pageslen = 1;
  3785. /* Disregard any system unsigned PD attribute from userspace */
  3786. uproc->attrs &= (~FASTRPC_MODE_SYSTEM_UNSIGNED_PD);
  3787. /* Untrusted apps are not allowed to offload to signedPD on DSP. */
  3788. if (fl->untrusted_process) {
  3789. VERIFY(err, fl->is_unsigned_pd);
  3790. if (err) {
  3791. err = -ECONNREFUSED;
  3792. ADSPRPC_ERR(
  3793. "untrusted app trying to offload to signed remote process\n");
  3794. goto bail;
  3795. }
  3796. } else {
  3797. /* Trusted apps will be launched as system unsigned PDs */
  3798. if (fl->is_unsigned_pd)
  3799. uproc->attrs |= FASTRPC_MODE_SYSTEM_UNSIGNED_PD;
  3800. }
  3801. /* Disregard any privilege bits from userspace */
  3802. uproc->attrs &= (~FASTRPC_MODE_PRIVILEGED);
  3803. /*
  3804. * Check if the primary or supplementary group(s) of the process is
  3805. * one of the 'privileged' fastrpc GIDs stored in the device-tree.
  3806. */
  3807. gid = sorted_lists_intersection(fl->gidlist.gids,
  3808. fl->gidlist.gidcount, gfa.gidlist.gids, gfa.gidlist.gidcount);
  3809. if (gid) {
  3810. ADSPRPC_INFO("PID %d, GID %u is a privileged process\n",
  3811. fl->tgid, gid);
  3812. uproc->attrs |= FASTRPC_MODE_PRIVILEGED;
  3813. }
  3814. /*
  3815. * Userspace client should try to allocate the initial memory donated
  3816. * to remote subsystem as only the kernel and DSP should have access
  3817. * to that memory.
  3818. */
  3819. VERIFY(err, !init->mem);
  3820. if (err) {
  3821. err = -EINVAL;
  3822. ADSPRPC_ERR("donated memory allocated in userspace\n");
  3823. goto bail;
  3824. }
  3825. /* Allocate DMA buffer in kernel for donating to remote process
  3826. * Unsigned PD requires additional memory because of the
  3827. * additional static heap initialized within the process.
  3828. */
  3829. if (fl->is_unsigned_pd)
  3830. dsp_userpd_memlen = 5*one_mb;
  3831. memlen = ALIGN(max(dsp_userpd_memlen, init->filelen * 4), one_mb);
  3832. imem_dma_attr = DMA_ATTR_DELAYED_UNMAP | DMA_ATTR_NO_KERNEL_MAPPING;
  3833. err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0,
  3834. INITMEM_BUF, &imem);
  3835. if (err)
  3836. goto bail;
  3837. fl->init_mem = imem;
  3838. inbuf.pageslen = 1;
  3839. if ((fl->sharedbuf_info.buf_fd != -1) && fl->sharedbuf_info.buf_size) {
  3840. mutex_lock(&fl->map_mutex);
  3841. err = fastrpc_mmap_create(fl, fl->sharedbuf_info.buf_fd, NULL, 0,
  3842. 0, fl->sharedbuf_info.buf_size, mflags, &sharedbuf_map);
  3843. mutex_unlock(&fl->map_mutex);
  3844. if (err)
  3845. goto bail;
  3846. /* if shared buff is available send this as the second page and set pageslen as 2 */
  3847. inbuf.pageslen = PAGESLEN_WITH_SHAREDBUF;
  3848. }
  3849. /*
  3850. * Prepare remote arguments for dynamic process create
  3851. * call to remote subsystem.
  3852. */
  3853. ra[0].buf.pv = (void *)&inbuf;
  3854. ra[0].buf.len = sizeof(inbuf);
  3855. fds[0] = -1;
  3856. ra[1].buf.pv = (void *)current->comm;
  3857. ra[1].buf.len = inbuf.namelen;
  3858. fds[1] = -1;
  3859. ra[2].buf.pv = (void *)init->file;
  3860. ra[2].buf.len = inbuf.filelen;
  3861. fds[2] = init->filefd;
  3862. pages[0].addr = imem->phys;
  3863. pages[0].size = imem->size;
  3864. /* Update IOVA of second page shared with DSP */
  3865. if (inbuf.pageslen > 1) {
  3866. pages[1].addr = sharedbuf_map->phys;
  3867. pages[1].size = sharedbuf_map->size;
  3868. }
  3869. ra[3].buf.pv = (void *)pages;
  3870. ra[3].buf.len = (inbuf.pageslen) * sizeof(*pages);
  3871. fds[3] = -1;
  3872. inbuf.attrs = uproc->attrs;
  3873. ra[4].buf.pv = (void *)&(inbuf.attrs);
  3874. ra[4].buf.len = sizeof(inbuf.attrs);
  3875. fds[4] = -1;
  3876. inbuf.siglen = uproc->siglen;
  3877. ra[5].buf.pv = (void *)&(inbuf.siglen);
  3878. ra[5].buf.len = sizeof(inbuf.siglen);
  3879. fds[5] = -1;
  3880. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  3881. /*
  3882. * Choose appropriate remote method ID depending on whether the
  3883. * HLOS process has any attributes enabled (like unsignedPD,
  3884. * critical process, adaptive QoS, CRC checks etc).
  3885. */
  3886. ioctl.inv.sc = REMOTE_SCALARS_MAKE(6, 4, 0);
  3887. if (uproc->attrs)
  3888. ioctl.inv.sc = REMOTE_SCALARS_MAKE(7, 4, 0);
  3889. ioctl.inv.pra = ra;
  3890. ioctl.fds = fds;
  3891. ioctl.attrs = NULL;
  3892. ioctl.crc = NULL;
  3893. ioctl.perf_kernel = NULL;
  3894. ioctl.perf_dsp = NULL;
  3895. ioctl.job = NULL;
  3896. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  3897. if (err)
  3898. goto bail;
  3899. bail:
  3900. /*
  3901. * Shell is loaded into the donated memory on remote subsystem. So, the
  3902. * original file buffer can be DMA unmapped. In case of a failure also,
  3903. * the mapping needs to be removed.
  3904. */
  3905. if (file) {
  3906. mutex_lock(&fl->map_mutex);
  3907. fastrpc_mmap_free(file, 0);
  3908. mutex_unlock(&fl->map_mutex);
  3909. }
  3910. spin_lock(&fl->hlock);
  3911. locked = 1;
  3912. if (err) {
  3913. ADSPRPC_ERR("failed with err %d\n", err);
  3914. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  3915. spin_unlock(&fl->hlock);
  3916. locked = 0;
  3917. spin_lock_irqsave(&me->hlock, irq_flags);
  3918. glocked = 1;
  3919. if (!IS_ERR_OR_NULL(fl->init_mem)) {
  3920. init_mem = fl->init_mem;
  3921. fl->init_mem = NULL;
  3922. spin_unlock_irqrestore(&me->hlock, irq_flags);
  3923. glocked = 0;
  3924. fastrpc_buf_free(init_mem, 0);
  3925. }
  3926. if (glocked) {
  3927. spin_unlock_irqrestore(&me->hlock, irq_flags);
  3928. glocked = 0;
  3929. }
  3930. } else {
  3931. fl->dsp_process_state = PROCESS_CREATE_SUCCESS;
  3932. }
  3933. if (locked) {
  3934. spin_unlock(&fl->hlock);
  3935. locked = 0;
  3936. }
  3937. return err;
  3938. }
  3939. /*
  3940. * This function makes a call to create a thread group in the static
  3941. * process on the remote subsystem.
  3942. * Example: audio daemon 'adsprpcd' on audioPD on ADSP
  3943. */
  3944. static int fastrpc_init_create_static_process(struct fastrpc_file *fl,
  3945. struct fastrpc_ioctl_init *init)
  3946. {
  3947. int err = 0, rh_hyp_done = 0;
  3948. struct fastrpc_apps *me = &gfa;
  3949. struct fastrpc_ioctl_invoke_async ioctl;
  3950. struct smq_phy_page pages[1];
  3951. struct fastrpc_mmap *mem = NULL;
  3952. char *proc_name = NULL;
  3953. remote_arg_t ra[3];
  3954. uint64_t phys = 0;
  3955. size_t size = 0;
  3956. int fds[3];
  3957. struct secure_vm *rhvm = &me->channel[fl->cid].rhvm;
  3958. struct {
  3959. int pgid;
  3960. unsigned int namelen;
  3961. unsigned int pageslen;
  3962. } inbuf;
  3963. unsigned long irq_flags = 0;
  3964. if (fl->dev_minor == MINOR_NUM_DEV) {
  3965. err = -ECONNREFUSED;
  3966. ADSPRPC_ERR(
  3967. "untrusted app trying to attach to audio PD\n");
  3968. return err;
  3969. }
  3970. VERIFY(err, init->memlen <= INIT_MEMLEN_MAX_STATIC);
  3971. if (err) {
  3972. ADSPRPC_ERR(
  3973. "init memory for static process %d is more than max allowed init len %d\n",
  3974. init->memlen, INIT_MEMLEN_MAX_STATIC);
  3975. err = -EFBIG;
  3976. goto bail;
  3977. }
  3978. if (!init->filelen)
  3979. goto bail;
  3980. proc_name = kzalloc(init->filelen + 1, GFP_KERNEL);
  3981. VERIFY(err, !IS_ERR_OR_NULL(proc_name));
  3982. if (err) {
  3983. err = -ENOMEM;
  3984. goto bail;
  3985. }
  3986. err = copy_from_user((void *)proc_name,
  3987. (void __user *)init->file, init->filelen);
  3988. if (err) {
  3989. err = -EFAULT;
  3990. goto bail;
  3991. }
  3992. fl->pd = FASTRPC_USER_PD;
  3993. inbuf.pgid = fl->tgid_frpc;
  3994. inbuf.namelen = init->filelen;
  3995. inbuf.pageslen = 0;
  3996. if (!strcmp(proc_name, "audiopd")) {
  3997. /*
  3998. * Remove any previous mappings in case process is trying
  3999. * to reconnect after a PD restart on remote subsystem.
  4000. */
  4001. err = fastrpc_mmap_remove_pdr(fl);
  4002. if (err)
  4003. goto bail;
  4004. } else {
  4005. ADSPRPC_ERR(
  4006. "Create static process is failed for proc_name %s",
  4007. proc_name);
  4008. goto bail;
  4009. }
  4010. if ((!me->staticpd_flags && !me->legacy_remote_heap)) {
  4011. inbuf.pageslen = 1;
  4012. if (!fastrpc_get_persistent_map(init->memlen, &mem)) {
  4013. mutex_lock(&fl->map_mutex);
  4014. err = fastrpc_mmap_create(fl, -1, NULL, 0, init->mem,
  4015. init->memlen, ADSP_MMAP_REMOTE_HEAP_ADDR, &mem);
  4016. if (mem)
  4017. mem->is_filemap = true;
  4018. mutex_unlock(&fl->map_mutex);
  4019. if (err || (!mem))
  4020. goto bail;
  4021. spin_lock_irqsave(&me->hlock, irq_flags);
  4022. mem->in_use = true;
  4023. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4024. }
  4025. VERIFY(err, mem);
  4026. if (err)
  4027. goto bail;
  4028. phys = mem->phys;
  4029. size = mem->size;
  4030. /*
  4031. * If remote-heap VMIDs are defined in DTSI, then do
  4032. * hyp_assign from HLOS to those VMs (LPASS, ADSP).
  4033. */
  4034. if (rhvm->vmid && mem->refs == 1 && size) {
  4035. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  4036. struct qcom_scm_vmperm *dst_perms;
  4037. uint32_t i = 0;
  4038. VERIFY(err, NULL != (dst_perms = kcalloc(rhvm->vmcount,
  4039. sizeof(struct qcom_scm_vmperm), GFP_KERNEL)));
  4040. if (err)
  4041. goto bail;
  4042. for (i = 0; i < rhvm->vmcount; i++) {
  4043. dst_perms[i].vmid = rhvm->vmid[i];
  4044. dst_perms[i].perm = rhvm->vmperm[i];
  4045. }
  4046. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4047. &src_perms, dst_perms, rhvm->vmcount);
  4048. kfree(dst_perms);
  4049. if (err) {
  4050. ADSPRPC_ERR(
  4051. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  4052. err, phys, size);
  4053. err = -EADDRNOTAVAIL;
  4054. goto bail;
  4055. }
  4056. rh_hyp_done = 1;
  4057. }
  4058. me->staticpd_flags = 1;
  4059. mem->is_persistent = true;
  4060. }
  4061. /*
  4062. * Prepare remote arguments for static process create
  4063. * call to remote subsystem.
  4064. */
  4065. ra[0].buf.pv = (void *)&inbuf;
  4066. ra[0].buf.len = sizeof(inbuf);
  4067. fds[0] = -1;
  4068. ra[1].buf.pv = (void *)proc_name;
  4069. ra[1].buf.len = inbuf.namelen;
  4070. fds[1] = -1;
  4071. pages[0].addr = phys;
  4072. pages[0].size = size;
  4073. ra[2].buf.pv = (void *)pages;
  4074. ra[2].buf.len = sizeof(*pages);
  4075. fds[2] = -1;
  4076. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4077. ioctl.inv.sc = REMOTE_SCALARS_MAKE(8, 3, 0);
  4078. ioctl.inv.pra = ra;
  4079. ioctl.fds = NULL;
  4080. ioctl.attrs = NULL;
  4081. ioctl.crc = NULL;
  4082. ioctl.perf_kernel = NULL;
  4083. ioctl.perf_dsp = NULL;
  4084. ioctl.job = NULL;
  4085. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4086. if (err)
  4087. goto bail;
  4088. bail:
  4089. kfree(proc_name);
  4090. if (err) {
  4091. me->staticpd_flags = 0;
  4092. if (rh_hyp_done) {
  4093. int hyp_err = 0;
  4094. u64 src_perms = 0;
  4095. struct qcom_scm_vmperm dst_perms;
  4096. uint32_t i = 0;
  4097. for (i = 0; i < rhvm->vmcount; i++) {
  4098. src_perms |= BIT(rhvm->vmid[i]);
  4099. }
  4100. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4101. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4102. /* Assign memory back to HLOS in case of errors */
  4103. hyp_err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4104. &src_perms, &dst_perms, 1);
  4105. if (hyp_err)
  4106. ADSPRPC_WARN(
  4107. "rh hyp unassign failed with %d for phys 0x%llx of size %zu\n",
  4108. hyp_err, phys, size);
  4109. }
  4110. mutex_lock(&fl->map_mutex);
  4111. fastrpc_mmap_free(mem, 0);
  4112. mutex_unlock(&fl->map_mutex);
  4113. }
  4114. return err;
  4115. }
  4116. /*
  4117. * This function sets fastrpc service location name
  4118. * based on ioctl init flags.
  4119. */
  4120. static void fastrpc_set_servloc(struct fastrpc_file *fl,
  4121. struct fastrpc_ioctl_init *init)
  4122. {
  4123. char *proc_name = NULL;
  4124. int err = 0;
  4125. if (init->flags == FASTRPC_INIT_ATTACH_SENSORS) {
  4126. if (fl->cid == ADSP_DOMAIN_ID)
  4127. fl->servloc_name =
  4128. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME;
  4129. else if (fl->cid == SDSP_DOMAIN_ID)
  4130. fl->servloc_name =
  4131. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME;
  4132. } else if (init->flags == FASTRPC_INIT_CREATE_STATIC) {
  4133. if (!init->filelen)
  4134. goto bail;
  4135. proc_name = kzalloc(init->filelen + 1, GFP_KERNEL);
  4136. VERIFY(err, !IS_ERR_OR_NULL(proc_name));
  4137. if (err) {
  4138. err = -ENOMEM;
  4139. goto bail;
  4140. }
  4141. err = copy_from_user((void *)proc_name,
  4142. (void __user *)init->file, init->filelen);
  4143. if (err) {
  4144. err = -EFAULT;
  4145. goto bail;
  4146. }
  4147. if (!strcmp(proc_name, "audiopd"))
  4148. fl->servloc_name = AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME;
  4149. }
  4150. bail:
  4151. kfree(proc_name);
  4152. }
  4153. int fastrpc_init_process(struct fastrpc_file *fl,
  4154. struct fastrpc_ioctl_init_attrs *uproc)
  4155. {
  4156. int err = 0;
  4157. struct fastrpc_ioctl_init *init = &uproc->init;
  4158. int cid = fl->cid;
  4159. struct fastrpc_apps *me = &gfa;
  4160. struct fastrpc_channel_ctx *chan = NULL;
  4161. VERIFY(err, init->filelen < INIT_FILELEN_MAX
  4162. && init->memlen <= INIT_MEMLEN_MAX_DYNAMIC);
  4163. if (err) {
  4164. ADSPRPC_ERR(
  4165. "file size 0x%x or init memory 0x%x is more than max allowed file size 0x%x or init len 0x%x\n",
  4166. init->filelen, init->memlen,
  4167. INIT_FILELEN_MAX, INIT_MEMLEN_MAX_DYNAMIC);
  4168. err = -EFBIG;
  4169. goto bail;
  4170. }
  4171. VERIFY(err, VALID_FASTRPC_CID(cid));
  4172. if (err) {
  4173. err = -ECHRNG;
  4174. goto bail;
  4175. }
  4176. chan = &me->channel[cid];
  4177. if (chan->unsigned_support && fl->dev_minor == MINOR_NUM_DEV) {
  4178. /* Make sure third party applications */
  4179. /* can spawn only unsigned PD when */
  4180. /* channel configured as secure. */
  4181. if (chan->secure && !(fl->is_unsigned_pd)) {
  4182. err = -ECONNREFUSED;
  4183. goto bail;
  4184. }
  4185. }
  4186. if (fl->sharedcb == 1) {
  4187. // Only attach sensors pd use cases can share CB
  4188. VERIFY(err, init->flags == FASTRPC_INIT_ATTACH_SENSORS);
  4189. if (err) {
  4190. err = -EACCES;
  4191. goto bail;
  4192. }
  4193. }
  4194. fastrpc_set_servloc(fl, init);
  4195. err = fastrpc_set_tvm_remote_domain(fl, init);
  4196. if (err)
  4197. goto bail;
  4198. err = fastrpc_channel_open(fl, init->flags);
  4199. if (err)
  4200. goto bail;
  4201. fl->proc_flags = init->flags;
  4202. switch (init->flags) {
  4203. case FASTRPC_INIT_ATTACH:
  4204. case FASTRPC_INIT_ATTACH_SENSORS:
  4205. err = fastrpc_init_attach_process(fl, init);
  4206. break;
  4207. case FASTRPC_INIT_CREATE:
  4208. err = fastrpc_init_create_dynamic_process(fl, uproc);
  4209. break;
  4210. case FASTRPC_INIT_CREATE_STATIC:
  4211. err = fastrpc_init_create_static_process(fl, init);
  4212. break;
  4213. default:
  4214. err = -ENOTTY;
  4215. break;
  4216. }
  4217. if (err)
  4218. goto bail;
  4219. fl->dsp_proc_init = 1;
  4220. VERIFY(err, 0 == (err = fastrpc_device_create(fl)));
  4221. if (err)
  4222. goto bail;
  4223. bail:
  4224. return err;
  4225. }
  4226. static int fastrpc_send_cpuinfo_to_dsp(struct fastrpc_file *fl)
  4227. {
  4228. int err = 0;
  4229. uint64_t cpuinfo = 0;
  4230. struct fastrpc_apps *me = &gfa;
  4231. struct fastrpc_ioctl_invoke_async ioctl;
  4232. remote_arg_t ra[1];
  4233. int cid = -1;
  4234. if (!fl) {
  4235. err = -EBADF;
  4236. goto bail;
  4237. }
  4238. cid = fl->cid;
  4239. VERIFY(err, VALID_FASTRPC_CID(cid));
  4240. if (err) {
  4241. err = -ECHRNG;
  4242. ADSPRPC_ERR(
  4243. "invalid channel 0x%zx set for session\n",
  4244. cid);
  4245. goto bail;
  4246. }
  4247. cpuinfo = me->channel[cid].cpuinfo_todsp;
  4248. /* return success if already updated to remote processor */
  4249. if (me->channel[cid].cpuinfo_status)
  4250. return 0;
  4251. ra[0].buf.pv = (void *)&cpuinfo;
  4252. ra[0].buf.len = sizeof(cpuinfo);
  4253. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
  4254. ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
  4255. ioctl.inv.pra = ra;
  4256. ioctl.fds = NULL;
  4257. ioctl.attrs = NULL;
  4258. ioctl.crc = NULL;
  4259. ioctl.perf_kernel = NULL;
  4260. ioctl.perf_dsp = NULL;
  4261. ioctl.job = NULL;
  4262. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4263. if (!err)
  4264. me->channel[cid].cpuinfo_status = true;
  4265. bail:
  4266. return err;
  4267. }
  4268. int fastrpc_get_info_from_dsp(struct fastrpc_file *fl,
  4269. uint32_t *dsp_attr_buf,
  4270. uint32_t dsp_attr_buf_len,
  4271. uint32_t domain)
  4272. {
  4273. int err = 0;
  4274. struct fastrpc_ioctl_invoke_async ioctl;
  4275. remote_arg_t ra[2];
  4276. dsp_attr_buf[0] = 0; // Capability filled in userspace
  4277. // Fastrpc to modem not supported
  4278. if (domain == MDSP_DOMAIN_ID)
  4279. goto bail;
  4280. err = fastrpc_channel_open(fl, FASTRPC_INIT_NO_CREATE);
  4281. if (err)
  4282. goto bail;
  4283. ra[0].buf.pv = (void *)&dsp_attr_buf_len;
  4284. ra[0].buf.len = sizeof(dsp_attr_buf_len);
  4285. ra[1].buf.pv = (void *)(&dsp_attr_buf[1]);
  4286. ra[1].buf.len = dsp_attr_buf_len * sizeof(uint32_t);
  4287. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
  4288. ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 1);
  4289. ioctl.inv.pra = ra;
  4290. ioctl.fds = NULL;
  4291. ioctl.attrs = NULL;
  4292. ioctl.crc = NULL;
  4293. ioctl.perf_kernel = NULL;
  4294. ioctl.perf_dsp = NULL;
  4295. ioctl.job = NULL;
  4296. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4297. bail:
  4298. if (err)
  4299. ADSPRPC_ERR("could not obtain dsp information, err val %d\n",
  4300. err);
  4301. return err;
  4302. }
  4303. int fastrpc_get_info_from_kernel(
  4304. struct fastrpc_ioctl_capability *cap,
  4305. struct fastrpc_file *fl)
  4306. {
  4307. int err = 0;
  4308. uint32_t domain = cap->domain, attribute_ID = cap->attribute_ID;
  4309. uint32_t async_capability = 0;
  4310. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  4311. VERIFY(err, domain < NUM_CHANNELS);
  4312. if (err) {
  4313. err = -ECHRNG;
  4314. goto bail;
  4315. }
  4316. /*
  4317. * Check if number of attribute IDs obtained from userspace
  4318. * is less than the number of attribute IDs supported by
  4319. * kernel
  4320. */
  4321. if (attribute_ID >= FASTRPC_MAX_ATTRIBUTES) {
  4322. err = -EOVERFLOW;
  4323. goto bail;
  4324. }
  4325. dsp_cap_ptr = &gcinfo[domain].dsp_cap_kernel;
  4326. if (attribute_ID >= FASTRPC_MAX_DSP_ATTRIBUTES) {
  4327. // Driver capability, pass it to user
  4328. memcpy(&cap->capability,
  4329. &kernel_capabilities[attribute_ID -
  4330. FASTRPC_MAX_DSP_ATTRIBUTES],
  4331. sizeof(cap->capability));
  4332. } else if (!dsp_cap_ptr->is_cached) {
  4333. /*
  4334. * Information not on kernel, query device for information
  4335. * and cache on kernel
  4336. */
  4337. err = fastrpc_get_info_from_dsp(fl,
  4338. dsp_cap_ptr->dsp_attributes,
  4339. FASTRPC_MAX_DSP_ATTRIBUTES - 1,
  4340. domain);
  4341. if (err)
  4342. goto bail;
  4343. /* Async capability support depends on both kernel and DSP */
  4344. async_capability = IS_ASYNC_FASTRPC_AVAILABLE &&
  4345. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP];
  4346. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP]
  4347. = async_capability;
  4348. memcpy(&cap->capability,
  4349. &dsp_cap_ptr->dsp_attributes[attribute_ID],
  4350. sizeof(cap->capability));
  4351. dsp_cap_ptr->is_cached = 1;
  4352. } else {
  4353. // Information on Kernel, pass it to user
  4354. memcpy(&cap->capability,
  4355. &dsp_cap_ptr->dsp_attributes[attribute_ID],
  4356. sizeof(cap->capability));
  4357. }
  4358. bail:
  4359. return err;
  4360. }
  4361. static int fastrpc_release_current_dsp_process(struct fastrpc_file *fl)
  4362. {
  4363. int err = 0;
  4364. struct fastrpc_ioctl_invoke_async ioctl;
  4365. remote_arg_t ra[1];
  4366. int tgid = 0;
  4367. int cid = -1;
  4368. unsigned long irq_flags = 0;
  4369. if (!fl) {
  4370. err = -EBADF;
  4371. goto bail;
  4372. }
  4373. cid = fl->cid;
  4374. VERIFY(err, VALID_FASTRPC_CID(cid));
  4375. if (err) {
  4376. err = -ECHRNG;
  4377. goto bail;
  4378. }
  4379. VERIFY(err, fl->sctx != NULL);
  4380. if (err) {
  4381. err = -EBADR;
  4382. goto bail;
  4383. }
  4384. err = verify_transport_device(cid, fl->tvm_remote_domain);
  4385. if (err)
  4386. goto bail;
  4387. VERIFY(err, fl->apps->channel[cid].subsystemstate != SUBSYSTEM_RESTARTING);
  4388. if (err) {
  4389. wait_for_completion(&fl->shutdown);
  4390. err = -ECONNRESET;
  4391. goto bail;
  4392. }
  4393. /* Send unique fastrpc process ID to dsp */
  4394. tgid = fl->tgid_frpc;
  4395. ra[0].buf.pv = (void *)&tgid;
  4396. ra[0].buf.len = sizeof(tgid);
  4397. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4398. ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
  4399. ioctl.inv.pra = ra;
  4400. ioctl.fds = NULL;
  4401. ioctl.attrs = NULL;
  4402. ioctl.crc = NULL;
  4403. ioctl.perf_kernel = NULL;
  4404. ioctl.perf_dsp = NULL;
  4405. ioctl.job = NULL;
  4406. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4407. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_INIT;
  4408. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4409. /*
  4410. * Pass 2 for "kernel" arg to send kernel msg to DSP
  4411. * with non-zero msg PID for the DSP to directly use
  4412. * that info to kill the remote process.
  4413. */
  4414. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4415. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_NONZERO_PID, &ioctl)));
  4416. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4417. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_COMPLETE;
  4418. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4419. if (err && fl->dsp_proc_init)
  4420. ADSPRPC_ERR(
  4421. "releasing DSP process failed with %d (0x%x) for %s\n",
  4422. err, err, current->comm);
  4423. bail:
  4424. if (err && fl && fl->apps) {
  4425. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4426. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_ERROR;
  4427. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4428. }
  4429. return err;
  4430. }
  4431. static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
  4432. uint32_t flags, uintptr_t va, uint64_t phys,
  4433. size_t size, uintptr_t *raddr)
  4434. {
  4435. struct fastrpc_ioctl_invoke_async ioctl;
  4436. struct smq_phy_page page;
  4437. remote_arg_t ra[4];
  4438. int err = 0;
  4439. struct {
  4440. int pid;
  4441. int fd;
  4442. int offset;
  4443. uint32_t flags;
  4444. uint64_t vaddrin;
  4445. int num;
  4446. int data_len;
  4447. } inargs;
  4448. struct {
  4449. uint64_t vaddrout;
  4450. } routargs;
  4451. /* Send unique fastrpc process ID to dsp */
  4452. inargs.pid = fl->tgid_frpc;
  4453. inargs.fd = fd;
  4454. inargs.offset = offset;
  4455. inargs.vaddrin = (uintptr_t)va;
  4456. inargs.flags = flags;
  4457. inargs.num = sizeof(page);
  4458. inargs.data_len = 0;
  4459. ra[0].buf.pv = (void *)&inargs;
  4460. ra[0].buf.len = sizeof(inargs);
  4461. page.addr = phys;
  4462. page.size = size;
  4463. ra[1].buf.pv = (void *)&page;
  4464. ra[1].buf.len = sizeof(page);
  4465. ra[2].buf.pv = (void *)&page;
  4466. ra[2].buf.len = 0;
  4467. ra[3].buf.pv = (void *)&routargs;
  4468. ra[3].buf.len = sizeof(routargs);
  4469. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4470. ioctl.inv.sc = REMOTE_SCALARS_MAKE(10, 3, 1);
  4471. ioctl.inv.pra = ra;
  4472. ioctl.fds = NULL;
  4473. ioctl.attrs = NULL;
  4474. ioctl.crc = NULL;
  4475. ioctl.perf_kernel = NULL;
  4476. ioctl.perf_dsp = NULL;
  4477. ioctl.job = NULL;
  4478. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4479. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4480. if (err)
  4481. goto bail;
  4482. if (raddr)
  4483. *raddr = (uintptr_t)routargs.vaddrout;
  4484. bail:
  4485. return err;
  4486. }
  4487. static int fastrpc_mem_unmap_to_dsp(struct fastrpc_file *fl, int fd,
  4488. uint32_t flags, uintptr_t va,
  4489. uint64_t phys, size_t size)
  4490. {
  4491. struct fastrpc_ioctl_invoke_async ioctl;
  4492. remote_arg_t ra[1];
  4493. int err = 0;
  4494. struct {
  4495. int pid;
  4496. int fd;
  4497. uint64_t vaddrin;
  4498. uint64_t len;
  4499. } inargs;
  4500. /* Send unique fastrpc process ID to dsp */
  4501. inargs.pid = fl->tgid_frpc;
  4502. inargs.fd = fd;
  4503. inargs.vaddrin = (uint64_t)va;
  4504. inargs.len = (uint64_t)size;
  4505. ra[0].buf.pv = (void *)&inargs;
  4506. ra[0].buf.len = sizeof(inargs);
  4507. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4508. ioctl.inv.sc = REMOTE_SCALARS_MAKE(11, 1, 0);
  4509. ioctl.inv.pra = ra;
  4510. ioctl.fds = NULL;
  4511. ioctl.attrs = NULL;
  4512. ioctl.crc = NULL;
  4513. ioctl.perf_kernel = NULL;
  4514. ioctl.perf_dsp = NULL;
  4515. ioctl.job = NULL;
  4516. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4517. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4518. if (err)
  4519. goto bail;
  4520. bail:
  4521. return err;
  4522. }
  4523. static int fastrpc_unmap_on_dsp(struct fastrpc_file *fl,
  4524. uintptr_t raddr, uint64_t phys, size_t size, uint32_t flags)
  4525. {
  4526. struct fastrpc_ioctl_invoke_async ioctl;
  4527. remote_arg_t ra[1] = {};
  4528. int err = 0;
  4529. struct {
  4530. int pid;
  4531. uintptr_t vaddrout;
  4532. size_t size;
  4533. } inargs;
  4534. /* Send unique fastrpc process ID to dsp */
  4535. inargs.pid = fl->tgid_frpc;
  4536. inargs.size = size;
  4537. inargs.vaddrout = raddr;
  4538. ra[0].buf.pv = (void *)&inargs;
  4539. ra[0].buf.len = sizeof(inargs);
  4540. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4541. if (fl->apps->compat)
  4542. ioctl.inv.sc = REMOTE_SCALARS_MAKE(5, 1, 0);
  4543. else
  4544. ioctl.inv.sc = REMOTE_SCALARS_MAKE(3, 1, 0);
  4545. ioctl.inv.pra = ra;
  4546. ioctl.fds = NULL;
  4547. ioctl.attrs = NULL;
  4548. ioctl.crc = NULL;
  4549. ioctl.perf_kernel = NULL;
  4550. ioctl.perf_dsp = NULL;
  4551. ioctl.job = NULL;
  4552. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4553. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4554. if (err)
  4555. goto bail;
  4556. bail:
  4557. return err;
  4558. }
  4559. static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags,
  4560. uintptr_t va, uint64_t phys,
  4561. size_t size, int refs, uintptr_t *raddr)
  4562. {
  4563. struct fastrpc_ioctl_invoke_async ioctl;
  4564. struct fastrpc_apps *me = &gfa;
  4565. struct smq_phy_page page;
  4566. int num = 1;
  4567. remote_arg_t ra[3];
  4568. int err = 0;
  4569. struct {
  4570. int pid;
  4571. uint32_t flags;
  4572. uintptr_t vaddrin;
  4573. int num;
  4574. } inargs;
  4575. struct {
  4576. uintptr_t vaddrout;
  4577. } routargs;
  4578. int cid = -1;
  4579. if (!fl) {
  4580. err = -EBADF;
  4581. goto bail;
  4582. }
  4583. cid = fl->cid;
  4584. /* Send unique fastrpc process ID to dsp */
  4585. inargs.pid = fl->tgid_frpc;
  4586. inargs.vaddrin = (uintptr_t)va;
  4587. inargs.flags = flags;
  4588. inargs.num = fl->apps->compat ? num * sizeof(page) : num;
  4589. ra[0].buf.pv = (void *)&inargs;
  4590. ra[0].buf.len = sizeof(inargs);
  4591. page.addr = phys;
  4592. page.size = size;
  4593. ra[1].buf.pv = (void *)&page;
  4594. ra[1].buf.len = num * sizeof(page);
  4595. ra[2].buf.pv = (void *)&routargs;
  4596. ra[2].buf.len = sizeof(routargs);
  4597. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4598. if (fl->apps->compat)
  4599. ioctl.inv.sc = REMOTE_SCALARS_MAKE(4, 2, 1);
  4600. else
  4601. ioctl.inv.sc = REMOTE_SCALARS_MAKE(2, 2, 1);
  4602. ioctl.inv.pra = ra;
  4603. ioctl.fds = NULL;
  4604. ioctl.attrs = NULL;
  4605. ioctl.crc = NULL;
  4606. ioctl.perf_kernel = NULL;
  4607. ioctl.perf_dsp = NULL;
  4608. ioctl.job = NULL;
  4609. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4610. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4611. *raddr = (uintptr_t)routargs.vaddrout;
  4612. if (err)
  4613. goto bail;
  4614. if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4615. VERIFY(err, VALID_FASTRPC_CID(cid));
  4616. if (err) {
  4617. err = -ECHRNG;
  4618. ADSPRPC_ERR(
  4619. "invalid channel 0x%zx set for session\n",
  4620. cid);
  4621. goto bail;
  4622. }
  4623. }
  4624. if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR
  4625. && me->channel[cid].rhvm.vmid && refs == 1) {
  4626. struct secure_vm *rhvm = &me->channel[cid].rhvm;
  4627. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  4628. struct qcom_scm_vmperm *dst_perms;
  4629. uint32_t i = 0;
  4630. VERIFY(err, NULL != (dst_perms = kcalloc(rhvm->vmcount,
  4631. sizeof(struct qcom_scm_vmperm), GFP_KERNEL)));
  4632. if (err)
  4633. goto bail;
  4634. for (i = 0; i < rhvm->vmcount; i++) {
  4635. dst_perms[i].vmid = rhvm->vmid[i];
  4636. dst_perms[i].perm = rhvm->vmperm[i];
  4637. }
  4638. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4639. &src_perms, dst_perms, rhvm->vmcount);
  4640. kfree(dst_perms);
  4641. if (err) {
  4642. int unmap_err = 0;
  4643. ADSPRPC_ERR(
  4644. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  4645. err, phys, size);
  4646. err = -EADDRNOTAVAIL;
  4647. unmap_err = fastrpc_unmap_on_dsp(fl,
  4648. *raddr, phys, size, flags);
  4649. if (unmap_err) {
  4650. ADSPRPC_ERR(
  4651. "failed to unmap %d for phys 0x%llx, size %zd\n",
  4652. unmap_err, phys, size);
  4653. }
  4654. goto bail;
  4655. }
  4656. }
  4657. bail:
  4658. return err;
  4659. }
  4660. static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl, uint64_t phys,
  4661. size_t size, uint32_t flags, int locked)
  4662. {
  4663. int err = 0;
  4664. int tgid = 0;
  4665. struct fastrpc_apps *me = &gfa;
  4666. int cid = -1;
  4667. struct fastrpc_ioctl_invoke_async ioctl;
  4668. remote_arg_t ra[2];
  4669. struct {
  4670. uint8_t skey;
  4671. } routargs;
  4672. if (!fl) {
  4673. err = -EBADF;
  4674. goto bail;
  4675. }
  4676. cid = fl->cid;
  4677. VERIFY(err, VALID_FASTRPC_CID(cid));
  4678. if (err) {
  4679. err = -ECHRNG;
  4680. ADSPRPC_ERR(
  4681. "invalid channel 0x%zx set for session\n",
  4682. cid);
  4683. goto bail;
  4684. }
  4685. /* Send unique fastrpc process ID to dsp */
  4686. tgid = fl->tgid_frpc;
  4687. ra[0].buf.pv = (void *)&tgid;
  4688. ra[0].buf.len = sizeof(tgid);
  4689. ra[1].buf.pv = (void *)&routargs;
  4690. ra[1].buf.len = sizeof(routargs);
  4691. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4692. ioctl.inv.sc = REMOTE_SCALARS_MAKE(9, 1, 1);
  4693. ioctl.inv.pra = ra;
  4694. ioctl.fds = NULL;
  4695. ioctl.attrs = NULL;
  4696. ioctl.crc = NULL;
  4697. ioctl.perf_kernel = NULL;
  4698. ioctl.perf_dsp = NULL;
  4699. ioctl.job = NULL;
  4700. if (locked) {
  4701. mutex_unlock(&fl->map_mutex);
  4702. mutex_unlock(&me->channel[cid].smd_mutex);
  4703. }
  4704. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4705. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4706. if (locked) {
  4707. mutex_lock(&me->channel[cid].smd_mutex);
  4708. mutex_lock(&fl->map_mutex);
  4709. }
  4710. if (err)
  4711. goto bail;
  4712. bail:
  4713. return err;
  4714. }
  4715. static int fastrpc_munmap_rh(uint64_t phys, size_t size,
  4716. uint32_t flags)
  4717. {
  4718. int err = 0;
  4719. struct fastrpc_apps *me = &gfa;
  4720. struct secure_vm *rhvm = &me->channel[RH_CID].rhvm;
  4721. if ((rhvm->vmid)
  4722. && (me->channel[RH_CID].in_hib == 0)) {
  4723. u64 src_perms = 0;
  4724. struct qcom_scm_vmperm dst_perms = {0};
  4725. uint32_t i = 0;
  4726. for (i = 0; i < rhvm->vmcount; i++) {
  4727. src_perms |= BIT(rhvm->vmid[i]);
  4728. }
  4729. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4730. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4731. err = qcom_scm_assign_mem(phys,
  4732. (uint64_t)size, &src_perms, &dst_perms, 1);
  4733. if (err) {
  4734. ADSPRPC_ERR(
  4735. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  4736. err, phys, size);
  4737. err = -EADDRNOTAVAIL;
  4738. return err;
  4739. }
  4740. }
  4741. return err;
  4742. }
  4743. static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, uintptr_t raddr,
  4744. uint64_t phys, size_t size, uint32_t flags)
  4745. {
  4746. int err = 0;
  4747. VERIFY(err, 0 == (err = fastrpc_unmap_on_dsp(fl, raddr, phys,
  4748. size, flags)));
  4749. if (err)
  4750. goto bail;
  4751. if (flags == ADSP_MMAP_HEAP_ADDR) {
  4752. VERIFY(err, !(err = fastrpc_munmap_on_dsp_rh(fl, phys,
  4753. size, flags, 0)));
  4754. if (err)
  4755. goto bail;
  4756. } else if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4757. VERIFY(err, !(err = fastrpc_munmap_rh(phys,
  4758. size, flags)));
  4759. if (err)
  4760. goto bail;
  4761. }
  4762. bail:
  4763. return err;
  4764. }
  4765. static int fastrpc_mmap_dump(struct fastrpc_mmap *map, struct fastrpc_file *fl, int locked, bool dump_req)
  4766. {
  4767. struct fastrpc_mmap *match = map;
  4768. int err = 0, ret = 0;
  4769. struct fastrpc_apps *me = &gfa;
  4770. struct qcom_dump_segment ramdump_segments_rh;
  4771. struct list_head head;
  4772. unsigned long irq_flags = 0;
  4773. if (map->is_persistent && map->in_use) {
  4774. struct secure_vm *rhvm = &me->channel[RH_CID].rhvm;
  4775. uint64_t phys = map->phys;
  4776. size_t size = map->size;
  4777. //scm assign it back to HLOS
  4778. if (rhvm->vmid) {
  4779. u64 src_perms = 0;
  4780. struct qcom_scm_vmperm dst_perms = {0};
  4781. uint32_t i = 0;
  4782. for (i = 0; i < rhvm->vmcount; i++) {
  4783. src_perms |= BIT(rhvm->vmid[i]);
  4784. }
  4785. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4786. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4787. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4788. &src_perms, &dst_perms, 1);
  4789. }
  4790. if (err) {
  4791. ADSPRPC_ERR(
  4792. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  4793. err, phys, size);
  4794. err = -EADDRNOTAVAIL;
  4795. return err;
  4796. }
  4797. spin_lock_irqsave(&me->hlock, irq_flags);
  4798. map->in_use = false;
  4799. /*
  4800. * decrementing refcount for persistent mappings
  4801. * as incrementing it in fastrpc_get_persistent_map
  4802. */
  4803. map->refs--;
  4804. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4805. }
  4806. if (!match->is_persistent) {
  4807. if (match->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4808. err = fastrpc_munmap_rh(match->phys,
  4809. match->size, match->flags);
  4810. } else if (match->flags == ADSP_MMAP_HEAP_ADDR) {
  4811. if (fl)
  4812. err = fastrpc_munmap_on_dsp_rh(fl, match->phys,
  4813. match->size, match->flags, 0);
  4814. else {
  4815. pr_err("Cannot communicate with DSP, ADSP is down\n");
  4816. fastrpc_mmap_add(match);
  4817. }
  4818. }
  4819. if (err)
  4820. return err;
  4821. }
  4822. if (dump_req) {
  4823. memset(&ramdump_segments_rh, 0, sizeof(ramdump_segments_rh));
  4824. ramdump_segments_rh.da = match->phys;
  4825. ramdump_segments_rh.va = (void *)page_address((struct page *)match->va);
  4826. ramdump_segments_rh.size = match->size;
  4827. INIT_LIST_HEAD(&head);
  4828. list_add(&ramdump_segments_rh.node, &head);
  4829. if (me->dev && dump_enabled()) {
  4830. ret = qcom_elf_dump(&head, me->dev, ELF_CLASS);
  4831. if (ret < 0)
  4832. pr_err("adsprpc: %s: unable to dump heap (err %d)\n",
  4833. __func__, ret);
  4834. }
  4835. }
  4836. if (!match->is_persistent) {
  4837. if (!locked && fl)
  4838. mutex_lock(&fl->map_mutex);
  4839. fastrpc_mmap_free(match, 0);
  4840. if (!locked && fl)
  4841. mutex_unlock(&fl->map_mutex);
  4842. }
  4843. return 0;
  4844. }
  4845. static int fastrpc_dsp_restart_handler(struct fastrpc_file *fl, int locked, bool dump_req)
  4846. {
  4847. struct fastrpc_mmap *match = NULL, *map = NULL;
  4848. struct hlist_node *n = NULL;
  4849. int err = 0;
  4850. struct fastrpc_apps *me = &gfa;
  4851. struct list_head head;
  4852. unsigned long irq_flags = 0;
  4853. INIT_LIST_HEAD(&head);
  4854. if (fl) {
  4855. VERIFY(err, fl->cid == RH_CID);
  4856. if (err) {
  4857. err = -EBADR;
  4858. goto bail;
  4859. }
  4860. }
  4861. do {
  4862. match = NULL;
  4863. spin_lock_irqsave(&me->hlock, irq_flags);
  4864. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  4865. if (!map->is_dumped && (!fl ||
  4866. (fl && map->servloc_name && fl->servloc_name &&
  4867. !strcmp(map->servloc_name, fl->servloc_name)))) {
  4868. map->is_dumped = true;
  4869. match = map;
  4870. if (!match->is_persistent)
  4871. hlist_del_init(&map->hn);
  4872. break;
  4873. }
  4874. }
  4875. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4876. if (match)
  4877. err = fastrpc_mmap_dump(match, fl, locked, dump_req);
  4878. } while (match && !err);
  4879. bail:
  4880. if (err && match) {
  4881. if (!locked && fl)
  4882. mutex_lock(&fl->map_mutex);
  4883. fastrpc_mmap_add(match);
  4884. if (!locked && fl)
  4885. mutex_unlock(&fl->map_mutex);
  4886. }
  4887. spin_lock_irqsave(&me->hlock, irq_flags);
  4888. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  4889. if (map->is_dumped && ((!fl && map->servloc_name) ||
  4890. (fl && map->servloc_name && fl->servloc_name &&
  4891. !strcmp(map->servloc_name, fl->servloc_name))))
  4892. map->is_dumped = false;
  4893. }
  4894. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4895. return err;
  4896. }
  4897. static int fastrpc_mmap_remove_pdr(struct fastrpc_file *fl)
  4898. {
  4899. struct fastrpc_apps *me = &gfa;
  4900. int session = 0, err = 0, cid = -1;
  4901. if (!fl) {
  4902. err = -EBADF;
  4903. goto bail;
  4904. }
  4905. err = fastrpc_get_spd_session(fl->servloc_name,
  4906. &session, &cid);
  4907. if (err)
  4908. goto bail;
  4909. VERIFY(err, cid == fl->cid);
  4910. if (err) {
  4911. err = -EBADR;
  4912. goto bail;
  4913. }
  4914. if (atomic_read(&me->channel[cid].spd[session].ispdup) == 0) {
  4915. err = -ENOTCONN;
  4916. goto bail;
  4917. }
  4918. if (me->channel[cid].spd[session].pdrcount !=
  4919. me->channel[cid].spd[session].prevpdrcount) {
  4920. err = fastrpc_dsp_restart_handler(fl, 0, false);
  4921. if (err)
  4922. ADSPRPC_WARN("failed to unmap remote heap (err %d)\n",
  4923. err);
  4924. me->channel[cid].spd[session].prevpdrcount =
  4925. me->channel[cid].spd[session].pdrcount;
  4926. }
  4927. bail:
  4928. return err;
  4929. }
  4930. static inline void get_fastrpc_ioctl_mmap_64(
  4931. struct fastrpc_ioctl_mmap_64 *mmap64,
  4932. struct fastrpc_ioctl_mmap *immap)
  4933. {
  4934. immap->fd = mmap64->fd;
  4935. immap->flags = mmap64->flags;
  4936. immap->vaddrin = (uintptr_t)mmap64->vaddrin;
  4937. immap->size = mmap64->size;
  4938. }
  4939. static inline void put_fastrpc_ioctl_mmap_64(
  4940. struct fastrpc_ioctl_mmap_64 *mmap64,
  4941. struct fastrpc_ioctl_mmap *immap)
  4942. {
  4943. mmap64->vaddrout = (uint64_t)immap->vaddrout;
  4944. }
  4945. static inline void get_fastrpc_ioctl_munmap_64(
  4946. struct fastrpc_ioctl_munmap_64 *munmap64,
  4947. struct fastrpc_ioctl_munmap *imunmap)
  4948. {
  4949. imunmap->vaddrout = (uintptr_t)munmap64->vaddrout;
  4950. imunmap->size = munmap64->size;
  4951. }
  4952. int fastrpc_internal_munmap(struct fastrpc_file *fl,
  4953. struct fastrpc_ioctl_munmap *ud)
  4954. {
  4955. int err = 0;
  4956. struct fastrpc_mmap *map = NULL;
  4957. struct fastrpc_buf *rbuf = NULL, *free = NULL;
  4958. struct hlist_node *n;
  4959. VERIFY(err, fl->dsp_proc_init == 1);
  4960. if (err) {
  4961. ADSPRPC_ERR(
  4962. "user application %s trying to unmap without initialization\n",
  4963. current->comm);
  4964. err = -EHOSTDOWN;
  4965. return err;
  4966. }
  4967. mutex_lock(&fl->internal_map_mutex);
  4968. spin_lock(&fl->hlock);
  4969. hlist_for_each_entry_safe(rbuf, n, &fl->remote_bufs, hn_rem) {
  4970. if (rbuf->raddr && ((rbuf->flags == ADSP_MMAP_ADD_PAGES) ||
  4971. (rbuf->flags == ADSP_MMAP_ADD_PAGES_LLC))) {
  4972. if ((rbuf->raddr == ud->vaddrout) &&
  4973. (rbuf->size == ud->size)) {
  4974. free = rbuf;
  4975. break;
  4976. }
  4977. }
  4978. }
  4979. spin_unlock(&fl->hlock);
  4980. if (free) {
  4981. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, free->raddr,
  4982. free->phys, free->size, free->flags)));
  4983. if (err)
  4984. goto bail;
  4985. fastrpc_buf_free(rbuf, 0);
  4986. mutex_unlock(&fl->internal_map_mutex);
  4987. return err;
  4988. }
  4989. mutex_lock(&fl->map_mutex);
  4990. VERIFY(err, !(err = fastrpc_mmap_remove(fl, -1, ud->vaddrout,
  4991. ud->size, &map)));
  4992. mutex_unlock(&fl->map_mutex);
  4993. if (err)
  4994. goto bail;
  4995. VERIFY(err, map != NULL);
  4996. if (err) {
  4997. err = -EINVAL;
  4998. goto bail;
  4999. }
  5000. if (!map->is_persistent) {
  5001. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, map->raddr,
  5002. map->phys, map->size, map->flags)));
  5003. }
  5004. if (err)
  5005. goto bail;
  5006. mutex_lock(&fl->map_mutex);
  5007. fastrpc_mmap_free(map, 0);
  5008. mutex_unlock(&fl->map_mutex);
  5009. bail:
  5010. if (err && map) {
  5011. mutex_lock(&fl->map_mutex);
  5012. fastrpc_mmap_add(map);
  5013. mutex_unlock(&fl->map_mutex);
  5014. }
  5015. mutex_unlock(&fl->internal_map_mutex);
  5016. return err;
  5017. }
  5018. /*
  5019. * fastrpc_internal_munmap_fd can only be used for buffers
  5020. * mapped with persist attributes. This can only be called
  5021. * once for any persist buffer
  5022. */
  5023. static int fastrpc_internal_munmap_fd(struct fastrpc_file *fl,
  5024. struct fastrpc_ioctl_munmap_fd *ud)
  5025. {
  5026. int err = 0;
  5027. struct fastrpc_mmap *map = NULL;
  5028. VERIFY(err, (fl && ud));
  5029. if (err) {
  5030. err = -EINVAL;
  5031. return err;
  5032. }
  5033. VERIFY(err, fl->dsp_proc_init == 1);
  5034. if (err) {
  5035. ADSPRPC_ERR(
  5036. "user application %s trying to unmap without initialization\n",
  5037. current->comm);
  5038. err = -EHOSTDOWN;
  5039. return err;
  5040. }
  5041. mutex_lock(&fl->internal_map_mutex);
  5042. mutex_lock(&fl->map_mutex);
  5043. err = fastrpc_mmap_find(fl, ud->fd, NULL, ud->va, ud->len, 0, 0, &map);
  5044. if (err) {
  5045. ADSPRPC_ERR(
  5046. "mapping not found to unmap fd 0x%x, va 0x%llx, len 0x%x, err %d\n",
  5047. ud->fd, (unsigned long long)ud->va,
  5048. (unsigned int)ud->len, err);
  5049. mutex_unlock(&fl->map_mutex);
  5050. goto bail;
  5051. }
  5052. if (map && (map->attr & FASTRPC_ATTR_KEEP_MAP)) {
  5053. map->attr = map->attr & (~FASTRPC_ATTR_KEEP_MAP);
  5054. fastrpc_mmap_free(map, 0);
  5055. }
  5056. mutex_unlock(&fl->map_mutex);
  5057. bail:
  5058. mutex_unlock(&fl->internal_map_mutex);
  5059. return err;
  5060. }
  5061. int fastrpc_internal_mem_map(struct fastrpc_file *fl,
  5062. struct fastrpc_ioctl_mem_map *ud)
  5063. {
  5064. int err = 0;
  5065. struct fastrpc_mmap *map = NULL;
  5066. mutex_lock(&fl->internal_map_mutex);
  5067. VERIFY(err, fl->dsp_proc_init == 1);
  5068. if (err) {
  5069. pr_err("adsprpc: ERROR: %s: user application %s trying to map without initialization\n",
  5070. __func__, current->comm);
  5071. err = EBADR;
  5072. goto bail;
  5073. }
  5074. /* create SMMU mapping */
  5075. mutex_lock(&fl->map_mutex);
  5076. VERIFY(err, !(err = fastrpc_mmap_create(fl, ud->m.fd, NULL, ud->m.attrs,
  5077. ud->m.vaddrin, ud->m.length,
  5078. ud->m.flags, &map)));
  5079. mutex_unlock(&fl->map_mutex);
  5080. if (err)
  5081. goto bail;
  5082. if (map->raddr) {
  5083. err = -EEXIST;
  5084. goto bail;
  5085. }
  5086. /* create DSP mapping */
  5087. VERIFY(err, !(err = fastrpc_mem_map_to_dsp(fl, ud->m.fd, ud->m.offset,
  5088. ud->m.flags, map->va, map->phys, map->size, &map->raddr)));
  5089. if (err)
  5090. goto bail;
  5091. ud->m.vaddrout = map->raddr;
  5092. bail:
  5093. if (err) {
  5094. ADSPRPC_ERR("failed to map fd %d, len 0x%x, flags %d, map %pK, err %d\n",
  5095. ud->m.fd, ud->m.length, ud->m.flags, map, err);
  5096. if (map) {
  5097. mutex_lock(&fl->map_mutex);
  5098. fastrpc_mmap_free(map, 0);
  5099. mutex_unlock(&fl->map_mutex);
  5100. }
  5101. }
  5102. mutex_unlock(&fl->internal_map_mutex);
  5103. return err;
  5104. }
  5105. int fastrpc_internal_mem_unmap(struct fastrpc_file *fl,
  5106. struct fastrpc_ioctl_mem_unmap *ud)
  5107. {
  5108. int err = 0;
  5109. struct fastrpc_mmap *map = NULL;
  5110. size_t map_size = 0;
  5111. mutex_lock(&fl->internal_map_mutex);
  5112. VERIFY(err, fl->dsp_proc_init == 1);
  5113. if (err) {
  5114. pr_err("adsprpc: ERROR: %s: user application %s trying to map without initialization\n",
  5115. __func__, current->comm);
  5116. err = EBADR;
  5117. goto bail;
  5118. }
  5119. mutex_lock(&fl->map_mutex);
  5120. VERIFY(err, !(err = fastrpc_mmap_remove(fl, ud->um.fd,
  5121. (uintptr_t)ud->um.vaddr, ud->um.length, &map)));
  5122. mutex_unlock(&fl->map_mutex);
  5123. if (err)
  5124. goto bail;
  5125. VERIFY(err, map->flags == FASTRPC_MAP_FD ||
  5126. map->flags == FASTRPC_MAP_FD_DELAYED ||
  5127. map->flags == FASTRPC_MAP_STATIC);
  5128. if (err) {
  5129. err = -EBADMSG;
  5130. goto bail;
  5131. }
  5132. map_size = map->size;
  5133. /* remove mapping on DSP */
  5134. VERIFY(err, !(err = fastrpc_mem_unmap_to_dsp(fl, map->fd, map->flags,
  5135. map->raddr, map->phys, map->size)));
  5136. if (err)
  5137. goto bail;
  5138. /* remove SMMU mapping */
  5139. mutex_lock(&fl->map_mutex);
  5140. fastrpc_mmap_free(map, 0);
  5141. mutex_unlock(&fl->map_mutex);
  5142. map = NULL;
  5143. bail:
  5144. if (err) {
  5145. ADSPRPC_ERR(
  5146. "failed to unmap fd %d addr 0x%llx length %zu map size %zu err 0x%x\n",
  5147. ud->um.fd, ud->um.vaddr, ud->um.length, map_size, err);
  5148. /* Add back to map list in case of error to unmap on DSP */
  5149. if (map) {
  5150. mutex_lock(&fl->map_mutex);
  5151. fastrpc_mmap_add(map);
  5152. mutex_unlock(&fl->map_mutex);
  5153. }
  5154. }
  5155. mutex_unlock(&fl->internal_map_mutex);
  5156. return err;
  5157. }
  5158. int fastrpc_internal_mmap(struct fastrpc_file *fl,
  5159. struct fastrpc_ioctl_mmap *ud)
  5160. {
  5161. struct fastrpc_mmap *map = NULL;
  5162. struct fastrpc_buf *rbuf = NULL;
  5163. unsigned long dma_attr = 0;
  5164. uintptr_t raddr = 0;
  5165. int err = 0;
  5166. VERIFY(err, fl->dsp_proc_init == 1);
  5167. if (err) {
  5168. ADSPRPC_ERR(
  5169. "user application %s trying to map without initialization\n",
  5170. current->comm);
  5171. err = -EHOSTDOWN;
  5172. return err;
  5173. }
  5174. mutex_lock(&fl->internal_map_mutex);
  5175. /* Pages for unsigned PD's user-heap should be allocated in userspace */
  5176. if (((ud->flags == ADSP_MMAP_ADD_PAGES) ||
  5177. (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)) && !fl->is_unsigned_pd) {
  5178. if (ud->vaddrin) {
  5179. err = -EINVAL;
  5180. ADSPRPC_ERR(
  5181. "adding user allocated pages is not supported\n");
  5182. goto bail;
  5183. }
  5184. dma_attr = DMA_ATTR_DELAYED_UNMAP | DMA_ATTR_NO_KERNEL_MAPPING;
  5185. if (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)
  5186. dma_attr |= DMA_ATTR_SYS_CACHE_ONLY;
  5187. err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,
  5188. USERHEAP_BUF, &rbuf);
  5189. if (err)
  5190. goto bail;
  5191. err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,
  5192. rbuf->phys, rbuf->size, 0, &raddr);
  5193. if (err)
  5194. goto bail;
  5195. rbuf->raddr = raddr;
  5196. } else {
  5197. uintptr_t va_to_dsp;
  5198. if (fl->is_unsigned_pd && ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  5199. err = -EINVAL;
  5200. ADSPRPC_ERR(
  5201. "Secure memory allocation is not supported in unsigned PD");
  5202. goto bail;
  5203. }
  5204. mutex_lock(&fl->map_mutex);
  5205. VERIFY(err, !(err = fastrpc_mmap_create(fl, ud->fd, NULL, 0,
  5206. (uintptr_t)ud->vaddrin, ud->size,
  5207. ud->flags, &map)));
  5208. mutex_unlock(&fl->map_mutex);
  5209. if (err)
  5210. goto bail;
  5211. if (ud->flags == ADSP_MMAP_HEAP_ADDR ||
  5212. ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
  5213. va_to_dsp = 0;
  5214. else
  5215. va_to_dsp = (uintptr_t)map->va;
  5216. VERIFY(err, 0 == (err = fastrpc_mmap_on_dsp(fl, ud->flags,
  5217. va_to_dsp, map->phys, map->size, map->refs, &raddr)));
  5218. if (err)
  5219. goto bail;
  5220. map->raddr = raddr;
  5221. }
  5222. ud->vaddrout = raddr;
  5223. bail:
  5224. if (err) {
  5225. if (map) {
  5226. mutex_lock(&fl->map_mutex);
  5227. fastrpc_mmap_free(map, 0);
  5228. mutex_unlock(&fl->map_mutex);
  5229. }
  5230. if (!IS_ERR_OR_NULL(rbuf))
  5231. fastrpc_buf_free(rbuf, 0);
  5232. }
  5233. mutex_unlock(&fl->internal_map_mutex);
  5234. return err;
  5235. }
  5236. static void fastrpc_context_list_dtor(struct fastrpc_file *fl);
  5237. static int fastrpc_session_alloc_locked(struct fastrpc_channel_ctx *chan,
  5238. int secure, int sharedcb, int pd_type, struct fastrpc_session_ctx **session)
  5239. {
  5240. struct fastrpc_apps *me = &gfa;
  5241. uint64_t idx = 0;
  5242. int err = 0;
  5243. /*
  5244. * PD type can be either unused(DEFAULT_UNUSED) (or) if PD type
  5245. * is used, choose the context bank with matching PD type.
  5246. */
  5247. if (chan->sesscount) {
  5248. for (idx = 0; idx < chan->sesscount; ++idx) {
  5249. if (!chan->session[idx].used &&
  5250. chan->session[idx].smmu.secure == secure &&
  5251. chan->session[idx].smmu.sharedcb == sharedcb &&
  5252. (pd_type == DEFAULT_UNUSED ||
  5253. chan->session[idx].smmu.pd_type == pd_type)) {
  5254. chan->session[idx].used = 1;
  5255. break;
  5256. }
  5257. }
  5258. if (idx >= chan->sesscount) {
  5259. err = -EUSERS;
  5260. goto bail;
  5261. }
  5262. chan->session[idx].smmu.faults = 0;
  5263. } else {
  5264. VERIFY(err, me->dev != NULL);
  5265. if (err) {
  5266. err = -ENODEV;
  5267. goto bail;
  5268. }
  5269. chan->session[0].dev = me->dev;
  5270. chan->session[0].smmu.dev = me->dev;
  5271. }
  5272. *session = &chan->session[idx];
  5273. bail:
  5274. return err;
  5275. }
  5276. static void handle_remote_signal(uint64_t msg, int cid)
  5277. {
  5278. struct fastrpc_apps *me = &gfa;
  5279. uint32_t pid = msg >> 32;
  5280. uint32_t signal_id = msg & 0xffffffff;
  5281. struct fastrpc_file *fl = NULL;
  5282. struct hlist_node *n = NULL;
  5283. unsigned long irq_flags = 0;
  5284. DSPSIGNAL_VERBOSE("Received queue signal %llx: PID %u, signal %u\n", msg, pid, signal_id);
  5285. if (signal_id >= DSPSIGNAL_NUM_SIGNALS) {
  5286. ADSPRPC_ERR("Received bad signal %u for PID %u\n", signal_id, pid);
  5287. return;
  5288. }
  5289. spin_lock_irqsave(&me->hlock, irq_flags);
  5290. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  5291. /* Response from DSP contains unique fastrpc process id, use unique fastrpc process ID to compare */
  5292. if ((fl->tgid_frpc == pid) && (fl->cid == cid)) {
  5293. unsigned long fflags = 0;
  5294. spin_lock_irqsave(&fl->dspsignals_lock, fflags);
  5295. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE]) {
  5296. struct fastrpc_dspsignal *group =
  5297. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  5298. struct fastrpc_dspsignal *sig =
  5299. &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  5300. if ((sig->state == DSPSIGNAL_STATE_PENDING) ||
  5301. (sig->state == DSPSIGNAL_STATE_SIGNALED)) {
  5302. DSPSIGNAL_VERBOSE("Signaling signal %u for PID %u\n",
  5303. signal_id, pid);
  5304. trace_fastrpc_dspsignal("complete", signal_id, sig->state, 0);
  5305. complete(&sig->comp);
  5306. sig->state = DSPSIGNAL_STATE_SIGNALED;
  5307. } else if (sig->state == DSPSIGNAL_STATE_UNUSED) {
  5308. ADSPRPC_ERR("Received unknown signal %u for PID %u\n",
  5309. signal_id, pid);
  5310. }
  5311. } else {
  5312. ADSPRPC_ERR("Received unknown signal %u for PID %u\n",
  5313. signal_id, pid);
  5314. }
  5315. spin_unlock_irqrestore(&fl->dspsignals_lock, fflags);
  5316. break;
  5317. }
  5318. }
  5319. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5320. }
  5321. int fastrpc_handle_rpc_response(void *data, int len, int cid)
  5322. {
  5323. struct smq_invoke_rsp *rsp = (struct smq_invoke_rsp *)data;
  5324. struct smq_notif_rspv3 *notif = (struct smq_notif_rspv3 *)data;
  5325. struct smq_invoke_rspv2 *rspv2 = NULL;
  5326. struct smq_invoke_ctx *ctx = NULL;
  5327. struct fastrpc_apps *me = &gfa;
  5328. uint32_t index, rsp_flags = 0, early_wake_time = 0, ver = 0;
  5329. int err = 0, ignore_rsp_err = 0;
  5330. struct fastrpc_channel_ctx *chan = NULL;
  5331. unsigned long irq_flags = 0;
  5332. int64_t ns = 0;
  5333. uint64_t xo_time_in_us = 0, dspsig_msg = 0;
  5334. xo_time_in_us = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  5335. if (len == sizeof(uint64_t)) {
  5336. /*
  5337. * dspsignal message from the DSP
  5338. */
  5339. dspsig_msg = *((uint64_t *)data);
  5340. trace_fastrpc_transport_response(cid, dspsig_msg, 0, 0, 0);
  5341. handle_remote_signal(dspsig_msg, cid);
  5342. goto bail;
  5343. }
  5344. chan = &me->channel[cid];
  5345. VERIFY(err, (rsp && len >= sizeof(*rsp)));
  5346. if (err) {
  5347. err = -EINVAL;
  5348. goto bail;
  5349. }
  5350. if (notif->ctx == FASTRPC_NOTIF_CTX_RESERVED) {
  5351. VERIFY(err, (notif->type == STATUS_RESPONSE &&
  5352. len >= sizeof(*notif)));
  5353. if (err)
  5354. goto bail;
  5355. fastrpc_notif_find_process(cid, notif);
  5356. goto bail;
  5357. }
  5358. if (len >= sizeof(struct smq_invoke_rspv2))
  5359. rspv2 = (struct smq_invoke_rspv2 *)data;
  5360. if (rspv2) {
  5361. early_wake_time = rspv2->early_wake_time;
  5362. rsp_flags = rspv2->flags;
  5363. ver = rspv2->version;
  5364. }
  5365. trace_fastrpc_transport_response(cid, rsp->ctx,
  5366. rsp->retval, rsp_flags, early_wake_time);
  5367. ns = get_timestamp_in_ns();
  5368. fastrpc_update_rxmsg_buf(chan, rsp->ctx, rsp->retval,
  5369. rsp_flags, early_wake_time, ver, ns, xo_time_in_us);
  5370. index = (uint32_t)GET_TABLE_IDX_FROM_CTXID(rsp->ctx);
  5371. VERIFY(err, index < FASTRPC_CTX_MAX);
  5372. if (err)
  5373. goto bail;
  5374. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  5375. ctx = chan->ctxtable[index];
  5376. VERIFY(err, !IS_ERR_OR_NULL(ctx) &&
  5377. (ctx->ctxid == GET_CTXID_FROM_RSP_CTX(rsp->ctx)) &&
  5378. ctx->magic == FASTRPC_CTX_MAGIC);
  5379. if (err) {
  5380. /*
  5381. * Received an anticipatory COMPLETE_SIGNAL from DSP for a
  5382. * context after CPU successfully polling on memory and
  5383. * completed processing of context. Ignore the message.
  5384. * Also ignore response for a call which was already
  5385. * completed by update of poll memory and the context was
  5386. * removed from the table and possibly reused for another call.
  5387. */
  5388. ignore_rsp_err = ((rsp_flags == COMPLETE_SIGNAL) || !ctx ||
  5389. (ctx && (ctx->ctxid != GET_CTXID_FROM_RSP_CTX(rsp->ctx)))) ? 1 : 0;
  5390. goto bail_unlock;
  5391. }
  5392. if (rspv2) {
  5393. VERIFY(err, rspv2->version == FASTRPC_RSP_VERSION2);
  5394. if (err)
  5395. goto bail_unlock;
  5396. }
  5397. VERIFY(err, VALID_FASTRPC_CID(ctx->fl->cid));
  5398. if (err) {
  5399. err = -ECHRNG;
  5400. goto bail_unlock;
  5401. }
  5402. context_notify_user(ctx, rsp->retval, rsp_flags, early_wake_time);
  5403. bail_unlock:
  5404. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  5405. bail:
  5406. if (err) {
  5407. err = -ENOKEY;
  5408. if (!ignore_rsp_err)
  5409. ADSPRPC_ERR(
  5410. "invalid response data %pK, len %d from remote subsystem err %d\n",
  5411. data, len, err);
  5412. else {
  5413. err = 0;
  5414. me->duplicate_rsp_err_cnt++;
  5415. }
  5416. }
  5417. return err;
  5418. }
  5419. static int fastrpc_session_alloc_secure_memory(
  5420. struct fastrpc_channel_ctx *chan, int secure,
  5421. int sharedcb, int pd_type, struct fastrpc_session_ctx **session)
  5422. {
  5423. int err = 0;
  5424. struct fastrpc_apps *me = &gfa;
  5425. /*
  5426. * If PD type is configured for context banks,
  5427. * Use CPZ_USERPD, to allocate secure context bank type.
  5428. */
  5429. if (pd_type != DEFAULT_UNUSED && me->cb_pd_type)
  5430. pd_type = CPZ_USERPD;
  5431. mutex_lock(&chan->smd_mutex);
  5432. if (!*session)
  5433. err = fastrpc_session_alloc_locked(chan, secure, sharedcb, pd_type, session);
  5434. mutex_unlock(&chan->smd_mutex);
  5435. if (err == -EUSERS) {
  5436. ADSPRPC_WARN(
  5437. "max concurrent sessions limit (%d) already reached on %s err %d\n",
  5438. chan->sesscount, chan->subsys, err);
  5439. }
  5440. return err;
  5441. }
  5442. static void fastrpc_session_free(struct fastrpc_channel_ctx *chan,
  5443. struct fastrpc_session_ctx *session)
  5444. {
  5445. mutex_lock(&chan->smd_mutex);
  5446. session->used = 0;
  5447. mutex_unlock(&chan->smd_mutex);
  5448. }
  5449. void fastrpc_file_free(struct kref *ref)
  5450. {
  5451. struct fastrpc_file *fl = NULL;
  5452. struct hlist_node *n = NULL;
  5453. struct fastrpc_mmap *map = NULL, *lmap = NULL;
  5454. unsigned long flags;
  5455. int cid;
  5456. struct fastrpc_apps *me = &gfa;
  5457. bool is_driver_closed = false;
  5458. int err = 0;
  5459. unsigned long irq_flags = 0;
  5460. bool is_locked = false;
  5461. int i;
  5462. struct fastrpc_buf *init_mem = NULL;
  5463. fl = container_of(ref, struct fastrpc_file, refcount);
  5464. if (!fl)
  5465. return;
  5466. cid = fl->cid;
  5467. spin_lock_irqsave(&me->hlock, irq_flags);
  5468. if (fl->device) {
  5469. fl->device->dev_close = true;
  5470. if (fl->device->refs == 0) {
  5471. is_driver_closed = true;
  5472. hlist_del_init(&fl->device->hn);
  5473. }
  5474. }
  5475. fl->file_close = FASTRPC_PROCESS_EXIT_START;
  5476. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5477. (void)fastrpc_release_current_dsp_process(fl);
  5478. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5479. is_locked = true;
  5480. if (!fl->is_dma_invoke_pend)
  5481. goto skip_dmainvoke_wait;
  5482. is_locked = false;
  5483. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5484. wait_for_completion(&fl->dma_invoke);
  5485. skip_dmainvoke_wait:
  5486. if (!is_locked) {
  5487. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5488. is_locked = true;
  5489. }
  5490. if (!fl->is_ramdump_pend)
  5491. goto skip_dump_wait;
  5492. is_locked = false;
  5493. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5494. wait_for_completion(&fl->work);
  5495. skip_dump_wait:
  5496. if (!is_locked) {
  5497. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5498. is_locked = true;
  5499. }
  5500. hlist_del_init(&fl->hn);
  5501. fl->is_ramdump_pend = false;
  5502. fl->is_dma_invoke_pend = false;
  5503. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  5504. is_locked = false;
  5505. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5506. if (!fl->sctx) {
  5507. spin_lock_irqsave(&me->hlock, irq_flags);
  5508. /* Reset the tgid usage to false */
  5509. if (fl->tgid_frpc != -1)
  5510. frpc_tgid_usage_array[fl->tgid_frpc] = false;
  5511. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5512. kfree(fl);
  5513. fl = NULL;
  5514. return;
  5515. }
  5516. //Dummy wake up to exit Async worker thread
  5517. spin_lock_irqsave(&fl->aqlock, flags);
  5518. atomic_add(1, &fl->async_queue_job_count);
  5519. wake_up_interruptible(&fl->async_wait_queue);
  5520. spin_unlock_irqrestore(&fl->aqlock, flags);
  5521. // Dummy wake up to exit notification worker thread
  5522. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  5523. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  5524. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  5525. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  5526. if (!is_locked) {
  5527. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5528. is_locked = true;
  5529. }
  5530. if (!IS_ERR_OR_NULL(fl->init_mem)) {
  5531. init_mem = fl->init_mem;
  5532. fl->init_mem = NULL;
  5533. is_locked = false;
  5534. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5535. fastrpc_buf_free(init_mem, 0);
  5536. }
  5537. if (is_locked) {
  5538. is_locked = false;
  5539. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5540. }
  5541. fastrpc_context_list_dtor(fl);
  5542. fastrpc_cached_buf_list_free(fl);
  5543. if (!IS_ERR_OR_NULL(fl->hdr_bufs))
  5544. kfree(fl->hdr_bufs);
  5545. if (!IS_ERR_OR_NULL(fl->pers_hdr_buf))
  5546. fastrpc_buf_free(fl->pers_hdr_buf, 0);
  5547. mutex_lock(&fl->internal_map_mutex);
  5548. mutex_lock(&fl->map_mutex);
  5549. do {
  5550. lmap = NULL;
  5551. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5552. hlist_del_init(&map->hn);
  5553. lmap = map;
  5554. break;
  5555. }
  5556. fastrpc_mmap_free(lmap, 1);
  5557. } while (lmap);
  5558. mutex_unlock(&fl->map_mutex);
  5559. mutex_unlock(&fl->internal_map_mutex);
  5560. fastrpc_pm_relax(fl, gcinfo[fl->cid].secure);
  5561. if (fl->device && is_driver_closed)
  5562. device_unregister(&fl->device->dev);
  5563. spin_lock_irqsave(&me->hlock, irq_flags);
  5564. /* Reset the tgid usage to false */
  5565. if (fl->tgid_frpc != -1)
  5566. frpc_tgid_usage_array[fl->tgid_frpc] = false;
  5567. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5568. VERIFY(err, VALID_FASTRPC_CID(cid));
  5569. if (!err && fl->sctx)
  5570. fastrpc_session_free(&fl->apps->channel[cid], fl->sctx);
  5571. if (!err && fl->secsctx)
  5572. fastrpc_session_free(&fl->apps->channel[cid], fl->secsctx);
  5573. for (i = 0; i < (DSPSIGNAL_NUM_SIGNALS / DSPSIGNAL_GROUP_SIZE); i++)
  5574. kfree(fl->signal_groups[i]);
  5575. mutex_destroy(&fl->signal_create_mutex);
  5576. fastrpc_remote_buf_list_free(fl);
  5577. mutex_destroy(&fl->map_mutex);
  5578. mutex_destroy(&fl->internal_map_mutex);
  5579. kfree(fl->dev_pm_qos_req);
  5580. kfree(fl->gidlist.gids);
  5581. kfree(fl);
  5582. fl = NULL;
  5583. }
  5584. static int fastrpc_file_get(struct fastrpc_file *fl)
  5585. {
  5586. if (!fl)
  5587. return -ENOENT;
  5588. return kref_get_unless_zero(&fl->refcount) ? 0 : -ENOENT;
  5589. }
  5590. static void fastrpc_file_put(struct fastrpc_file *fl)
  5591. {
  5592. if (fl)
  5593. kref_put(&fl->refcount, fastrpc_file_free);
  5594. }
  5595. static int fastrpc_device_release(struct inode *inode, struct file *file)
  5596. {
  5597. struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
  5598. struct fastrpc_apps *me = &gfa;
  5599. unsigned int ii;
  5600. if (!fl)
  5601. return 0;
  5602. if (fl->qos_request && fl->dev_pm_qos_req) {
  5603. for (ii = 0; ii < me->lowest_capacity_core_count; ii++) {
  5604. if (!dev_pm_qos_request_active(&fl->dev_pm_qos_req[ii]))
  5605. continue;
  5606. dev_pm_qos_remove_request(&fl->dev_pm_qos_req[ii]);
  5607. }
  5608. }
  5609. debugfs_remove(fl->debugfs_file);
  5610. fastrpc_file_put(fl);
  5611. file->private_data = NULL;
  5612. return 0;
  5613. }
  5614. static ssize_t fastrpc_debugfs_read(struct file *filp, char __user *buffer,
  5615. size_t count, loff_t *position)
  5616. {
  5617. struct fastrpc_apps *me = &gfa;
  5618. struct fastrpc_file *fl = filp->private_data;
  5619. struct hlist_node *n;
  5620. struct fastrpc_buf *buf = NULL;
  5621. struct fastrpc_mmap *map = NULL;
  5622. struct fastrpc_mmap *gmaps = NULL;
  5623. struct smq_invoke_ctx *ictx = NULL;
  5624. struct fastrpc_channel_ctx *chan = NULL;
  5625. unsigned int len = 0;
  5626. int i, j, sess_used = 0, ret = 0;
  5627. char *fileinfo = NULL;
  5628. char single_line[] = "----------------";
  5629. char title[] = "=========================";
  5630. unsigned long irq_flags = 0;
  5631. fileinfo = kzalloc(DEBUGFS_SIZE, GFP_KERNEL);
  5632. if (!fileinfo) {
  5633. ret = -ENOMEM;
  5634. goto bail;
  5635. }
  5636. if (fl == NULL) {
  5637. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5638. "\n%s %s %s\n", title, " CHANNEL INFO ", title);
  5639. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5640. "%-7s|%-10s|%-15s|%-9s|%-13s\n",
  5641. "subsys", "sesscount", "subsystemstate",
  5642. "ssrcount", "session_used");
  5643. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5644. "-%s%s%s%s-\n", single_line, single_line,
  5645. single_line, single_line);
  5646. for (i = 0; i < NUM_CHANNELS; i++) {
  5647. sess_used = 0;
  5648. chan = &gcinfo[i];
  5649. len += scnprintf(fileinfo + len,
  5650. DEBUGFS_SIZE - len, "%-7s", chan->subsys);
  5651. len += scnprintf(fileinfo + len,
  5652. DEBUGFS_SIZE - len, "|%-10u",
  5653. chan->sesscount);
  5654. len += scnprintf(fileinfo + len,
  5655. DEBUGFS_SIZE - len, "|%-15d",
  5656. chan->subsystemstate);
  5657. len += scnprintf(fileinfo + len,
  5658. DEBUGFS_SIZE - len, "|%-9u",
  5659. chan->ssrcount);
  5660. for (j = 0; j < chan->sesscount; j++)
  5661. sess_used += chan->session[j].used;
  5662. len += scnprintf(fileinfo + len,
  5663. DEBUGFS_SIZE - len, "|%-13d\n", sess_used);
  5664. }
  5665. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5666. "\n%s%s%s\n", "=============",
  5667. " CMA HEAP ", "==============");
  5668. len += scnprintf(fileinfo + len,
  5669. DEBUGFS_SIZE - len, "%-20s|%-20s\n", "addr", "size");
  5670. len += scnprintf(fileinfo + len,
  5671. DEBUGFS_SIZE - len, "--%s%s---\n",
  5672. single_line, single_line);
  5673. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5674. "\n==========%s %s %s===========\n",
  5675. title, " GMAPS ", title);
  5676. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5677. "%-20s|%-20s|%-20s|%-20s\n",
  5678. "fd", "phys", "size", "va");
  5679. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5680. "%s%s%s%s%s\n", single_line, single_line,
  5681. single_line, single_line, single_line);
  5682. spin_lock_irqsave(&me->hlock, irq_flags);
  5683. hlist_for_each_entry_safe(gmaps, n, &me->maps, hn) {
  5684. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5685. "%-20d|0x%-18llX|0x%-18X|0x%-20lX\n\n",
  5686. gmaps->fd, gmaps->phys,
  5687. (uint32_t)gmaps->size,
  5688. gmaps->va);
  5689. }
  5690. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5691. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5692. "%-20s|%-20s|%-20s|%-20s\n",
  5693. "len", "refs", "raddr", "flags");
  5694. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5695. "%s%s%s%s%s\n", single_line, single_line,
  5696. single_line, single_line, single_line);
  5697. spin_lock_irqsave(&me->hlock, irq_flags);
  5698. hlist_for_each_entry_safe(gmaps, n, &me->maps, hn) {
  5699. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5700. "0x%-18X|%-20d|%-20lu|%-20u\n",
  5701. (uint32_t)gmaps->len, gmaps->refs,
  5702. gmaps->raddr, gmaps->flags);
  5703. }
  5704. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5705. } else {
  5706. ret = fastrpc_file_get(fl);
  5707. if (ret) {
  5708. ADSPRPC_ERR("Failed to get user process reference\n");
  5709. goto bail;
  5710. }
  5711. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5712. "\n%s %13s %d\n", "cid", ":", fl->cid);
  5713. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5714. "%s %12s %d\n", "tgid", ":", fl->tgid);
  5715. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5716. "%s %14s %d\n", "tgid_frpc", ":", fl->tgid_frpc);
  5717. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5718. "%s %7s %d\n", "sessionid", ":", fl->sessionid);
  5719. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5720. "%s %8s %u\n", "ssrcount", ":", fl->ssrcount);
  5721. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5722. "%s %14s %d\n", "pd", ":", fl->pd);
  5723. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5724. "%s %9s %s\n", "servloc_name", ":", fl->servloc_name);
  5725. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5726. "%s %6s %d\n", "file_close", ":", fl->file_close);
  5727. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5728. "%s %9s %d\n", "profile", ":", fl->profile);
  5729. if (fl->sctx) {
  5730. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5731. "%s %3s %d\n", "smmu.coherent", ":",
  5732. fl->sctx->smmu.coherent);
  5733. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5734. "%s %4s %d\n", "smmu.enabled", ":",
  5735. fl->sctx->smmu.enabled);
  5736. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5737. "%s %9s %d\n", "smmu.cb", ":", fl->sctx->smmu.cb);
  5738. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5739. "%s %5s %d\n", "smmu.secure", ":",
  5740. fl->sctx->smmu.secure);
  5741. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5742. "%s %5s %d\n", "smmu.faults", ":",
  5743. fl->sctx->smmu.faults);
  5744. }
  5745. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5746. "\n=======%s %s %s======\n", title,
  5747. " LIST OF MAPS ", title);
  5748. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5749. "%-20s|%-20s|%-20s|%-20s\n", "va", "phys", "size", "flags");
  5750. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5751. "%s%s%s%s%s\n",
  5752. single_line, single_line, single_line,
  5753. single_line, single_line);
  5754. mutex_lock(&fl->map_mutex);
  5755. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5756. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5757. "0x%-20lX|0x%-20llX|0x%-20zu|0x%-17llX\n\n",
  5758. map->va, map->phys,
  5759. map->size, map->flags);
  5760. }
  5761. mutex_unlock(&fl->map_mutex);
  5762. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5763. "%-20s|%-20s|%-20s\n",
  5764. "len", "refs",
  5765. "raddr");
  5766. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5767. "%s%s%s%s%s\n",
  5768. single_line, single_line, single_line,
  5769. single_line, single_line);
  5770. mutex_lock(&fl->map_mutex);
  5771. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5772. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5773. "%-20zu|%-20d|0x%-20lX\n\n",
  5774. map->len, map->refs, map->raddr);
  5775. }
  5776. mutex_unlock(&fl->map_mutex);
  5777. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5778. "%-20s|%-20s\n", "secure", "attr");
  5779. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5780. "%s%s%s%s%s\n",
  5781. single_line, single_line, single_line,
  5782. single_line, single_line);
  5783. mutex_lock(&fl->map_mutex);
  5784. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5785. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5786. "%-20d|0x%-20lX\n\n",
  5787. map->secure, map->attr);
  5788. }
  5789. mutex_unlock(&fl->map_mutex);
  5790. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5791. "\n======%s %s %s======\n", title,
  5792. " LIST OF BUFS ", title);
  5793. spin_lock(&fl->hlock);
  5794. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5795. "%-19s|%-19s|%-19s|%-19s\n",
  5796. "virt", "phys", "size", "flags");
  5797. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5798. "%s%s%s%s%s\n", single_line, single_line,
  5799. single_line, single_line, single_line);
  5800. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  5801. len += scnprintf(fileinfo + len,
  5802. DEBUGFS_SIZE - len,
  5803. "0x%-17p|0x%-17llX|%-19zu|0x%-17llX\n",
  5804. buf->virt, (uint64_t)buf->phys, buf->size, buf->flags);
  5805. }
  5806. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5807. "\n======%s %s %s======\n", title,
  5808. " LIST OF REMOTE BUFS ", title);
  5809. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5810. "%-19s|%-19s|%-19s|%-19s\n",
  5811. "virt", "phys", "size", "flags");
  5812. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5813. "%s%s%s%s%s\n", single_line, single_line,
  5814. single_line, single_line, single_line);
  5815. hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {
  5816. len += scnprintf(fileinfo + len,
  5817. DEBUGFS_SIZE - len,
  5818. "0x%-17p|0x%-17llX|%-19zu|0x%-17llX\n",
  5819. buf->virt, (uint64_t)buf->phys, buf->size, buf->flags);
  5820. }
  5821. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5822. "\n%s %s %s\n", title,
  5823. " LIST OF PENDING SMQCONTEXTS ", title);
  5824. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5825. "%-20s|%-10s|%-10s|%-10s|%-20s\n",
  5826. "sc", "pid", "tgid", "used", "ctxid");
  5827. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5828. "%s%s%s%s%s\n", single_line, single_line,
  5829. single_line, single_line, single_line);
  5830. hlist_for_each_entry_safe(ictx, n, &fl->clst.pending, hn) {
  5831. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5832. "0x%-18X|%-10d|%-10d|%-10zu|0x%-20llX\n\n",
  5833. ictx->sc, ictx->pid, ictx->tgid,
  5834. ictx->used, ictx->ctxid);
  5835. }
  5836. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5837. "\n%s %s %s\n", title,
  5838. " LIST OF INTERRUPTED SMQCONTEXTS ", title);
  5839. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5840. "%-20s|%-10s|%-10s|%-10s|%-20s\n",
  5841. "sc", "pid", "tgid", "used", "ctxid");
  5842. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5843. "%s%s%s%s%s\n", single_line, single_line,
  5844. single_line, single_line, single_line);
  5845. hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {
  5846. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5847. "%-20u|%-20d|%-20d|%-20zu|0x%-20llX\n\n",
  5848. ictx->sc, ictx->pid, ictx->tgid,
  5849. ictx->used, ictx->ctxid);
  5850. }
  5851. spin_unlock(&fl->hlock);
  5852. fastrpc_file_put(fl);
  5853. }
  5854. if (len > DEBUGFS_SIZE)
  5855. len = DEBUGFS_SIZE;
  5856. ret = simple_read_from_buffer(buffer, count, position, fileinfo, len);
  5857. kfree(fileinfo);
  5858. bail:
  5859. return ret;
  5860. }
  5861. static const struct file_operations debugfs_fops = {
  5862. .open = simple_open,
  5863. .read = fastrpc_debugfs_read,
  5864. };
  5865. static int fastrpc_channel_open(struct fastrpc_file *fl, uint32_t flags)
  5866. {
  5867. struct fastrpc_apps *me = &gfa;
  5868. int cid = -1, err = 0;
  5869. VERIFY(err, fl && fl->sctx && fl->cid >= 0 && fl->cid < NUM_CHANNELS);
  5870. if (err) {
  5871. ADSPRPC_ERR("kernel session not initialized yet for %s\n",
  5872. current->comm);
  5873. err = -EBADR;
  5874. return err;
  5875. }
  5876. cid = fl->cid;
  5877. err = fastrpc_wait_for_transport_interrupt(cid, flags);
  5878. if (err)
  5879. goto bail;
  5880. err = verify_transport_device(cid, fl->tvm_remote_domain);
  5881. if (err)
  5882. goto bail;
  5883. mutex_lock(&me->channel[cid].smd_mutex);
  5884. if (me->channel[cid].ssrcount !=
  5885. me->channel[cid].prevssrcount) {
  5886. if (me->channel[cid].subsystemstate != SUBSYSTEM_UP) {
  5887. err = -ECONNREFUSED;
  5888. mutex_unlock(&me->channel[cid].smd_mutex);
  5889. goto bail;
  5890. }
  5891. }
  5892. fl->ssrcount = me->channel[cid].ssrcount;
  5893. if (cid == ADSP_DOMAIN_ID && me->channel[cid].ssrcount !=
  5894. me->channel[cid].prevssrcount) {
  5895. mutex_unlock(&me->channel[cid].smd_mutex);
  5896. mutex_lock(&fl->map_mutex);
  5897. err = fastrpc_dsp_restart_handler(fl, 1, true);
  5898. mutex_unlock(&fl->map_mutex);
  5899. if (err)
  5900. ADSPRPC_WARN(
  5901. "failed to unmap remote heap for %s (err %d)\n",
  5902. me->channel[cid].subsys, err);
  5903. mutex_lock(&me->channel[cid].smd_mutex);
  5904. me->channel[cid].prevssrcount =
  5905. me->channel[cid].ssrcount;
  5906. }
  5907. me->channel[cid].in_hib = 0;
  5908. mutex_unlock(&me->channel[cid].smd_mutex);
  5909. bail:
  5910. return err;
  5911. }
  5912. static inline void fastrpc_register_wakeup_source(struct device *dev,
  5913. const char *client_name, struct wakeup_source **device_wake_source)
  5914. {
  5915. struct wakeup_source *wake_source = NULL;
  5916. wake_source = wakeup_source_register(dev, client_name);
  5917. if (IS_ERR_OR_NULL(wake_source)) {
  5918. ADSPRPC_ERR(
  5919. "wakeup_source_register failed for dev %s, client %s with err %ld\n",
  5920. dev_name(dev), client_name, PTR_ERR(wake_source));
  5921. return;
  5922. }
  5923. *device_wake_source = wake_source;
  5924. }
  5925. static int fastrpc_device_open(struct inode *inode, struct file *filp)
  5926. {
  5927. int err = 0;
  5928. struct fastrpc_file *fl = NULL;
  5929. struct fastrpc_apps *me = &gfa;
  5930. unsigned long irq_flags = 0;
  5931. /*
  5932. * Indicates the device node opened
  5933. * MINOR_NUM_DEV or MINOR_NUM_SECURE_DEV
  5934. */
  5935. int dev_minor = MINOR(inode->i_rdev);
  5936. VERIFY(err, ((dev_minor == MINOR_NUM_DEV) ||
  5937. (dev_minor == MINOR_NUM_SECURE_DEV)));
  5938. if (err) {
  5939. ADSPRPC_ERR("Invalid dev minor num %d\n",
  5940. dev_minor);
  5941. return err;
  5942. }
  5943. VERIFY(err, NULL != (fl = kzalloc(sizeof(*fl), GFP_KERNEL)));
  5944. if (err) {
  5945. err = -ENOMEM;
  5946. return err;
  5947. }
  5948. context_list_ctor(&fl->clst);
  5949. spin_lock_init(&fl->hlock);
  5950. spin_lock_init(&fl->aqlock);
  5951. spin_lock_init(&fl->proc_state_notif.nqlock);
  5952. INIT_HLIST_HEAD(&fl->maps);
  5953. INIT_HLIST_HEAD(&fl->cached_bufs);
  5954. fl->num_cached_buf = 0;
  5955. INIT_HLIST_HEAD(&fl->remote_bufs);
  5956. init_waitqueue_head(&fl->async_wait_queue);
  5957. init_waitqueue_head(&fl->proc_state_notif.notif_wait_queue);
  5958. INIT_HLIST_NODE(&fl->hn);
  5959. fl->sessionid = 0;
  5960. fl->tgid_open = current->tgid;
  5961. /* PD type is not known, when device is opened */
  5962. fl->pd_type = DEFAULT_UNUSED;
  5963. fl->apps = me;
  5964. fl->mode = FASTRPC_MODE_SERIAL;
  5965. fl->cid = -1;
  5966. fl->tgid_frpc = -1;
  5967. fl->tvm_remote_domain = -1;
  5968. fl->dev_minor = dev_minor;
  5969. fl->init_mem = NULL;
  5970. fl->qos_request = 0;
  5971. fl->dsp_proc_init = 0;
  5972. fl->is_ramdump_pend = false;
  5973. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  5974. fl->is_unsigned_pd = false;
  5975. fl->is_compat = false;
  5976. fl->exit_notif = false;
  5977. fl->exit_async = false;
  5978. fl->multi_session_support = false;
  5979. fl->set_session_info = false;
  5980. init_completion(&fl->work);
  5981. init_completion(&fl->dma_invoke);
  5982. fl->file_close = FASTRPC_PROCESS_DEFAULT_STATE;
  5983. filp->private_data = fl;
  5984. fl->sharedbuf_info.buf_fd = -1;
  5985. mutex_init(&fl->internal_map_mutex);
  5986. mutex_init(&fl->map_mutex);
  5987. spin_lock_irqsave(&me->hlock, irq_flags);
  5988. hlist_add_head(&fl->hn, &me->drivers);
  5989. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5990. kref_init(&fl->refcount);
  5991. if (me->lowest_capacity_core_count)
  5992. fl->dev_pm_qos_req = kzalloc((me->lowest_capacity_core_count) *
  5993. sizeof(struct dev_pm_qos_request),
  5994. GFP_KERNEL);
  5995. spin_lock_init(&fl->dspsignals_lock);
  5996. mutex_init(&fl->signal_create_mutex);
  5997. init_completion(&fl->shutdown);
  5998. return 0;
  5999. }
  6000. static int fastrpc_get_process_gids(struct gid_list *gidlist)
  6001. {
  6002. struct group_info *group_info = get_current_groups();
  6003. int i = 0, err = 0, num_gids = group_info->ngroups + 1;
  6004. unsigned int *gids = NULL;
  6005. gids = kcalloc(num_gids, sizeof(unsigned int), GFP_KERNEL);
  6006. if (!gids) {
  6007. err = -ENOMEM;
  6008. goto bail;
  6009. }
  6010. /* Get the real GID */
  6011. gids[0] = __kgid_val(current_gid());
  6012. /* Get the supplemental GIDs */
  6013. for (i = 1; i < num_gids; i++)
  6014. gids[i] = __kgid_val(group_info->gid[i - 1]);
  6015. sort(gids, num_gids, sizeof(*gids), uint_cmp_func, NULL);
  6016. gidlist->gids = gids;
  6017. gidlist->gidcount = num_gids;
  6018. bail:
  6019. if (err)
  6020. kfree(gids);
  6021. return err;
  6022. }
  6023. // Generate a unique process ID to DSP process
  6024. static int get_unique_hlos_process_id(void)
  6025. {
  6026. int tgid_frpc = -1, tgid_index = 1;
  6027. struct fastrpc_apps *me = &gfa;
  6028. unsigned long irq_flags = 0;
  6029. spin_lock_irqsave(&me->hlock, irq_flags);
  6030. for (tgid_index = 1; tgid_index < MAX_FRPC_TGID; tgid_index++) {
  6031. if (!frpc_tgid_usage_array[tgid_index]) {
  6032. tgid_frpc = tgid_index;
  6033. /* Set the tgid usage to false */
  6034. frpc_tgid_usage_array[tgid_index] = true;
  6035. break;
  6036. }
  6037. }
  6038. spin_unlock_irqrestore(&me->hlock, irq_flags);
  6039. return tgid_frpc;
  6040. }
  6041. static int fastrpc_set_process_info(struct fastrpc_file *fl, uint32_t cid)
  6042. {
  6043. int err = 0, buf_size = 0;
  6044. char strpid[PID_SIZE];
  6045. char cur_comm[TASK_COMM_LEN];
  6046. memcpy(cur_comm, current->comm, TASK_COMM_LEN);
  6047. cur_comm[TASK_COMM_LEN-1] = '\0';
  6048. fl->tgid = current->tgid;
  6049. fl->tgid_frpc = get_unique_hlos_process_id();
  6050. VERIFY(err, fl->tgid_frpc != -1);
  6051. if (err) {
  6052. ADSPRPC_ERR("too many fastrpc clients, max %u allowed\n", MAX_FRPC_TGID);
  6053. err = -EUSERS;
  6054. goto bail;
  6055. }
  6056. ADSPRPC_INFO("HLOS pid %d, cid %d is mapped to unique sessions pid %d",
  6057. fl->tgid, cid, fl->tgid_frpc);
  6058. /*
  6059. * Third-party apps don't have permission to open the fastrpc device, so
  6060. * it is opened on their behalf by DSP HAL. This is detected by
  6061. * comparing current PID with the one stored during device open.
  6062. */
  6063. if (current->tgid != fl->tgid_open)
  6064. fl->untrusted_process = true;
  6065. snprintf(strpid, PID_SIZE, "%d", current->pid);
  6066. if (debugfs_root) {
  6067. VERIFY(err, VALID_FASTRPC_CID(cid));
  6068. if (err) {
  6069. err = -ECHRNG;
  6070. goto bail;
  6071. }
  6072. buf_size = strlen(cur_comm) + strlen("_") + strlen(strpid)
  6073. + strlen("_") + strlen(__TOSTR__(NUM_CHANNELS)) + 1;
  6074. spin_lock(&fl->hlock);
  6075. if (fl->debug_buf_alloced_attempted) {
  6076. spin_unlock(&fl->hlock);
  6077. return err;
  6078. }
  6079. fl->debug_buf_alloced_attempted = 1;
  6080. spin_unlock(&fl->hlock);
  6081. fl->debug_buf = kzalloc(buf_size, GFP_KERNEL);
  6082. if (!fl->debug_buf) {
  6083. err = -ENOMEM;
  6084. return err;
  6085. }
  6086. /* Use HLOS PID, unique fastrpc PID, CID in debugfs filename,
  6087. * for better ability to debug
  6088. */
  6089. snprintf(fl->debug_buf, buf_size, "%.10s%s%d%s%d%s%d",
  6090. cur_comm, "_", current->pid, "_", fl->tgid_frpc, "_", cid);
  6091. fl->debugfs_file = debugfs_create_file(fl->debug_buf, 0644,
  6092. debugfs_root, fl, &debugfs_fops);
  6093. if (IS_ERR_OR_NULL(fl->debugfs_file)) {
  6094. pr_warn("Error: %s: %s: failed to create debugfs file %s\n",
  6095. cur_comm, __func__, fl->debug_buf);
  6096. fl->debugfs_file = NULL;
  6097. }
  6098. kfree(fl->debug_buf);
  6099. fl->debug_buf = NULL;
  6100. }
  6101. bail:
  6102. return err;
  6103. }
  6104. int fastrpc_get_info(struct fastrpc_file *fl, uint32_t *info)
  6105. {
  6106. int err = 0;
  6107. uint32_t cid = *info;
  6108. struct fastrpc_apps *me = &gfa;
  6109. VERIFY(err, fl != NULL);
  6110. if (err) {
  6111. err = -EBADF;
  6112. goto bail;
  6113. }
  6114. spin_lock(&fl->hlock);
  6115. if (fl->set_session_info) {
  6116. spin_unlock(&fl->hlock);
  6117. ADSPRPC_ERR("Set session info invoked multiple times\n");
  6118. err = -EBADR;
  6119. goto bail;
  6120. }
  6121. // Set set_session_info to true
  6122. fl->set_session_info = true;
  6123. spin_unlock(&fl->hlock);
  6124. VERIFY(err, VALID_FASTRPC_CID(cid));
  6125. if (err) {
  6126. err = -ECHRNG;
  6127. goto bail;
  6128. }
  6129. fastrpc_get_process_gids(&fl->gidlist);
  6130. err = fastrpc_set_process_info(fl, cid);
  6131. if (err)
  6132. goto bail;
  6133. if (fl->cid == -1) {
  6134. struct fastrpc_channel_ctx *chan = NULL;
  6135. chan = &me->channel[cid];
  6136. /* Check to see if the device node is non-secure */
  6137. if (fl->dev_minor == MINOR_NUM_DEV) {
  6138. /*
  6139. * If an app is trying to offload to a secure remote
  6140. * channel by opening the non-secure device node, allow
  6141. * the access if the subsystem supports unsigned
  6142. * offload. Untrusted apps will be restricted from
  6143. * offloading to signed PD using DSP HAL.
  6144. */
  6145. if (chan->secure == SECURE_CHANNEL
  6146. && !chan->unsigned_support) {
  6147. ADSPRPC_ERR(
  6148. "cannot use domain %d with non-secure device\n",
  6149. cid);
  6150. err = -EACCES;
  6151. goto bail;
  6152. }
  6153. }
  6154. fl->cid = cid;
  6155. fl->ssrcount = fl->apps->channel[cid].ssrcount;
  6156. mutex_lock(&fl->apps->channel[cid].smd_mutex);
  6157. err = fastrpc_session_alloc_locked(&fl->apps->channel[cid],
  6158. 0, fl->sharedcb, fl->pd_type, &fl->sctx);
  6159. mutex_unlock(&fl->apps->channel[cid].smd_mutex);
  6160. if (err == -EUSERS) {
  6161. ADSPRPC_WARN(
  6162. "max concurrent sessions limit (%d) already reached on %s err %d\n",
  6163. chan->sesscount, chan->subsys, err);
  6164. }
  6165. if (err)
  6166. goto bail;
  6167. }
  6168. VERIFY(err, fl->sctx != NULL);
  6169. if (err) {
  6170. err = -EBADR;
  6171. goto bail;
  6172. }
  6173. *info = (fl->sctx->smmu.enabled ? 1 : 0);
  6174. bail:
  6175. return err;
  6176. }
  6177. static int fastrpc_manage_poll_mode(struct fastrpc_file *fl, uint32_t enable, uint32_t timeout)
  6178. {
  6179. int err = 0;
  6180. const unsigned int MAX_POLL_TIMEOUT_US = 10000;
  6181. if ((fl->cid != CDSP_DOMAIN_ID) || (fl->proc_flags != FASTRPC_INIT_CREATE)) {
  6182. err = -EPERM;
  6183. ADSPRPC_ERR("flags %d, cid %d, poll mode allowed only for dynamic CDSP process\n",
  6184. fl->proc_flags, fl->cid);
  6185. goto bail;
  6186. }
  6187. if (timeout > MAX_POLL_TIMEOUT_US) {
  6188. err = -EBADMSG;
  6189. ADSPRPC_ERR("poll timeout %u is greater than max allowed value %u\n",
  6190. timeout, MAX_POLL_TIMEOUT_US);
  6191. goto bail;
  6192. }
  6193. spin_lock(&fl->hlock);
  6194. if (enable) {
  6195. fl->poll_mode = true;
  6196. fl->poll_timeout = timeout;
  6197. } else {
  6198. fl->poll_mode = false;
  6199. fl->poll_timeout = 0;
  6200. }
  6201. spin_unlock(&fl->hlock);
  6202. ADSPRPC_INFO("updated poll mode to %d, timeout %u\n", enable, timeout);
  6203. bail:
  6204. return err;
  6205. }
  6206. int fastrpc_internal_control(struct fastrpc_file *fl,
  6207. struct fastrpc_ioctl_control *cp)
  6208. {
  6209. int err = 0;
  6210. unsigned int latency;
  6211. struct fastrpc_apps *me = &gfa;
  6212. unsigned int cpu;
  6213. unsigned long flags = 0;
  6214. VERIFY(err, !IS_ERR_OR_NULL(fl) && !IS_ERR_OR_NULL(fl->apps));
  6215. if (err) {
  6216. err = -EBADF;
  6217. goto bail;
  6218. }
  6219. VERIFY(err, !IS_ERR_OR_NULL(cp));
  6220. if (err) {
  6221. err = -EINVAL;
  6222. goto bail;
  6223. }
  6224. switch (cp->req) {
  6225. case FASTRPC_CONTROL_LATENCY:
  6226. latency = cp->lp.enable == FASTRPC_LATENCY_CTRL_ENB ?
  6227. fl->apps->latency : PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
  6228. VERIFY(err, latency != 0);
  6229. if (err) {
  6230. err = -EINVAL;
  6231. goto bail;
  6232. }
  6233. VERIFY(err, (me->lowest_capacity_core_count && fl->dev_pm_qos_req));
  6234. if (err) {
  6235. ADSPRPC_INFO("Skipping PM QoS latency voting, core count: %u\n",
  6236. me->lowest_capacity_core_count);
  6237. err = -EINVAL;
  6238. goto bail;
  6239. }
  6240. /*
  6241. * Add voting request for all possible cores corresponding to cluster
  6242. * id 0. If DT property 'qcom,single-core-latency-vote' is enabled
  6243. * then add voting request for only one core of cluster id 0.
  6244. */
  6245. for (cpu = 0; cpu < me->lowest_capacity_core_count; cpu++) {
  6246. if (!fl->qos_request) {
  6247. err = dev_pm_qos_add_request(
  6248. get_cpu_device(cpu),
  6249. &fl->dev_pm_qos_req[cpu],
  6250. DEV_PM_QOS_RESUME_LATENCY,
  6251. latency);
  6252. } else {
  6253. err = dev_pm_qos_update_request(
  6254. &fl->dev_pm_qos_req[cpu],
  6255. latency);
  6256. }
  6257. /* PM QoS request APIs return 0 or 1 on success */
  6258. if (err < 0) {
  6259. ADSPRPC_WARN("QoS with lat %u failed for CPU %d, err %d, req %d\n",
  6260. latency, cpu, err, fl->qos_request);
  6261. break;
  6262. }
  6263. }
  6264. if (err >= 0) {
  6265. fl->qos_request = 1;
  6266. err = 0;
  6267. }
  6268. /* Ensure CPU feature map updated to DSP for early WakeUp */
  6269. fastrpc_send_cpuinfo_to_dsp(fl);
  6270. break;
  6271. case FASTRPC_CONTROL_KALLOC:
  6272. cp->kalloc.kalloc_support = 1;
  6273. break;
  6274. case FASTRPC_CONTROL_WAKELOCK:
  6275. if (fl->dev_minor != MINOR_NUM_SECURE_DEV) {
  6276. ADSPRPC_ERR(
  6277. "PM voting not allowed for non-secure device node %d\n",
  6278. fl->dev_minor);
  6279. err = -EPERM;
  6280. goto bail;
  6281. }
  6282. fl->wake_enable = cp->wp.enable;
  6283. break;
  6284. case FASTRPC_CONTROL_PM:
  6285. if (!fl->wake_enable) {
  6286. /* Kernel PM voting not requested by this application */
  6287. err = -EACCES;
  6288. goto bail;
  6289. }
  6290. if (cp->pm.timeout > MAX_PM_TIMEOUT_MS)
  6291. fl->ws_timeout = MAX_PM_TIMEOUT_MS;
  6292. else
  6293. fl->ws_timeout = cp->pm.timeout;
  6294. VERIFY(err, VALID_FASTRPC_CID(fl->cid));
  6295. if (err) {
  6296. err = -ECHRNG;
  6297. goto bail;
  6298. }
  6299. fastrpc_pm_awake(fl, gcinfo[fl->cid].secure);
  6300. break;
  6301. case FASTRPC_CONTROL_DSPPROCESS_CLEAN:
  6302. (void)fastrpc_release_current_dsp_process(fl);
  6303. fastrpc_queue_pd_status(fl, fl->cid, FASTRPC_USER_PD_FORCE_KILL, fl->sessionid);
  6304. break;
  6305. case FASTRPC_CONTROL_RPC_POLL:
  6306. err = fastrpc_manage_poll_mode(fl, cp->lp.enable, cp->lp.latency);
  6307. if (err)
  6308. goto bail;
  6309. break;
  6310. case FASTRPC_CONTROL_SMMU:
  6311. fl->sharedcb = cp->smmu.sharedcb;
  6312. break;
  6313. case FASTRPC_CONTROL_ASYNC_WAKE:
  6314. fl->exit_async = true;
  6315. spin_lock_irqsave(&fl->aqlock, flags);
  6316. atomic_add(1, &fl->async_queue_job_count);
  6317. wake_up_interruptible(&fl->async_wait_queue);
  6318. spin_unlock_irqrestore(&fl->aqlock, flags);
  6319. break;
  6320. case FASTRPC_CONTROL_NOTIF_WAKE:
  6321. fl->exit_notif = true;
  6322. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  6323. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  6324. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  6325. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  6326. break;
  6327. default:
  6328. err = -EBADRQC;
  6329. break;
  6330. }
  6331. bail:
  6332. return err;
  6333. }
  6334. /* Wait for PD to be up before audio or sensors daemons try connecting */
  6335. static int fastrpc_check_pd_status(struct fastrpc_file *fl, char *sloc_name)
  6336. {
  6337. int err = 0, session = -1, cid = -1;
  6338. struct fastrpc_apps *me = &gfa;
  6339. if (fl->servloc_name && sloc_name
  6340. && !strcmp(fl->servloc_name, sloc_name)) {
  6341. err = fastrpc_get_spd_session(sloc_name, &session, &cid);
  6342. if (err || cid != fl->cid)
  6343. goto bail;
  6344. /*
  6345. * Audio PD attachment is not allowed after PDR.
  6346. * Allow kill message after PDR to clean DSP guestOS resources.
  6347. */
  6348. if ((!strcmp(fl->servloc_name,
  6349. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME)) &&
  6350. (me->channel[cid].spd[session].pdrcount !=
  6351. me->channel[cid].spd[session].prevpdrcount) &&
  6352. !fl->dsp_proc_init) {
  6353. err = -ECONNRESET;
  6354. goto bail;
  6355. }
  6356. #if IS_ENABLED(CONFIG_QCOM_PDR_HELPERS)
  6357. if (!strcmp(fl->servloc_name,
  6358. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME) || !strcmp(fl->servloc_name,
  6359. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME) ||
  6360. !strcmp(fl->servloc_name,
  6361. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME)) {
  6362. err = wait_event_interruptible(
  6363. me->channel[cid].spd[session].wait_for_pdup,
  6364. atomic_read(&me->channel[cid].spd[session].ispdup));
  6365. goto bail;
  6366. }
  6367. #else
  6368. (void)me;
  6369. #endif
  6370. }
  6371. bail:
  6372. return err;
  6373. }
  6374. int fastrpc_setmode(unsigned long ioctl_param,
  6375. struct fastrpc_file *fl)
  6376. {
  6377. int err = 0;
  6378. switch ((uint32_t)ioctl_param) {
  6379. case FASTRPC_MODE_PARALLEL:
  6380. case FASTRPC_MODE_SERIAL:
  6381. fl->mode = (uint32_t)ioctl_param;
  6382. break;
  6383. case FASTRPC_MODE_PROFILE:
  6384. fl->profile = (uint32_t)ioctl_param;
  6385. break;
  6386. case FASTRPC_MODE_SESSION:
  6387. if (!fl->multi_session_support)
  6388. fl->sessionid = 1;
  6389. break;
  6390. default:
  6391. err = -ENOTTY;
  6392. break;
  6393. }
  6394. return err;
  6395. }
  6396. int fastrpc_control(struct fastrpc_ioctl_control *cp,
  6397. void *param, struct fastrpc_file *fl)
  6398. {
  6399. int err = 0;
  6400. K_COPY_FROM_USER(err, 0, cp, param,
  6401. sizeof(*cp));
  6402. if (err) {
  6403. err = -EFAULT;
  6404. goto bail;
  6405. }
  6406. VERIFY(err, 0 == (err = fastrpc_internal_control(fl, cp)));
  6407. if (err)
  6408. goto bail;
  6409. if (cp->req == FASTRPC_CONTROL_KALLOC) {
  6410. K_COPY_TO_USER(err, 0, param, cp, sizeof(*cp));
  6411. if (err) {
  6412. err = -EFAULT;
  6413. goto bail;
  6414. }
  6415. }
  6416. bail:
  6417. return err;
  6418. }
  6419. static int fastrpc_get_dsp_info(
  6420. struct fastrpc_ioctl_capability *cap,
  6421. void *param, struct fastrpc_file *fl)
  6422. {
  6423. int err = 0;
  6424. K_COPY_FROM_USER(err, 0, cap, param,
  6425. sizeof(struct fastrpc_ioctl_capability));
  6426. VERIFY(err, cap->domain < NUM_CHANNELS);
  6427. if (err) {
  6428. err = -ECHRNG;
  6429. goto bail;
  6430. }
  6431. cap->capability = 0;
  6432. err = fastrpc_get_info_from_kernel(cap, fl);
  6433. if (err)
  6434. goto bail;
  6435. K_COPY_TO_USER(err, 0, &((struct fastrpc_ioctl_capability *)
  6436. param)->capability, &cap->capability, sizeof(cap->capability));
  6437. bail:
  6438. return err;
  6439. }
  6440. int fastrpc_dspsignal_signal(struct fastrpc_file *fl,
  6441. struct fastrpc_ioctl_dspsignal_signal *sig)
  6442. {
  6443. int err = 0, cid = -1;
  6444. struct fastrpc_channel_ctx *channel_ctx = NULL;
  6445. uint64_t msg = 0;
  6446. // We don't check if the signal has even been allocated since we don't
  6447. // track outgoing signals in the driver. The userspace library does a
  6448. // basic sanity check and any security validation needs to be done by
  6449. // the recipient.
  6450. DSPSIGNAL_VERBOSE("Send signal PID %u, unique fastrpc pid %u signal %u\n",
  6451. (unsigned int)fl->tgid, (unsigned int)fl->tgid_frpc,
  6452. (unsigned int)sig->signal_id);
  6453. VERIFY(err, sig->signal_id < DSPSIGNAL_NUM_SIGNALS);
  6454. if (err) {
  6455. ADSPRPC_ERR("Sending bad signal %u for PID %u",
  6456. sig->signal_id, (unsigned int)fl->tgid);
  6457. err = -EBADR;
  6458. goto bail;
  6459. }
  6460. cid = fl->cid;
  6461. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6462. if (err) {
  6463. err = -EBADR;
  6464. goto bail;
  6465. }
  6466. channel_ctx = &fl->apps->channel[cid];
  6467. mutex_lock(&channel_ctx->smd_mutex);
  6468. if (fl->ssrcount != channel_ctx->ssrcount) {
  6469. err = -ECONNRESET;
  6470. mutex_unlock(&channel_ctx->smd_mutex);
  6471. goto bail;
  6472. }
  6473. /* Use unique fastrpc pid, to signal DSP process */
  6474. msg = (((uint64_t)fl->tgid_frpc) << 32) | ((uint64_t)sig->signal_id);
  6475. err = fastrpc_transport_send(cid, (void *)&msg, sizeof(msg), fl->tvm_remote_domain);
  6476. mutex_unlock(&channel_ctx->smd_mutex);
  6477. trace_fastrpc_dspsignal("signal", sig->signal_id, 0, 0);
  6478. bail:
  6479. return err;
  6480. }
  6481. int fastrpc_dspsignal_wait(struct fastrpc_file *fl,
  6482. struct fastrpc_ioctl_dspsignal_wait *wait)
  6483. {
  6484. int err = 0, cid = -1;
  6485. unsigned long timeout = usecs_to_jiffies(wait->timeout_usec);
  6486. uint32_t signal_id = wait->signal_id;
  6487. struct fastrpc_dspsignal *s = NULL;
  6488. long ret = 0;
  6489. unsigned long irq_flags = 0;
  6490. DSPSIGNAL_VERBOSE("Wait for signal %u\n", signal_id);
  6491. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6492. if (err) {
  6493. ADSPRPC_ERR("Waiting on bad signal %u", signal_id);
  6494. err = -EINVAL;
  6495. goto bail;
  6496. }
  6497. cid = fl->cid;
  6498. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6499. if (err) {
  6500. err = -EBADR;
  6501. goto bail;
  6502. }
  6503. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6504. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6505. struct fastrpc_dspsignal *group =
  6506. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6507. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6508. }
  6509. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6510. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6511. ADSPRPC_ERR("Unknown signal id %u\n", signal_id);
  6512. err = -ENOENT;
  6513. goto bail;
  6514. }
  6515. if (s->state != DSPSIGNAL_STATE_PENDING) {
  6516. if ((s->state == DSPSIGNAL_STATE_CANCELED) || (s->state == DSPSIGNAL_STATE_UNUSED))
  6517. err = -EINTR;
  6518. if (s->state == DSPSIGNAL_STATE_SIGNALED) {
  6519. /* Signal already received from DSP. Reset signal state and return */
  6520. s->state = DSPSIGNAL_STATE_PENDING;
  6521. reinit_completion(&s->comp);
  6522. }
  6523. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6524. DSPSIGNAL_VERBOSE("Signal %u in state %u, complete wait immediately",
  6525. signal_id, s->state);
  6526. goto bail;
  6527. }
  6528. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6529. trace_fastrpc_dspsignal("wait", signal_id, s->state, wait->timeout_usec);
  6530. if (timeout != 0xffffffff)
  6531. ret = wait_for_completion_interruptible_timeout(&s->comp, timeout);
  6532. else
  6533. ret = wait_for_completion_interruptible(&s->comp);
  6534. trace_fastrpc_dspsignal("wakeup", signal_id, s->state, wait->timeout_usec);
  6535. if (ret == 0) {
  6536. DSPSIGNAL_VERBOSE("Wait for signal %u timed out\n", signal_id);
  6537. err = -ETIMEDOUT;
  6538. goto bail;
  6539. } else if (ret < 0) {
  6540. ADSPRPC_ERR("Wait for signal %u failed %d\n", signal_id, (int)ret);
  6541. err = ret;
  6542. goto bail;
  6543. }
  6544. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6545. if (s->state == DSPSIGNAL_STATE_SIGNALED) {
  6546. s->state = DSPSIGNAL_STATE_PENDING;
  6547. DSPSIGNAL_VERBOSE("Signal %u completed\n", signal_id);
  6548. } else if ((s->state == DSPSIGNAL_STATE_CANCELED) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6549. DSPSIGNAL_VERBOSE("Signal %u cancelled or destroyed\n", signal_id);
  6550. err = -EINTR;
  6551. }
  6552. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6553. bail:
  6554. return err;
  6555. }
  6556. int fastrpc_dspsignal_create(struct fastrpc_file *fl,
  6557. struct fastrpc_ioctl_dspsignal_create *create)
  6558. {
  6559. int err = 0, cid = -1;
  6560. uint32_t signal_id = create->signal_id;
  6561. struct fastrpc_dspsignal *group, *sig;
  6562. unsigned long irq_flags = 0;
  6563. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6564. if (err) {
  6565. err = -EINVAL;
  6566. goto bail;
  6567. }
  6568. cid = fl->cid;
  6569. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6570. if (err) {
  6571. err = -EBADR;
  6572. goto bail;
  6573. }
  6574. // Use a separate mutex for creating signals. This avoids holding on
  6575. // to a spinlock if we need to allocate a whole group of signals. The
  6576. // mutex ensures nobody else will allocate the same group.
  6577. mutex_lock(&fl->signal_create_mutex);
  6578. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6579. group = fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6580. if (group == NULL) {
  6581. int i;
  6582. // Release the spinlock while we allocate a new group but take
  6583. // it back before taking the group into use. No other code
  6584. // allocates groups so the mutex is sufficient.
  6585. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6586. VERIFY(err, (group = kzalloc(DSPSIGNAL_GROUP_SIZE * sizeof(*group),
  6587. GFP_KERNEL)) != NULL);
  6588. if (err) {
  6589. ADSPRPC_ERR("Unable to allocate signal group\n");
  6590. err = -ENOMEM;
  6591. mutex_unlock(&fl->signal_create_mutex);
  6592. goto bail;
  6593. }
  6594. for (i = 0; i < DSPSIGNAL_GROUP_SIZE; i++) {
  6595. sig = &group[i];
  6596. init_completion(&sig->comp);
  6597. sig->state = DSPSIGNAL_STATE_UNUSED;
  6598. }
  6599. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6600. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] = group;
  6601. }
  6602. sig = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6603. if (sig->state != DSPSIGNAL_STATE_UNUSED) {
  6604. err = -EBUSY;
  6605. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6606. mutex_unlock(&fl->signal_create_mutex);
  6607. ADSPRPC_ERR("Attempting to create signal %u already in use (state %u)\n",
  6608. signal_id, sig->state);
  6609. goto bail;
  6610. }
  6611. sig->state = DSPSIGNAL_STATE_PENDING;
  6612. reinit_completion(&sig->comp);
  6613. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6614. mutex_unlock(&fl->signal_create_mutex);
  6615. DSPSIGNAL_VERBOSE("Signal %u created\n", signal_id);
  6616. bail:
  6617. return err;
  6618. }
  6619. int fastrpc_dspsignal_destroy(struct fastrpc_file *fl,
  6620. struct fastrpc_ioctl_dspsignal_destroy *destroy)
  6621. {
  6622. int err = 0, cid = -1;
  6623. uint32_t signal_id = destroy->signal_id;
  6624. struct fastrpc_dspsignal *s = NULL;
  6625. unsigned long irq_flags = 0;
  6626. DSPSIGNAL_VERBOSE("Destroy signal %u\n", signal_id);
  6627. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6628. if (err) {
  6629. err = -EINVAL;
  6630. goto bail;
  6631. }
  6632. cid = fl->cid;
  6633. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6634. if (err) {
  6635. err = -EBADR;
  6636. goto bail;
  6637. }
  6638. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6639. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6640. struct fastrpc_dspsignal *group =
  6641. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6642. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6643. }
  6644. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6645. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6646. ADSPRPC_ERR("Attempting to destroy unused signal %u\n", signal_id);
  6647. err = -ENOENT;
  6648. goto bail;
  6649. }
  6650. s->state = DSPSIGNAL_STATE_UNUSED;
  6651. complete_all(&s->comp);
  6652. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6653. DSPSIGNAL_VERBOSE("Signal %u destroyed\n", signal_id);
  6654. bail:
  6655. return err;
  6656. }
  6657. int fastrpc_dspsignal_cancel_wait(struct fastrpc_file *fl,
  6658. struct fastrpc_ioctl_dspsignal_cancel_wait *cancel)
  6659. {
  6660. int err = 0, cid = -1;
  6661. uint32_t signal_id = cancel->signal_id;
  6662. struct fastrpc_dspsignal *s = NULL;
  6663. unsigned long irq_flags = 0;
  6664. DSPSIGNAL_VERBOSE("Cancel wait for signal %u\n", signal_id);
  6665. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6666. if (err) {
  6667. err = -EINVAL;
  6668. goto bail;
  6669. }
  6670. cid = fl->cid;
  6671. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6672. if (err) {
  6673. err = -EBADR;
  6674. goto bail;
  6675. }
  6676. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6677. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6678. struct fastrpc_dspsignal *group =
  6679. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6680. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6681. }
  6682. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6683. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6684. ADSPRPC_ERR("Attempting to cancel unused signal %u\n", signal_id);
  6685. err = -ENOENT;
  6686. goto bail;
  6687. }
  6688. if (s->state != DSPSIGNAL_STATE_CANCELED) {
  6689. s->state = DSPSIGNAL_STATE_CANCELED;
  6690. trace_fastrpc_dspsignal("cancel", signal_id, s->state, 0);
  6691. complete_all(&s->comp);
  6692. }
  6693. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6694. DSPSIGNAL_VERBOSE("Signal %u cancelled\n", signal_id);
  6695. bail:
  6696. return err;
  6697. }
  6698. static inline int fastrpc_mmap_device_ioctl(struct fastrpc_file *fl,
  6699. unsigned int ioctl_num, union fastrpc_ioctl_param *p,
  6700. void *param)
  6701. {
  6702. union {
  6703. struct fastrpc_ioctl_mmap mmap;
  6704. struct fastrpc_ioctl_munmap munmap;
  6705. } i;
  6706. int err = 0;
  6707. switch (ioctl_num) {
  6708. case FASTRPC_IOCTL_MEM_MAP:
  6709. K_COPY_FROM_USER(err, 0, &p->mem_map, param,
  6710. sizeof(p->mem_map));
  6711. if (err) {
  6712. err = -EFAULT;
  6713. goto bail;
  6714. }
  6715. VERIFY(err, 0 == (err = fastrpc_internal_mem_map(fl,
  6716. &p->mem_map)));
  6717. if (err)
  6718. goto bail;
  6719. K_COPY_TO_USER(err, 0, param, &p->mem_map, sizeof(p->mem_map));
  6720. if (err) {
  6721. err = -EFAULT;
  6722. goto bail;
  6723. }
  6724. break;
  6725. case FASTRPC_IOCTL_MEM_UNMAP:
  6726. K_COPY_FROM_USER(err, 0, &p->mem_unmap, param,
  6727. sizeof(p->mem_unmap));
  6728. if (err) {
  6729. err = -EFAULT;
  6730. goto bail;
  6731. }
  6732. VERIFY(err, 0 == (err = fastrpc_internal_mem_unmap(fl,
  6733. &p->mem_unmap)));
  6734. if (err)
  6735. goto bail;
  6736. K_COPY_TO_USER(err, 0, param, &p->mem_unmap,
  6737. sizeof(p->mem_unmap));
  6738. if (err) {
  6739. err = -EFAULT;
  6740. goto bail;
  6741. }
  6742. break;
  6743. case FASTRPC_IOCTL_MMAP:
  6744. K_COPY_FROM_USER(err, 0, &p->mmap, param,
  6745. sizeof(p->mmap));
  6746. if (err) {
  6747. err = -EFAULT;
  6748. goto bail;
  6749. }
  6750. VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &p->mmap)));
  6751. if (err)
  6752. goto bail;
  6753. K_COPY_TO_USER(err, 0, param, &p->mmap, sizeof(p->mmap));
  6754. if (err) {
  6755. err = -EFAULT;
  6756. goto bail;
  6757. }
  6758. break;
  6759. case FASTRPC_IOCTL_MUNMAP:
  6760. K_COPY_FROM_USER(err, 0, &p->munmap, param,
  6761. sizeof(p->munmap));
  6762. if (err) {
  6763. err = -EFAULT;
  6764. goto bail;
  6765. }
  6766. VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl,
  6767. &p->munmap)));
  6768. if (err)
  6769. goto bail;
  6770. break;
  6771. case FASTRPC_IOCTL_MMAP_64:
  6772. K_COPY_FROM_USER(err, 0, &p->mmap64, param,
  6773. sizeof(p->mmap64));
  6774. if (err) {
  6775. err = -EFAULT;
  6776. goto bail;
  6777. }
  6778. get_fastrpc_ioctl_mmap_64(&p->mmap64, &i.mmap);
  6779. VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &i.mmap)));
  6780. if (err)
  6781. goto bail;
  6782. put_fastrpc_ioctl_mmap_64(&p->mmap64, &i.mmap);
  6783. K_COPY_TO_USER(err, 0, param, &p->mmap64, sizeof(p->mmap64));
  6784. if (err) {
  6785. err = -EFAULT;
  6786. goto bail;
  6787. }
  6788. break;
  6789. case FASTRPC_IOCTL_MUNMAP_64:
  6790. K_COPY_FROM_USER(err, 0, &p->munmap64, param,
  6791. sizeof(p->munmap64));
  6792. if (err) {
  6793. err = -EFAULT;
  6794. goto bail;
  6795. }
  6796. get_fastrpc_ioctl_munmap_64(&p->munmap64, &i.munmap);
  6797. VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl,
  6798. &i.munmap)));
  6799. if (err)
  6800. goto bail;
  6801. break;
  6802. case FASTRPC_IOCTL_MUNMAP_FD:
  6803. K_COPY_FROM_USER(err, 0, &p->munmap_fd, param,
  6804. sizeof(p->munmap_fd));
  6805. if (err) {
  6806. err = -EFAULT;
  6807. goto bail;
  6808. }
  6809. VERIFY(err, 0 == (err = fastrpc_internal_munmap_fd(fl,
  6810. &p->munmap_fd)));
  6811. if (err)
  6812. goto bail;
  6813. break;
  6814. default:
  6815. err = -ENOTTY;
  6816. pr_info("bad ioctl: %d\n", ioctl_num);
  6817. break;
  6818. }
  6819. bail:
  6820. return err;
  6821. }
  6822. static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num,
  6823. unsigned long ioctl_param)
  6824. {
  6825. union fastrpc_ioctl_param p;
  6826. void *param = (char *)ioctl_param;
  6827. struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
  6828. int size = 0, err = 0;
  6829. uint32_t info;
  6830. p.inv.fds = NULL;
  6831. p.inv.attrs = NULL;
  6832. p.inv.crc = NULL;
  6833. p.inv.perf_kernel = NULL;
  6834. p.inv.perf_dsp = NULL;
  6835. p.inv.job = NULL;
  6836. err = fastrpc_file_get(fl);
  6837. if (err) {
  6838. ADSPRPC_ERR("Failed to get user process reference\n");
  6839. goto bail;
  6840. }
  6841. spin_lock(&fl->hlock);
  6842. if (fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  6843. err = -ESHUTDOWN;
  6844. pr_warn("adsprpc: fastrpc_device_release is happening, So not sending any new requests to DSP\n");
  6845. spin_unlock(&fl->hlock);
  6846. goto bail;
  6847. }
  6848. spin_unlock(&fl->hlock);
  6849. switch (ioctl_num) {
  6850. case FASTRPC_IOCTL_INVOKE:
  6851. size = sizeof(struct fastrpc_ioctl_invoke);
  6852. fallthrough;
  6853. case FASTRPC_IOCTL_INVOKE_FD:
  6854. if (!size)
  6855. size = sizeof(struct fastrpc_ioctl_invoke_fd);
  6856. fallthrough;
  6857. case FASTRPC_IOCTL_INVOKE_ATTRS:
  6858. if (!size)
  6859. size = sizeof(struct fastrpc_ioctl_invoke_attrs);
  6860. fallthrough;
  6861. case FASTRPC_IOCTL_INVOKE_CRC:
  6862. if (!size)
  6863. size = sizeof(struct fastrpc_ioctl_invoke_crc);
  6864. fallthrough;
  6865. case FASTRPC_IOCTL_INVOKE_PERF:
  6866. if (!size)
  6867. size = sizeof(struct fastrpc_ioctl_invoke_perf);
  6868. trace_fastrpc_msg("invoke: begin");
  6869. K_COPY_FROM_USER(err, 0, &p.inv, param, size);
  6870. if (err) {
  6871. err = -EFAULT;
  6872. goto bail;
  6873. }
  6874. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode,
  6875. USER_MSG, &p.inv)));
  6876. trace_fastrpc_msg("invoke: end");
  6877. if (err)
  6878. goto bail;
  6879. break;
  6880. case FASTRPC_IOCTL_INVOKE2:
  6881. K_COPY_FROM_USER(err, 0, &p.inv2, param,
  6882. sizeof(struct fastrpc_ioctl_invoke2));
  6883. if (err) {
  6884. err = -EFAULT;
  6885. goto bail;
  6886. }
  6887. VERIFY(err, 0 == (err = fastrpc_internal_invoke2(fl, &p.inv2)));
  6888. if (err)
  6889. goto bail;
  6890. break;
  6891. case FASTRPC_IOCTL_SETMODE:
  6892. err = fastrpc_setmode(ioctl_param, fl);
  6893. break;
  6894. case FASTRPC_IOCTL_CONTROL:
  6895. err = fastrpc_control(&p.cp, param, fl);
  6896. break;
  6897. case FASTRPC_IOCTL_GETINFO:
  6898. K_COPY_FROM_USER(err, 0, &info, param, sizeof(info));
  6899. if (err) {
  6900. err = -EFAULT;
  6901. goto bail;
  6902. }
  6903. VERIFY(err, 0 == (err = fastrpc_get_info(fl, &info)));
  6904. if (err)
  6905. goto bail;
  6906. K_COPY_TO_USER(err, 0, param, &info, sizeof(info));
  6907. if (err) {
  6908. err = -EFAULT;
  6909. goto bail;
  6910. }
  6911. break;
  6912. case FASTRPC_IOCTL_INIT:
  6913. p.init.attrs = 0;
  6914. p.init.siglen = 0;
  6915. size = sizeof(struct fastrpc_ioctl_init);
  6916. fallthrough;
  6917. case FASTRPC_IOCTL_INIT_ATTRS:
  6918. if (!size)
  6919. size = sizeof(struct fastrpc_ioctl_init_attrs);
  6920. K_COPY_FROM_USER(err, 0, &p.init, param, size);
  6921. if (err) {
  6922. err = -EFAULT;
  6923. goto bail;
  6924. }
  6925. VERIFY(err, 0 == (err = fastrpc_init_process(fl, &p.init)));
  6926. if (err)
  6927. goto bail;
  6928. break;
  6929. case FASTRPC_IOCTL_GET_DSP_INFO:
  6930. err = fastrpc_get_dsp_info(&p.cap, param, fl);
  6931. break;
  6932. case FASTRPC_IOCTL_MEM_MAP:
  6933. fallthrough;
  6934. case FASTRPC_IOCTL_MEM_UNMAP:
  6935. fallthrough;
  6936. case FASTRPC_IOCTL_MMAP:
  6937. fallthrough;
  6938. case FASTRPC_IOCTL_MUNMAP:
  6939. fallthrough;
  6940. case FASTRPC_IOCTL_MMAP_64:
  6941. fallthrough;
  6942. case FASTRPC_IOCTL_MUNMAP_64:
  6943. fallthrough;
  6944. case FASTRPC_IOCTL_MUNMAP_FD:
  6945. err = fastrpc_mmap_device_ioctl(fl, ioctl_num, &p, param);
  6946. break;
  6947. case FASTRPC_IOCTL_DSPSIGNAL_SIGNAL:
  6948. K_COPY_FROM_USER(err, 0, &p.sig, param,
  6949. sizeof(struct fastrpc_ioctl_dspsignal_signal));
  6950. if (err) {
  6951. err = -EFAULT;
  6952. goto bail;
  6953. }
  6954. VERIFY(err, 0 == (err = fastrpc_dspsignal_signal(fl, &p.sig)));
  6955. if (err)
  6956. goto bail;
  6957. break;
  6958. case FASTRPC_IOCTL_DSPSIGNAL_WAIT:
  6959. K_COPY_FROM_USER(err, 0, &p.wait, param,
  6960. sizeof(struct fastrpc_ioctl_dspsignal_wait));
  6961. if (err) {
  6962. err = -EFAULT;
  6963. goto bail;
  6964. }
  6965. VERIFY(err, 0 == (err = fastrpc_dspsignal_wait(fl, &p.wait)));
  6966. if (err)
  6967. goto bail;
  6968. break;
  6969. case FASTRPC_IOCTL_DSPSIGNAL_CREATE:
  6970. K_COPY_FROM_USER(err, 0, &p.cre, param,
  6971. sizeof(struct fastrpc_ioctl_dspsignal_create));
  6972. if (err) {
  6973. err = -EFAULT;
  6974. goto bail;
  6975. }
  6976. VERIFY(err, 0 == (err = fastrpc_dspsignal_create(fl, &p.cre)));
  6977. if (err)
  6978. goto bail;
  6979. break;
  6980. case FASTRPC_IOCTL_DSPSIGNAL_DESTROY:
  6981. K_COPY_FROM_USER(err, 0, &p.des, param,
  6982. sizeof(struct fastrpc_ioctl_dspsignal_destroy));
  6983. if (err) {
  6984. err = -EFAULT;
  6985. goto bail;
  6986. }
  6987. VERIFY(err, 0 == (err = fastrpc_dspsignal_destroy(fl, &p.des)));
  6988. if (err)
  6989. goto bail;
  6990. break;
  6991. case FASTRPC_IOCTL_DSPSIGNAL_CANCEL_WAIT:
  6992. K_COPY_FROM_USER(err, 0, &p.canc, param,
  6993. sizeof(struct fastrpc_ioctl_dspsignal_cancel_wait));
  6994. if (err) {
  6995. err = -EFAULT;
  6996. goto bail;
  6997. }
  6998. VERIFY(err, 0 == (err = fastrpc_dspsignal_cancel_wait(fl, &p.canc)));
  6999. if (err)
  7000. goto bail;
  7001. break;
  7002. default:
  7003. err = -ENOTTY;
  7004. pr_info("bad ioctl: %d\n", ioctl_num);
  7005. break;
  7006. }
  7007. bail:
  7008. fastrpc_file_put(fl);
  7009. return err;
  7010. }
  7011. /*
  7012. * fastrpc_smq_ctx_detail : Store smq_invoke_ctx structure parameter.
  7013. * Input :
  7014. * structure smq_invoke_ctx
  7015. * void* mini_dump_buff
  7016. */
  7017. static void fastrpc_smq_ctx_detail(struct smq_invoke_ctx *smq_ctx, int cid, void *mini_dump_buff)
  7018. {
  7019. int i = 0;
  7020. remote_arg64_t *rpra = NULL;
  7021. struct fastrpc_mmap *map = NULL;
  7022. if (!smq_ctx)
  7023. return;
  7024. if (smq_ctx->buf && smq_ctx->buf->virt)
  7025. rpra = smq_ctx->buf->virt;
  7026. for (i = 0; rpra &&
  7027. i < (REMOTE_SCALARS_INBUFS(smq_ctx->sc) + REMOTE_SCALARS_OUTBUFS(smq_ctx->sc));
  7028. ++i) {
  7029. map = smq_ctx->maps[i];
  7030. if (map) {
  7031. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7032. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7033. smq_invoke_ctx_params,
  7034. smq_ctx->pid, smq_ctx->tgid, smq_ctx->handle,
  7035. smq_ctx->sc, smq_ctx->fl, smq_ctx->fds,
  7036. smq_ctx->magic);
  7037. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7038. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7039. fastrpc_mmap_params,
  7040. map->fd, map->flags, map->buf,
  7041. map->phys, map->size, map->va,
  7042. map->raddr, map->len, map->refs,
  7043. map->secure);
  7044. } else {
  7045. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7046. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7047. smq_invoke_ctx_params, smq_ctx->pid, smq_ctx->tgid,
  7048. smq_ctx->handle, smq_ctx->sc, smq_ctx->fl, smq_ctx->fds,
  7049. smq_ctx->magic);
  7050. }
  7051. break;
  7052. }
  7053. }
  7054. /*
  7055. * fastrpc_print_fastrpcbuf : Print fastrpc_buf structure parameter.
  7056. * Input :
  7057. * structure fastrpc_buf
  7058. * void* buffer
  7059. */
  7060. static void fastrpc_print_fastrpcbuf(struct fastrpc_buf *buf, void *buffer)
  7061. {
  7062. if (!buf || !buffer)
  7063. return;
  7064. scnprintf(buffer + strlen(buffer),
  7065. MINI_DUMP_DBG_SIZE - strlen(buffer),
  7066. fastrpc_buf_params, buf->fl, buf->phys,
  7067. buf->virt, buf->size, buf->dma_attr, buf->raddr,
  7068. buf->flags, buf->type, buf->in_use);
  7069. }
  7070. /*
  7071. * fastrpc_print_debug_data : Print debug structure variable in CMA memory.
  7072. * Input cid: Channel id
  7073. */
  7074. static void fastrpc_print_debug_data(int cid)
  7075. {
  7076. unsigned int i = 0, count = 0, gmsg_log_iter = 3, err = 0, len = 0;
  7077. unsigned int tx_index = 0, rx_index = 0;
  7078. unsigned long flags = 0;
  7079. char *gmsg_log_tx = NULL;
  7080. char *gmsg_log_rx = NULL;
  7081. void *mini_dump_buff = NULL;
  7082. struct fastrpc_apps *me = &gfa;
  7083. struct smq_invoke_rspv2 *rsp = NULL;
  7084. struct fastrpc_file *fl = NULL;
  7085. struct fastrpc_channel_ctx *chan = NULL;
  7086. struct hlist_node *n = NULL;
  7087. struct smq_invoke_ctx *ictx = NULL;
  7088. struct fastrpc_tx_msg *tx_msg = NULL;
  7089. struct fastrpc_buf *buf = NULL;
  7090. struct fastrpc_mmap *map = NULL;
  7091. unsigned long irq_flags = 0;
  7092. VERIFY(err, NULL != (gmsg_log_tx = kzalloc(MD_GMSG_BUFFER, GFP_KERNEL)));
  7093. if (err) {
  7094. err = -ENOMEM;
  7095. goto free_buf;
  7096. }
  7097. VERIFY(err, NULL != (gmsg_log_rx = kzalloc(MD_GMSG_BUFFER, GFP_KERNEL)));
  7098. if (err) {
  7099. err = -ENOMEM;
  7100. goto free_buf;
  7101. }
  7102. chan = &me->channel[cid];
  7103. if ((!chan) || (!chan->buf))
  7104. goto free_buf;
  7105. mini_dump_buff = chan->buf->virt;
  7106. if (!mini_dump_buff)
  7107. goto free_buf;
  7108. if (chan) {
  7109. tx_index = chan->gmsg_log.tx_index;
  7110. rx_index = chan->gmsg_log.rx_index;
  7111. }
  7112. spin_lock_irqsave(&me->hlock, irq_flags);
  7113. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  7114. err = fastrpc_file_get(fl);
  7115. if (err) {
  7116. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7117. ADSPRPC_ERR("Failed to get user process reference\n");
  7118. goto free_buf;
  7119. }
  7120. if (fl->cid == cid && fl->is_ramdump_pend) {
  7121. scnprintf(mini_dump_buff +
  7122. strlen(mini_dump_buff),
  7123. MINI_DUMP_DBG_SIZE -
  7124. strlen(mini_dump_buff),
  7125. "\nfastrpc_file : %p\n", fl);
  7126. scnprintf(mini_dump_buff +
  7127. strlen(mini_dump_buff),
  7128. MINI_DUMP_DBG_SIZE -
  7129. strlen(mini_dump_buff),
  7130. fastrpc_file_params, fl->tgid,
  7131. fl->cid, fl->ssrcount, fl->pd,
  7132. fl->profile, fl->mode,
  7133. fl->tgid_open, fl->num_cached_buf,
  7134. fl->num_pers_hdrs, fl->sessionid,
  7135. fl->servloc_name, fl->file_close,
  7136. fl->dsp_proc_init, fl->apps,
  7137. fl->qos_request, fl->dev_minor,
  7138. fl->debug_buf,
  7139. fl->debug_buf_alloced_attempted,
  7140. fl->wake_enable,
  7141. fl->ws_timeout,
  7142. fl->untrusted_process);
  7143. scnprintf(mini_dump_buff +
  7144. strlen(mini_dump_buff),
  7145. MINI_DUMP_DBG_SIZE -
  7146. strlen(mini_dump_buff),
  7147. "\nSession Maps\n");
  7148. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  7149. scnprintf(mini_dump_buff +
  7150. strlen(mini_dump_buff),
  7151. MINI_DUMP_DBG_SIZE -
  7152. strlen(mini_dump_buff),
  7153. fastrpc_mmap_params,
  7154. map->fd,
  7155. map->flags, map->buf,
  7156. map->phys, map->size,
  7157. map->va, map->raddr,
  7158. map->len, map->refs,
  7159. map->secure);
  7160. }
  7161. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7162. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7163. "\ncached_bufs\n");
  7164. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  7165. fastrpc_print_fastrpcbuf(buf, mini_dump_buff);
  7166. }
  7167. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7168. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7169. "\ninit_mem: %p\n", fl->init_mem);
  7170. fastrpc_print_fastrpcbuf(fl->init_mem, mini_dump_buff);
  7171. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7172. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7173. "\npers_hdr_buf: %p\n", fl->pers_hdr_buf);
  7174. fastrpc_print_fastrpcbuf(fl->pers_hdr_buf, mini_dump_buff);
  7175. snprintf(mini_dump_buff + strlen(mini_dump_buff),
  7176. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7177. "\nhdr_bufs: %p\n", fl->hdr_bufs);
  7178. fastrpc_print_fastrpcbuf(fl->hdr_bufs, mini_dump_buff);
  7179. if (fl->debugfs_file) {
  7180. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7181. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7182. "\nfl->debugfs_file.d_iname : %s\n",
  7183. fl->debugfs_file->d_iname);
  7184. }
  7185. if (fl->sctx) {
  7186. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7187. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7188. "\nfl->sctx->smmu.cb : %d\n",
  7189. fl->sctx->smmu.cb);
  7190. }
  7191. if (fl->secsctx) {
  7192. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7193. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7194. "\nfl->secsctx->smmu.cb : %d\n",
  7195. fl->secsctx->smmu.cb);
  7196. }
  7197. spin_lock(&fl->hlock);
  7198. scnprintf(mini_dump_buff +
  7199. strlen(mini_dump_buff),
  7200. MINI_DUMP_DBG_SIZE -
  7201. strlen(mini_dump_buff),
  7202. "\nPending Ctx:\n");
  7203. hlist_for_each_entry_safe(ictx, n, &fl->clst.pending, hn) {
  7204. fastrpc_smq_ctx_detail(ictx,
  7205. cid, mini_dump_buff);
  7206. }
  7207. scnprintf(mini_dump_buff +
  7208. strlen(mini_dump_buff),
  7209. MINI_DUMP_DBG_SIZE -
  7210. strlen(mini_dump_buff),
  7211. "\nInterrupted Ctx:\n");
  7212. hlist_for_each_entry_safe(ictx, n,
  7213. &fl->clst.interrupted,
  7214. hn) {
  7215. fastrpc_smq_ctx_detail(ictx,
  7216. cid, mini_dump_buff);
  7217. }
  7218. spin_unlock(&fl->hlock);
  7219. fastrpc_file_put(fl);
  7220. }
  7221. }
  7222. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7223. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  7224. if (rx_index) {
  7225. for (i = rx_index, count = 0, len = 0 ; i > 0 &&
  7226. count <= gmsg_log_iter; i--, count++) {
  7227. rsp = &chan->gmsg_log.rx_msgs[i].rsp;
  7228. len += scnprintf(gmsg_log_rx + len, MD_GMSG_BUFFER - len,
  7229. "ctx: 0x%x, retval: %d, flags: %d, early_wake_time: %d, version: %d\n",
  7230. rsp->ctx, rsp->retval, rsp->flags,
  7231. rsp->early_wake_time, rsp->version);
  7232. }
  7233. }
  7234. if (tx_index) {
  7235. for (i = tx_index, count = 0, len = 0;
  7236. i > 0 && count <= gmsg_log_iter;
  7237. i--, count++) {
  7238. tx_msg = &chan->gmsg_log.tx_msgs[i];
  7239. len += scnprintf(gmsg_log_tx + len, MD_GMSG_BUFFER - len,
  7240. "pid: %d, tid: %d, ctx: 0x%x, handle: 0x%x, sc: 0x%x, addr: 0x%x, size:%d\n",
  7241. tx_msg->msg.pid,
  7242. tx_msg->msg.tid,
  7243. tx_msg->msg.invoke.header.ctx,
  7244. tx_msg->msg.invoke.header.handle,
  7245. tx_msg->msg.invoke.header.sc,
  7246. tx_msg->msg.invoke.page.addr,
  7247. tx_msg->msg.invoke.page.size);
  7248. }
  7249. }
  7250. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  7251. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7252. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7253. "gmsg_log_tx:\n%s\n", gmsg_log_tx);
  7254. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7255. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7256. "gmsg_log_rx:\n %s\n", gmsg_log_rx);
  7257. if (chan && chan->buf)
  7258. chan->buf->size = strlen(mini_dump_buff);
  7259. free_buf:
  7260. kfree(gmsg_log_tx);
  7261. kfree(gmsg_log_rx);
  7262. }
  7263. void fastrpc_restart_drivers(int cid)
  7264. {
  7265. struct fastrpc_apps *me = &gfa;
  7266. fastrpc_notify_drivers(me, cid);
  7267. me->channel[cid].ssrcount++;
  7268. }
  7269. static int fastrpc_restart_notifier_cb(struct notifier_block *nb,
  7270. unsigned long code,
  7271. void *data)
  7272. {
  7273. struct fastrpc_apps *me = &gfa;
  7274. struct fastrpc_channel_ctx *ctx;
  7275. struct fastrpc_file *fl;
  7276. struct hlist_node *n;
  7277. int cid = -1;
  7278. struct timespec64 startT = {0};
  7279. unsigned long irq_flags = 0;
  7280. ctx = container_of(nb, struct fastrpc_channel_ctx, nb);
  7281. cid = ctx - &me->channel[0];
  7282. switch (code) {
  7283. case QCOM_SSR_BEFORE_SHUTDOWN:
  7284. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7285. "QCOM_SSR_BEFORE_SHUTDOWN", "fastrpc_restart_notifier-enter");
  7286. pr_info("adsprpc: %s: %s subsystem is restarting\n",
  7287. __func__, gcinfo[cid].subsys);
  7288. mutex_lock(&me->channel[cid].smd_mutex);
  7289. ctx->ssrcount++;
  7290. ctx->subsystemstate = SUBSYSTEM_RESTARTING;
  7291. mutex_unlock(&me->channel[cid].smd_mutex);
  7292. if (cid == RH_CID)
  7293. me->staticpd_flags = 0;
  7294. break;
  7295. case QCOM_SSR_AFTER_SHUTDOWN:
  7296. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7297. "QCOM_SSR_AFTER_SHUTDOWN", "fastrpc_restart_notifier-enter");
  7298. spin_lock_irqsave(&me->hlock, irq_flags);
  7299. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  7300. if (fl->cid != cid)
  7301. continue;
  7302. complete(&fl->shutdown);
  7303. }
  7304. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7305. ctx->subsystemstate = SUBSYSTEM_DOWN;
  7306. pr_info("adsprpc: %s: received RAMDUMP notification for %s\n",
  7307. __func__, gcinfo[cid].subsys);
  7308. break;
  7309. case QCOM_SSR_BEFORE_POWERUP:
  7310. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7311. "QCOM_SSR_BEFORE_POWERUP", "fastrpc_restart_notifier-enter");
  7312. pr_info("adsprpc: %s: subsystem %s is about to start\n",
  7313. __func__, gcinfo[cid].subsys);
  7314. if (cid == CDSP_DOMAIN_ID && dump_enabled() &&
  7315. ctx->ssrcount)
  7316. fastrpc_update_ramdump_status(cid);
  7317. fastrpc_notify_drivers(me, cid);
  7318. /* Skip ram dump collection in first boot */
  7319. if (cid == CDSP_DOMAIN_ID && dump_enabled() &&
  7320. ctx->ssrcount) {
  7321. mutex_lock(&me->channel[cid].smd_mutex);
  7322. fastrpc_print_debug_data(cid);
  7323. mutex_unlock(&me->channel[cid].smd_mutex);
  7324. ktime_get_real_ts64(&startT);
  7325. fastrpc_ramdump_collection(cid);
  7326. pr_info("adsprpc: %s: fastrpc ramdump finished in %lu (us)\n",
  7327. __func__, getnstimediff(&startT));
  7328. }
  7329. break;
  7330. case QCOM_SSR_AFTER_POWERUP:
  7331. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7332. "QCOM_SSR_AFTER_POWERUP", "fastrpc_restart_notifier-enter");
  7333. pr_info("adsprpc: %s: %s subsystem is up\n",
  7334. __func__, gcinfo[cid].subsys);
  7335. ctx->subsystemstate = SUBSYSTEM_UP;
  7336. break;
  7337. default:
  7338. break;
  7339. }
  7340. fastrpc_rproc_trace_events(dev_name(me->dev), "fastrpc_restart_notifier", "exit");
  7341. return NOTIFY_DONE;
  7342. }
  7343. static void fastrpc_pdr_cb(int state, char *service_path, void *priv)
  7344. {
  7345. struct fastrpc_apps *me = &gfa;
  7346. struct fastrpc_static_pd *spd;
  7347. int err = 0;
  7348. spd = priv;
  7349. VERIFY(err, spd);
  7350. if (err)
  7351. goto bail;
  7352. switch (state) {
  7353. case SERVREG_SERVICE_STATE_DOWN:
  7354. pr_info("adsprpc: %s: %s (%s) is down for PDR on %s\n",
  7355. __func__, spd->spdname,
  7356. spd->servloc_name,
  7357. gcinfo[spd->cid].subsys);
  7358. mutex_lock(&me->channel[spd->cid].smd_mutex);
  7359. spd->pdrcount++;
  7360. atomic_set(&spd->ispdup, 0);
  7361. mutex_unlock(&me->channel[spd->cid].smd_mutex);
  7362. if (!strcmp(spd->servloc_name,
  7363. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME))
  7364. me->staticpd_flags = 0;
  7365. fastrpc_notify_pdr_drivers(me, spd->servloc_name);
  7366. break;
  7367. case SERVREG_SERVICE_STATE_UP:
  7368. pr_info("adsprpc: %s: %s (%s) is up for PDR on %s\n",
  7369. __func__, spd->spdname,
  7370. spd->servloc_name,
  7371. gcinfo[spd->cid].subsys);
  7372. atomic_set(&spd->ispdup, 1);
  7373. wake_up_interruptible(&spd->wait_for_pdup);
  7374. break;
  7375. default:
  7376. break;
  7377. }
  7378. bail:
  7379. if (err) {
  7380. pr_err("adsprpc: %s: failed for path %s, state %d, spd %pK\n",
  7381. __func__, service_path, state, spd);
  7382. }
  7383. }
  7384. static const struct file_operations fops = {
  7385. .open = fastrpc_device_open,
  7386. .release = fastrpc_device_release,
  7387. .unlocked_ioctl = fastrpc_device_ioctl,
  7388. /* Only DSP service 64-bit app will interface with fastrpc TVM driver.
  7389. * There is not need to support 32-bit fastrpc driver on TVM.
  7390. */
  7391. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  7392. .compat_ioctl = NULL,
  7393. #else
  7394. .compat_ioctl = compat_fastrpc_device_ioctl,
  7395. #endif
  7396. };
  7397. static const struct of_device_id fastrpc_match_table[] = {
  7398. { .compatible = "qcom,msm-fastrpc-adsp", },
  7399. { .compatible = "qcom,msm-fastrpc-compute", },
  7400. { .compatible = "qcom,msm-fastrpc-compute-cb", },
  7401. { .compatible = "qcom,msm-adsprpc-mem-region", },
  7402. {}
  7403. };
  7404. static int fastrpc_cb_probe(struct device *dev)
  7405. {
  7406. struct fastrpc_channel_ctx *chan = NULL;
  7407. struct fastrpc_session_ctx *sess = NULL;
  7408. struct of_phandle_args iommuspec;
  7409. struct fastrpc_apps *me = &gfa;
  7410. struct fastrpc_buf *buf = NULL;
  7411. struct gen_pool *gen_pool = NULL;
  7412. struct iommu_domain *domain = NULL;
  7413. const char *name;
  7414. int err = 0, cid = -1, i = 0;
  7415. u32 sharedcb_count = 0, j = 0;
  7416. uint32_t dma_addr_pool[2] = {0, 0};
  7417. VERIFY(err, NULL != (name = of_get_property(dev->of_node,
  7418. "label", NULL)));
  7419. if (err) {
  7420. err = -EINVAL;
  7421. goto bail;
  7422. }
  7423. for (i = 0; i < NUM_CHANNELS; i++) {
  7424. if (!gcinfo[i].name)
  7425. continue;
  7426. if (!strcmp(name, gcinfo[i].name))
  7427. break;
  7428. }
  7429. VERIFY(err, i < NUM_CHANNELS);
  7430. if (err) {
  7431. err = -ECHRNG;
  7432. goto bail;
  7433. }
  7434. cid = i;
  7435. chan = &gcinfo[i];
  7436. VERIFY(err, chan->sesscount < NUM_SESSIONS);
  7437. if (err) {
  7438. err = -EINVAL;
  7439. goto bail;
  7440. }
  7441. err = of_parse_phandle_with_args(dev->of_node, "iommus",
  7442. "#iommu-cells", 0, &iommuspec);
  7443. if (err) {
  7444. pr_err("Error: adsprpc: %s: parsing iommu arguments failed for %s with err %d\n",
  7445. __func__, dev_name(dev), err);
  7446. goto bail;
  7447. }
  7448. sess = &chan->session[chan->sesscount];
  7449. sess->used = 0;
  7450. sess->smmu.coherent = of_property_read_bool(dev->of_node,
  7451. "dma-coherent");
  7452. sess->smmu.secure = of_property_read_bool(dev->of_node,
  7453. "qcom,secure-context-bank");
  7454. sess->smmu.cb = iommuspec.args[0] & 0xf;
  7455. sess->smmu.dev = dev;
  7456. sess->smmu.dev_name = dev_name(dev);
  7457. sess->smmu.enabled = 1;
  7458. if (!sess->smmu.dev->dma_parms)
  7459. sess->smmu.dev->dma_parms = devm_kzalloc(sess->smmu.dev,
  7460. sizeof(*sess->smmu.dev->dma_parms), GFP_KERNEL);
  7461. dma_set_max_seg_size(sess->smmu.dev, DMA_BIT_MASK(32));
  7462. dma_set_seg_boundary(sess->smmu.dev, (unsigned long)DMA_BIT_MASK(64));
  7463. of_property_read_u32_array(dev->of_node, "qcom,iommu-dma-addr-pool",
  7464. dma_addr_pool, 2);
  7465. me->max_size_limit = (dma_addr_pool[1] == 0 ? 0x78000000 :
  7466. dma_addr_pool[1]);
  7467. if (of_get_property(dev->of_node, "pd-type", NULL) != NULL) {
  7468. err = of_property_read_u32(dev->of_node, "pd-type",
  7469. &(sess->smmu.pd_type));
  7470. /* Set cb_pd_type, if the process type is set for context banks */
  7471. me->cb_pd_type = true;
  7472. if (err)
  7473. goto bail;
  7474. }
  7475. if (of_get_property(dev->of_node, "shared-cb", NULL) != NULL) {
  7476. sess->smmu.sharedcb = 1;
  7477. // Set share_securecb, if the secure context bank is shared
  7478. if (sess->smmu.secure)
  7479. me->share_securecb = 1;
  7480. err = of_property_read_u32(dev->of_node, "shared-cb",
  7481. &sharedcb_count);
  7482. if (err)
  7483. goto bail;
  7484. if (sharedcb_count > 0) {
  7485. struct fastrpc_session_ctx *dup_sess;
  7486. for (j = 1; j < sharedcb_count &&
  7487. chan->sesscount < NUM_SESSIONS; j++) {
  7488. chan->sesscount++;
  7489. dup_sess = &chan->session[chan->sesscount];
  7490. memcpy(dup_sess, sess,
  7491. sizeof(struct fastrpc_session_ctx));
  7492. }
  7493. }
  7494. }
  7495. if (of_get_property(dev->of_node, "qrtr-gen-pool", NULL) != NULL) {
  7496. u32 frpc_gen_addr_pool[2] = {0, 0};
  7497. struct sg_table sgt;
  7498. err = of_property_read_u32_array(dev->of_node, "frpc-gen-addr-pool",
  7499. frpc_gen_addr_pool, 2);
  7500. if (err) {
  7501. pr_err("Error: adsprpc: %s: parsing frpc-gen-addr-pool arguments failed for %s with err %d\n",
  7502. __func__, dev_name(dev), err);
  7503. goto bail;
  7504. }
  7505. sess->smmu.genpool_iova = frpc_gen_addr_pool[0];
  7506. sess->smmu.genpool_size = frpc_gen_addr_pool[1];
  7507. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  7508. if (err) {
  7509. err = -ENOMEM;
  7510. ADSPRPC_ERR(
  7511. "allocation failed for size 0x%zx\n", sizeof(*buf));
  7512. goto bail;
  7513. }
  7514. INIT_HLIST_NODE(&buf->hn);
  7515. buf->virt = NULL;
  7516. buf->phys = 0;
  7517. buf->size = frpc_gen_addr_pool[1];
  7518. buf->dma_attr = DMA_ATTR_DELAYED_UNMAP;
  7519. /* Allocate memory for adding to genpool */
  7520. buf->virt = dma_alloc_attrs(sess->smmu.dev, buf->size,
  7521. (dma_addr_t *)&buf->phys,
  7522. GFP_KERNEL, buf->dma_attr);
  7523. if (IS_ERR_OR_NULL(buf->virt)) {
  7524. ADSPRPC_ERR(
  7525. "dma_alloc_attrs failed for size 0x%zx, returned %pK\n",
  7526. buf->size, buf->virt);
  7527. err = -ENOBUFS;
  7528. goto dma_alloc_bail;
  7529. }
  7530. err = dma_get_sgtable_attrs(sess->smmu.dev, &sgt, buf->virt,
  7531. buf->phys, buf->size, buf->dma_attr);
  7532. if (err) {
  7533. ADSPRPC_ERR("dma_get_sgtable_attrs failed with err %d", err);
  7534. goto iommu_map_bail;
  7535. }
  7536. domain = iommu_get_domain_for_dev(sess->smmu.dev);
  7537. if (!domain) {
  7538. ADSPRPC_ERR("iommu_get_domain_for_dev failed ");
  7539. goto iommu_map_bail;
  7540. }
  7541. /* Map the allocated memory with fixed IOVA and is shared to remote subsystem */
  7542. err = iommu_map_sg(domain, frpc_gen_addr_pool[0], sgt.sgl,
  7543. sgt.nents, IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE);
  7544. if (err < 0) {
  7545. ADSPRPC_ERR("iommu_map_sg failed with err %d", err);
  7546. goto iommu_map_bail;
  7547. }
  7548. /* Create genpool using SMMU device */
  7549. gen_pool = devm_gen_pool_create(sess->smmu.dev, 0,
  7550. NUMA_NO_NODE, NULL);
  7551. if (IS_ERR(gen_pool)) {
  7552. err = PTR_ERR(gen_pool);
  7553. ADSPRPC_ERR("devm_gen_pool_create failed with err %d", err);
  7554. goto genpool_create_bail;
  7555. }
  7556. /* Add allocated memory to genpool */
  7557. err = gen_pool_add_virt(gen_pool, (unsigned long)buf->virt,
  7558. buf->phys, buf->size, NUMA_NO_NODE);
  7559. if (err) {
  7560. ADSPRPC_ERR("gen_pool_add_virt failed with err %d", err);
  7561. goto genpool_add_bail;
  7562. }
  7563. sess->smmu.frpc_genpool = gen_pool;
  7564. sess->smmu.frpc_genpool_buf = buf;
  7565. }
  7566. chan->sesscount++;
  7567. if (debugfs_root && !debugfs_global_file) {
  7568. debugfs_global_file = debugfs_create_file("global", 0644,
  7569. debugfs_root, NULL, &debugfs_fops);
  7570. if (IS_ERR_OR_NULL(debugfs_global_file)) {
  7571. pr_warn("Error: %s: %s: failed to create debugfs global file\n",
  7572. current->comm, __func__);
  7573. debugfs_global_file = NULL;
  7574. }
  7575. }
  7576. bail:
  7577. return err;
  7578. genpool_add_bail:
  7579. gen_pool_destroy(gen_pool);
  7580. genpool_create_bail:
  7581. iommu_unmap(domain, sess->smmu.genpool_iova,
  7582. sess->smmu.genpool_size);
  7583. iommu_map_bail:
  7584. dma_free_attrs(sess->smmu.dev, buf->size, buf->virt,
  7585. buf->phys, buf->dma_attr);
  7586. dma_alloc_bail:
  7587. kfree(buf);
  7588. return err;
  7589. }
  7590. static void init_secure_vmid_list(struct device *dev, char *prop_name,
  7591. struct secure_vm *destvm)
  7592. {
  7593. int err = 0;
  7594. u32 len = 0, i = 0;
  7595. u32 *rhvmlist = NULL;
  7596. u32 *rhvmpermlist = NULL;
  7597. if (!of_find_property(dev->of_node, prop_name, &len))
  7598. goto bail;
  7599. if (len == 0)
  7600. goto bail;
  7601. len /= sizeof(u32);
  7602. VERIFY(err, NULL != (rhvmlist = kcalloc(len, sizeof(u32), GFP_KERNEL)));
  7603. if (err)
  7604. goto bail;
  7605. VERIFY(err, NULL != (rhvmpermlist = kcalloc(len, sizeof(u32),
  7606. GFP_KERNEL)));
  7607. if (err)
  7608. goto bail;
  7609. for (i = 0; i < len; i++) {
  7610. err = of_property_read_u32_index(dev->of_node, prop_name, i,
  7611. &rhvmlist[i]);
  7612. if (err) {
  7613. pr_err("Error: adsprpc: %s: failed to read VMID\n",
  7614. __func__);
  7615. goto bail;
  7616. }
  7617. ADSPRPC_INFO("secure VMID = %d\n",
  7618. rhvmlist[i]);
  7619. rhvmpermlist[i] = QCOM_SCM_PERM_RWX;
  7620. }
  7621. destvm->vmid = rhvmlist;
  7622. destvm->vmperm = rhvmpermlist;
  7623. destvm->vmcount = len;
  7624. bail:
  7625. if (err) {
  7626. kfree(rhvmlist);
  7627. kfree(rhvmpermlist);
  7628. }
  7629. }
  7630. static void fastrpc_init_privileged_gids(struct device *dev, char *prop_name,
  7631. struct gid_list *gidlist)
  7632. {
  7633. int err = 0;
  7634. u32 len = 0, i = 0;
  7635. u32 *gids = NULL;
  7636. if (!of_find_property(dev->of_node, prop_name, &len))
  7637. goto bail;
  7638. if (len == 0)
  7639. goto bail;
  7640. len /= sizeof(u32);
  7641. gids = kcalloc(len, sizeof(u32), GFP_KERNEL);
  7642. if (!gids) {
  7643. err = ENOMEM;
  7644. goto bail;
  7645. }
  7646. for (i = 0; i < len; i++) {
  7647. err = of_property_read_u32_index(dev->of_node, prop_name,
  7648. i, &gids[i]);
  7649. if (err) {
  7650. pr_err("Error: adsprpc: %s: failed to read GID %u\n",
  7651. __func__, i);
  7652. goto bail;
  7653. }
  7654. pr_info("adsprpc: %s: privileged GID: %u\n", __func__, gids[i]);
  7655. }
  7656. sort(gids, len, sizeof(*gids), uint_cmp_func, NULL);
  7657. gidlist->gids = gids;
  7658. gidlist->gidcount = len;
  7659. bail:
  7660. if (err)
  7661. kfree(gids);
  7662. }
  7663. static void configure_secure_channels(uint32_t secure_domains)
  7664. {
  7665. struct fastrpc_apps *me = &gfa;
  7666. int ii = 0;
  7667. /*
  7668. * secure_domains contains the bitmask of the secure channels
  7669. * Bit 0 - ADSP
  7670. * Bit 1 - MDSP
  7671. * Bit 2 - SLPI
  7672. * Bit 3 - CDSP
  7673. */
  7674. for (ii = ADSP_DOMAIN_ID; ii <= CDSP_DOMAIN_ID; ++ii) {
  7675. int secure = (secure_domains >> ii) & 0x01;
  7676. me->channel[ii].secure = secure;
  7677. ADSPRPC_INFO("domain %d configured as secure %d\n", ii, secure);
  7678. }
  7679. }
  7680. /*
  7681. * This function is used to create the service locator required for
  7682. * registering for remote process restart (PDR) notifications if that
  7683. * PDR property has been enabled in the fastrpc node on the DTSI.
  7684. */
  7685. static int fastrpc_setup_service_locator(struct device *dev,
  7686. const char *propname,
  7687. char *client_name, char *service_name,
  7688. char *service_path)
  7689. {
  7690. int err = 0, session = -1, cid = -1;
  7691. struct fastrpc_apps *me = &gfa;
  7692. struct pdr_handle *handle = NULL;
  7693. struct pdr_service *service = NULL;
  7694. if (of_property_read_bool(dev->of_node, propname)) {
  7695. err = fastrpc_get_spd_session(client_name, &session, &cid);
  7696. if (err)
  7697. goto bail;
  7698. /* Register the service locator's callback function */
  7699. handle = pdr_handle_alloc(fastrpc_pdr_cb, &me->channel[cid].spd[session]);
  7700. if (IS_ERR_OR_NULL(handle)) {
  7701. err = PTR_ERR(handle);
  7702. goto bail;
  7703. }
  7704. me->channel[cid].spd[session].pdrhandle = handle;
  7705. service = pdr_add_lookup(handle, service_name, service_path);
  7706. if (IS_ERR_OR_NULL(service)) {
  7707. err = PTR_ERR(service);
  7708. goto bail;
  7709. }
  7710. pr_info("adsprpc: %s: pdr_add_lookup enabled for %s (%s, %s), DTSI (%s)\n",
  7711. __func__, service_name, client_name, service_path, propname);
  7712. }
  7713. bail:
  7714. if (err) {
  7715. pr_warn("adsprpc: %s: failed for %s (%s, %s), DTSI (%s) with err %d\n",
  7716. __func__, service_name, client_name, service_path, propname, err);
  7717. }
  7718. return err;
  7719. }
  7720. /*
  7721. * fastrpc_nsp_status_show() - Updates the buffer with remote nsp status
  7722. * by reading the fastrpc node.
  7723. * @dev : pointer to device node.
  7724. * @attr: pointer to device attribute.
  7725. * @buf : Output parameter to be updated with remote nsp status.
  7726. * Return : bytes written to buffer.
  7727. */
  7728. static ssize_t fastrpc_nsp_status_show(struct device *dev,
  7729. struct device_attribute *attr, char *buf)
  7730. {
  7731. struct fastrpc_apps *me = &gfa;
  7732. /*
  7733. * Default remote DSP status: 0
  7734. * driver possibly not probed yet or not the main device.
  7735. */
  7736. if (!dev || !dev->driver ||
  7737. !of_device_is_compatible(dev->of_node, "qcom,msm-fastrpc-compute")) {
  7738. ADSPRPC_ERR("Driver not probed yet or not the main device\n");
  7739. return 0;
  7740. }
  7741. return scnprintf(buf, PAGE_SIZE, "%d",
  7742. me->fastrpc_nsp_status);
  7743. }
  7744. /* Remote nsp status attribute declaration as read only */
  7745. static DEVICE_ATTR_RO(fastrpc_nsp_status);
  7746. /* Declaring attribute for remote dsp */
  7747. static struct attribute *msm_remote_dsp_attrs[] = {
  7748. &dev_attr_fastrpc_nsp_status.attr,
  7749. NULL
  7750. };
  7751. /* Defining remote dsp attributes in attributes group */
  7752. static struct attribute_group msm_remote_dsp_attr_group = {
  7753. .attrs = msm_remote_dsp_attrs,
  7754. };
  7755. static int fastrpc_probe(struct platform_device *pdev)
  7756. {
  7757. int err = 0;
  7758. struct fastrpc_apps *me = &gfa;
  7759. struct device *dev = &pdev->dev;
  7760. int ret = 0;
  7761. uint32_t secure_domains = 0;
  7762. if (of_device_is_compatible(dev->of_node,
  7763. "qcom,msm-fastrpc-compute")) {
  7764. err = sysfs_create_group(&pdev->dev.kobj, &msm_remote_dsp_attr_group);
  7765. if (err) {
  7766. ADSPRPC_ERR(
  7767. "Initialization of sysfs create group failed with %d\n",
  7768. err);
  7769. goto bail;
  7770. }
  7771. init_secure_vmid_list(dev, "qcom,adsp-remoteheap-vmid",
  7772. &gcinfo[0].rhvm);
  7773. fastrpc_init_privileged_gids(dev, "qcom,fastrpc-gids",
  7774. &me->gidlist);
  7775. /*
  7776. * Check if latency voting for only one core
  7777. * is enabled for the platform
  7778. */
  7779. me->single_core_latency_vote = of_property_read_bool(dev->of_node,
  7780. "qcom,single-core-latency-vote");
  7781. if (me->single_core_latency_vote)
  7782. me->lowest_capacity_core_count = 1;
  7783. of_property_read_u32(dev->of_node, "qcom,rpc-latency-us",
  7784. &me->latency);
  7785. of_property_read_u32(dev->of_node, "qcom,max-sessions",
  7786. &me->max_sess_per_proc);
  7787. if (of_get_property(dev->of_node,
  7788. "qcom,secure-domains", NULL) != NULL) {
  7789. VERIFY(err, !of_property_read_u32(dev->of_node,
  7790. "qcom,secure-domains",
  7791. &secure_domains));
  7792. if (!err)
  7793. configure_secure_channels(secure_domains);
  7794. else
  7795. pr_info("adsprpc: unable to read the domain configuration from dts\n");
  7796. }
  7797. }
  7798. if (of_device_is_compatible(dev->of_node,
  7799. "qcom,msm-fastrpc-compute-cb"))
  7800. return fastrpc_cb_probe(dev);
  7801. if (of_device_is_compatible(dev->of_node,
  7802. "qcom,msm-adsprpc-mem-region")) {
  7803. me->dev = dev;
  7804. ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0);
  7805. if (ret) {
  7806. pr_warn("adsprpc: Error: %s: initialization of memory region adsp_mem failed with %d\n",
  7807. __func__, ret);
  7808. }
  7809. goto bail;
  7810. }
  7811. me->legacy_remote_heap = of_property_read_bool(dev->of_node,
  7812. "qcom,fastrpc-legacy-remote-heap");
  7813. err = fastrpc_setup_service_locator(dev, AUDIO_PDR_ADSP_DTSI_PROPERTY_NAME,
  7814. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME,
  7815. AUDIO_PDR_ADSP_SERVICE_NAME, ADSP_AUDIOPD_NAME);
  7816. if (err)
  7817. goto bail;
  7818. err = fastrpc_setup_service_locator(dev, SENSORS_PDR_ADSP_DTSI_PROPERTY_NAME,
  7819. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME,
  7820. SENSORS_PDR_ADSP_SERVICE_NAME, ADSP_SENSORPD_NAME);
  7821. if (err)
  7822. goto bail;
  7823. err = fastrpc_setup_service_locator(dev, SENSORS_PDR_SLPI_DTSI_PROPERTY_NAME,
  7824. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME,
  7825. SENSORS_PDR_SLPI_SERVICE_NAME, SLPI_SENSORPD_NAME);
  7826. if (err)
  7827. goto bail;
  7828. err = of_platform_populate(pdev->dev.of_node,
  7829. fastrpc_match_table,
  7830. NULL, &pdev->dev);
  7831. if (err)
  7832. goto bail;
  7833. bail:
  7834. return err;
  7835. }
  7836. /*
  7837. * Function to free fastrpc genpool buffer
  7838. */
  7839. static void fastrpc_genpool_free(struct fastrpc_session_ctx *sess)
  7840. {
  7841. struct fastrpc_buf *buf = NULL;
  7842. struct iommu_domain *domain = NULL;
  7843. if (!sess)
  7844. goto bail;
  7845. buf = sess->smmu.frpc_genpool_buf;
  7846. if (sess->smmu.frpc_genpool) {
  7847. gen_pool_destroy(sess->smmu.frpc_genpool);
  7848. sess->smmu.frpc_genpool = NULL;
  7849. }
  7850. if (buf && sess->smmu.dev) {
  7851. domain = iommu_get_domain_for_dev(sess->smmu.dev);
  7852. iommu_unmap(domain, sess->smmu.genpool_iova,
  7853. sess->smmu.genpool_size);
  7854. if (buf->phys)
  7855. dma_free_attrs(sess->smmu.dev, buf->size, buf->virt,
  7856. buf->phys, buf->dma_attr);
  7857. kfree(buf);
  7858. sess->smmu.frpc_genpool_buf = NULL;
  7859. }
  7860. bail:
  7861. return;
  7862. }
  7863. static void fastrpc_deinit(void)
  7864. {
  7865. struct fastrpc_channel_ctx *chan = gcinfo;
  7866. struct fastrpc_apps *me = &gfa;
  7867. int i, j;
  7868. for (i = 0; i < NUM_CHANNELS; i++, chan++) {
  7869. for (j = 0; j < NUM_SESSIONS; j++) {
  7870. struct fastrpc_session_ctx *sess = &chan->session[j];
  7871. fastrpc_genpool_free(sess);
  7872. if (sess->smmu.dev)
  7873. sess->smmu.dev = NULL;
  7874. }
  7875. kfree(chan->rhvm.vmid);
  7876. kfree(chan->rhvm.vmperm);
  7877. fastrpc_transport_session_deinit(i);
  7878. mutex_destroy(&chan->smd_mutex);
  7879. }
  7880. if (me->transport_initialized)
  7881. fastrpc_transport_deinit();
  7882. me->transport_initialized = 0;
  7883. mutex_destroy(&me->mut_uid);
  7884. }
  7885. #ifdef CONFIG_HIBERNATION
  7886. static bool hibernation;
  7887. static int fastrpc_hibernation_notifier(struct notifier_block *nb,
  7888. unsigned long event, void *dummy)
  7889. {
  7890. if (event == PM_HIBERNATION_PREPARE)
  7891. hibernation = true;
  7892. else if (event == PM_POST_HIBERNATION)
  7893. hibernation = false;
  7894. return NOTIFY_OK;
  7895. }
  7896. static struct notifier_block fastrpc_notif_block = {
  7897. .notifier_call = fastrpc_hibernation_notifier,
  7898. };
  7899. #endif
  7900. #ifdef CONFIG_PM_SLEEP
  7901. static int fastrpc_hibernation_suspend(struct device *dev)
  7902. {
  7903. int err = 0;
  7904. if (of_device_is_compatible(dev->of_node,
  7905. "qcom,msm-fastrpc-compute")) {
  7906. err = fastrpc_dsp_restart_handler(NULL, 0, true);
  7907. if (err)
  7908. ADSPRPC_WARN("failed to unmap remote heap (err %d)\n",
  7909. err);
  7910. }
  7911. return err;
  7912. }
  7913. static int fastrpc_restore(struct device *dev)
  7914. {
  7915. struct fastrpc_apps *me = &gfa;
  7916. int cid;
  7917. pr_info("adsprpc: restore enter\n");
  7918. for (cid = 0; cid < NUM_CHANNELS; cid++)
  7919. me->channel[cid].in_hib = 1;
  7920. pr_info("adsprpc: restore exit\n");
  7921. return 0;
  7922. }
  7923. static const struct dev_pm_ops fastrpc_pm = {
  7924. .freeze = fastrpc_hibernation_suspend,
  7925. .restore = fastrpc_restore,
  7926. };
  7927. #endif
  7928. static struct platform_driver fastrpc_driver = {
  7929. .probe = fastrpc_probe,
  7930. .driver = {
  7931. .name = "fastrpc",
  7932. .of_match_table = fastrpc_match_table,
  7933. .suppress_bind_attrs = true,
  7934. #ifdef CONFIG_PM_SLEEP
  7935. .pm = &fastrpc_pm,
  7936. #endif
  7937. },
  7938. };
  7939. union fastrpc_dev_param {
  7940. struct fastrpc_dev_map_dma *map;
  7941. struct fastrpc_dev_unmap_dma *unmap;
  7942. struct fastrpc_dev_get_hlos_pid *hpid;
  7943. };
  7944. long fastrpc_dev_map_dma(struct fastrpc_device *dev, unsigned long invoke_param)
  7945. {
  7946. int err = 0;
  7947. union fastrpc_dev_param p;
  7948. struct fastrpc_file *fl = NULL;
  7949. struct fastrpc_mmap *map = NULL;
  7950. struct fastrpc_apps *me = &gfa;
  7951. uintptr_t raddr = 0;
  7952. unsigned long irq_flags = 0;
  7953. bool reftaken = 0;
  7954. p.map = (struct fastrpc_dev_map_dma *)invoke_param;
  7955. spin_lock_irqsave(&me->hlock, irq_flags);
  7956. /* Verify if fastrpc device is closed*/
  7957. VERIFY(err, dev && !dev->dev_close);
  7958. if (err) {
  7959. err = -ESRCH;
  7960. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7961. return err;
  7962. }
  7963. fl = dev->fl;
  7964. /* Verify if fastrpc file is not NULL*/
  7965. if (!fl) {
  7966. err = -EBADF;
  7967. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7968. return err;
  7969. }
  7970. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7971. err = fastrpc_file_get(fl);
  7972. if (err) {
  7973. ADSPRPC_ERR("Failed to get user process reference\n");
  7974. goto bail;
  7975. }
  7976. reftaken = 1;
  7977. mutex_lock(&fl->internal_map_mutex);
  7978. spin_lock_irqsave(&me->hlock, irq_flags);
  7979. /* Verify if fastrpc file is being closed, holding device lock*/
  7980. if (fl->file_close) {
  7981. err = -ESRCH;
  7982. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7983. goto bail;
  7984. }
  7985. fl->is_dma_invoke_pend = true;
  7986. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7987. mutex_lock(&fl->map_mutex);
  7988. /* Map DMA buffer on SMMU device*/
  7989. err = fastrpc_mmap_create(fl, -1, p.map->buf,
  7990. p.map->attrs, 0, p.map->size,
  7991. ADSP_MMAP_DMA_BUFFER, &map);
  7992. mutex_unlock(&fl->map_mutex);
  7993. if (err)
  7994. goto bail;
  7995. /* Map DMA buffer on DSP*/
  7996. VERIFY(err, 0 == (err = fastrpc_mmap_on_dsp(fl,
  7997. map->flags, 0, map->phys, map->size, map->refs, &raddr)));
  7998. if (err)
  7999. goto bail;
  8000. map->raddr = raddr;
  8001. p.map->v_dsp_addr = raddr;
  8002. bail:
  8003. if (err && map) {
  8004. mutex_lock(&fl->map_mutex);
  8005. fastrpc_mmap_free(map, 0);
  8006. mutex_unlock(&fl->map_mutex);
  8007. }
  8008. if (fl) {
  8009. spin_lock_irqsave(&me->hlock, irq_flags);
  8010. if (fl->file_close && fl->is_dma_invoke_pend)
  8011. complete(&fl->dma_invoke);
  8012. fl->is_dma_invoke_pend = false;
  8013. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8014. }
  8015. mutex_unlock(&fl->internal_map_mutex);
  8016. if (reftaken)
  8017. fastrpc_file_put(fl);
  8018. return err;
  8019. }
  8020. long fastrpc_dev_unmap_dma(struct fastrpc_device *dev, unsigned long invoke_param)
  8021. {
  8022. int err = 0;
  8023. union fastrpc_dev_param p;
  8024. struct fastrpc_file *fl = NULL;
  8025. struct fastrpc_mmap *map = NULL;
  8026. struct fastrpc_apps *me = &gfa;
  8027. unsigned long irq_flags = 0;
  8028. bool reftaken = 0;
  8029. p.unmap = (struct fastrpc_dev_unmap_dma *)invoke_param;
  8030. spin_lock_irqsave(&me->hlock, irq_flags);
  8031. /* Verify if fastrpc device is closed*/
  8032. VERIFY(err, dev && !dev->dev_close);
  8033. if (err) {
  8034. err = -ESRCH;
  8035. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8036. return err;
  8037. }
  8038. fl = dev->fl;
  8039. /* Verify if fastrpc file is not NULL*/
  8040. if (!fl) {
  8041. err = -EBADF;
  8042. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8043. return err;
  8044. }
  8045. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8046. err = fastrpc_file_get(fl);
  8047. if (err) {
  8048. ADSPRPC_ERR("Failed to get user process reference\n");
  8049. goto bail;
  8050. }
  8051. reftaken = 1;
  8052. mutex_lock(&fl->internal_map_mutex);
  8053. spin_lock_irqsave(&me->hlock, irq_flags);
  8054. /* Verify if fastrpc file is being closed, holding device lock*/
  8055. if (fl->file_close) {
  8056. err = -ESRCH;
  8057. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8058. goto bail;
  8059. }
  8060. fl->is_dma_invoke_pend = true;
  8061. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8062. mutex_lock(&fl->map_mutex);
  8063. if (!fastrpc_mmap_find(fl, -1, p.unmap->buf, 0, 0, ADSP_MMAP_DMA_BUFFER, 0, &map)) {
  8064. mutex_unlock(&fl->map_mutex);
  8065. if (err)
  8066. goto bail;
  8067. /* Un-map DMA buffer on DSP*/
  8068. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, map->raddr,
  8069. map->phys, map->size, map->flags)));
  8070. if (err)
  8071. goto bail;
  8072. mutex_lock(&fl->map_mutex);
  8073. fastrpc_mmap_free(map, 0);
  8074. }
  8075. mutex_unlock(&fl->map_mutex);
  8076. bail:
  8077. if (fl) {
  8078. spin_lock_irqsave(&me->hlock, irq_flags);
  8079. if (fl->file_close && fl->is_dma_invoke_pend)
  8080. complete(&fl->dma_invoke);
  8081. fl->is_dma_invoke_pend = false;
  8082. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8083. }
  8084. mutex_unlock(&fl->internal_map_mutex);
  8085. if (reftaken)
  8086. fastrpc_file_put(fl);
  8087. return err;
  8088. }
  8089. long fastrpc_dev_get_hlos_pid(struct fastrpc_device *dev, unsigned long invoke_param)
  8090. {
  8091. int err = 0;
  8092. union fastrpc_dev_param p;
  8093. struct fastrpc_file *fl = NULL;
  8094. struct fastrpc_apps *me = &gfa;
  8095. unsigned long irq_flags = 0;
  8096. p.hpid = (struct fastrpc_dev_get_hlos_pid *)invoke_param;
  8097. spin_lock_irqsave(&me->hlock, irq_flags);
  8098. /* Verify if fastrpc device is closed*/
  8099. VERIFY(err, dev && !dev->dev_close);
  8100. if (err) {
  8101. err = -ESRCH;
  8102. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8103. return err;
  8104. }
  8105. fl = dev->fl;
  8106. /* Verify if fastrpc file is not NULL*/
  8107. if (!fl) {
  8108. err = -EBADF;
  8109. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8110. return err;
  8111. }
  8112. p.hpid->hlos_pid = fl->tgid;
  8113. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8114. return err;
  8115. }
  8116. long fastrpc_driver_invoke(struct fastrpc_device *dev, unsigned int invoke_num,
  8117. unsigned long invoke_param)
  8118. {
  8119. int err = 0;
  8120. switch (invoke_num) {
  8121. case FASTRPC_DEV_MAP_DMA:
  8122. err = fastrpc_dev_map_dma(dev, invoke_param);
  8123. break;
  8124. case FASTRPC_DEV_UNMAP_DMA:
  8125. err = fastrpc_dev_unmap_dma(dev, invoke_param);
  8126. break;
  8127. case FASTRPC_DEV_GET_HLOS_PID:
  8128. err = fastrpc_dev_get_hlos_pid(dev, invoke_param);
  8129. break;
  8130. default:
  8131. err = -ENOTTY;
  8132. break;
  8133. }
  8134. return err;
  8135. }
  8136. EXPORT_SYMBOL(fastrpc_driver_invoke);
  8137. static struct device fastrpc_bus = {
  8138. .init_name = "fastrpc"
  8139. };
  8140. static int fastrpc_bus_match(struct device *dev, struct device_driver *driver)
  8141. {
  8142. struct fastrpc_apps *me = &gfa;
  8143. struct fastrpc_driver *frpc_driver = to_fastrpc_driver(driver);
  8144. struct fastrpc_device *frpc_device = to_fastrpc_device(dev);
  8145. unsigned long irq_flags = 0;
  8146. if (frpc_device->handle == frpc_driver->handle) {
  8147. spin_lock_irqsave(&me->hlock, irq_flags);
  8148. /* If device is being closed, fail the match */
  8149. if (frpc_device->dev_close) {
  8150. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8151. return 0;
  8152. }
  8153. frpc_device->refs++;
  8154. frpc_driver->device = dev;
  8155. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8156. return 1;
  8157. }
  8158. return 0;
  8159. }
  8160. static int fastrpc_bus_probe(struct device *dev)
  8161. {
  8162. struct fastrpc_device *frpc_dev = to_fastrpc_device(dev);
  8163. struct fastrpc_driver *frpc_drv = to_fastrpc_driver(dev->driver);
  8164. if (frpc_drv && frpc_drv->probe)
  8165. return frpc_drv->probe(frpc_dev);
  8166. return 0;
  8167. }
  8168. static void fastrpc_bus_remove(struct device *dev)
  8169. {
  8170. struct fastrpc_driver *frpc_drv = to_fastrpc_driver(dev->driver);
  8171. if (frpc_drv && frpc_drv->callback)
  8172. frpc_drv->callback(to_fastrpc_device(dev), FASTRPC_PROC_DOWN);
  8173. }
  8174. static struct bus_type fastrpc_bus_type = {
  8175. .name = "fastrpc",
  8176. .match = fastrpc_bus_match,
  8177. .probe = fastrpc_bus_probe,
  8178. .remove = fastrpc_bus_remove,
  8179. };
  8180. static void fastrpc_dev_release(struct device *dev)
  8181. {
  8182. kfree(to_fastrpc_device(dev));
  8183. }
  8184. static int fastrpc_device_create(struct fastrpc_file *fl)
  8185. {
  8186. int err = 0;
  8187. struct fastrpc_device *frpc_dev;
  8188. struct fastrpc_apps *me = &gfa;
  8189. unsigned long irq_flags = 0;
  8190. frpc_dev = kzalloc(sizeof(*frpc_dev), GFP_KERNEL);
  8191. if (!frpc_dev) {
  8192. err = -ENOMEM;
  8193. goto bail;
  8194. }
  8195. frpc_dev->dev.parent = &fastrpc_bus;
  8196. frpc_dev->dev.bus = &fastrpc_bus_type;
  8197. /* Use HLOS PID, unique fastrpc process ID and CID to create device file,
  8198. * Else names would conflict for multiple sessions
  8199. * And also for better ability to debug
  8200. */
  8201. dev_set_name(&frpc_dev->dev, "%s-%d-%d-%d",
  8202. dev_name(frpc_dev->dev.parent), fl->tgid, fl->tgid_frpc, fl->cid);
  8203. frpc_dev->dev.release = fastrpc_dev_release;
  8204. frpc_dev->fl = fl;
  8205. /* Use unique fastrpc tgid as handle */
  8206. frpc_dev->handle = fl->tgid_frpc;
  8207. err = device_register(&frpc_dev->dev);
  8208. if (err) {
  8209. put_device(&frpc_dev->dev);
  8210. ADSPRPC_ERR(
  8211. "fastrpc device register failed for process %d unique fastrpc tgid %d session %d with error %d\n",
  8212. fl->tgid, fl->tgid_frpc, fl->sessionid, err);
  8213. goto bail;
  8214. }
  8215. fl->device = frpc_dev;
  8216. spin_lock_irqsave(&me->hlock, irq_flags);
  8217. hlist_add_head(&frpc_dev->hn, &me->frpc_devices);
  8218. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8219. bail:
  8220. return err;
  8221. }
  8222. void fastrpc_driver_unregister(struct fastrpc_driver *frpc_driver)
  8223. {
  8224. struct fastrpc_apps *me = &gfa;
  8225. struct device *dev = NULL;
  8226. struct fastrpc_device *frpc_dev = NULL;
  8227. bool is_device_closed = false;
  8228. unsigned long irq_flags = 0;
  8229. spin_lock_irqsave(&me->hlock, irq_flags);
  8230. dev = frpc_driver->device;
  8231. if (dev) {
  8232. frpc_dev = to_fastrpc_device(dev);
  8233. if (frpc_dev->refs > 0)
  8234. frpc_dev->refs--;
  8235. else
  8236. ADSPRPC_ERR("Fastrpc device for driver %s is already freed\n",
  8237. frpc_driver->driver.name);
  8238. if (frpc_dev->dev_close) {
  8239. hlist_del_init(&frpc_dev->hn);
  8240. is_device_closed = true;
  8241. }
  8242. }
  8243. hlist_del_init(&frpc_driver->hn);
  8244. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8245. if (is_device_closed) {
  8246. ADSPRPC_INFO("un-registering fastrpc device with handle %d\n",
  8247. frpc_dev->handle);
  8248. device_unregister(dev);
  8249. }
  8250. driver_unregister(&frpc_driver->driver);
  8251. ADSPRPC_INFO("Un-registering fastrpc driver %s with handle %d\n",
  8252. frpc_driver->driver.name, frpc_driver->handle);
  8253. }
  8254. EXPORT_SYMBOL(fastrpc_driver_unregister);
  8255. int fastrpc_driver_register(struct fastrpc_driver *frpc_driver)
  8256. {
  8257. int err = 0;
  8258. struct fastrpc_apps *me = &gfa;
  8259. unsigned long irq_flags = 0;
  8260. frpc_driver->driver.bus = &fastrpc_bus_type;
  8261. frpc_driver->driver.owner = THIS_MODULE;
  8262. err = driver_register(&frpc_driver->driver);
  8263. if (err) {
  8264. ADSPRPC_ERR("fastrpc driver %s failed to register with error %d\n",
  8265. frpc_driver->driver.name, err);
  8266. goto bail;
  8267. }
  8268. ADSPRPC_INFO("fastrpc driver %s registered with handle %d\n",
  8269. frpc_driver->driver.name, frpc_driver->handle);
  8270. spin_lock_irqsave(&me->hlock, irq_flags);
  8271. hlist_add_head(&frpc_driver->hn, &me->frpc_drivers);
  8272. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8273. bail:
  8274. return err;
  8275. }
  8276. EXPORT_SYMBOL(fastrpc_driver_register);
  8277. static int __init fastrpc_device_init(void)
  8278. {
  8279. struct fastrpc_apps *me = &gfa;
  8280. int err = 0, i;
  8281. uintptr_t attr = 0;
  8282. dma_addr_t region_phys = 0;
  8283. void *region_vaddr = NULL;
  8284. struct fastrpc_buf *buf = NULL;
  8285. debugfs_root = debugfs_create_dir("adsprpc", NULL);
  8286. if (IS_ERR_OR_NULL(debugfs_root)) {
  8287. pr_warn("Error: %s: %s: failed to create debugfs root dir\n",
  8288. current->comm, __func__);
  8289. debugfs_remove_recursive(debugfs_root);
  8290. debugfs_root = NULL;
  8291. }
  8292. memset(me, 0, sizeof(*me));
  8293. fastrpc_init(me);
  8294. fastrpc_get_nsp_status(me);
  8295. me->dev = NULL;
  8296. me->legacy_remote_heap = false;
  8297. err = bus_register(&fastrpc_bus_type);
  8298. if (err) {
  8299. ADSPRPC_ERR("fastrpc bus register failed with err %d\n",
  8300. err);
  8301. goto bus_register_bail;
  8302. }
  8303. err = device_register(&fastrpc_bus);
  8304. if (err) {
  8305. ADSPRPC_ERR("fastrpc bus device register failed with err %d\n",
  8306. err);
  8307. goto bus_device_register_bail;
  8308. }
  8309. me->fastrpc_bus_register = true;
  8310. fastrpc_lowest_capacity_corecount(me);
  8311. VERIFY(err, 0 == platform_driver_register(&fastrpc_driver));
  8312. if (err)
  8313. goto register_bail;
  8314. VERIFY(err, 0 == alloc_chrdev_region(&me->dev_no, 0, NUM_CHANNELS,
  8315. DEVICE_NAME));
  8316. if (err)
  8317. goto alloc_chrdev_bail;
  8318. cdev_init(&me->cdev, &fops);
  8319. me->cdev.owner = THIS_MODULE;
  8320. VERIFY(err, 0 == cdev_add(&me->cdev, MKDEV(MAJOR(me->dev_no), 0),
  8321. NUM_DEVICES));
  8322. if (err)
  8323. goto cdev_init_bail;
  8324. me->class = class_create(THIS_MODULE, "fastrpc");
  8325. VERIFY(err, !IS_ERR(me->class));
  8326. if (err)
  8327. goto class_create_bail;
  8328. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  8329. me->compat = 1;
  8330. #else
  8331. me->compat = (fops.compat_ioctl == NULL) ? 0 : 1;
  8332. #endif
  8333. /*
  8334. * Create devices and register with sysfs
  8335. * Create first device with minor number 0
  8336. */
  8337. me->non_secure_dev = device_create(me->class, NULL,
  8338. MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV),
  8339. NULL, DEVICE_NAME);
  8340. VERIFY(err, !IS_ERR_OR_NULL(me->non_secure_dev));
  8341. if (err) {
  8342. err = -ENODEV;
  8343. goto device_create_bail;
  8344. }
  8345. /* Create secure device with minor number for secure device */
  8346. me->secure_dev = device_create(me->class, NULL,
  8347. MKDEV(MAJOR(me->dev_no), MINOR_NUM_SECURE_DEV),
  8348. NULL, DEVICE_NAME_SECURE);
  8349. VERIFY(err, !IS_ERR_OR_NULL(me->secure_dev));
  8350. if (err)
  8351. goto device_create_bail;
  8352. for (i = 0; i < NUM_CHANNELS; i++) {
  8353. me->jobid[i] = 1;
  8354. me->channel[i].dev = me->secure_dev;
  8355. me->channel[i].ssrcount = 0;
  8356. me->channel[i].in_hib = 0;
  8357. me->channel[i].prevssrcount = 0;
  8358. me->channel[i].subsystemstate = SUBSYSTEM_UP;
  8359. me->channel[i].rh_dump_dev = NULL;
  8360. me->channel[i].nb.notifier_call = fastrpc_restart_notifier_cb;
  8361. me->channel[i].handle = qcom_register_ssr_notifier(
  8362. gcinfo[i].subsys,
  8363. &me->channel[i].nb);
  8364. if (i == CDSP_DOMAIN_ID) {
  8365. me->channel[i].dev = me->non_secure_dev;
  8366. #if !IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  8367. /*
  8368. * Allocate CMA memory for mini dump.
  8369. * Ignore error as CMA node may not be available on all targets.
  8370. */
  8371. err = fastrpc_alloc_cma_memory(&region_phys,
  8372. &region_vaddr,
  8373. MINI_DUMP_DBG_SIZE,
  8374. (unsigned long)attr);
  8375. #endif
  8376. if (err) {
  8377. ADSPRPC_WARN("CMA alloc failed err 0x%x\n", err);
  8378. err = 0;
  8379. }
  8380. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  8381. if (err) {
  8382. err = -ENOMEM;
  8383. ADSPRPC_WARN("kzalloc failed err 0x%x\n", err);
  8384. err = 0;
  8385. } else {
  8386. INIT_HLIST_NODE(&buf->hn);
  8387. buf->virt = region_vaddr;
  8388. buf->phys = (uintptr_t)region_phys;
  8389. buf->size = MINI_DUMP_DBG_SIZE;
  8390. buf->dma_attr = attr;
  8391. buf->raddr = 0;
  8392. me->channel[i].buf = buf;
  8393. }
  8394. }
  8395. if (IS_ERR_OR_NULL(me->channel[i].handle))
  8396. pr_warn("adsprpc: %s: SSR notifier register failed for %s with err %d\n",
  8397. __func__, gcinfo[i].subsys,
  8398. PTR_ERR(me->channel[i].handle));
  8399. else
  8400. pr_info("adsprpc: %s: SSR notifier registered for %s\n",
  8401. __func__, gcinfo[i].subsys);
  8402. }
  8403. err = fastrpc_transport_init();
  8404. if (err)
  8405. goto device_create_bail;
  8406. me->transport_initialized = 1;
  8407. #ifdef CONFIG_HIBERNATION
  8408. err = register_pm_notifier(&fastrpc_notif_block);
  8409. if (err)
  8410. goto device_create_bail;
  8411. #endif
  8412. fastrpc_register_wakeup_source(me->non_secure_dev,
  8413. FASTRPC_NON_SECURE_WAKE_SOURCE_CLIENT_NAME,
  8414. &me->wake_source);
  8415. fastrpc_register_wakeup_source(me->secure_dev,
  8416. FASTRPC_SECURE_WAKE_SOURCE_CLIENT_NAME,
  8417. &me->wake_source_secure);
  8418. return 0;
  8419. device_create_bail:
  8420. for (i = 0; i < NUM_CHANNELS; i++) {
  8421. if (me->channel[i].handle)
  8422. qcom_unregister_ssr_notifier(me->channel[i].handle,
  8423. &me->channel[i].nb);
  8424. }
  8425. if (!IS_ERR_OR_NULL(me->non_secure_dev))
  8426. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8427. MINOR_NUM_DEV));
  8428. if (!IS_ERR_OR_NULL(me->secure_dev))
  8429. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8430. MINOR_NUM_SECURE_DEV));
  8431. class_destroy(me->class);
  8432. class_create_bail:
  8433. cdev_del(&me->cdev);
  8434. cdev_init_bail:
  8435. unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
  8436. alloc_chrdev_bail:
  8437. platform_driver_unregister(&fastrpc_driver);
  8438. register_bail:
  8439. device_unregister(&fastrpc_bus);
  8440. bus_device_register_bail:
  8441. bus_unregister(&fastrpc_bus_type);
  8442. bus_register_bail:
  8443. fastrpc_deinit();
  8444. return err;
  8445. }
  8446. static void __exit fastrpc_device_exit(void)
  8447. {
  8448. struct fastrpc_apps *me = &gfa;
  8449. int i;
  8450. fastrpc_file_list_dtor(me);
  8451. fastrpc_deinit();
  8452. wakeup_source_unregister(me->wake_source);
  8453. wakeup_source_unregister(me->wake_source_secure);
  8454. for (i = 0; i < NUM_CHANNELS; i++) {
  8455. if (i == CDSP_DOMAIN_ID)
  8456. kfree(me->channel[i].buf);
  8457. if (!gcinfo[i].name)
  8458. continue;
  8459. qcom_unregister_ssr_notifier(me->channel[i].handle,
  8460. &me->channel[i].nb);
  8461. }
  8462. /* Destroy the secure and non secure devices */
  8463. device_destroy(me->class, MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV));
  8464. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8465. MINOR_NUM_SECURE_DEV));
  8466. of_reserved_mem_device_release(me->dev);
  8467. class_destroy(me->class);
  8468. cdev_del(&me->cdev);
  8469. unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
  8470. if (me->transport_initialized)
  8471. fastrpc_transport_deinit();
  8472. me->transport_initialized = 0;
  8473. if (me->fastrpc_bus_register) {
  8474. bus_unregister(&fastrpc_bus_type);
  8475. device_unregister(&fastrpc_bus);
  8476. }
  8477. kfree(me->gidlist.gids);
  8478. debugfs_remove_recursive(debugfs_root);
  8479. }
  8480. module_init(fastrpc_device_init);
  8481. module_exit(fastrpc_device_exit);
  8482. MODULE_LICENSE("GPL v2");