adsprpc.c 243 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. /* Uncomment this block to log an error on every VERIFY failure */
  7. /*
  8. * #ifndef VERIFY_PRINT_ERROR
  9. * #define VERIFY_PRINT_ERROR
  10. * #endif
  11. */
  12. #include <linux/dma-buf.h>
  13. #include <linux/dma-mapping.h>
  14. #include <linux/qcom-dma-mapping.h>
  15. #include <linux/slab.h>
  16. #include <linux/completion.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/mm.h>
  19. #include <linux/wait.h>
  20. #include <linux/sched.h>
  21. #include <linux/module.h>
  22. #include <linux/list.h>
  23. #include <linux/arch_topology.h>
  24. #include <linux/hash.h>
  25. #include <linux/msm_ion.h>
  26. #include <linux/qcom_scm.h>
  27. #include <linux/ipc_logging.h>
  28. #include <linux/remoteproc/qcom_rproc.h>
  29. #include <linux/scatterlist.h>
  30. #include <linux/uaccess.h>
  31. #include <linux/device.h>
  32. #include <linux/of.h>
  33. #include <linux/of_address.h>
  34. #include <linux/of_platform.h>
  35. #include <linux/dma-map-ops.h>
  36. #include <linux/cma.h>
  37. #include <linux/sort.h>
  38. #include <linux/cred.h>
  39. #include <linux/msm_dma_iommu_mapping.h>
  40. #include "adsprpc_compat.h"
  41. #include "adsprpc_shared.h"
  42. #include <soc/qcom/qcom_ramdump.h>
  43. #include <soc/qcom/minidump.h>
  44. #include <soc/qcom/secure_buffer.h>
  45. #include <linux/delay.h>
  46. #include <linux/debugfs.h>
  47. #include <linux/pm_qos.h>
  48. #include <linux/stat.h>
  49. #include <linux/preempt.h>
  50. #include <linux/of_reserved_mem.h>
  51. #include <linux/soc/qcom/pdr.h>
  52. #include <linux/soc/qcom/qmi.h>
  53. #include <linux/mem-buf.h>
  54. #include <linux/iommu.h>
  55. #include <asm/arch_timer.h>
  56. #include <linux/genalloc.h>
  57. #include <soc/qcom/socinfo.h>
  58. #ifdef CONFIG_HIBERNATION
  59. #include <linux/suspend.h>
  60. #include <linux/notifier.h>
  61. #endif
  62. #define CREATE_TRACE_POINTS
  63. #include "fastrpc_trace.h"
  64. #ifdef CONFIG_MSM_ADSPRPC_TRUSTED
  65. #include "../include/linux/fastrpc.h"
  66. #else
  67. #include "fastrpc.h"
  68. #endif
  69. #define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C
  70. #define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D
  71. #define TZ_PIL_AUTH_QDSP6_PROC 1
  72. #define FASTRPC_ENOSUCH 39
  73. #define VMID_SSC_Q6 5
  74. #define VMID_ADSP_Q6 6
  75. #define DEBUGFS_SIZE 3072
  76. #define PID_SIZE 10
  77. #define AUDIO_PDR_ADSP_DTSI_PROPERTY_NAME "qcom,fastrpc-adsp-audio-pdr"
  78. #define AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME "audio_pdr_adsprpc"
  79. #define AUDIO_PDR_ADSP_SERVICE_NAME "avs/audio"
  80. #define ADSP_AUDIOPD_NAME "msm/adsp/audio_pd"
  81. #define SENSORS_PDR_ADSP_DTSI_PROPERTY_NAME "qcom,fastrpc-adsp-sensors-pdr"
  82. #define SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME "sensors_pdr_adsprpc"
  83. #define SENSORS_PDR_ADSP_SERVICE_NAME "tms/servreg"
  84. #define ADSP_SENSORPD_NAME "msm/adsp/sensor_pd"
  85. #define SENSORS_PDR_SLPI_DTSI_PROPERTY_NAME "qcom,fastrpc-slpi-sensors-pdr"
  86. #define SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME "sensors_pdr_sdsprpc"
  87. #define SENSORS_PDR_SLPI_SERVICE_NAME SENSORS_PDR_ADSP_SERVICE_NAME
  88. #define SLPI_SENSORPD_NAME "msm/slpi/sensor_pd"
  89. #define FASTRPC_SECURE_WAKE_SOURCE_CLIENT_NAME "adsprpc-secure"
  90. #define FASTRPC_NON_SECURE_WAKE_SOURCE_CLIENT_NAME "adsprpc-non_secure"
  91. #define RPC_TIMEOUT (5 * HZ)
  92. #define BALIGN 128
  93. #define M_FDLIST (16)
  94. #define M_CRCLIST (64)
  95. #define M_KERNEL_PERF_LIST (PERF_KEY_MAX)
  96. #define M_DSP_PERF_LIST (12)
  97. #define FASTRPC_CTX_MAGIC (0xbeeddeed)
  98. /* Process status notifications from DSP will be sent with this unique context */
  99. #define FASTRPC_NOTIF_CTX_RESERVED 0xABCDABCD
  100. #define FASTRPC_CTX_JOB_TYPE_POS (4)
  101. #define FASTRPC_CTX_TABLE_IDX_POS (6)
  102. #define FASTRPC_CTX_JOBID_POS (16)
  103. #define FASTRPC_CTX_TABLE_IDX_MASK \
  104. ((FASTRPC_CTX_MAX - 1) << FASTRPC_CTX_TABLE_IDX_POS)
  105. #define FASTRPC_ASYNC_JOB_MASK (1)
  106. #define GET_TABLE_IDX_FROM_CTXID(ctxid) \
  107. ((ctxid & FASTRPC_CTX_TABLE_IDX_MASK) >> FASTRPC_CTX_TABLE_IDX_POS)
  108. /* Reserve few entries in context table for critical kernel and static RPC
  109. * calls to avoid user invocations from exhausting all entries.
  110. */
  111. #define NUM_KERNEL_AND_STATIC_ONLY_CONTEXTS (70)
  112. /* Maximum number of pending contexts per remote session */
  113. #define MAX_PENDING_CTX_PER_SESSION (64)
  114. #define NUM_DEVICES 2 /* adsprpc-smd, adsprpc-smd-secure */
  115. #define MINOR_NUM_DEV 0
  116. #define MINOR_NUM_SECURE_DEV 1
  117. #define NON_SECURE_CHANNEL 0
  118. #define SECURE_CHANNEL 1
  119. #define IS_CACHE_ALIGNED(x) (((x) & ((L1_CACHE_BYTES)-1)) == 0)
  120. #ifndef ION_FLAG_CACHED
  121. #define ION_FLAG_CACHED (1)
  122. #endif
  123. #ifndef topology_cluster_id
  124. #define topology_cluster_id(cpu) topology_physical_package_id(cpu)
  125. #endif
  126. /*
  127. * ctxid of every message is OR-ed with fastrpc_remote_pd_type before
  128. * it is sent to DSP. So mask 2 LSBs to retrieve actual context
  129. */
  130. #define CONTEXT_PD_CHECK (3)
  131. #define GET_CTXID_FROM_RSP_CTX(rsp_ctx) (rsp_ctx & ~CONTEXT_PD_CHECK)
  132. #define RH_CID ADSP_DOMAIN_ID
  133. #define FASTRPC_STATIC_HANDLE_PROCESS_GROUP (1)
  134. #define FASTRPC_STATIC_HANDLE_DSP_UTILITIES (2)
  135. #define FASTRPC_STATIC_HANDLE_LISTENER (3)
  136. #define FASTRPC_STATIC_HANDLE_MAX (20)
  137. #define FASTRPC_LATENCY_CTRL_ENB (1)
  138. /* Maximum PM timeout that can be voted through fastrpc */
  139. #define MAX_PM_TIMEOUT_MS 50
  140. /* timeout in us for busy polling after early response from remote processor */
  141. #define FASTRPC_POLL_TIME (4000)
  142. /* timeout in us for polling until memory barrier */
  143. #define FASTRPC_POLL_TIME_MEM_UPDATE (500)
  144. /* timeout in us for polling completion signal after user early hint */
  145. #define FASTRPC_USER_EARLY_HINT_TIMEOUT (500)
  146. /* Early wake up poll completion number received from remote processor */
  147. #define FASTRPC_EARLY_WAKEUP_POLL (0xabbccdde)
  148. /* Poll response number from remote processor for call completion */
  149. #define FASTRPC_POLL_RESPONSE (0xdecaf)
  150. /* latency in us, early wake up signal used below this value */
  151. #define FASTRPC_EARLY_WAKEUP_LATENCY (200)
  152. /* response version number */
  153. #define FASTRPC_RSP_VERSION2 (2)
  154. /* CPU feature information to DSP */
  155. #define FASTRPC_CPUINFO_DEFAULT (0)
  156. #define FASTRPC_CPUINFO_EARLY_WAKEUP (1)
  157. #define INIT_FILELEN_MAX (2*1024*1024)
  158. #define INIT_MEMLEN_MAX_STATIC (8*1024*1024)
  159. #define INIT_MEMLEN_MAX_DYNAMIC (200*1024*1024)
  160. #define INIT_MEMLEN_MIN_DYNAMIC (3*1024*1024)
  161. #define MAX_CACHE_BUF_SIZE (8*1024*1024)
  162. /* Maximum buffers cached in cached buffer list */
  163. #define MAX_CACHED_BUFS (32)
  164. /* Max no. of persistent headers pre-allocated per process */
  165. #define MAX_PERSISTENT_HEADERS (25)
  166. /* Max value of unique fastrpc tgid */
  167. #define MAX_FRPC_TGID 256
  168. #define PERF_CAPABILITY_SUPPORT (1 << 1)
  169. #define KERNEL_ERROR_CODE_V1_SUPPORT 1
  170. #define USERSPACE_ALLOCATION_SUPPORT 1
  171. #define DSPSIGNAL_SUPPORT 1
  172. #define MD_GMSG_BUFFER (1000)
  173. #define MINI_DUMP_DBG_SIZE (200*1024)
  174. /* Max number of region supported */
  175. #define MAX_UNIQUE_ID 5
  176. /* Convert the 19.2MHz clock count to micro-seconds */
  177. #define CONVERT_CNT_TO_US(CNT) (CNT * 10ull / 192ull)
  178. #define FASTRPC_USER_PD_FORCE_KILL 2
  179. /*
  180. * No. of pages shared with DSP during process init
  181. * First page for init-mem and second page for proc-attrs
  182. */
  183. #define PAGESLEN_WITH_SHAREDBUF 2
  184. /* Unique index flag used for mini dump */
  185. static int md_unique_index_flag[MAX_UNIQUE_ID] = { 0, 0, 0, 0, 0 };
  186. /* Array to keep track unique tgid_frpc usage */
  187. static bool frpc_tgid_usage_array[MAX_FRPC_TGID] = {0};
  188. /* Fastrpc remote process attributes */
  189. enum fastrpc_proc_attr {
  190. /* Macro for Debug attr */
  191. FASTRPC_MODE_DEBUG = 1 << 0,
  192. /* Macro for Ptrace */
  193. FASTRPC_MODE_PTRACE = 1 << 1,
  194. /* Macro for CRC Check */
  195. FASTRPC_MODE_CRC = 1 << 2,
  196. /* Macro for Unsigned PD */
  197. FASTRPC_MODE_UNSIGNED_MODULE = 1 << 3,
  198. /* Macro for Adaptive QoS */
  199. FASTRPC_MODE_ADAPTIVE_QOS = 1 << 4,
  200. /* Macro for System Process */
  201. FASTRPC_MODE_SYSTEM_PROCESS = 1 << 5,
  202. /* Macro for Prvileged Process */
  203. FASTRPC_MODE_PRIVILEGED = (1 << 6),
  204. /* Macro for system unsigned PD */
  205. FASTRPC_MODE_SYSTEM_UNSIGNED_PD = 1 << 17,
  206. };
  207. /* FastRPC remote subsystem state*/
  208. enum fastrpc_remote_subsys_state {
  209. SUBSYSTEM_RESTARTING = 0,
  210. SUBSYSTEM_DOWN,
  211. SUBSYSTEM_UP,
  212. };
  213. #define PERF_END ((void)0)
  214. #define PERF(enb, cnt, ff) \
  215. {\
  216. struct timespec64 startT = {0};\
  217. uint64_t *counter = cnt;\
  218. if (enb && counter) {\
  219. ktime_get_real_ts64(&startT);\
  220. } \
  221. ff ;\
  222. if (enb && counter) {\
  223. *counter += getnstimediff(&startT);\
  224. } \
  225. }
  226. #define GET_COUNTER(perf_ptr, offset) \
  227. (perf_ptr != NULL ?\
  228. (((offset >= 0) && (offset < PERF_KEY_MAX)) ?\
  229. (uint64_t *)(perf_ptr + offset)\
  230. : (uint64_t *)NULL) : (uint64_t *)NULL)
  231. /* Macro for comparing local client and PD names with those from callback */
  232. #define COMPARE_SERVICE_LOCATOR_NAMES(cb_client, local_client, \
  233. cb_pdname, local_pdname) \
  234. ((!strcmp(cb_client, local_client)) \
  235. && (!strcmp(cb_pdname, local_pdname)))
  236. #define IS_ASYNC_FASTRPC_AVAILABLE (1)
  237. /* Use the second definition to enable additional dspsignal debug logging */
  238. #define DSPSIGNAL_VERBOSE(x, ...)
  239. /*#define DSPSIGNAL_VERBOSE ADSPRPC_INFO*/
  240. MODULE_IMPORT_NS(DMA_BUF);
  241. static struct dentry *debugfs_root;
  242. static struct dentry *debugfs_global_file;
  243. static inline uint64_t buf_page_start(uint64_t buf)
  244. {
  245. uint64_t start = (uint64_t) buf & PAGE_MASK;
  246. return start;
  247. }
  248. static inline uint64_t buf_page_offset(uint64_t buf)
  249. {
  250. uint64_t offset = (uint64_t) buf & (PAGE_SIZE - 1);
  251. return offset;
  252. }
  253. static inline uint64_t buf_num_pages(uint64_t buf, size_t len)
  254. {
  255. uint64_t start = buf_page_start(buf) >> PAGE_SHIFT;
  256. uint64_t end = (((uint64_t) buf + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
  257. uint64_t nPages = end - start + 1;
  258. return nPages;
  259. }
  260. static inline uint64_t buf_page_size(uint32_t size)
  261. {
  262. uint64_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;
  263. return sz > PAGE_SIZE ? sz : PAGE_SIZE;
  264. }
  265. static inline void *uint64_to_ptr(uint64_t addr)
  266. {
  267. void *ptr = (void *)((uintptr_t)addr);
  268. return ptr;
  269. }
  270. static inline uint64_t ptr_to_uint64(void *ptr)
  271. {
  272. uint64_t addr = (uint64_t)((uintptr_t)ptr);
  273. return addr;
  274. }
  275. static struct fastrpc_apps gfa;
  276. static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
  277. {
  278. .name = "adsprpc-smd",
  279. .subsys = "lpass",
  280. .spd = {
  281. {
  282. .servloc_name =
  283. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME,
  284. .spdname = ADSP_AUDIOPD_NAME,
  285. .cid = ADSP_DOMAIN_ID,
  286. },
  287. {
  288. .servloc_name =
  289. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME,
  290. .spdname = ADSP_SENSORPD_NAME,
  291. .cid = ADSP_DOMAIN_ID,
  292. }
  293. },
  294. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  295. .cpuinfo_status = false,
  296. },
  297. {
  298. .name = "mdsprpc-smd",
  299. .subsys = "mpss",
  300. .spd = {
  301. {
  302. .cid = MDSP_DOMAIN_ID,
  303. }
  304. },
  305. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  306. .cpuinfo_status = false,
  307. },
  308. {
  309. .name = "sdsprpc-smd",
  310. .subsys = "dsps",
  311. .spd = {
  312. {
  313. .servloc_name =
  314. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME,
  315. .spdname = SLPI_SENSORPD_NAME,
  316. .cid = SDSP_DOMAIN_ID,
  317. }
  318. },
  319. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  320. .cpuinfo_status = false,
  321. },
  322. {
  323. .name = "cdsprpc-smd",
  324. .subsys = "cdsp",
  325. .spd = {
  326. {
  327. .cid = CDSP_DOMAIN_ID,
  328. }
  329. },
  330. .cpuinfo_todsp = FASTRPC_CPUINFO_EARLY_WAKEUP,
  331. .cpuinfo_status = false,
  332. },
  333. };
  334. static uint32_t kernel_capabilities[FASTRPC_MAX_ATTRIBUTES -
  335. FASTRPC_MAX_DSP_ATTRIBUTES] = {
  336. PERF_CAPABILITY_SUPPORT,
  337. /* PERF_LOGGING_V2_SUPPORT feature is supported, unsupported = 0 */
  338. KERNEL_ERROR_CODE_V1_SUPPORT,
  339. /* Fastrpc Driver error code changes present */
  340. USERSPACE_ALLOCATION_SUPPORT,
  341. /* Userspace allocation allowed for DSP memory request*/
  342. DSPSIGNAL_SUPPORT
  343. /* Lightweight driver-based signaling */
  344. };
  345. static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type);
  346. static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
  347. uint32_t flags, uintptr_t va, uint64_t phys,
  348. size_t size, uintptr_t *raddr);
  349. static inline void fastrpc_update_rxmsg_buf(struct fastrpc_channel_ctx *chan,
  350. uint64_t ctx, int retval, uint32_t rsp_flags,
  351. uint32_t early_wake_time, uint32_t ver, int64_t ns, uint64_t xo_time_in_us);
  352. /**
  353. * fastrpc_device_create - Create device for the fastrpc process file
  354. * @fl : Fastrpc process file
  355. * Returns: 0 on Success
  356. */
  357. static int fastrpc_device_create(struct fastrpc_file *fl);
  358. static inline int64_t getnstimediff(struct timespec64 *start)
  359. {
  360. int64_t ns;
  361. struct timespec64 ts, b;
  362. ktime_get_real_ts64(&ts);
  363. b = timespec64_sub(ts, *start);
  364. ns = timespec64_to_ns(&b);
  365. return ns;
  366. }
  367. /**
  368. * get_timestamp_in_ns - Gets time of day in nanoseconds
  369. *
  370. * Returns: Timestamp in nanoseconds
  371. */
  372. static inline int64_t get_timestamp_in_ns(void)
  373. {
  374. int64_t ns = 0;
  375. struct timespec64 ts;
  376. ktime_get_real_ts64(&ts);
  377. ns = timespec64_to_ns(&ts);
  378. return ns;
  379. }
  380. static inline int poll_for_remote_response(struct smq_invoke_ctx *ctx, uint32_t timeout)
  381. {
  382. int err = -EIO;
  383. uint32_t sc = ctx->sc, ii = 0, jj = 0;
  384. struct smq_invoke_buf *list;
  385. struct smq_phy_page *pages;
  386. uint64_t *fdlist = NULL;
  387. uint32_t *crclist = NULL, *poll = NULL;
  388. unsigned int inbufs, outbufs, handles;
  389. /* calculate poll memory location */
  390. inbufs = REMOTE_SCALARS_INBUFS(sc);
  391. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  392. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  393. list = smq_invoke_buf_start(ctx->rpra, sc);
  394. pages = smq_phy_page_start(sc, list);
  395. fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
  396. crclist = (uint32_t *)(fdlist + M_FDLIST);
  397. poll = (uint32_t *)(crclist + M_CRCLIST);
  398. /* poll on memory for DSP response. Return failure on timeout */
  399. for (ii = 0, jj = 0; ii < timeout; ii++, jj++) {
  400. if (*poll == FASTRPC_EARLY_WAKEUP_POLL) {
  401. /* Remote processor sent early response */
  402. err = 0;
  403. break;
  404. } else if (*poll == FASTRPC_POLL_RESPONSE) {
  405. /* Remote processor sent poll response to complete the call */
  406. err = 0;
  407. ctx->is_work_done = true;
  408. ctx->retval = 0;
  409. /* Update DSP response history */
  410. fastrpc_update_rxmsg_buf(&gfa.channel[ctx->fl->cid],
  411. ctx->msg.invoke.header.ctx, 0, POLL_MODE, 0,
  412. FASTRPC_RSP_VERSION2, get_timestamp_in_ns(),
  413. CONVERT_CNT_TO_US(__arch_counter_get_cntvct()));
  414. break;
  415. }
  416. if (jj == FASTRPC_POLL_TIME_MEM_UPDATE) {
  417. /* Wait for DSP to finish updating poll memory */
  418. rmb();
  419. jj = 0;
  420. }
  421. udelay(1);
  422. }
  423. return err;
  424. }
  425. enum interrupted_state {
  426. DEFAULT_STATE = 0,
  427. INTERRUPTED_STATE = 1,
  428. RESTORED_STATE = 2,
  429. };
  430. /**
  431. * fastrpc_update_txmsg_buf - Update history of sent glink messages
  432. * @msg : Pointer to RPC message to remote subsystem
  433. * @transport_send_err : Error from transport
  434. * @ns : Timestamp (in ns) of sent message
  435. * @xo_time_in_us : XO Timestamp (in us) of sent message
  436. * @ctx : invoke ctx
  437. * @interrupted : 0/1/2 (default/interrupted/restored)
  438. *
  439. * Returns none
  440. */
  441. static inline void fastrpc_update_txmsg_buf(struct smq_msg *msg,
  442. int transport_send_err, int64_t ns, uint64_t xo_time_in_us,
  443. struct smq_invoke_ctx *ctx, enum interrupted_state interrupted)
  444. {
  445. unsigned long flags = 0;
  446. unsigned int tx_index = 0;
  447. struct fastrpc_tx_msg *tx_msg = NULL;
  448. struct fastrpc_channel_ctx *chan = NULL;
  449. struct fastrpc_file *fl = ctx->fl;
  450. int err = 0, cid = -1;
  451. if (!fl) {
  452. err = -EBADF;
  453. goto bail;
  454. }
  455. cid = fl->cid;
  456. VERIFY(err, VALID_FASTRPC_CID(cid));
  457. if (err) {
  458. err = -ECHRNG;
  459. goto bail;
  460. }
  461. chan = &fl->apps->channel[cid];
  462. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  463. if (interrupted){
  464. if (ctx->tx_index >= 0 && ctx->tx_index < GLINK_MSG_HISTORY_LEN) {
  465. tx_msg = &chan->gmsg_log.tx_msgs[ctx->tx_index];
  466. if (tx_msg->msg.invoke.header.ctx == ctx->msg.invoke.header.ctx) {
  467. tx_msg->xo_time_in_us_interrupted = ctx->xo_time_in_us_interrupted;
  468. tx_msg->xo_time_in_us_restored = ctx->xo_time_in_us_restored;
  469. }
  470. }
  471. } else {
  472. tx_index = chan->gmsg_log.tx_index;
  473. ctx->tx_index = tx_index;
  474. tx_msg = &chan->gmsg_log.tx_msgs[tx_index];
  475. memcpy(&tx_msg->msg, msg, sizeof(struct smq_msg));
  476. tx_msg->transport_send_err = transport_send_err;
  477. tx_msg->ns = ns;
  478. tx_msg->xo_time_in_us = xo_time_in_us;
  479. tx_index++;
  480. chan->gmsg_log.tx_index =
  481. (tx_index > (GLINK_MSG_HISTORY_LEN - 1)) ? 0 : tx_index;
  482. }
  483. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  484. bail:
  485. if (err)
  486. ADSPRPC_ERR("adsprpc: %s: unable to update txmsg buf (err %d) for ctx: 0x%x\n",
  487. __func__, err, ctx->msg.invoke.header.ctx);
  488. }
  489. /**
  490. * fastrpc_update_rxmsg_buf - Update history of received glink responses
  491. * @chan : Channel context
  492. * @ctx : Context of received response from DSP
  493. * @retval : Return value for RPC call
  494. * @rsp_flags : Response type
  495. * @early_wake_time : Poll time for early wakeup
  496. * @ver : Version of response
  497. * @ns : Timestamp (in ns) of response
  498. * @xo_time_in_us : XO Timestamp (in us) of response
  499. *
  500. * Returns none
  501. */
  502. static inline void fastrpc_update_rxmsg_buf(struct fastrpc_channel_ctx *chan,
  503. uint64_t ctx, int retval, uint32_t rsp_flags,
  504. uint32_t early_wake_time, uint32_t ver, int64_t ns, uint64_t xo_time_in_us)
  505. {
  506. unsigned long flags = 0;
  507. unsigned int rx_index = 0;
  508. struct fastrpc_rx_msg *rx_msg = NULL;
  509. struct smq_invoke_rspv2 *rsp = NULL;
  510. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  511. rx_index = chan->gmsg_log.rx_index;
  512. rx_msg = &chan->gmsg_log.rx_msgs[rx_index];
  513. rsp = &rx_msg->rsp;
  514. rsp->ctx = ctx;
  515. rsp->retval = retval;
  516. rsp->flags = rsp_flags;
  517. rsp->early_wake_time = early_wake_time;
  518. rsp->version = ver;
  519. rx_msg->ns = ns;
  520. rx_msg->xo_time_in_us = xo_time_in_us;
  521. rx_index++;
  522. chan->gmsg_log.rx_index =
  523. (rx_index > (GLINK_MSG_HISTORY_LEN - 1)) ? 0 : rx_index;
  524. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  525. }
  526. static inline int get_unique_index(void)
  527. {
  528. int index = -1;
  529. mutex_lock(&gfa.mut_uid);
  530. for (index = 0; index < MAX_UNIQUE_ID; index++) {
  531. if (md_unique_index_flag[index] == 0) {
  532. md_unique_index_flag[index] = 1;
  533. mutex_unlock(&gfa.mut_uid);
  534. return index;
  535. }
  536. }
  537. mutex_unlock(&gfa.mut_uid);
  538. return index;
  539. }
  540. static inline void reset_unique_index(int index)
  541. {
  542. mutex_lock(&gfa.mut_uid);
  543. if (index > -1 && index < MAX_UNIQUE_ID)
  544. md_unique_index_flag[index] = 0;
  545. mutex_unlock(&gfa.mut_uid);
  546. }
  547. /**
  548. * fastrpc_minidump_add_region - Add mini dump region
  549. * @fastrpc_mmap : Input structure mmap
  550. *
  551. * Returns int
  552. */
  553. static int fastrpc_minidump_add_region(struct fastrpc_mmap *map)
  554. {
  555. int err = 0, ret_val = 0, md_index = 0;
  556. struct md_region md_entry;
  557. md_index = get_unique_index();
  558. if (md_index > -1 && md_index < MAX_UNIQUE_ID) {
  559. scnprintf(md_entry.name, MAX_NAME_LENGTH, "FRPC_%d", md_index);
  560. md_entry.virt_addr = map->va;
  561. md_entry.phys_addr = map->phys;
  562. md_entry.size = map->size;
  563. ret_val = msm_minidump_add_region(&md_entry);
  564. if (ret_val < 0) {
  565. ADSPRPC_ERR(
  566. "Failed to add/update CMA to Minidump for phys: 0x%llx, size: %zu, md_index %d, md_entry.name %s\n",
  567. map->phys,
  568. map->size, md_index,
  569. md_entry.name);
  570. reset_unique_index(md_index);
  571. err = ret_val;
  572. } else {
  573. map->frpc_md_index = md_index;
  574. }
  575. } else {
  576. pr_warn("failed to generate valid unique id for mini dump : %d\n", md_index);
  577. }
  578. return err;
  579. }
  580. /**
  581. * fastrpc_minidump_remove_region - Remove mini dump region if added
  582. * @fastrpc_mmap : Input structure mmap
  583. *
  584. * Returns int
  585. */
  586. static int fastrpc_minidump_remove_region(struct fastrpc_mmap *map)
  587. {
  588. int err = -EINVAL;
  589. struct md_region md_entry;
  590. if (map->frpc_md_index > -1 && map->frpc_md_index < MAX_UNIQUE_ID) {
  591. scnprintf(md_entry.name, MAX_NAME_LENGTH, "FRPC_%d",
  592. map->frpc_md_index);
  593. md_entry.virt_addr = map->va;
  594. md_entry.phys_addr = map->phys;
  595. md_entry.size = map->size;
  596. err = msm_minidump_remove_region(&md_entry);
  597. if (err < 0) {
  598. ADSPRPC_ERR(
  599. "Failed to remove CMA from Minidump for phys: 0x%llx, size: %zu index = %d\n",
  600. map->phys, map->size, map->frpc_md_index);
  601. } else {
  602. reset_unique_index(map->frpc_md_index);
  603. map->frpc_md_index = -1;
  604. }
  605. } else {
  606. ADSPRPC_WARN("mini-dump enabled with invalid unique id: %d\n", map->frpc_md_index);
  607. }
  608. return err;
  609. }
  610. static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache)
  611. {
  612. struct fastrpc_file *fl = buf == NULL ? NULL : buf->fl;
  613. int vmid, err = 0, cid = -1;
  614. if (!fl)
  615. return;
  616. if (buf->in_use) {
  617. /* Don't free persistent header buf. Just mark as available */
  618. spin_lock(&fl->hlock);
  619. buf->in_use = false;
  620. spin_unlock(&fl->hlock);
  621. return;
  622. }
  623. if (cache && buf->size < MAX_CACHE_BUF_SIZE) {
  624. spin_lock(&fl->hlock);
  625. if (fl->num_cached_buf > MAX_CACHED_BUFS) {
  626. spin_unlock(&fl->hlock);
  627. goto skip_buf_cache;
  628. }
  629. hlist_add_head(&buf->hn, &fl->cached_bufs);
  630. fl->num_cached_buf++;
  631. buf->type = -1;
  632. spin_unlock(&fl->hlock);
  633. return;
  634. }
  635. skip_buf_cache:
  636. if (buf->type == USERHEAP_BUF) {
  637. spin_lock(&fl->hlock);
  638. hlist_del_init(&buf->hn_rem);
  639. spin_unlock(&fl->hlock);
  640. buf->raddr = 0;
  641. }
  642. if (!IS_ERR_OR_NULL(buf->virt)) {
  643. VERIFY(err, fl->sctx != NULL);
  644. if (err)
  645. goto bail;
  646. if (fl->sctx->smmu.cb)
  647. buf->phys &= ~((uint64_t)fl->sctx->smmu.cb << 32);
  648. cid = fl->cid;
  649. VERIFY(err, VALID_FASTRPC_CID(cid));
  650. if (err) {
  651. err = -ECHRNG;
  652. ADSPRPC_ERR(
  653. "invalid channel 0x%zx set for session\n",
  654. cid);
  655. goto bail;
  656. }
  657. vmid = fl->apps->channel[cid].vmid;
  658. if ((vmid) && (fl->apps->channel[cid].in_hib == 0)) {
  659. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS)| BIT(vmid);
  660. struct qcom_scm_vmperm dest_perms = {0};
  661. int hyp_err = 0;
  662. dest_perms.vmid = QCOM_SCM_VMID_HLOS;
  663. dest_perms.perm = QCOM_SCM_PERM_RWX;
  664. hyp_err = qcom_scm_assign_mem(buf->phys,
  665. buf_page_size(buf->size),
  666. &src_perms, &dest_perms, 1);
  667. if (hyp_err) {
  668. ADSPRPC_ERR(
  669. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  670. hyp_err, buf->phys, buf->size);
  671. }
  672. }
  673. trace_fastrpc_dma_free(cid, buf->phys, buf->size);
  674. dma_free_attrs(fl->sctx->smmu.dev, buf->size, buf->virt,
  675. buf->phys, buf->dma_attr);
  676. }
  677. bail:
  678. kfree(buf);
  679. }
  680. static void fastrpc_cached_buf_list_free(struct fastrpc_file *fl)
  681. {
  682. struct fastrpc_buf *buf, *free;
  683. do {
  684. struct hlist_node *n;
  685. free = NULL;
  686. spin_lock(&fl->hlock);
  687. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  688. hlist_del_init(&buf->hn);
  689. fl->num_cached_buf--;
  690. free = buf;
  691. break;
  692. }
  693. spin_unlock(&fl->hlock);
  694. if (free)
  695. fastrpc_buf_free(free, 0);
  696. } while (free);
  697. }
  698. static void fastrpc_remote_buf_list_free(struct fastrpc_file *fl)
  699. {
  700. struct fastrpc_buf *buf, *free;
  701. do {
  702. struct hlist_node *n;
  703. free = NULL;
  704. spin_lock(&fl->hlock);
  705. hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {
  706. free = buf;
  707. break;
  708. }
  709. spin_unlock(&fl->hlock);
  710. if (free)
  711. fastrpc_buf_free(free, 0);
  712. } while (free);
  713. }
  714. static void fastrpc_mmap_add(struct fastrpc_mmap *map)
  715. {
  716. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  717. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  718. struct fastrpc_apps *me = &gfa;
  719. unsigned long irq_flags = 0;
  720. spin_lock_irqsave(&me->hlock, irq_flags);
  721. hlist_add_head(&map->hn, &me->maps);
  722. spin_unlock_irqrestore(&me->hlock, irq_flags);
  723. } else {
  724. struct fastrpc_file *fl = map->fl;
  725. hlist_add_head(&map->hn, &fl->maps);
  726. }
  727. }
  728. static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd,
  729. struct dma_buf *buf, uintptr_t va, size_t len, int mflags, int refs,
  730. struct fastrpc_mmap **ppmap)
  731. {
  732. struct fastrpc_apps *me = &gfa;
  733. struct fastrpc_mmap *match = NULL, *map = NULL;
  734. struct hlist_node *n;
  735. unsigned long irq_flags = 0;
  736. if ((va + len) < va)
  737. return -EFAULT;
  738. if (mflags == ADSP_MMAP_HEAP_ADDR ||
  739. mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  740. spin_lock_irqsave(&me->hlock, irq_flags);
  741. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  742. if (va >= map->va &&
  743. va + len <= map->va + map->len &&
  744. map->fd == fd) {
  745. if (refs) {
  746. if (map->refs + 1 == INT_MAX) {
  747. spin_unlock_irqrestore(&me->hlock, irq_flags);
  748. return -ETOOMANYREFS;
  749. }
  750. map->refs++;
  751. }
  752. match = map;
  753. break;
  754. }
  755. }
  756. spin_unlock_irqrestore(&me->hlock, irq_flags);
  757. } else if (mflags == ADSP_MMAP_DMA_BUFFER) {
  758. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  759. if (map->buf == buf) {
  760. if (refs) {
  761. if (map->refs + 1 == INT_MAX)
  762. return -ETOOMANYREFS;
  763. map->refs++;
  764. }
  765. match = map;
  766. break;
  767. }
  768. }
  769. } else {
  770. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  771. if (va >= map->va &&
  772. va + len <= map->va + map->len &&
  773. map->fd == fd) {
  774. if (refs) {
  775. if (map->refs + 1 == INT_MAX)
  776. return -ETOOMANYREFS;
  777. map->refs++;
  778. }
  779. match = map;
  780. break;
  781. }
  782. }
  783. }
  784. if (match) {
  785. *ppmap = match;
  786. return 0;
  787. }
  788. return -ENXIO;
  789. }
  790. static int fastrpc_alloc_cma_memory(dma_addr_t *region_phys, void **vaddr,
  791. size_t size, unsigned long dma_attr)
  792. {
  793. int err = 0;
  794. struct fastrpc_apps *me = &gfa;
  795. if (me->dev == NULL) {
  796. ADSPRPC_ERR(
  797. "failed to allocate CMA memory, device adsprpc-mem is not initialized\n");
  798. return -ENODEV;
  799. }
  800. VERIFY(err, size > 0 && size < me->max_size_limit);
  801. if (err) {
  802. err = -EFAULT;
  803. pr_err("adsprpc: %s: invalid allocation size 0x%zx\n",
  804. __func__, size);
  805. return err;
  806. }
  807. *vaddr = dma_alloc_attrs(me->dev, size, region_phys,
  808. GFP_KERNEL, dma_attr);
  809. if (IS_ERR_OR_NULL(*vaddr)) {
  810. ADSPRPC_ERR(
  811. "dma_alloc_attrs failed for device %s size 0x%zx dma_attr %lu, returned %ld\n",
  812. dev_name(me->dev), size, dma_attr, PTR_ERR(*vaddr));
  813. return -ENOBUFS;
  814. }
  815. return 0;
  816. }
  817. static int fastrpc_mmap_remove(struct fastrpc_file *fl, int fd, uintptr_t va,
  818. size_t len, struct fastrpc_mmap **ppmap)
  819. {
  820. struct fastrpc_mmap *match = NULL, *map;
  821. struct hlist_node *n;
  822. struct fastrpc_apps *me = &gfa;
  823. unsigned long irq_flags = 0;
  824. /*
  825. * Search for a mapping by matching fd, remote address and length.
  826. * For backward compatibility, search for a mapping by matching is
  827. * limited to remote address and length when passed fd < 0.
  828. */
  829. spin_lock_irqsave(&me->hlock, irq_flags);
  830. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  831. if ((fd < 0 || map->fd == fd) && map->raddr == va &&
  832. map->raddr + map->len == va + len &&
  833. map->refs == 1 && !map->is_persistent &&
  834. /* Skip unmap if it is fastrpc shell memory */
  835. !map->is_filemap) {
  836. match = map;
  837. hlist_del_init(&map->hn);
  838. break;
  839. }
  840. }
  841. spin_unlock_irqrestore(&me->hlock, irq_flags);
  842. if (match) {
  843. *ppmap = match;
  844. return 0;
  845. }
  846. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  847. if ((fd < 0 || map->fd == fd) && map->raddr == va &&
  848. map->raddr + map->len == va + len &&
  849. map->refs == 1 &&
  850. /* Remove map only if it isn't being used in any pending RPC calls */
  851. !map->ctx_refs &&
  852. /* Skip unmap if it is fastrpc shell memory */
  853. !map->is_filemap) {
  854. match = map;
  855. hlist_del_init(&map->hn);
  856. break;
  857. }
  858. }
  859. if (match) {
  860. *ppmap = match;
  861. return 0;
  862. }
  863. return -ETOOMANYREFS;
  864. }
  865. static void fastrpc_mmap_free(struct fastrpc_mmap *map, uint32_t flags)
  866. {
  867. struct fastrpc_apps *me = &gfa;
  868. struct fastrpc_file *fl;
  869. int vmid, cid = -1, err = 0;
  870. struct fastrpc_session_ctx *sess;
  871. unsigned long irq_flags = 0;
  872. if (!map)
  873. return;
  874. fl = map->fl;
  875. if (fl && !(map->flags == ADSP_MMAP_HEAP_ADDR ||
  876. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)) {
  877. cid = fl->cid;
  878. VERIFY(err, VALID_FASTRPC_CID(cid));
  879. if (err) {
  880. err = -ECHRNG;
  881. pr_err("adsprpc: ERROR:%s, Invalid channel id: %d, err:%d\n",
  882. __func__, cid, err);
  883. return;
  884. }
  885. }
  886. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  887. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  888. spin_lock_irqsave(&me->hlock, irq_flags);
  889. map->refs--;
  890. if (!map->refs && !map->is_persistent && !map->ctx_refs)
  891. hlist_del_init(&map->hn);
  892. if (map->refs > 0) {
  893. ADSPRPC_WARN(
  894. "multiple references for remote heap size %zu va 0x%lx ref count is %d\n",
  895. map->size, map->va, map->refs);
  896. spin_unlock_irqrestore(&me->hlock, irq_flags);
  897. return;
  898. }
  899. if (map->is_persistent && map->in_use)
  900. map->in_use = false;
  901. spin_unlock_irqrestore(&me->hlock, irq_flags);
  902. } else {
  903. map->refs--;
  904. if (!map->refs && !map->ctx_refs)
  905. hlist_del_init(&map->hn);
  906. if (map->refs > 0 && !flags)
  907. return;
  908. }
  909. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  910. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  911. if (me->dev == NULL) {
  912. ADSPRPC_ERR(
  913. "failed to free remote heap allocation, device is not initialized\n");
  914. return;
  915. }
  916. if (msm_minidump_enabled() && !map->is_persistent)
  917. err = fastrpc_minidump_remove_region(map);
  918. if (map->phys && !map->is_persistent) {
  919. trace_fastrpc_dma_free(-1, map->phys, map->size);
  920. dma_free_attrs(me->dev, map->size, (void *)map->va,
  921. (dma_addr_t)map->phys, (unsigned long)map->attr);
  922. }
  923. } else if (map->flags == FASTRPC_MAP_FD_NOMAP) {
  924. trace_fastrpc_dma_unmap(cid, map->phys, map->size);
  925. if (!IS_ERR_OR_NULL(map->table))
  926. dma_buf_unmap_attachment(map->attach, map->table,
  927. DMA_BIDIRECTIONAL);
  928. if (!IS_ERR_OR_NULL(map->attach))
  929. dma_buf_detach(map->buf, map->attach);
  930. if (!IS_ERR_OR_NULL(map->buf))
  931. dma_buf_put(map->buf);
  932. } else {
  933. if (!fl)
  934. goto bail;
  935. if (map->secure)
  936. sess = fl->secsctx;
  937. else
  938. sess = fl->sctx;
  939. vmid = fl->apps->channel[cid].vmid;
  940. if (vmid && map->phys && (me->channel[cid].in_hib == 0)) {
  941. int hyp_err = 0;
  942. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
  943. struct qcom_scm_vmperm dst_perms = {0};
  944. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  945. dst_perms.perm = QCOM_SCM_PERM_RWX;
  946. hyp_err = qcom_scm_assign_mem(map->phys,
  947. buf_page_size(map->size),
  948. &src_perms, &dst_perms, 1);
  949. if (hyp_err) {
  950. ADSPRPC_ERR(
  951. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  952. hyp_err, map->phys, map->size);
  953. }
  954. }
  955. trace_fastrpc_dma_unmap(cid, map->phys, map->size);
  956. if (!IS_ERR_OR_NULL(map->table))
  957. dma_buf_unmap_attachment(map->attach, map->table,
  958. DMA_BIDIRECTIONAL);
  959. if (!IS_ERR_OR_NULL(map->attach))
  960. dma_buf_detach(map->buf, map->attach);
  961. if (!IS_ERR_OR_NULL(map->buf))
  962. dma_buf_put(map->buf);
  963. }
  964. if (fl) {
  965. spin_lock(&fl->hlock);
  966. if ((map->flags == ADSP_MMAP_ADD_PAGES) || (map->flags == ADSP_MMAP_ADD_PAGES_LLC))
  967. fl->mem_snap.heap_bufs_size -= map->size;
  968. else
  969. fl->mem_snap.nonheap_bufs_size -= map->size;
  970. spin_unlock(&fl->hlock);
  971. }
  972. bail:
  973. if (!map->is_persistent)
  974. kfree(map);
  975. }
  976. static int fastrpc_session_alloc_secure_memory(
  977. struct fastrpc_channel_ctx *chan, int secure,
  978. int sharedcb, int pd_type, struct fastrpc_session_ctx **session);
  979. static inline bool fastrpc_get_persistent_map(size_t len, struct fastrpc_mmap **pers_map)
  980. {
  981. struct fastrpc_apps *me = &gfa;
  982. struct fastrpc_mmap *map = NULL;
  983. struct hlist_node *n = NULL;
  984. bool found = false;
  985. unsigned long irq_flags = 0;
  986. spin_lock_irqsave(&me->hlock, irq_flags);
  987. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  988. if (len == map->len &&
  989. map->is_persistent && !map->in_use) {
  990. *pers_map = map;
  991. map->in_use = true;
  992. /*
  993. * Incrementing map reference count when getting
  994. * the map to avoid negative reference count when
  995. * freeing the map.
  996. */
  997. map->refs++;
  998. found = true;
  999. break;
  1000. }
  1001. }
  1002. spin_unlock_irqrestore(&me->hlock, irq_flags);
  1003. return found;
  1004. }
  1005. static int fastrpc_mmap_create_remote_heap(struct fastrpc_file *fl,
  1006. struct fastrpc_mmap *map, size_t len, int mflags)
  1007. {
  1008. int err = 0;
  1009. struct fastrpc_apps *me = &gfa;
  1010. dma_addr_t region_phys = 0;
  1011. void *region_vaddr = NULL;
  1012. map->apps = me;
  1013. map->fl = NULL;
  1014. map->attr |= DMA_ATTR_NO_KERNEL_MAPPING;
  1015. err = fastrpc_alloc_cma_memory(&region_phys, &region_vaddr,
  1016. len, (unsigned long) map->attr);
  1017. if (err)
  1018. goto bail;
  1019. trace_fastrpc_dma_alloc(fl->cid, (uint64_t)region_phys, len,
  1020. (unsigned long)map->attr, mflags);
  1021. map->phys = (uintptr_t)region_phys;
  1022. map->size = len;
  1023. map->va = (uintptr_t)region_vaddr;
  1024. map->servloc_name = fl->servloc_name;
  1025. bail:
  1026. return err;
  1027. }
  1028. static int get_buffer_attr(struct dma_buf *buf, bool *exclusive_access, bool *hlos_access)
  1029. {
  1030. const int *vmids_list = NULL, *perms = NULL;
  1031. int err = 0, vmids_list_len = 0;
  1032. *exclusive_access = false;
  1033. *hlos_access = false;
  1034. err = mem_buf_dma_buf_get_vmperm(buf, &vmids_list, &perms, &vmids_list_len);
  1035. if (err)
  1036. goto bail;
  1037. /*
  1038. * If one VM has access to buffer and is the current VM,
  1039. * then VM has exclusive access to buffer
  1040. */
  1041. if (vmids_list_len == 1 && vmids_list[0] == mem_buf_current_vmid())
  1042. *exclusive_access = true;
  1043. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1044. for (int ii = 0; ii < vmids_list_len; ii++) {
  1045. if (vmids_list[ii] == VMID_HLOS) {
  1046. *hlos_access = true;
  1047. break;
  1048. }
  1049. }
  1050. #endif
  1051. bail:
  1052. return err;
  1053. }
  1054. static int set_buffer_secure_type(struct fastrpc_mmap *map)
  1055. {
  1056. int err = 0;
  1057. bool hlos_access = false, exclusive_access = false;
  1058. VERIFY(err, 0 == (err = get_buffer_attr(map->buf, &exclusive_access, &hlos_access)));
  1059. if (err) {
  1060. ADSPRPC_ERR("failed to obtain buffer attributes for fd %d ret %d\n", map->fd, err);
  1061. err = -EBADFD;
  1062. goto bail;
  1063. }
  1064. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1065. /*
  1066. * PVM (HLOS) can share buffers with TVM, in case buffers are to be shared to secure PD,
  1067. * PVM is expected to relinquish its ownership to those buffers before sharing.
  1068. * If PVM still retains access, then those buffers cannot be shared to secure PD.
  1069. */
  1070. if (hlos_access) {
  1071. ADSPRPC_ERR("Buffers with HLOS access (fd %d) are not allowed on TVM\n", map->fd);
  1072. err = -EACCES;
  1073. goto bail;
  1074. }
  1075. #endif
  1076. /*
  1077. * Secure buffers would always be owned by multiple VMs.
  1078. * If current VM is the exclusive owner of a buffer, it is considered non-secure.
  1079. * In PVM:
  1080. * - CPZ buffers are secure
  1081. * - All other buffers are non-secure
  1082. * In TVM:
  1083. * - Since it is a secure environment by default, there are no explicit "secure" buffers
  1084. * - All buffers are marked "non-secure"
  1085. */
  1086. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1087. map->secure = 0;
  1088. #else
  1089. map->secure = (exclusive_access) ? 0 : 1;
  1090. #endif
  1091. bail:
  1092. return err;
  1093. }
  1094. static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, struct dma_buf *buf,
  1095. unsigned int attr, uintptr_t va, size_t len, int mflags,
  1096. struct fastrpc_mmap **ppmap)
  1097. {
  1098. struct fastrpc_apps *me = &gfa;
  1099. struct fastrpc_session_ctx *sess;
  1100. struct fastrpc_apps *apps = NULL;
  1101. int cid = -1;
  1102. struct fastrpc_channel_ctx *chan = NULL;
  1103. struct fastrpc_mmap *map = NULL;
  1104. int err = 0, vmid, sgl_index = 0;
  1105. struct scatterlist *sgl = NULL;
  1106. bool dma_attach_fail = false;
  1107. size_t tot_bufs_size = 0;
  1108. if (!fl) {
  1109. err = -EBADF;
  1110. goto bail;
  1111. }
  1112. apps = fl->apps;
  1113. cid = fl->cid;
  1114. VERIFY(err, VALID_FASTRPC_CID(cid));
  1115. if (err) {
  1116. err = -ECHRNG;
  1117. goto bail;
  1118. }
  1119. chan = &apps->channel[cid];
  1120. if (!fastrpc_mmap_find(fl, fd, NULL, va, len, mflags, 1, ppmap))
  1121. return 0;
  1122. map = kzalloc(sizeof(*map), GFP_KERNEL);
  1123. VERIFY(err, !IS_ERR_OR_NULL(map));
  1124. if (err) {
  1125. err = -ENOMEM;
  1126. goto bail;
  1127. }
  1128. INIT_HLIST_NODE(&map->hn);
  1129. map->flags = mflags;
  1130. map->refs = 1;
  1131. map->fl = fl;
  1132. map->fd = fd;
  1133. map->attr = attr;
  1134. map->buf = buf;
  1135. map->frpc_md_index = -1;
  1136. map->is_filemap = false;
  1137. ktime_get_real_ts64(&map->map_start_time);
  1138. if (mflags == ADSP_MMAP_HEAP_ADDR ||
  1139. mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  1140. VERIFY(err, 0 == (err = fastrpc_mmap_create_remote_heap(fl, map,
  1141. len, mflags)));
  1142. if (err)
  1143. goto bail;
  1144. if (msm_minidump_enabled()) {
  1145. err = fastrpc_minidump_add_region(map);
  1146. if (err)
  1147. goto bail;
  1148. }
  1149. } else if (mflags == FASTRPC_MAP_FD_NOMAP) {
  1150. VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));
  1151. if (err) {
  1152. ADSPRPC_ERR("dma_buf_get failed for fd %d ret %ld\n",
  1153. fd, PTR_ERR(map->buf));
  1154. err = -EBADFD;
  1155. goto bail;
  1156. }
  1157. err = set_buffer_secure_type(map);
  1158. if (err)
  1159. goto bail;
  1160. map->va = 0;
  1161. map->phys = 0;
  1162. VERIFY(err, !IS_ERR_OR_NULL(map->attach =
  1163. dma_buf_attach(map->buf, me->dev)));
  1164. if (err) {
  1165. ADSPRPC_ERR(
  1166. "dma_buf_attach for fd %d for len 0x%zx failed to map buffer on SMMU device %s ret %ld\n",
  1167. fd, len, dev_name(me->dev), PTR_ERR(map->attach));
  1168. dma_attach_fail = true;
  1169. err = -EFAULT;
  1170. goto bail;
  1171. }
  1172. map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  1173. VERIFY(err, !IS_ERR_OR_NULL(map->table =
  1174. dma_buf_map_attachment(map->attach,
  1175. DMA_BIDIRECTIONAL)));
  1176. if (err) {
  1177. ADSPRPC_ERR(
  1178. "dma_buf_map_attachment for fd %d for len 0x%zx failed on device %s ret %ld\n",
  1179. fd, len, dev_name(me->dev), PTR_ERR(map->table));
  1180. err = -EFAULT;
  1181. goto bail;
  1182. }
  1183. VERIFY(err, map->table->nents == 1);
  1184. if (err) {
  1185. ADSPRPC_ERR(
  1186. "multiple scatter-gather entries (%u) present for NOMAP fd %d\n",
  1187. map->table->nents, fd);
  1188. err = -EFAULT;
  1189. goto bail;
  1190. }
  1191. map->phys = sg_dma_address(map->table->sgl);
  1192. map->size = len;
  1193. map->flags = FASTRPC_MAP_FD_DELAYED;
  1194. trace_fastrpc_dma_map(cid, fd, map->phys, map->size,
  1195. len, map->attach->dma_map_attrs, mflags);
  1196. } else {
  1197. if (map->attr && (map->attr & FASTRPC_ATTR_KEEP_MAP)) {
  1198. ADSPRPC_INFO("buffer mapped with persist attr 0x%x\n",
  1199. (unsigned int)map->attr);
  1200. map->refs = 2;
  1201. }
  1202. if (mflags == ADSP_MMAP_DMA_BUFFER) {
  1203. VERIFY(err, !IS_ERR_OR_NULL(map->buf));
  1204. if (err) {
  1205. ADSPRPC_ERR("Invalid DMA buffer address %pK\n",
  1206. map->buf);
  1207. err = -EFAULT;
  1208. goto bail;
  1209. }
  1210. /* Increment DMA buffer ref count,
  1211. * so that client cannot unmap DMA buffer, before freeing buffer
  1212. */
  1213. get_dma_buf(map->buf);
  1214. } else {
  1215. VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));
  1216. if (err) {
  1217. ADSPRPC_ERR("dma_buf_get failed for fd %d ret %ld\n",
  1218. fd, PTR_ERR(map->buf));
  1219. err = -EBADFD;
  1220. goto bail;
  1221. }
  1222. }
  1223. err = set_buffer_secure_type(map);
  1224. if (err)
  1225. goto bail;
  1226. if (map->secure) {
  1227. if (!fl->secsctx)
  1228. err = fastrpc_session_alloc_secure_memory(chan, 1,
  1229. me->share_securecb, fl->pd_type, &fl->secsctx);
  1230. if (err) {
  1231. ADSPRPC_ERR(
  1232. "fastrpc_session_alloc_secure_memory failed for fd %d ret %d\n",
  1233. fd, err);
  1234. err = -ENOSR;
  1235. goto bail;
  1236. }
  1237. }
  1238. if (map->secure)
  1239. sess = fl->secsctx;
  1240. else
  1241. sess = fl->sctx;
  1242. VERIFY(err, !IS_ERR_OR_NULL(sess));
  1243. if (err) {
  1244. ADSPRPC_ERR(
  1245. "session is invalid for fd %d, secure flag %d\n",
  1246. fd, map->secure);
  1247. err = -EBADR;
  1248. goto bail;
  1249. }
  1250. VERIFY(err, !IS_ERR_OR_NULL(map->attach =
  1251. dma_buf_attach(map->buf, sess->smmu.dev)));
  1252. if (err) {
  1253. ADSPRPC_ERR(
  1254. "dma_buf_attach for fd %d failed for len 0x%zx to map buffer on SMMU device %s ret %ld\n",
  1255. fd, len, dev_name(sess->smmu.dev),
  1256. PTR_ERR(map->attach));
  1257. dma_attach_fail = true;
  1258. err = -EFAULT;
  1259. goto bail;
  1260. }
  1261. map->attach->dma_map_attrs |= DMA_ATTR_DELAYED_UNMAP;
  1262. /*
  1263. * Skip CPU sync if IO Cohernecy is not supported
  1264. */
  1265. if (!sess->smmu.coherent)
  1266. map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  1267. VERIFY(err, !IS_ERR_OR_NULL(map->table =
  1268. dma_buf_map_attachment(map->attach,
  1269. DMA_BIDIRECTIONAL)));
  1270. if (err) {
  1271. ADSPRPC_ERR(
  1272. "dma_buf_map_attachment for fd %d failed for len 0x%zx on device %s ret %ld\n",
  1273. fd, len, dev_name(sess->smmu.dev),
  1274. PTR_ERR(map->table));
  1275. err = -EFAULT;
  1276. goto bail;
  1277. }
  1278. if (!sess->smmu.enabled) {
  1279. VERIFY(err, map->table->nents == 1);
  1280. if (err) {
  1281. ADSPRPC_ERR(
  1282. "multiple scatter-gather entries (%u) present for fd %d mapped on SMMU disabled device\n",
  1283. map->table->nents, fd);
  1284. err = -EFAULT;
  1285. goto bail;
  1286. }
  1287. }
  1288. map->phys = sg_dma_address(map->table->sgl);
  1289. if (sess->smmu.cb) {
  1290. map->phys += ((uint64_t)sess->smmu.cb << 32);
  1291. for_each_sg(map->table->sgl, sgl, map->table->nents,
  1292. sgl_index)
  1293. map->size += sg_dma_len(sgl);
  1294. } else {
  1295. map->size = buf_page_size(len);
  1296. }
  1297. trace_fastrpc_dma_map(cid, fd, map->phys, map->size,
  1298. len, map->attach->dma_map_attrs, mflags);
  1299. VERIFY(err, map->size >= len && map->size < me->max_size_limit);
  1300. if (err) {
  1301. err = -EFAULT;
  1302. pr_err("adsprpc: %s: invalid map size 0x%zx len 0x%zx\n",
  1303. __func__, map->size, len);
  1304. goto bail;
  1305. }
  1306. vmid = fl->apps->channel[cid].vmid;
  1307. if (vmid) {
  1308. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  1309. struct qcom_scm_vmperm dst_perms[2] = {0};
  1310. dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
  1311. dst_perms[0].perm = QCOM_SCM_PERM_RW;
  1312. dst_perms[1].vmid = vmid;
  1313. dst_perms[1].perm = QCOM_SCM_PERM_RWX;
  1314. err = qcom_scm_assign_mem(map->phys,
  1315. buf_page_size(map->size),
  1316. &src_perms, dst_perms, 2);
  1317. if (err) {
  1318. ADSPRPC_ERR(
  1319. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  1320. err, map->phys, map->size);
  1321. err = -EADDRNOTAVAIL;
  1322. goto bail;
  1323. }
  1324. }
  1325. map->va = va;
  1326. }
  1327. map->len = len;
  1328. spin_lock(&fl->hlock);
  1329. if ((mflags == ADSP_MMAP_ADD_PAGES) || (mflags == ADSP_MMAP_ADD_PAGES_LLC))
  1330. fl->mem_snap.heap_bufs_size += map->size;
  1331. else
  1332. fl->mem_snap.nonheap_bufs_size += map->size;
  1333. spin_unlock(&fl->hlock);
  1334. fastrpc_mmap_add(map);
  1335. *ppmap = map;
  1336. bail:
  1337. if (dma_attach_fail && fl) {
  1338. tot_bufs_size = fl->mem_snap.heap_bufs_size
  1339. + fl->mem_snap.nonheap_bufs_size;
  1340. ADSPRPC_INFO("Heapbufs size: %zu, non-heapbufs size: %zu, total size: %zu\n",
  1341. fl->mem_snap.heap_bufs_size, fl->mem_snap.nonheap_bufs_size,
  1342. tot_bufs_size);
  1343. }
  1344. if (map)
  1345. ktime_get_real_ts64(&map->map_end_time);
  1346. if (err && map)
  1347. fastrpc_mmap_free(map, 0);
  1348. return err;
  1349. }
  1350. static inline bool fastrpc_get_cached_buf(struct fastrpc_file *fl,
  1351. size_t size, int buf_type, struct fastrpc_buf **obuf)
  1352. {
  1353. bool found = false;
  1354. struct fastrpc_buf *buf = NULL, *fr = NULL;
  1355. struct hlist_node *n = NULL;
  1356. if (buf_type == USERHEAP_BUF)
  1357. goto bail;
  1358. /* find the smallest buffer that fits in the cache */
  1359. spin_lock(&fl->hlock);
  1360. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  1361. if (buf->size >= size && (!fr || fr->size > buf->size))
  1362. fr = buf;
  1363. }
  1364. if (fr) {
  1365. hlist_del_init(&fr->hn);
  1366. fl->num_cached_buf--;
  1367. }
  1368. spin_unlock(&fl->hlock);
  1369. if (fr) {
  1370. fr->type = buf_type;
  1371. *obuf = fr;
  1372. found = true;
  1373. }
  1374. bail:
  1375. return found;
  1376. }
  1377. static inline bool fastrpc_get_persistent_buf(struct fastrpc_file *fl,
  1378. size_t size, int buf_type, struct fastrpc_buf **obuf)
  1379. {
  1380. unsigned int i = 0;
  1381. bool found = false;
  1382. struct fastrpc_buf *buf = NULL;
  1383. spin_lock(&fl->hlock);
  1384. if (!fl->num_pers_hdrs)
  1385. goto bail;
  1386. /*
  1387. * Persistent header buffer can be used only if
  1388. * metadata length is less than 1 page size.
  1389. */
  1390. if (buf_type != METADATA_BUF || size > PAGE_SIZE)
  1391. goto bail;
  1392. for (i = 0; i < fl->num_pers_hdrs; i++) {
  1393. buf = &fl->hdr_bufs[i];
  1394. /* If buffer not in use, then assign it for requested alloc */
  1395. if (!buf->in_use) {
  1396. buf->in_use = true;
  1397. *obuf = buf;
  1398. found = true;
  1399. break;
  1400. }
  1401. }
  1402. bail:
  1403. spin_unlock(&fl->hlock);
  1404. return found;
  1405. }
  1406. static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
  1407. unsigned long dma_attr, uint32_t rflags,
  1408. int buf_type, struct fastrpc_buf **obuf)
  1409. {
  1410. int err = 0, vmid;
  1411. struct fastrpc_apps *me = &gfa;
  1412. struct fastrpc_buf *buf = NULL;
  1413. int cid = -1;
  1414. VERIFY(err, fl && fl->sctx != NULL);
  1415. if (err) {
  1416. err = -EBADR;
  1417. goto bail;
  1418. }
  1419. cid = fl->cid;
  1420. VERIFY(err, VALID_FASTRPC_CID(cid));
  1421. if (err) {
  1422. err = -ECHRNG;
  1423. goto bail;
  1424. }
  1425. VERIFY(err, size > 0 && size < me->max_size_limit);
  1426. if (err) {
  1427. err = -EFAULT;
  1428. pr_err("adsprpc: %s: invalid allocation size 0x%zx\n",
  1429. __func__, size);
  1430. goto bail;
  1431. }
  1432. VERIFY(err, size > 0 && fl->sctx->smmu.dev);
  1433. if (err) {
  1434. err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;
  1435. goto bail;
  1436. }
  1437. if (fastrpc_get_persistent_buf(fl, size, buf_type, obuf))
  1438. return err;
  1439. if (fastrpc_get_cached_buf(fl, size, buf_type, obuf))
  1440. return err;
  1441. /* If unable to get persistent or cached buf, allocate new buffer */
  1442. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  1443. if (err) {
  1444. err = -ENOMEM;
  1445. goto bail;
  1446. }
  1447. INIT_HLIST_NODE(&buf->hn);
  1448. buf->fl = fl;
  1449. buf->virt = NULL;
  1450. buf->phys = 0;
  1451. buf->size = size;
  1452. buf->dma_attr = dma_attr;
  1453. buf->flags = rflags;
  1454. buf->raddr = 0;
  1455. buf->type = buf_type;
  1456. ktime_get_real_ts64(&buf->buf_start_time);
  1457. buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
  1458. (dma_addr_t *)&buf->phys,
  1459. GFP_KERNEL, buf->dma_attr);
  1460. if (IS_ERR_OR_NULL(buf->virt)) {
  1461. /* free cache and retry */
  1462. fastrpc_cached_buf_list_free(fl);
  1463. buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
  1464. (dma_addr_t *)&buf->phys, GFP_KERNEL,
  1465. buf->dma_attr);
  1466. VERIFY(err, !IS_ERR_OR_NULL(buf->virt));
  1467. }
  1468. if (err) {
  1469. ADSPRPC_ERR(
  1470. "dma_alloc_attrs failed for size 0x%zx, returned %pK\n",
  1471. size, buf->virt);
  1472. err = -ENOBUFS;
  1473. goto bail;
  1474. }
  1475. if (fl->sctx->smmu.cb)
  1476. buf->phys += ((uint64_t)fl->sctx->smmu.cb << 32);
  1477. trace_fastrpc_dma_alloc(cid, buf->phys, size,
  1478. dma_attr, (int)rflags);
  1479. vmid = fl->apps->channel[cid].vmid;
  1480. if (vmid) {
  1481. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  1482. struct qcom_scm_vmperm dst_perms[2] = {0};
  1483. dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
  1484. dst_perms[0].perm = QCOM_SCM_PERM_RW;
  1485. dst_perms[1].vmid = vmid;
  1486. dst_perms[1].perm = QCOM_SCM_PERM_RWX;
  1487. err = qcom_scm_assign_mem(buf->phys, buf_page_size(size),
  1488. &src_perms, dst_perms, 2);
  1489. if (err) {
  1490. ADSPRPC_DEBUG(
  1491. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  1492. err, buf->phys, size);
  1493. err = -EADDRNOTAVAIL;
  1494. goto bail;
  1495. }
  1496. }
  1497. if (buf_type == USERHEAP_BUF) {
  1498. INIT_HLIST_NODE(&buf->hn_rem);
  1499. spin_lock(&fl->hlock);
  1500. hlist_add_head(&buf->hn_rem, &fl->remote_bufs);
  1501. spin_unlock(&fl->hlock);
  1502. }
  1503. *obuf = buf;
  1504. bail:
  1505. if (buf)
  1506. ktime_get_real_ts64(&buf->buf_end_time);
  1507. if (err && buf)
  1508. fastrpc_buf_free(buf, 0);
  1509. return err;
  1510. }
  1511. static int context_restore_interrupted(struct fastrpc_file *fl,
  1512. struct fastrpc_ioctl_invoke_async *inv,
  1513. struct smq_invoke_ctx **po)
  1514. {
  1515. int err = 0;
  1516. struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;
  1517. struct hlist_node *n;
  1518. struct fastrpc_ioctl_invoke *invoke = &inv->inv;
  1519. spin_lock(&fl->hlock);
  1520. hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {
  1521. if (ictx->pid == current->pid) {
  1522. if (invoke->sc != ictx->sc || ictx->fl != fl) {
  1523. err = -EINVAL;
  1524. ictx->sc_interrupted = invoke->sc;
  1525. ictx->fl_interrupted = fl;
  1526. ictx->handle_interrupted = invoke->handle;
  1527. ADSPRPC_ERR(
  1528. "interrupted sc (0x%x) or fl (%pK) does not match with invoke sc (0x%x) or fl (%pK)\n",
  1529. ictx->sc, ictx->fl, invoke->sc, fl);
  1530. } else {
  1531. ictx->xo_time_in_us_restored = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1532. fastrpc_update_txmsg_buf(NULL, 0, 0, 0, ictx, RESTORED_STATE);
  1533. ADSPRPC_DEBUG(
  1534. "restored sc (0x%x) of fl (%pK), interrupt ts 0x%llx, restore ts 0x%llx \n",
  1535. ictx->sc, ictx->fl, ictx->xo_time_in_us_interrupted, ictx->xo_time_in_us_restored);
  1536. ctx = ictx;
  1537. hlist_del_init(&ctx->hn);
  1538. hlist_add_head(&ctx->hn, &fl->clst.pending);
  1539. }
  1540. break;
  1541. }
  1542. }
  1543. spin_unlock(&fl->hlock);
  1544. if (ctx)
  1545. *po = ctx;
  1546. return err;
  1547. }
  1548. static unsigned int sorted_lists_intersection(unsigned int *listA,
  1549. unsigned int lenA, unsigned int *listB, unsigned int lenB)
  1550. {
  1551. unsigned int i = 0, j = 0;
  1552. while (i < lenA && j < lenB) {
  1553. if (listA[i] < listB[j])
  1554. i++;
  1555. else if (listA[i] > listB[j])
  1556. j++;
  1557. else
  1558. return listA[i];
  1559. }
  1560. return 0;
  1561. }
  1562. #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
  1563. static int uint_cmp_func(const void *p1, const void *p2)
  1564. {
  1565. unsigned int a1 = *((unsigned int *)p1);
  1566. unsigned int a2 = *((unsigned int *)p2);
  1567. return CMP(a1, a2);
  1568. }
  1569. static int overlap_ptr_cmp(const void *a, const void *b)
  1570. {
  1571. struct overlap *pa = *((struct overlap **)a);
  1572. struct overlap *pb = *((struct overlap **)b);
  1573. /* sort with lowest starting buffer first */
  1574. int st = CMP(pa->start, pb->start);
  1575. /* sort with highest ending buffer first */
  1576. int ed = CMP(pb->end, pa->end);
  1577. return st == 0 ? ed : st;
  1578. }
  1579. static int context_build_overlap(struct smq_invoke_ctx *ctx)
  1580. {
  1581. int i, err = 0;
  1582. remote_arg_t *lpra = ctx->lpra;
  1583. int inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
  1584. int outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
  1585. int nbufs = inbufs + outbufs;
  1586. struct overlap max;
  1587. for (i = 0; i < nbufs; ++i) {
  1588. ctx->overs[i].start = (uintptr_t)lpra[i].buf.pv;
  1589. ctx->overs[i].end = ctx->overs[i].start + lpra[i].buf.len;
  1590. if (lpra[i].buf.len) {
  1591. VERIFY(err, ctx->overs[i].end > ctx->overs[i].start);
  1592. if (err) {
  1593. err = -EFAULT;
  1594. ADSPRPC_ERR(
  1595. "Invalid address 0x%llx and size %zu\n",
  1596. (uintptr_t)lpra[i].buf.pv,
  1597. lpra[i].buf.len);
  1598. goto bail;
  1599. }
  1600. }
  1601. ctx->overs[i].raix = i;
  1602. ctx->overps[i] = &ctx->overs[i];
  1603. }
  1604. sort(ctx->overps, nbufs, sizeof(*ctx->overps), overlap_ptr_cmp, NULL);
  1605. max.start = 0;
  1606. max.end = 0;
  1607. for (i = 0; i < nbufs; ++i) {
  1608. if (ctx->overps[i]->start < max.end) {
  1609. ctx->overps[i]->mstart = max.end;
  1610. ctx->overps[i]->mend = ctx->overps[i]->end;
  1611. ctx->overps[i]->offset = max.end -
  1612. ctx->overps[i]->start;
  1613. if (ctx->overps[i]->end > max.end) {
  1614. max.end = ctx->overps[i]->end;
  1615. } else {
  1616. if ((max.raix < inbufs &&
  1617. ctx->overps[i]->raix + 1 > inbufs) ||
  1618. (ctx->overps[i]->raix < inbufs &&
  1619. max.raix + 1 > inbufs))
  1620. ctx->overps[i]->do_cmo = 1;
  1621. ctx->overps[i]->mend = 0;
  1622. ctx->overps[i]->mstart = 0;
  1623. }
  1624. } else {
  1625. ctx->overps[i]->mend = ctx->overps[i]->end;
  1626. ctx->overps[i]->mstart = ctx->overps[i]->start;
  1627. ctx->overps[i]->offset = 0;
  1628. max = *ctx->overps[i];
  1629. }
  1630. }
  1631. bail:
  1632. return err;
  1633. }
  1634. #define K_COPY_FROM_USER(err, kernel, dst, src, size) \
  1635. do {\
  1636. if (!(kernel))\
  1637. err = copy_from_user((dst),\
  1638. (void const __user *)(src),\
  1639. (size));\
  1640. else\
  1641. memmove((dst), (src), (size));\
  1642. } while (0)
  1643. #define K_COPY_TO_USER(err, kernel, dst, src, size) \
  1644. do {\
  1645. if (!(kernel))\
  1646. err = copy_to_user((void __user *)(dst),\
  1647. (src), (size));\
  1648. else\
  1649. memmove((dst), (src), (size));\
  1650. } while (0)
  1651. static void context_free(struct smq_invoke_ctx *ctx);
  1652. static int context_alloc(struct fastrpc_file *fl, uint32_t kernel,
  1653. struct fastrpc_ioctl_invoke_async *invokefd,
  1654. struct smq_invoke_ctx **po)
  1655. {
  1656. struct fastrpc_apps *me = &gfa;
  1657. int err = 0, bufs, ii, size = 0, cid = fl->cid;
  1658. struct smq_invoke_ctx *ctx = NULL;
  1659. struct fastrpc_ctx_lst *clst = &fl->clst;
  1660. struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;
  1661. struct fastrpc_channel_ctx *chan = NULL;
  1662. unsigned long irq_flags = 0;
  1663. uint32_t is_kernel_memory = 0;
  1664. spin_lock(&fl->hlock);
  1665. if (fl->clst.num_active_ctxs > MAX_PENDING_CTX_PER_SESSION &&
  1666. !(kernel || invoke->handle < FASTRPC_STATIC_HANDLE_MAX)) {
  1667. err = -EDQUOT;
  1668. spin_unlock(&fl->hlock);
  1669. goto bail;
  1670. }
  1671. spin_unlock(&fl->hlock);
  1672. bufs = REMOTE_SCALARS_LENGTH(invoke->sc);
  1673. size = bufs * sizeof(*ctx->lpra) + bufs * sizeof(*ctx->maps) +
  1674. sizeof(*ctx->fds) * (bufs) +
  1675. sizeof(*ctx->attrs) * (bufs) +
  1676. sizeof(*ctx->overs) * (bufs) +
  1677. sizeof(*ctx->overps) * (bufs);
  1678. VERIFY(err, NULL != (ctx = kzalloc(sizeof(*ctx) + size, GFP_KERNEL)));
  1679. if (err) {
  1680. err = -ENOMEM;
  1681. goto bail;
  1682. }
  1683. INIT_HLIST_NODE(&ctx->hn);
  1684. INIT_HLIST_NODE(&ctx->asyncn);
  1685. hlist_add_fake(&ctx->hn);
  1686. hlist_add_fake(&ctx->asyncn);
  1687. ctx->fl = fl;
  1688. ctx->maps = (struct fastrpc_mmap **)(&ctx[1]);
  1689. ctx->lpra = (remote_arg_t *)(&ctx->maps[bufs]);
  1690. ctx->fds = (int *)(&ctx->lpra[bufs]);
  1691. ctx->attrs = (unsigned int *)(&ctx->fds[bufs]);
  1692. ctx->overs = (struct overlap *)(&ctx->attrs[bufs]);
  1693. ctx->overps = (struct overlap **)(&ctx->overs[bufs]);
  1694. /* If user message, do not use copy_from_user to copy buffers for
  1695. * compat driver,as memory is already copied to kernel memory
  1696. * for compat driver
  1697. */
  1698. is_kernel_memory = ((kernel == USER_MSG) ? (fl->is_compat) : kernel);
  1699. K_COPY_FROM_USER(err, is_kernel_memory, (void *)ctx->lpra, invoke->pra,
  1700. bufs * sizeof(*ctx->lpra));
  1701. if (err) {
  1702. ADSPRPC_ERR(
  1703. "copy from user failed with %d for remote arguments list\n",
  1704. err);
  1705. err = -EFAULT;
  1706. goto bail;
  1707. }
  1708. if (invokefd->fds) {
  1709. K_COPY_FROM_USER(err, kernel, ctx->fds, invokefd->fds,
  1710. bufs * sizeof(*ctx->fds));
  1711. if (err) {
  1712. ADSPRPC_ERR(
  1713. "copy from user failed with %d for fd list\n",
  1714. err);
  1715. err = -EFAULT;
  1716. goto bail;
  1717. }
  1718. } else {
  1719. ctx->fds = NULL;
  1720. }
  1721. if (invokefd->attrs) {
  1722. K_COPY_FROM_USER(err, kernel, ctx->attrs, invokefd->attrs,
  1723. bufs * sizeof(*ctx->attrs));
  1724. if (err) {
  1725. ADSPRPC_ERR(
  1726. "copy from user failed with %d for attribute list\n",
  1727. err);
  1728. err = -EFAULT;
  1729. goto bail;
  1730. }
  1731. }
  1732. ctx->crc = (uint32_t *)invokefd->crc;
  1733. ctx->perf_dsp = (uint64_t *)invokefd->perf_dsp;
  1734. ctx->perf_kernel = (uint64_t *)invokefd->perf_kernel;
  1735. ctx->handle = invoke->handle;
  1736. ctx->sc = invoke->sc;
  1737. if (bufs) {
  1738. VERIFY(err, 0 == (err = context_build_overlap(ctx)));
  1739. if (err)
  1740. goto bail;
  1741. }
  1742. ctx->retval = -1;
  1743. ctx->pid = current->pid;
  1744. /* Store HLOS PID in context, it is not being sent to DSP */
  1745. ctx->tgid = fl->tgid;
  1746. init_completion(&ctx->work);
  1747. ctx->magic = FASTRPC_CTX_MAGIC;
  1748. ctx->rsp_flags = NORMAL_RESPONSE;
  1749. ctx->is_work_done = false;
  1750. ctx->copybuf = NULL;
  1751. ctx->is_early_wakeup = false;
  1752. ctx->is_job_sent_to_remote_ss = false;
  1753. if (ctx->fl->profile) {
  1754. ctx->perf = kzalloc(sizeof(*(ctx->perf)), GFP_KERNEL);
  1755. VERIFY(err, !IS_ERR_OR_NULL(ctx->perf));
  1756. if (err) {
  1757. kfree(ctx->perf);
  1758. err = -ENOMEM;
  1759. goto bail;
  1760. }
  1761. memset(ctx->perf, 0, sizeof(*(ctx->perf)));
  1762. /* Use HLOS PID, as perf tid is not being sent to DSP and is used to log in traces */
  1763. ctx->perf->tid = fl->tgid;
  1764. }
  1765. if (invokefd->job) {
  1766. K_COPY_FROM_USER(err, kernel, &ctx->asyncjob, invokefd->job,
  1767. sizeof(ctx->asyncjob));
  1768. if (err)
  1769. goto bail;
  1770. }
  1771. VERIFY(err, VALID_FASTRPC_CID(cid));
  1772. if (err) {
  1773. err = -ECHRNG;
  1774. goto bail;
  1775. }
  1776. chan = &me->channel[cid];
  1777. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  1778. me->jobid[cid]++;
  1779. for (ii = ((kernel || ctx->handle < FASTRPC_STATIC_HANDLE_MAX)
  1780. ? 0 : NUM_KERNEL_AND_STATIC_ONLY_CONTEXTS);
  1781. ii < FASTRPC_CTX_MAX; ii++) {
  1782. if (!chan->ctxtable[ii]) {
  1783. chan->ctxtable[ii] = ctx;
  1784. ctx->ctxid = (me->jobid[cid] << FASTRPC_CTX_JOBID_POS)
  1785. | (ii << FASTRPC_CTX_TABLE_IDX_POS)
  1786. | ((ctx->asyncjob.isasyncjob &&
  1787. FASTRPC_ASYNC_JOB_MASK) << FASTRPC_CTX_JOB_TYPE_POS);
  1788. break;
  1789. }
  1790. }
  1791. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  1792. VERIFY(err, ii < FASTRPC_CTX_MAX);
  1793. if (err) {
  1794. ADSPRPC_ERR(
  1795. "adsprpc: out of context table entries for handle 0x%x, sc 0x%x\n",
  1796. ctx->handle, ctx->sc);
  1797. err = -ENOKEY;
  1798. goto bail;
  1799. }
  1800. ctx->xo_time_in_us_created = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1801. spin_lock(&fl->hlock);
  1802. hlist_add_head(&ctx->hn, &clst->pending);
  1803. clst->num_active_ctxs++;
  1804. spin_unlock(&fl->hlock);
  1805. trace_fastrpc_context_alloc((uint64_t)ctx,
  1806. ctx->ctxid | fl->pd, ctx->handle, ctx->sc);
  1807. *po = ctx;
  1808. bail:
  1809. if (ctx && err)
  1810. context_free(ctx);
  1811. return err;
  1812. }
  1813. static void context_save_interrupted(struct smq_invoke_ctx *ctx)
  1814. {
  1815. struct fastrpc_ctx_lst *clst = &ctx->fl->clst;
  1816. ctx->xo_time_in_us_interrupted = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1817. fastrpc_update_txmsg_buf(NULL, 0, 0, 0, ctx, INTERRUPTED_STATE);
  1818. spin_lock(&ctx->fl->hlock);
  1819. hlist_del_init(&ctx->hn);
  1820. hlist_add_head(&ctx->hn, &clst->interrupted);
  1821. spin_unlock(&ctx->fl->hlock);
  1822. }
  1823. static void context_free(struct smq_invoke_ctx *ctx)
  1824. {
  1825. uint32_t i = 0;
  1826. struct fastrpc_apps *me = &gfa;
  1827. int nbufs = REMOTE_SCALARS_INBUFS(ctx->sc) +
  1828. REMOTE_SCALARS_OUTBUFS(ctx->sc);
  1829. int cid = ctx->fl->cid;
  1830. struct fastrpc_channel_ctx *chan = NULL;
  1831. unsigned long irq_flags = 0;
  1832. int err = 0;
  1833. VERIFY(err, VALID_FASTRPC_CID(cid));
  1834. if (err) {
  1835. ADSPRPC_ERR(
  1836. "invalid channel 0x%zx set for session\n",
  1837. cid);
  1838. return;
  1839. }
  1840. chan = &me->channel[cid];
  1841. i = (uint32_t)GET_TABLE_IDX_FROM_CTXID(ctx->ctxid);
  1842. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  1843. if (i < FASTRPC_CTX_MAX && chan->ctxtable[i] == ctx) {
  1844. chan->ctxtable[i] = NULL;
  1845. } else {
  1846. for (i = 0; i < FASTRPC_CTX_MAX; i++) {
  1847. if (chan->ctxtable[i] == ctx) {
  1848. chan->ctxtable[i] = NULL;
  1849. break;
  1850. }
  1851. }
  1852. }
  1853. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  1854. spin_lock(&ctx->fl->hlock);
  1855. if (!hlist_unhashed(&ctx->hn)) {
  1856. hlist_del_init(&ctx->hn);
  1857. ctx->fl->clst.num_active_ctxs--;
  1858. }
  1859. spin_unlock(&ctx->fl->hlock);
  1860. mutex_lock(&ctx->fl->map_mutex);
  1861. for (i = 0; i < nbufs; ++i) {
  1862. /*
  1863. * Decrement ctx refs count before mmap free,
  1864. * indicate remote call no longer using it
  1865. */
  1866. if (ctx->maps[i] && ctx->maps[i]->ctx_refs)
  1867. ctx->maps[i]->ctx_refs--;
  1868. fastrpc_mmap_free(ctx->maps[i], 0);
  1869. }
  1870. mutex_unlock(&ctx->fl->map_mutex);
  1871. fastrpc_buf_free(ctx->buf, 1);
  1872. if (ctx->copybuf != ctx->buf)
  1873. fastrpc_buf_free(ctx->copybuf, 1);
  1874. kfree(ctx->lrpra);
  1875. ctx->lrpra = NULL;
  1876. ctx->magic = 0;
  1877. ctx->ctxid = 0;
  1878. if (ctx->fl->profile)
  1879. kfree(ctx->perf);
  1880. trace_fastrpc_context_free((uint64_t)ctx,
  1881. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  1882. kfree(ctx);
  1883. }
  1884. static void fastrpc_queue_completed_async_job(struct smq_invoke_ctx *ctx)
  1885. {
  1886. struct fastrpc_file *fl = ctx->fl;
  1887. unsigned long flags;
  1888. spin_lock_irqsave(&fl->aqlock, flags);
  1889. if (ctx->is_early_wakeup)
  1890. goto bail;
  1891. if (!hlist_unhashed(&ctx->asyncn)) {
  1892. hlist_add_head(&ctx->asyncn, &fl->clst.async_queue);
  1893. atomic_add(1, &fl->async_queue_job_count);
  1894. ctx->is_early_wakeup = true;
  1895. wake_up_interruptible(&fl->async_wait_queue);
  1896. }
  1897. bail:
  1898. spin_unlock_irqrestore(&fl->aqlock, flags);
  1899. }
  1900. static void fastrpc_queue_pd_status(struct fastrpc_file *fl, int domain, int status, int sessionid)
  1901. {
  1902. struct smq_notif_rsp *notif_rsp = NULL;
  1903. unsigned long flags;
  1904. int err = 0;
  1905. VERIFY(err, NULL != (notif_rsp = kzalloc(sizeof(*notif_rsp), GFP_ATOMIC)));
  1906. if (err) {
  1907. ADSPRPC_ERR(
  1908. "allocation failed for size 0x%zx\n",
  1909. sizeof(*notif_rsp));
  1910. return;
  1911. }
  1912. notif_rsp->status = status;
  1913. notif_rsp->domain = domain;
  1914. notif_rsp->session = sessionid;
  1915. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  1916. list_add_tail(&notif_rsp->notifn, &fl->clst.notif_queue);
  1917. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  1918. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  1919. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  1920. }
  1921. static void fastrpc_notif_find_process(int domain, struct smq_notif_rspv3 *notif)
  1922. {
  1923. struct fastrpc_apps *me = &gfa;
  1924. struct fastrpc_file *fl = NULL;
  1925. struct hlist_node *n;
  1926. bool is_process_found = false;
  1927. unsigned long irq_flags = 0;
  1928. spin_lock_irqsave(&me->hlock, irq_flags);
  1929. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  1930. if (fl->tgid_frpc == notif->pid) {
  1931. is_process_found = true;
  1932. break;
  1933. }
  1934. }
  1935. spin_unlock_irqrestore(&me->hlock, irq_flags);
  1936. if (!is_process_found)
  1937. return;
  1938. fastrpc_queue_pd_status(fl, domain, notif->status, fl->sessionid);
  1939. }
  1940. static void context_notify_user(struct smq_invoke_ctx *ctx,
  1941. int retval, uint32_t rsp_flags, uint32_t early_wake_time)
  1942. {
  1943. fastrpc_pm_awake(ctx->fl, gcinfo[ctx->fl->cid].secure);
  1944. ctx->retval = retval;
  1945. ctx->rsp_flags = (enum fastrpc_response_flags)rsp_flags;
  1946. trace_fastrpc_context_complete(ctx->fl->cid, (uint64_t)ctx, retval,
  1947. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  1948. switch (rsp_flags) {
  1949. case NORMAL_RESPONSE:
  1950. fallthrough;
  1951. case COMPLETE_SIGNAL:
  1952. /* normal and complete response with return value */
  1953. ctx->is_work_done = true;
  1954. if (ctx->asyncjob.isasyncjob)
  1955. fastrpc_queue_completed_async_job(ctx);
  1956. trace_fastrpc_msg("wakeup_task: begin");
  1957. complete(&ctx->work);
  1958. trace_fastrpc_msg("wakeup_task: end");
  1959. break;
  1960. case USER_EARLY_SIGNAL:
  1961. /* user hint of approximate time of completion */
  1962. ctx->early_wake_time = early_wake_time;
  1963. if (ctx->asyncjob.isasyncjob)
  1964. break;
  1965. fallthrough;
  1966. case EARLY_RESPONSE:
  1967. /* rpc framework early response with return value */
  1968. if (ctx->asyncjob.isasyncjob)
  1969. fastrpc_queue_completed_async_job(ctx);
  1970. else {
  1971. trace_fastrpc_msg("wakeup_task: begin");
  1972. complete(&ctx->work);
  1973. trace_fastrpc_msg("wakeup_task: end");
  1974. }
  1975. break;
  1976. default:
  1977. break;
  1978. }
  1979. }
  1980. static void fastrpc_notify_users(struct fastrpc_file *me)
  1981. {
  1982. struct smq_invoke_ctx *ictx;
  1983. struct hlist_node *n;
  1984. unsigned long irq_flags = 0;
  1985. spin_lock_irqsave(&me->hlock, irq_flags);
  1986. hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
  1987. ictx->is_work_done = true;
  1988. ictx->retval = -ECONNRESET;
  1989. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  1990. ictx->retval, ictx->msg.invoke.header.ctx,
  1991. ictx->handle, ictx->sc);
  1992. if (ictx->asyncjob.isasyncjob && ictx->is_job_sent_to_remote_ss)
  1993. fastrpc_queue_completed_async_job(ictx);
  1994. else
  1995. complete(&ictx->work);
  1996. }
  1997. hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
  1998. ictx->is_work_done = true;
  1999. ictx->retval = -ECONNRESET;
  2000. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2001. ictx->retval, ictx->msg.invoke.header.ctx,
  2002. ictx->handle, ictx->sc);
  2003. complete(&ictx->work);
  2004. }
  2005. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2006. }
  2007. static void fastrpc_notify_users_staticpd_pdr(struct fastrpc_file *me)
  2008. {
  2009. struct smq_invoke_ctx *ictx;
  2010. struct hlist_node *n;
  2011. unsigned long irq_flags = 0;
  2012. spin_lock_irqsave(&me->hlock, irq_flags);
  2013. hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
  2014. if (ictx->msg.pid) {
  2015. ictx->is_work_done = true;
  2016. ictx->retval = -ECONNRESET;
  2017. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2018. ictx->retval, ictx->msg.invoke.header.ctx,
  2019. ictx->handle, ictx->sc);
  2020. if (ictx->asyncjob.isasyncjob && ictx->is_job_sent_to_remote_ss)
  2021. fastrpc_queue_completed_async_job(ictx);
  2022. else
  2023. complete(&ictx->work);
  2024. }
  2025. }
  2026. hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
  2027. if (ictx->msg.pid) {
  2028. ictx->is_work_done = true;
  2029. ictx->retval = -ECONNRESET;
  2030. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2031. ictx->retval, ictx->msg.invoke.header.ctx,
  2032. ictx->handle, ictx->sc);
  2033. complete(&ictx->work);
  2034. }
  2035. }
  2036. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2037. }
  2038. static void fastrpc_update_ramdump_status(int cid)
  2039. {
  2040. struct fastrpc_file *fl = NULL;
  2041. struct hlist_node *n = NULL;
  2042. struct fastrpc_apps *me = &gfa;
  2043. struct fastrpc_channel_ctx *chan = &me->channel[cid];
  2044. unsigned long irq_flags = 0;
  2045. spin_lock_irqsave(&me->hlock, irq_flags);
  2046. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2047. if (fl->cid == cid && fl->init_mem &&
  2048. fl->file_close < FASTRPC_PROCESS_DSP_EXIT_COMPLETE &&
  2049. fl->dsp_proc_init) {
  2050. hlist_add_head(&fl->init_mem->hn_init, &chan->initmems);
  2051. fl->is_ramdump_pend = true;
  2052. }
  2053. }
  2054. if (chan->buf)
  2055. hlist_add_head(&chan->buf->hn_init, &chan->initmems);
  2056. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2057. }
  2058. static void fastrpc_ramdump_collection(int cid)
  2059. {
  2060. struct fastrpc_file *fl = NULL;
  2061. struct hlist_node *n = NULL;
  2062. struct fastrpc_apps *me = &gfa;
  2063. struct fastrpc_channel_ctx *chan = &me->channel[cid];
  2064. struct qcom_dump_segment ramdump_entry;
  2065. struct fastrpc_buf *buf = NULL;
  2066. int ret = 0;
  2067. unsigned long irq_flags = 0;
  2068. struct list_head head;
  2069. hlist_for_each_entry_safe(buf, n, &chan->initmems, hn_init) {
  2070. fl = buf->fl;
  2071. memset(&ramdump_entry, 0, sizeof(ramdump_entry));
  2072. ramdump_entry.da = buf->phys;
  2073. ramdump_entry.va = (void *)buf->virt;
  2074. ramdump_entry.size = buf->size;
  2075. INIT_LIST_HEAD(&head);
  2076. list_add(&ramdump_entry.node, &head);
  2077. if (fl && fl->sctx && fl->sctx->smmu.dev)
  2078. ret = qcom_elf_dump(&head, fl->sctx->smmu.dev, ELF_CLASS);
  2079. else {
  2080. if (me->dev != NULL)
  2081. ret = qcom_elf_dump(&head, me->dev, ELF_CLASS);
  2082. }
  2083. if (ret < 0)
  2084. ADSPRPC_ERR("adsprpc: %s: unable to dump PD memory (err %d)\n",
  2085. __func__, ret);
  2086. hlist_del_init(&buf->hn_init);
  2087. if (fl) {
  2088. spin_lock_irqsave(&me->hlock, irq_flags);
  2089. if (fl->file_close)
  2090. complete(&fl->work);
  2091. fl->is_ramdump_pend = false;
  2092. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2093. }
  2094. }
  2095. }
  2096. static void fastrpc_notify_drivers(struct fastrpc_apps *me, int cid)
  2097. {
  2098. struct fastrpc_file *fl;
  2099. struct hlist_node *n;
  2100. unsigned long irq_flags = 0;
  2101. spin_lock_irqsave(&me->hlock, irq_flags);
  2102. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2103. if (fl->cid == cid) {
  2104. fastrpc_queue_pd_status(fl, cid, FASTRPC_DSP_SSR, fl->sessionid);
  2105. fastrpc_notify_users(fl);
  2106. }
  2107. }
  2108. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2109. }
  2110. static void fastrpc_notify_pdr_drivers(struct fastrpc_apps *me,
  2111. char *servloc_name)
  2112. {
  2113. struct fastrpc_file *fl;
  2114. struct hlist_node *n;
  2115. unsigned long irq_flags = 0;
  2116. spin_lock_irqsave(&me->hlock, irq_flags);
  2117. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2118. if (fl->servloc_name && !strcmp(servloc_name, fl->servloc_name))
  2119. fastrpc_notify_users_staticpd_pdr(fl);
  2120. }
  2121. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2122. }
  2123. static void context_list_ctor(struct fastrpc_ctx_lst *me)
  2124. {
  2125. INIT_HLIST_HEAD(&me->interrupted);
  2126. INIT_HLIST_HEAD(&me->pending);
  2127. me->num_active_ctxs = 0;
  2128. INIT_HLIST_HEAD(&me->async_queue);
  2129. INIT_LIST_HEAD(&me->notif_queue);
  2130. }
  2131. static void fastrpc_context_list_dtor(struct fastrpc_file *fl)
  2132. {
  2133. struct fastrpc_ctx_lst *clst = &fl->clst;
  2134. struct smq_invoke_ctx *ictx = NULL, *ctxfree;
  2135. struct hlist_node *n;
  2136. do {
  2137. ctxfree = NULL;
  2138. spin_lock(&fl->hlock);
  2139. hlist_for_each_entry_safe(ictx, n, &clst->interrupted, hn) {
  2140. hlist_del_init(&ictx->hn);
  2141. clst->num_active_ctxs--;
  2142. ctxfree = ictx;
  2143. break;
  2144. }
  2145. spin_unlock(&fl->hlock);
  2146. if (ctxfree)
  2147. context_free(ctxfree);
  2148. } while (ctxfree);
  2149. do {
  2150. ctxfree = NULL;
  2151. spin_lock(&fl->hlock);
  2152. hlist_for_each_entry_safe(ictx, n, &clst->pending, hn) {
  2153. hlist_del_init(&ictx->hn);
  2154. clst->num_active_ctxs--;
  2155. ctxfree = ictx;
  2156. break;
  2157. }
  2158. spin_unlock(&fl->hlock);
  2159. if (ctxfree)
  2160. context_free(ctxfree);
  2161. } while (ctxfree);
  2162. }
  2163. static int fastrpc_file_free(struct fastrpc_file *fl);
  2164. static void fastrpc_file_list_dtor(struct fastrpc_apps *me)
  2165. {
  2166. struct fastrpc_file *fl, *free;
  2167. struct hlist_node *n;
  2168. unsigned long irq_flags = 0;
  2169. do {
  2170. free = NULL;
  2171. spin_lock_irqsave(&me->hlock, irq_flags);
  2172. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2173. hlist_del_init(&fl->hn);
  2174. free = fl;
  2175. break;
  2176. }
  2177. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2178. if (free)
  2179. fastrpc_file_free(free);
  2180. } while (free);
  2181. }
  2182. static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
  2183. {
  2184. remote_arg64_t *rpra, *lrpra;
  2185. remote_arg_t *lpra = ctx->lpra;
  2186. struct smq_invoke_buf *list;
  2187. struct smq_phy_page *pages, *ipage;
  2188. uint32_t sc = ctx->sc;
  2189. int inbufs = REMOTE_SCALARS_INBUFS(sc);
  2190. int outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2191. int handles, bufs = inbufs + outbufs;
  2192. uintptr_t args = 0;
  2193. size_t rlen = 0, copylen = 0, metalen = 0, lrpralen = 0, templen = 0;
  2194. size_t totallen = 0; //header and non ion copy buf len
  2195. int i, oix;
  2196. int err = 0, j = 0;
  2197. int mflags = 0;
  2198. uint64_t *fdlist = NULL;
  2199. uint32_t *crclist = NULL;
  2200. uint32_t early_hint;
  2201. uint64_t *perf_counter = NULL;
  2202. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  2203. if (ctx->fl->profile)
  2204. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  2205. /* calculate size of the metadata */
  2206. rpra = NULL;
  2207. lrpra = NULL;
  2208. list = smq_invoke_buf_start(rpra, sc);
  2209. pages = smq_phy_page_start(sc, list);
  2210. ipage = pages;
  2211. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_MAP),
  2212. for (i = 0; i < bufs; ++i) {
  2213. uintptr_t buf = (uintptr_t)lpra[i].buf.pv;
  2214. size_t len = lpra[i].buf.len;
  2215. mutex_lock(&ctx->fl->map_mutex);
  2216. if (ctx->fds && (ctx->fds[i] != -1))
  2217. err = fastrpc_mmap_create(ctx->fl, ctx->fds[i], NULL,
  2218. ctx->attrs[i], buf, len,
  2219. mflags, &ctx->maps[i]);
  2220. /*
  2221. * Increment ctx refs count for in/out buffer if map created,
  2222. * indicate map under use in remote call
  2223. */
  2224. if (ctx->maps[i])
  2225. ctx->maps[i]->ctx_refs++;
  2226. mutex_unlock(&ctx->fl->map_mutex);
  2227. if (err)
  2228. goto bail;
  2229. ipage += 1;
  2230. }
  2231. PERF_END);
  2232. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  2233. mutex_lock(&ctx->fl->map_mutex);
  2234. for (i = bufs; i < bufs + handles; i++) {
  2235. int dmaflags = 0;
  2236. if (ctx->attrs && (ctx->attrs[i] & FASTRPC_ATTR_NOMAP))
  2237. dmaflags = FASTRPC_MAP_FD_NOMAP;
  2238. VERIFY(err, VALID_FASTRPC_CID(ctx->fl->cid));
  2239. if (err) {
  2240. err = -ECHRNG;
  2241. mutex_unlock(&ctx->fl->map_mutex);
  2242. goto bail;
  2243. }
  2244. dsp_cap_ptr = &gcinfo[ctx->fl->cid].dsp_cap_kernel;
  2245. // Skip cpu mapping if DMA_HANDLE_REVERSE_RPC_CAP is true.
  2246. if (!dsp_cap_ptr->dsp_attributes[DMA_HANDLE_REVERSE_RPC_CAP] &&
  2247. ctx->fds && (ctx->fds[i] != -1))
  2248. err = fastrpc_mmap_create(ctx->fl, ctx->fds[i], NULL,
  2249. FASTRPC_ATTR_NOVA, 0, 0, dmaflags,
  2250. &ctx->maps[i]);
  2251. if (err) {
  2252. for (j = bufs; j < i; j++) {
  2253. /*
  2254. * Due to error decrement ctx refs count before mmap free
  2255. * for each in/out handle, if map created
  2256. */
  2257. if (ctx->maps[j] && ctx->maps[j]->ctx_refs)
  2258. ctx->maps[j]->ctx_refs--;
  2259. fastrpc_mmap_free(ctx->maps[j], 0);
  2260. }
  2261. mutex_unlock(&ctx->fl->map_mutex);
  2262. goto bail;
  2263. } else if (ctx->maps[i]) {
  2264. /*
  2265. * Increment ctx refs count for in/out handle if map created
  2266. * and no error, indicate map under use in remote call
  2267. */
  2268. ctx->maps[i]->ctx_refs++;
  2269. }
  2270. ipage += 1;
  2271. }
  2272. mutex_unlock(&ctx->fl->map_mutex);
  2273. /* metalen includes meta data, fds, crc, dsp perf and early wakeup hint */
  2274. metalen = totallen = (size_t)&ipage[0] + (sizeof(uint64_t) * M_FDLIST) +
  2275. (sizeof(uint32_t) * M_CRCLIST) + (sizeof(uint64_t) * M_DSP_PERF_LIST) +
  2276. sizeof(early_hint);
  2277. if (metalen) {
  2278. err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0,
  2279. METADATA_BUF, &ctx->buf);
  2280. if (err)
  2281. goto bail;
  2282. VERIFY(err, !IS_ERR_OR_NULL(ctx->buf->virt));
  2283. if (err)
  2284. goto bail;
  2285. memset(ctx->buf->virt, 0, ctx->buf->size);
  2286. }
  2287. ctx->used = metalen;
  2288. /* allocate new local rpra buffer */
  2289. lrpralen = (size_t)&list[0];
  2290. if (lrpralen) {
  2291. lrpra = kzalloc(lrpralen, GFP_KERNEL);
  2292. VERIFY(err, !IS_ERR_OR_NULL(lrpra));
  2293. if (err) {
  2294. err = -ENOMEM;
  2295. goto bail;
  2296. }
  2297. }
  2298. ctx->lrpra = lrpra;
  2299. /* calculate len required for copying */
  2300. for (oix = 0; oix < inbufs + outbufs; ++oix) {
  2301. int i = ctx->overps[oix]->raix;
  2302. uintptr_t mstart, mend;
  2303. size_t len = lpra[i].buf.len;
  2304. if (!len)
  2305. continue;
  2306. if (ctx->maps[i])
  2307. continue;
  2308. if (ctx->overps[oix]->offset == 0)
  2309. copylen = ALIGN(copylen, BALIGN);
  2310. mstart = ctx->overps[oix]->mstart;
  2311. mend = ctx->overps[oix]->mend;
  2312. templen = mend - mstart;
  2313. VERIFY(err, ((templen <= LONG_MAX) && (copylen <= (LONG_MAX - templen))));
  2314. if (err) {
  2315. err = -EFAULT;
  2316. goto bail;
  2317. }
  2318. if (templen > DEBUG_PRINT_SIZE_LIMIT)
  2319. ADSPRPC_WARN(
  2320. "user passed non ion buffer size %zu, mend 0x%llx mstart 0x%llx, sc 0x%x handle 0x%x\n",
  2321. templen, mend, mstart, sc, ctx->handle);
  2322. copylen += templen;
  2323. }
  2324. totallen = ALIGN(totallen, BALIGN) + copylen;
  2325. /* allocate non -ion copy buffer */
  2326. /* Checking if copylen can be accomodated in metalen*/
  2327. /*if not allocating new buffer */
  2328. if (totallen <= (size_t)buf_page_size(metalen)) {
  2329. args = (uintptr_t)ctx->buf->virt + metalen;
  2330. ctx->copybuf = ctx->buf;
  2331. rlen = totallen - metalen;
  2332. } else if (copylen) {
  2333. err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, COPYDATA_BUF,
  2334. &ctx->copybuf);
  2335. if (err)
  2336. goto bail;
  2337. memset(ctx->copybuf->virt, 0, copylen);
  2338. args = (uintptr_t)ctx->copybuf->virt;
  2339. rlen = copylen;
  2340. totallen = copylen;
  2341. }
  2342. /* copy metadata */
  2343. rpra = ctx->buf->virt;
  2344. ctx->rpra = rpra;
  2345. list = smq_invoke_buf_start(rpra, sc);
  2346. pages = smq_phy_page_start(sc, list);
  2347. ipage = pages;
  2348. for (i = 0; i < bufs + handles; ++i) {
  2349. if (lpra[i].buf.len)
  2350. list[i].num = 1;
  2351. else
  2352. list[i].num = 0;
  2353. list[i].pgidx = ipage - pages;
  2354. ipage++;
  2355. }
  2356. /* map ion buffers */
  2357. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_MAP),
  2358. for (i = 0; rpra && i < inbufs + outbufs; ++i) {
  2359. struct fastrpc_mmap *map = ctx->maps[i];
  2360. uint64_t buf = ptr_to_uint64(lpra[i].buf.pv);
  2361. size_t len = lpra[i].buf.len;
  2362. uint64_t buf_start = 0;
  2363. rpra[i].buf.pv = 0;
  2364. rpra[i].buf.len = len;
  2365. if (!len)
  2366. continue;
  2367. if (map) {
  2368. struct vm_area_struct *vma;
  2369. uintptr_t offset;
  2370. uint64_t num = buf_num_pages(buf, len);
  2371. int idx = list[i].pgidx;
  2372. if (map->attr & FASTRPC_ATTR_NOVA) {
  2373. offset = 0;
  2374. } else {
  2375. down_read(&current->mm->mmap_lock);
  2376. VERIFY(err, NULL != (vma = find_vma(current->mm,
  2377. map->va)));
  2378. if (err) {
  2379. up_read(&current->mm->mmap_lock);
  2380. goto bail;
  2381. }
  2382. buf_start = buf_page_start(buf);
  2383. VERIFY(err, vma->vm_start <= buf_start);
  2384. if (err) {
  2385. up_read(&current->mm->mmap_lock);
  2386. ADSPRPC_ERR(
  2387. "buffer VA invalid for fd %d, IPA 0x%llx, VA 0x%llx, vma start 0x%llx\n",
  2388. map->fd, map->phys, map->va, vma->vm_start);
  2389. err = -EFAULT;
  2390. goto bail;
  2391. }
  2392. offset = buf_start - vma->vm_start;
  2393. up_read(&current->mm->mmap_lock);
  2394. VERIFY(err, offset + len <= (uintptr_t)map->size);
  2395. if (err) {
  2396. ADSPRPC_ERR(
  2397. "buffer address is invalid for the fd passed for %d address 0x%llx and size %zu\n",
  2398. i, (uintptr_t)lpra[i].buf.pv,
  2399. lpra[i].buf.len);
  2400. err = -EFAULT;
  2401. goto bail;
  2402. }
  2403. }
  2404. pages[idx].addr = map->phys + offset;
  2405. pages[idx].size = num << PAGE_SHIFT;
  2406. }
  2407. rpra[i].buf.pv = buf;
  2408. }
  2409. PERF_END);
  2410. for (i = bufs; i < bufs + handles; ++i) {
  2411. struct fastrpc_mmap *map = ctx->maps[i];
  2412. if (map) {
  2413. pages[i].addr = map->phys;
  2414. pages[i].size = map->size;
  2415. }
  2416. }
  2417. fdlist = (uint64_t *)&pages[bufs + handles];
  2418. crclist = (uint32_t *)&fdlist[M_FDLIST];
  2419. /* reset fds, crc and early wakeup hint memory */
  2420. /* remote process updates these values before responding */
  2421. memset(fdlist, 0, sizeof(uint64_t)*M_FDLIST + sizeof(uint32_t)*M_CRCLIST +
  2422. (sizeof(uint64_t) * M_DSP_PERF_LIST) + sizeof(early_hint));
  2423. /* copy non ion buffers */
  2424. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_COPY),
  2425. for (oix = 0; rpra && oix < inbufs + outbufs; ++oix) {
  2426. int i = ctx->overps[oix]->raix;
  2427. struct fastrpc_mmap *map = ctx->maps[i];
  2428. size_t mlen;
  2429. uint64_t buf;
  2430. size_t len = lpra[i].buf.len;
  2431. if (!len)
  2432. continue;
  2433. if (map)
  2434. continue;
  2435. if (ctx->overps[oix]->offset == 0) {
  2436. rlen -= ALIGN(args, BALIGN) - args;
  2437. args = ALIGN(args, BALIGN);
  2438. }
  2439. mlen = ctx->overps[oix]->mend - ctx->overps[oix]->mstart;
  2440. VERIFY(err, rlen >= mlen);
  2441. if (err) {
  2442. err = -EFAULT;
  2443. goto bail;
  2444. }
  2445. rpra[i].buf.pv =
  2446. (args - ctx->overps[oix]->offset);
  2447. pages[list[i].pgidx].addr = ctx->copybuf->phys -
  2448. ctx->overps[oix]->offset +
  2449. (totallen - rlen);
  2450. pages[list[i].pgidx].addr =
  2451. buf_page_start(pages[list[i].pgidx].addr);
  2452. buf = rpra[i].buf.pv;
  2453. pages[list[i].pgidx].size = buf_num_pages(buf, len) * PAGE_SIZE;
  2454. if (i < inbufs) {
  2455. K_COPY_FROM_USER(err, kernel, uint64_to_ptr(buf),
  2456. lpra[i].buf.pv, len);
  2457. if (err) {
  2458. ADSPRPC_ERR(
  2459. "copy from user failed with %d for dst 0x%llx, src %pK, size 0x%zx, arg %d\n",
  2460. err, buf, lpra[i].buf.pv, len, i+1);
  2461. err = -EFAULT;
  2462. goto bail;
  2463. }
  2464. }
  2465. if (len > DEBUG_PRINT_SIZE_LIMIT)
  2466. ADSPRPC_DEBUG(
  2467. "copied non ion buffer sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu\n",
  2468. sc, rpra[i].buf.pv,
  2469. ctx->overps[oix]->mend,
  2470. ctx->overps[oix]->mstart, len);
  2471. args = args + mlen;
  2472. rlen -= mlen;
  2473. }
  2474. PERF_END);
  2475. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_FLUSH),
  2476. for (oix = 0; oix < inbufs + outbufs; ++oix) {
  2477. int i = ctx->overps[oix]->raix;
  2478. struct fastrpc_mmap *map = ctx->maps[i];
  2479. if (i+1 > inbufs) // Avoiding flush for outbufs
  2480. continue;
  2481. if (ctx->fl->sctx && ctx->fl->sctx->smmu.coherent)
  2482. continue;
  2483. if (map && (map->attr & FASTRPC_ATTR_FORCE_NOFLUSH))
  2484. continue;
  2485. if (rpra && rpra[i].buf.len && (ctx->overps[oix]->mstart ||
  2486. ctx->overps[oix]->do_cmo == 1)) {
  2487. if (map && map->buf) {
  2488. if (((buf_page_size(ctx->overps[oix]->mend -
  2489. ctx->overps[oix]->mstart)) == map->size) ||
  2490. ctx->overps[oix]->do_cmo) {
  2491. dma_buf_begin_cpu_access(map->buf,
  2492. DMA_TO_DEVICE);
  2493. dma_buf_end_cpu_access(map->buf,
  2494. DMA_TO_DEVICE);
  2495. ADSPRPC_DEBUG(
  2496. "sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2497. sc, rpra[i].buf.pv,
  2498. ctx->overps[oix]->mend,
  2499. ctx->overps[oix]->mstart,
  2500. rpra[i].buf.len, map->size);
  2501. } else {
  2502. uintptr_t offset;
  2503. uint64_t flush_len;
  2504. struct vm_area_struct *vma;
  2505. down_read(&current->mm->mmap_lock);
  2506. VERIFY(err, NULL != (vma = find_vma(
  2507. current->mm, rpra[i].buf.pv)));
  2508. if (err) {
  2509. up_read(&current->mm->mmap_lock);
  2510. goto bail;
  2511. }
  2512. if (ctx->overps[oix]->do_cmo) {
  2513. offset = rpra[i].buf.pv -
  2514. vma->vm_start;
  2515. flush_len = rpra[i].buf.len;
  2516. } else {
  2517. offset =
  2518. ctx->overps[oix]->mstart
  2519. - vma->vm_start;
  2520. flush_len =
  2521. ctx->overps[oix]->mend -
  2522. ctx->overps[oix]->mstart;
  2523. }
  2524. up_read(&current->mm->mmap_lock);
  2525. dma_buf_begin_cpu_access_partial(
  2526. map->buf, DMA_TO_DEVICE, offset,
  2527. flush_len);
  2528. dma_buf_end_cpu_access_partial(
  2529. map->buf, DMA_TO_DEVICE, offset,
  2530. flush_len);
  2531. ADSPRPC_DEBUG(
  2532. "sc 0x%x vm_start 0x%llx pv 0x%llx, offset 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2533. sc, vma->vm_start,
  2534. rpra[i].buf.pv, offset,
  2535. ctx->overps[oix]->mend,
  2536. ctx->overps[oix]->mstart,
  2537. rpra[i].buf.len, map->size);
  2538. }
  2539. }
  2540. }
  2541. }
  2542. PERF_END);
  2543. for (i = bufs; ctx->fds && rpra && i < bufs + handles; i++) {
  2544. rpra[i].dma.fd = ctx->fds[i];
  2545. rpra[i].dma.len = (uint32_t)lpra[i].buf.len;
  2546. rpra[i].dma.offset =
  2547. (uint32_t)(uintptr_t)lpra[i].buf.pv;
  2548. }
  2549. /* Copy rpra to local buffer */
  2550. if (ctx->lrpra && rpra && lrpralen > 0)
  2551. memcpy(ctx->lrpra, rpra, lrpralen);
  2552. bail:
  2553. return err;
  2554. }
  2555. static int put_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
  2556. remote_arg_t *upra)
  2557. {
  2558. uint32_t sc = ctx->sc;
  2559. struct smq_invoke_buf *list;
  2560. struct smq_phy_page *pages;
  2561. struct fastrpc_mmap *mmap;
  2562. uint64_t *fdlist;
  2563. uint32_t *crclist = NULL, *poll = NULL;
  2564. uint64_t *perf_dsp_list = NULL;
  2565. remote_arg64_t *rpra = ctx->lrpra;
  2566. int i, inbufs, outbufs, handles;
  2567. int err = 0, perfErr = 0;
  2568. inbufs = REMOTE_SCALARS_INBUFS(sc);
  2569. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2570. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  2571. list = smq_invoke_buf_start(ctx->rpra, sc);
  2572. pages = smq_phy_page_start(sc, list);
  2573. fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
  2574. crclist = (uint32_t *)(fdlist + M_FDLIST);
  2575. poll = (uint32_t *)(crclist + M_CRCLIST);
  2576. perf_dsp_list = (uint64_t *)(poll + 1);
  2577. for (i = inbufs; i < inbufs + outbufs; ++i) {
  2578. if (!ctx->maps[i]) {
  2579. K_COPY_TO_USER(err, kernel,
  2580. ctx->lpra[i].buf.pv,
  2581. uint64_to_ptr(rpra[i].buf.pv),
  2582. rpra[i].buf.len);
  2583. if (err) {
  2584. ADSPRPC_ERR(
  2585. "Invalid size 0x%llx for output argument %d ret %ld\n",
  2586. rpra[i].buf.len, i+1, err);
  2587. err = -EFAULT;
  2588. goto bail;
  2589. }
  2590. } else {
  2591. mutex_lock(&ctx->fl->map_mutex);
  2592. /*
  2593. * Decrement ctx refs count before mmap free,
  2594. * indicate remote call no longer using it
  2595. */
  2596. if (ctx->maps[i]->ctx_refs)
  2597. ctx->maps[i]->ctx_refs--;
  2598. fastrpc_mmap_free(ctx->maps[i], 0);
  2599. mutex_unlock(&ctx->fl->map_mutex);
  2600. ctx->maps[i] = NULL;
  2601. }
  2602. }
  2603. mutex_lock(&ctx->fl->map_mutex);
  2604. for (i = 0; i < M_FDLIST; i++) {
  2605. if (!fdlist[i])
  2606. break;
  2607. if (!fastrpc_mmap_find(ctx->fl, (int)fdlist[i], NULL, 0, 0,
  2608. 0, 0, &mmap)) {
  2609. /*
  2610. * Decrement ctx refs count before mmap free,
  2611. * indicate remote call no longer using it
  2612. */
  2613. if (mmap && mmap->ctx_refs)
  2614. mmap->ctx_refs--;
  2615. fastrpc_mmap_free(mmap, 0);
  2616. }
  2617. }
  2618. mutex_unlock(&ctx->fl->map_mutex);
  2619. if (ctx->crc && crclist && rpra)
  2620. K_COPY_TO_USER(err, kernel, ctx->crc,
  2621. crclist, M_CRCLIST*sizeof(uint32_t));
  2622. if (ctx->perf_dsp && perf_dsp_list) {
  2623. K_COPY_TO_USER(perfErr, kernel, ctx->perf_dsp,
  2624. perf_dsp_list, M_DSP_PERF_LIST*sizeof(uint64_t));
  2625. if (perfErr)
  2626. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  2627. }
  2628. bail:
  2629. return err;
  2630. }
  2631. static void inv_args(struct smq_invoke_ctx *ctx)
  2632. {
  2633. int i, inbufs, outbufs;
  2634. uint32_t sc = ctx->sc;
  2635. remote_arg64_t *rpra = ctx->lrpra;
  2636. int err = 0;
  2637. inbufs = REMOTE_SCALARS_INBUFS(sc);
  2638. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2639. for (i = 0; i < inbufs + outbufs; ++i) {
  2640. int over = ctx->overps[i]->raix;
  2641. struct fastrpc_mmap *map = ctx->maps[over];
  2642. if ((over + 1 <= inbufs))
  2643. continue;
  2644. if (!rpra[over].buf.len)
  2645. continue;
  2646. if (ctx->fl && ctx->fl->sctx && ctx->fl->sctx->smmu.coherent)
  2647. continue;
  2648. if (map && (map->attr & FASTRPC_ATTR_FORCE_NOINVALIDATE))
  2649. continue;
  2650. if (buf_page_start(ptr_to_uint64((void *)rpra)) ==
  2651. buf_page_start(rpra[over].buf.pv)) {
  2652. continue;
  2653. }
  2654. if (ctx->overps[i]->mstart || ctx->overps[i]->do_cmo == 1) {
  2655. if (map && map->buf) {
  2656. if (((buf_page_size(ctx->overps[i]->mend -
  2657. ctx->overps[i]->mstart)) == map->size) ||
  2658. ctx->overps[i]->do_cmo) {
  2659. dma_buf_begin_cpu_access(map->buf,
  2660. DMA_FROM_DEVICE);
  2661. dma_buf_end_cpu_access(map->buf,
  2662. DMA_TO_DEVICE);
  2663. ADSPRPC_DEBUG(
  2664. "sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2665. sc, rpra[over].buf.pv,
  2666. ctx->overps[i]->mend,
  2667. ctx->overps[i]->mstart,
  2668. rpra[over].buf.len, map->size);
  2669. } else {
  2670. uintptr_t offset;
  2671. uint64_t inv_len;
  2672. struct vm_area_struct *vma;
  2673. down_read(&current->mm->mmap_lock);
  2674. VERIFY(err, NULL != (vma = find_vma(
  2675. current->mm,
  2676. rpra[over].buf.pv)));
  2677. if (err) {
  2678. up_read(&current->mm->mmap_lock);
  2679. goto bail;
  2680. }
  2681. if (ctx->overps[i]->do_cmo) {
  2682. offset = rpra[over].buf.pv -
  2683. vma->vm_start;
  2684. inv_len = rpra[over].buf.len;
  2685. } else {
  2686. offset =
  2687. ctx->overps[i]->mstart -
  2688. vma->vm_start;
  2689. inv_len =
  2690. ctx->overps[i]->mend -
  2691. ctx->overps[i]->mstart;
  2692. }
  2693. up_read(&current->mm->mmap_lock);
  2694. dma_buf_begin_cpu_access_partial(
  2695. map->buf, DMA_FROM_DEVICE, offset,
  2696. inv_len);
  2697. dma_buf_end_cpu_access_partial(map->buf,
  2698. DMA_TO_DEVICE, offset,
  2699. inv_len);
  2700. ADSPRPC_DEBUG(
  2701. "sc 0x%x vm_start 0x%llx pv 0x%llx, offset 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2702. sc, vma->vm_start,
  2703. rpra[over].buf.pv,
  2704. offset, ctx->overps[i]->mend,
  2705. ctx->overps[i]->mstart,
  2706. rpra[over].buf.len, map->size);
  2707. }
  2708. }
  2709. }
  2710. }
  2711. bail:
  2712. return;
  2713. }
  2714. static int fastrpc_invoke_send(struct smq_invoke_ctx *ctx,
  2715. uint32_t kernel, uint32_t handle)
  2716. {
  2717. struct smq_msg *msg = &ctx->msg;
  2718. struct smq_msg msg_temp;
  2719. struct smq_invoke_ctx ctx_temp;
  2720. struct fastrpc_file *fl = ctx->fl;
  2721. struct fastrpc_channel_ctx *channel_ctx = NULL;
  2722. int err = 0, cid = -1;
  2723. uint32_t sc = ctx->sc;
  2724. int64_t ns = 0;
  2725. uint64_t xo_time_in_us = 0;
  2726. int isasync = (ctx->asyncjob.isasyncjob ? true : false);
  2727. unsigned long irq_flags = 0;
  2728. uint32_t index = 0;
  2729. if (!fl) {
  2730. err = -EBADF;
  2731. goto bail;
  2732. }
  2733. cid = fl->cid;
  2734. VERIFY(err, VALID_FASTRPC_CID(cid));
  2735. if (err) {
  2736. err = -ECHRNG;
  2737. goto bail;
  2738. }
  2739. channel_ctx = &fl->apps->channel[cid];
  2740. mutex_lock(&channel_ctx->smd_mutex);
  2741. /* Send unique fastrpc process ID to dsp */
  2742. msg->pid = fl->tgid_frpc;
  2743. msg->tid = current->pid;
  2744. if (kernel == KERNEL_MSG_WITH_ZERO_PID)
  2745. msg->pid = 0;
  2746. msg->invoke.header.ctx = ctx->ctxid | fl->pd;
  2747. msg->invoke.header.handle = handle;
  2748. msg->invoke.header.sc = sc;
  2749. msg->invoke.page.addr = ctx->buf ? ctx->buf->phys : 0;
  2750. msg->invoke.page.size = buf_page_size(ctx->used);
  2751. if (fl->ssrcount != channel_ctx->ssrcount) {
  2752. err = -ECONNRESET;
  2753. mutex_unlock(&channel_ctx->smd_mutex);
  2754. goto bail;
  2755. }
  2756. mutex_unlock(&channel_ctx->smd_mutex);
  2757. xo_time_in_us = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  2758. if (isasync) {
  2759. /*
  2760. * After message is sent to DSP, async response thread could immediately
  2761. * get the response and free context, which will result in a use-after-free
  2762. * in this function. So use a local variable for message and context.
  2763. */
  2764. memcpy(&msg_temp, msg, sizeof(struct smq_msg));
  2765. msg = &msg_temp;
  2766. memcpy(&ctx_temp, ctx, sizeof(struct smq_invoke_ctx));
  2767. index = (uint32_t)GET_TABLE_IDX_FROM_CTXID(ctx->ctxid);
  2768. }
  2769. err = fastrpc_transport_send(cid, (void *)msg, sizeof(*msg), fl->tvm_remote_domain);
  2770. if (isasync) {
  2771. if (!err) {
  2772. /*
  2773. * Validate the ctx as this could have been already
  2774. * freed by async response.
  2775. */
  2776. spin_lock_irqsave(&channel_ctx->ctxlock, irq_flags);
  2777. if (index < FASTRPC_CTX_MAX && channel_ctx->ctxtable[index] == ctx)
  2778. ctx->is_job_sent_to_remote_ss = true;
  2779. spin_unlock_irqrestore(&channel_ctx->ctxlock, irq_flags);
  2780. }
  2781. ctx = &ctx_temp;
  2782. }
  2783. trace_fastrpc_transport_send(cid, (uint64_t)ctx, msg->invoke.header.ctx,
  2784. handle, sc, msg->invoke.page.addr, msg->invoke.page.size);
  2785. ns = get_timestamp_in_ns();
  2786. fastrpc_update_txmsg_buf(msg, err, ns, xo_time_in_us, ctx, DEFAULT_STATE);
  2787. bail:
  2788. return err;
  2789. }
  2790. /* fastrpc_get_nsp_status() - Reads the property string from soc_info
  2791. * denoted for nsp part, and updates the nsp device avialbility status
  2792. * if the nsp is not defective.
  2793. * @me : pointer to fastrpc_apps.
  2794. */
  2795. static void fastrpc_get_nsp_status(struct fastrpc_apps *me)
  2796. {
  2797. if (socinfo_get_part_info(PART_NSP)) {
  2798. me->fastrpc_nsp_status = 0;
  2799. ADSPRPC_ERR(
  2800. "nsp part defective with status:%x\n", me->fastrpc_nsp_status);
  2801. } else {
  2802. me->fastrpc_nsp_status = 1;
  2803. ADSPRPC_INFO("nsp available with status: %x\n", me->fastrpc_nsp_status);
  2804. }
  2805. }
  2806. /*
  2807. * Counts number of cores corresponding
  2808. * to cluster id 0. If a core is defective or unavailable, skip counting
  2809. * that core.
  2810. * @me : pointer to fastrpc_apps.
  2811. */
  2812. static void fastrpc_lowest_capacity_corecount(struct fastrpc_apps *me)
  2813. {
  2814. unsigned int cpu = 0;
  2815. cpu = cpumask_first(cpu_possible_mask);
  2816. for_each_cpu(cpu, cpu_possible_mask) {
  2817. if (topology_cluster_id(cpu) == 0)
  2818. me->lowest_capacity_core_count++;
  2819. }
  2820. ADSPRPC_INFO("lowest capacity core count: %u\n",
  2821. me->lowest_capacity_core_count);
  2822. }
  2823. static void fastrpc_init(struct fastrpc_apps *me)
  2824. {
  2825. int i, jj;
  2826. INIT_HLIST_HEAD(&me->drivers);
  2827. INIT_HLIST_HEAD(&me->maps);
  2828. spin_lock_init(&me->hlock);
  2829. me->channel = &gcinfo[0];
  2830. mutex_init(&me->mut_uid);
  2831. me->max_sess_per_proc = DEFAULT_MAX_SESS_PER_PROC;
  2832. for (i = 0; i < NUM_CHANNELS; i++) {
  2833. init_completion(&me->channel[i].work);
  2834. init_completion(&me->channel[i].workport);
  2835. me->channel[i].sesscount = 0;
  2836. /* All channels are secure by default except CDSP */
  2837. me->channel[i].secure = SECURE_CHANNEL;
  2838. me->channel[i].unsigned_support = false;
  2839. mutex_init(&me->channel[i].smd_mutex);
  2840. fastrpc_transport_session_init(i, me->channel[i].subsys);
  2841. spin_lock_init(&me->channel[i].ctxlock);
  2842. spin_lock_init(&me->channel[i].gmsg_log.lock);
  2843. INIT_HLIST_HEAD(&me->channel[i].initmems);
  2844. for (jj = 0; jj < NUM_SESSIONS; jj++)
  2845. init_waitqueue_head(&me->channel[i].spd[jj].wait_for_pdup);
  2846. }
  2847. /* Set CDSP channel to non secure */
  2848. me->channel[CDSP_DOMAIN_ID].secure = NON_SECURE_CHANNEL;
  2849. me->channel[CDSP_DOMAIN_ID].unsigned_support = true;
  2850. }
  2851. static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type)
  2852. {
  2853. struct fastrpc_apps *me = &gfa;
  2854. struct wakeup_source *wake_source = NULL;
  2855. if (!fl->wake_enable)
  2856. return;
  2857. /*
  2858. * Vote with PM to abort any suspend in progress and
  2859. * keep system awake for specified timeout
  2860. */
  2861. if (channel_type == SECURE_CHANNEL)
  2862. wake_source = me->wake_source_secure;
  2863. else if (channel_type == NON_SECURE_CHANNEL)
  2864. wake_source = me->wake_source;
  2865. if (wake_source)
  2866. pm_wakeup_ws_event(wake_source, fl->ws_timeout, true);
  2867. }
  2868. static inline void fastrpc_pm_relax(struct fastrpc_file *fl, int channel_type)
  2869. {
  2870. struct fastrpc_apps *me = &gfa;
  2871. struct wakeup_source *wake_source = NULL;
  2872. if (!fl->wake_enable)
  2873. return;
  2874. if (channel_type == SECURE_CHANNEL)
  2875. wake_source = me->wake_source_secure;
  2876. else if (channel_type == NON_SECURE_CHANNEL)
  2877. wake_source = me->wake_source;
  2878. ADSPRPC_INFO("done for tgid %d\n", fl->tgid);
  2879. if (wake_source)
  2880. __pm_relax(wake_source);
  2881. }
  2882. static inline int fastrpc_wait_for_response(struct smq_invoke_ctx *ctx,
  2883. uint32_t kernel)
  2884. {
  2885. int interrupted = 0;
  2886. if (kernel)
  2887. wait_for_completion(&ctx->work);
  2888. else
  2889. interrupted = wait_for_completion_interruptible(&ctx->work);
  2890. return interrupted;
  2891. }
  2892. static void fastrpc_wait_for_completion(struct smq_invoke_ctx *ctx,
  2893. int *ptr_interrupted, uint32_t kernel, uint32_t async,
  2894. bool *ptr_isworkdone)
  2895. {
  2896. int interrupted = 0, err = 0;
  2897. int jj;
  2898. bool wait_resp;
  2899. uint32_t wTimeout = FASTRPC_USER_EARLY_HINT_TIMEOUT;
  2900. uint32_t wakeTime = 0;
  2901. unsigned long flags;
  2902. if (!ctx) {
  2903. /* This failure is not expected */
  2904. err = *ptr_interrupted = EFAULT;
  2905. *ptr_isworkdone = false;
  2906. ADSPRPC_ERR("ctx is NULL, cannot wait for response err %d\n",
  2907. err);
  2908. return;
  2909. }
  2910. wakeTime = ctx->early_wake_time;
  2911. do {
  2912. switch (ctx->rsp_flags) {
  2913. /* try polling on completion with timeout */
  2914. case USER_EARLY_SIGNAL:
  2915. /* try wait if completion time is less than timeout */
  2916. /* disable preempt to avoid context switch latency */
  2917. preempt_disable();
  2918. jj = 0;
  2919. wait_resp = false;
  2920. for (; wakeTime < wTimeout && jj < wTimeout; jj++) {
  2921. wait_resp = try_wait_for_completion(&ctx->work);
  2922. if (wait_resp)
  2923. break;
  2924. udelay(1);
  2925. }
  2926. preempt_enable();
  2927. if (async) {
  2928. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2929. if (!ctx->is_work_done) {
  2930. ctx->is_early_wakeup = false;
  2931. *ptr_isworkdone = false;
  2932. } else
  2933. *ptr_isworkdone = true;
  2934. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2935. goto bail;
  2936. } else if (!wait_resp) {
  2937. interrupted = fastrpc_wait_for_response(ctx,
  2938. kernel);
  2939. *ptr_interrupted = interrupted;
  2940. if (interrupted || ctx->is_work_done)
  2941. goto bail;
  2942. }
  2943. break;
  2944. /* busy poll on memory for actual job done */
  2945. case EARLY_RESPONSE:
  2946. trace_fastrpc_msg("early_response: poll_begin");
  2947. err = poll_for_remote_response(ctx, FASTRPC_POLL_TIME);
  2948. /* Mark job done if poll on memory successful */
  2949. /* Wait for completion if poll on memory timoeut */
  2950. if (!err) {
  2951. ctx->is_work_done = true;
  2952. *ptr_isworkdone = true;
  2953. goto bail;
  2954. }
  2955. trace_fastrpc_msg("early_response: poll_timeout");
  2956. ADSPRPC_INFO("early rsp poll timeout (%u us) for handle 0x%x, sc 0x%x\n",
  2957. FASTRPC_POLL_TIME, ctx->handle, ctx->sc);
  2958. if (async) {
  2959. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2960. if (!ctx->is_work_done) {
  2961. ctx->is_early_wakeup = false;
  2962. *ptr_isworkdone = false;
  2963. } else
  2964. *ptr_isworkdone = true;
  2965. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2966. goto bail;
  2967. } else if (!ctx->is_work_done) {
  2968. interrupted = fastrpc_wait_for_response(ctx,
  2969. kernel);
  2970. *ptr_interrupted = interrupted;
  2971. if (interrupted || ctx->is_work_done)
  2972. goto bail;
  2973. }
  2974. break;
  2975. case COMPLETE_SIGNAL:
  2976. case NORMAL_RESPONSE:
  2977. if (!async) {
  2978. interrupted = fastrpc_wait_for_response(ctx,
  2979. kernel);
  2980. *ptr_interrupted = interrupted;
  2981. if (interrupted || ctx->is_work_done)
  2982. goto bail;
  2983. } else {
  2984. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2985. if (!ctx->is_work_done) {
  2986. ctx->is_early_wakeup = false;
  2987. *ptr_isworkdone = false;
  2988. } else
  2989. *ptr_isworkdone = true;
  2990. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2991. goto bail;
  2992. }
  2993. break;
  2994. case POLL_MODE:
  2995. trace_fastrpc_msg("poll_mode: begin");
  2996. err = poll_for_remote_response(ctx, ctx->fl->poll_timeout);
  2997. /* If polling timed out, move to normal response state */
  2998. if (err) {
  2999. trace_fastrpc_msg("poll_mode: timeout");
  3000. ADSPRPC_INFO("poll mode timeout (%u us) for handle 0x%x, sc 0x%x\n",
  3001. ctx->fl->poll_timeout, ctx->handle, ctx->sc);
  3002. ctx->rsp_flags = NORMAL_RESPONSE;
  3003. } else {
  3004. *ptr_interrupted = 0;
  3005. *ptr_isworkdone = true;
  3006. }
  3007. break;
  3008. default:
  3009. *ptr_interrupted = EBADR;
  3010. *ptr_isworkdone = false;
  3011. ADSPRPC_ERR(
  3012. "unsupported response flags 0x%x for handle 0x%x, sc 0x%x\n",
  3013. ctx->rsp_flags, ctx->handle, ctx->sc);
  3014. goto bail;
  3015. } /* end of switch */
  3016. } while (!ctx->is_work_done);
  3017. bail:
  3018. return;
  3019. }
  3020. static void fastrpc_update_invoke_count(uint32_t handle, uint64_t *perf_counter,
  3021. struct timespec64 *invoket)
  3022. {
  3023. /* update invoke count for dynamic handles */
  3024. if (handle != FASTRPC_STATIC_HANDLE_LISTENER) {
  3025. uint64_t *count = GET_COUNTER(perf_counter, PERF_INVOKE);
  3026. if (count)
  3027. *count += getnstimediff(invoket);
  3028. }
  3029. if (handle > FASTRPC_STATIC_HANDLE_MAX) {
  3030. uint64_t *count = GET_COUNTER(perf_counter, PERF_COUNT);
  3031. if (count)
  3032. *count += 1;
  3033. }
  3034. }
  3035. static int fastrpc_check_pd_status(struct fastrpc_file *fl, char *sloc_name);
  3036. int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
  3037. uint32_t kernel,
  3038. struct fastrpc_ioctl_invoke_async *inv)
  3039. {
  3040. struct smq_invoke_ctx *ctx = NULL;
  3041. struct fastrpc_ioctl_invoke *invoke = &inv->inv;
  3042. int err = 0, interrupted = 0, cid = -1, perfErr = 0;
  3043. struct timespec64 invoket = {0};
  3044. uint64_t *perf_counter = NULL;
  3045. bool isasyncinvoke = false, isworkdone = false;
  3046. cid = fl->cid;
  3047. VERIFY(err, VALID_FASTRPC_CID(cid) &&
  3048. fl->sctx != NULL);
  3049. if (err) {
  3050. ADSPRPC_ERR("kernel session not initialized yet for %s\n",
  3051. current->comm);
  3052. err = -EBADR;
  3053. goto bail;
  3054. }
  3055. if (fl->profile)
  3056. ktime_get_real_ts64(&invoket);
  3057. if (!kernel) {
  3058. VERIFY(err, invoke->handle !=
  3059. FASTRPC_STATIC_HANDLE_PROCESS_GROUP);
  3060. VERIFY(err, invoke->handle !=
  3061. FASTRPC_STATIC_HANDLE_DSP_UTILITIES);
  3062. if (err) {
  3063. err = -EINVAL;
  3064. ADSPRPC_ERR(
  3065. "user application trying to send a kernel RPC message to channel %d, handle 0x%x\n",
  3066. cid, invoke->handle);
  3067. goto bail;
  3068. }
  3069. }
  3070. if (!kernel) {
  3071. VERIFY(err, 0 == (err = context_restore_interrupted(fl,
  3072. inv, &ctx)));
  3073. if (err)
  3074. goto bail;
  3075. if (fl->sctx->smmu.faults)
  3076. err = -FASTRPC_ENOSUCH;
  3077. if (err)
  3078. goto bail;
  3079. if (ctx) {
  3080. trace_fastrpc_context_restore(cid, (uint64_t)ctx,
  3081. ctx->msg.invoke.header.ctx,
  3082. ctx->handle, ctx->sc);
  3083. goto wait;
  3084. }
  3085. }
  3086. trace_fastrpc_msg("context_alloc: begin");
  3087. VERIFY(err, 0 == (err = context_alloc(fl, kernel, inv, &ctx)));
  3088. trace_fastrpc_msg("context_alloc: end");
  3089. if (err)
  3090. goto bail;
  3091. if (fl->servloc_name) {
  3092. err = fastrpc_check_pd_status(fl,
  3093. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME);
  3094. err |= fastrpc_check_pd_status(fl,
  3095. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME);
  3096. err |= fastrpc_check_pd_status(fl,
  3097. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME);
  3098. if (err)
  3099. goto bail;
  3100. }
  3101. isasyncinvoke = (ctx->asyncjob.isasyncjob ? true : false);
  3102. if (fl->profile)
  3103. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  3104. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_GETARGS),
  3105. VERIFY(err, 0 == (err = get_args(kernel, ctx)));
  3106. PERF_END);
  3107. trace_fastrpc_msg("get_args: end");
  3108. if (err)
  3109. goto bail;
  3110. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3111. inv_args(ctx);
  3112. PERF_END);
  3113. trace_fastrpc_msg("inv_args_1: end");
  3114. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_LINK),
  3115. VERIFY(err, 0 == (err = fastrpc_invoke_send(ctx,
  3116. kernel, invoke->handle)));
  3117. PERF_END);
  3118. trace_fastrpc_msg("invoke_send: end");
  3119. if (err)
  3120. goto bail;
  3121. if (isasyncinvoke)
  3122. goto invoke_end;
  3123. wait:
  3124. /* Poll mode allowed only for non-static handle calls to dynamic CDSP process */
  3125. if (fl->poll_mode && (invoke->handle > FASTRPC_STATIC_HANDLE_MAX)
  3126. && (cid == CDSP_DOMAIN_ID)
  3127. && (fl->proc_flags == FASTRPC_INIT_CREATE))
  3128. ctx->rsp_flags = POLL_MODE;
  3129. fastrpc_wait_for_completion(ctx, &interrupted, kernel, 0, &isworkdone);
  3130. trace_fastrpc_msg("wait_for_completion: end");
  3131. VERIFY(err, 0 == (err = interrupted));
  3132. if (err)
  3133. goto bail;
  3134. if (!ctx->is_work_done) {
  3135. err = -ETIMEDOUT;
  3136. ADSPRPC_ERR(
  3137. "WorkDone state is invalid for handle 0x%x, sc 0x%x\n",
  3138. invoke->handle, ctx->sc);
  3139. goto bail;
  3140. }
  3141. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3142. inv_args(ctx);
  3143. PERF_END);
  3144. trace_fastrpc_msg("inv_args_2: end");
  3145. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_PUTARGS),
  3146. VERIFY(err, 0 == (err = put_args(kernel, ctx, invoke->pra)));
  3147. PERF_END);
  3148. trace_fastrpc_msg("put_args: end");
  3149. if (err)
  3150. goto bail;
  3151. VERIFY(err, 0 == (err = ctx->retval));
  3152. if (err)
  3153. goto bail;
  3154. bail:
  3155. if (ctx && interrupted == -ERESTARTSYS) {
  3156. trace_fastrpc_context_interrupt(cid, (uint64_t)ctx,
  3157. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  3158. context_save_interrupted(ctx);
  3159. } else if (ctx) {
  3160. if (fl->profile && !interrupted)
  3161. fastrpc_update_invoke_count(invoke->handle,
  3162. perf_counter, &invoket);
  3163. if (fl->profile && ctx->perf && ctx->handle > FASTRPC_STATIC_HANDLE_MAX) {
  3164. trace_fastrpc_perf_counters(ctx->handle, ctx->sc,
  3165. ctx->perf->count, ctx->perf->flush, ctx->perf->map,
  3166. ctx->perf->copy, ctx->perf->link, ctx->perf->getargs,
  3167. ctx->perf->putargs, ctx->perf->invargs,
  3168. ctx->perf->invoke, ctx->perf->tid);
  3169. if (ctx->perf_kernel) {
  3170. K_COPY_TO_USER(perfErr, kernel, ctx->perf_kernel,
  3171. ctx->perf, M_KERNEL_PERF_LIST*sizeof(uint64_t));
  3172. if (perfErr)
  3173. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  3174. }
  3175. }
  3176. context_free(ctx);
  3177. trace_fastrpc_msg("context_free: end");
  3178. }
  3179. if (!kernel) {
  3180. if (VALID_FASTRPC_CID(cid)
  3181. && (fl->ssrcount != fl->apps->channel[cid].ssrcount))
  3182. err = -ECONNRESET;
  3183. }
  3184. invoke_end:
  3185. if (fl->profile && !interrupted && isasyncinvoke)
  3186. fastrpc_update_invoke_count(invoke->handle, perf_counter,
  3187. &invoket);
  3188. return err;
  3189. }
  3190. static int fastrpc_wait_on_async_queue(
  3191. struct fastrpc_ioctl_async_response *async_res,
  3192. struct fastrpc_file *fl)
  3193. {
  3194. int err = 0, ierr = 0, interrupted = 0, perfErr = 0;
  3195. struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;
  3196. unsigned long flags;
  3197. uint64_t *perf_counter = NULL;
  3198. bool isworkdone = false;
  3199. struct hlist_node *n;
  3200. read_async_job:
  3201. if (!fl) {
  3202. err = -EBADF;
  3203. goto bail;
  3204. }
  3205. interrupted = wait_event_interruptible(fl->async_wait_queue,
  3206. atomic_read(&fl->async_queue_job_count));
  3207. if (fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  3208. err = -EBADF;
  3209. goto bail;
  3210. }
  3211. if (fl->exit_async) {
  3212. err = -EFAULT;
  3213. goto bail;
  3214. }
  3215. VERIFY(err, 0 == (err = interrupted));
  3216. if (err)
  3217. goto bail;
  3218. spin_lock_irqsave(&fl->aqlock, flags);
  3219. hlist_for_each_entry_safe(ictx, n, &fl->clst.async_queue, asyncn) {
  3220. hlist_del_init(&ictx->asyncn);
  3221. atomic_sub(1, &fl->async_queue_job_count);
  3222. ctx = ictx;
  3223. break;
  3224. }
  3225. spin_unlock_irqrestore(&fl->aqlock, flags);
  3226. if (ctx) {
  3227. if (fl->profile)
  3228. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  3229. fastrpc_wait_for_completion(ctx, &interrupted, 0, 1,
  3230. &isworkdone);
  3231. if (!isworkdone) {//In valid workdone state
  3232. ADSPRPC_DEBUG(
  3233. "Async early wake response did not reach on time for thread %d handle 0x%x, sc 0x%x\n",
  3234. ctx->pid, ctx->handle, ctx->sc);
  3235. goto read_async_job;
  3236. }
  3237. async_res->jobid = ctx->asyncjob.jobid;
  3238. async_res->result = ctx->retval;
  3239. async_res->handle = ctx->handle;
  3240. async_res->sc = ctx->sc;
  3241. async_res->perf_dsp = (uint64_t *)ctx->perf_dsp;
  3242. async_res->perf_kernel = (uint64_t *)ctx->perf_kernel;
  3243. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3244. inv_args(ctx);
  3245. PERF_END);
  3246. if (ctx->retval != 0)
  3247. goto bail;
  3248. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_PUTARGS),
  3249. VERIFY(ierr, 0 == (ierr = put_args(0, ctx, NULL)));
  3250. PERF_END);
  3251. if (ierr)
  3252. goto bail;
  3253. } else { // Go back to wait if ctx is invalid
  3254. ADSPRPC_ERR("Invalid async job wake up\n");
  3255. goto read_async_job;
  3256. }
  3257. bail:
  3258. if (ierr)
  3259. async_res->result = ierr;
  3260. if (ctx) {
  3261. if (fl->profile && ctx->perf && ctx->handle > FASTRPC_STATIC_HANDLE_MAX) {
  3262. trace_fastrpc_perf_counters(ctx->handle, ctx->sc,
  3263. ctx->perf->count, ctx->perf->flush, ctx->perf->map,
  3264. ctx->perf->copy, ctx->perf->link, ctx->perf->getargs,
  3265. ctx->perf->putargs, ctx->perf->invargs,
  3266. ctx->perf->invoke, ctx->perf->tid);
  3267. if (ctx->perf_kernel) {
  3268. K_COPY_TO_USER(perfErr, 0, ctx->perf_kernel,
  3269. ctx->perf, M_KERNEL_PERF_LIST*sizeof(uint64_t));
  3270. if (perfErr)
  3271. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  3272. }
  3273. }
  3274. context_free(ctx);
  3275. }
  3276. return err;
  3277. }
  3278. static int fastrpc_wait_on_notif_queue(
  3279. struct fastrpc_ioctl_notif_rsp *notif_rsp,
  3280. struct fastrpc_file *fl)
  3281. {
  3282. int err = 0, interrupted = 0;
  3283. unsigned long flags;
  3284. struct smq_notif_rsp *notif = NULL, *inotif = NULL, *n = NULL;
  3285. read_notif_status:
  3286. if (!fl) {
  3287. err = -EBADF;
  3288. goto bail;
  3289. }
  3290. interrupted = wait_event_interruptible(fl->proc_state_notif.notif_wait_queue,
  3291. atomic_read(&fl->proc_state_notif.notif_queue_count));
  3292. if (fl->exit_notif) {
  3293. err = -EFAULT;
  3294. goto bail;
  3295. }
  3296. VERIFY(err, 0 == (err = interrupted));
  3297. if (err)
  3298. goto bail;
  3299. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  3300. list_for_each_entry_safe(inotif, n, &fl->clst.notif_queue, notifn) {
  3301. list_del_init(&inotif->notifn);
  3302. atomic_sub(1, &fl->proc_state_notif.notif_queue_count);
  3303. notif = inotif;
  3304. break;
  3305. }
  3306. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  3307. if (notif) {
  3308. notif_rsp->status = notif->status;
  3309. notif_rsp->domain = notif->domain;
  3310. notif_rsp->session = notif->session;
  3311. } else {// Go back to wait if ctx is invalid
  3312. ADSPRPC_ERR("Invalid status notification response\n");
  3313. goto read_notif_status;
  3314. }
  3315. bail:
  3316. kfree(notif);
  3317. return err;
  3318. }
  3319. static int fastrpc_get_async_response(
  3320. struct fastrpc_ioctl_async_response *async_res,
  3321. void *param, struct fastrpc_file *fl)
  3322. {
  3323. int err = 0;
  3324. err = fastrpc_wait_on_async_queue(async_res, fl);
  3325. if (err)
  3326. goto bail;
  3327. K_COPY_TO_USER(err, 0, param, async_res,
  3328. sizeof(struct fastrpc_ioctl_async_response));
  3329. bail:
  3330. return err;
  3331. }
  3332. static int fastrpc_get_notif_response(
  3333. struct fastrpc_ioctl_notif_rsp *notif,
  3334. void *param, struct fastrpc_file *fl)
  3335. {
  3336. int err = 0;
  3337. err = fastrpc_wait_on_notif_queue(notif, fl);
  3338. if (err)
  3339. goto bail;
  3340. K_COPY_TO_USER(err, 0, param, notif,
  3341. sizeof(struct fastrpc_ioctl_notif_rsp));
  3342. bail:
  3343. return err;
  3344. }
  3345. static int fastrpc_set_session_info(
  3346. struct fastrpc_proc_sess_info *sess_info,
  3347. void *param, struct fastrpc_file *fl)
  3348. {
  3349. int err = 0;
  3350. struct fastrpc_apps *me = &gfa;
  3351. if (fl->set_session_info) {
  3352. ADSPRPC_ERR("Set session info invoked multiple times\n");
  3353. err = -EBADR;
  3354. goto bail;
  3355. }
  3356. /*
  3357. * Third-party apps don't have permission to open the fastrpc device, so
  3358. * it is opened on their behalf by DSP HAL. This is detected by
  3359. * comparing current PID with the one stored during device open.
  3360. */
  3361. if (current->tgid != fl->tgid_open)
  3362. fl->untrusted_process = true;
  3363. VERIFY(err, sess_info->pd_type > DEFAULT_UNUSED &&
  3364. sess_info->pd_type < MAX_PD_TYPE);
  3365. if (err) {
  3366. ADSPRPC_ERR(
  3367. "Session PD type %u is invalid for the process\n",
  3368. sess_info->pd_type);
  3369. err = -EBADR;
  3370. goto bail;
  3371. }
  3372. if (fl->untrusted_process && sess_info->pd_type != USERPD) {
  3373. ADSPRPC_ERR(
  3374. "Session PD type %u not allowed for untrusted process\n",
  3375. sess_info->pd_type);
  3376. err = -EBADR;
  3377. goto bail;
  3378. }
  3379. /*
  3380. * If PD type is not configured for context banks,
  3381. * ignore PD type passed by the user, leave pd_type set to DEFAULT_UNUSED(0)
  3382. */
  3383. if (me->cb_pd_type)
  3384. fl->pd_type = sess_info->pd_type;
  3385. // Processes attaching to Sensor Static PD, share context bank.
  3386. if (sess_info->pd_type == SENSORS_STATICPD)
  3387. fl->sharedcb = 1;
  3388. if (sess_info->session_id >= me->max_sess_per_proc) {
  3389. ADSPRPC_ERR(
  3390. "Session ID %u cannot be beyond %u\n",
  3391. sess_info->session_id, me->max_sess_per_proc);
  3392. err = -EBADR;
  3393. goto bail;
  3394. }
  3395. fl->sessionid = sess_info->session_id;
  3396. // Set multi_session_support, to disable old way of setting session_id
  3397. fl->multi_session_support = true;
  3398. VERIFY(err, 0 == (err = fastrpc_get_info(fl, &(sess_info->domain_id))));
  3399. if (err)
  3400. goto bail;
  3401. K_COPY_TO_USER(err, 0, param, sess_info,
  3402. sizeof(struct fastrpc_proc_sess_info));
  3403. bail:
  3404. return err;
  3405. }
  3406. static int fastrpc_create_persistent_headers(struct fastrpc_file *fl,
  3407. uint32_t user_concurrency)
  3408. {
  3409. int err = 0, i = 0;
  3410. uint64_t virtb = 0;
  3411. struct fastrpc_buf *pers_hdr_buf = NULL, *hdr_bufs = NULL, *buf = NULL;
  3412. unsigned int num_pers_hdrs = 0;
  3413. size_t hdr_buf_alloc_len = 0;
  3414. if (fl->pers_hdr_buf || !user_concurrency)
  3415. goto bail;
  3416. /*
  3417. * Pre-allocate memory for persistent header buffers based
  3418. * on concurrency info passed by user. Upper limit enforced.
  3419. */
  3420. num_pers_hdrs = (user_concurrency > MAX_PERSISTENT_HEADERS) ?
  3421. MAX_PERSISTENT_HEADERS : user_concurrency;
  3422. hdr_buf_alloc_len = num_pers_hdrs*PAGE_SIZE;
  3423. err = fastrpc_buf_alloc(fl, hdr_buf_alloc_len, 0, 0,
  3424. METADATA_BUF, &pers_hdr_buf);
  3425. if (err)
  3426. goto bail;
  3427. virtb = ptr_to_uint64(pers_hdr_buf->virt);
  3428. /* Map entire buffer on remote subsystem in single RPC call */
  3429. err = fastrpc_mem_map_to_dsp(fl, -1, 0, ADSP_MMAP_PERSIST_HDR, 0,
  3430. pers_hdr_buf->phys, pers_hdr_buf->size,
  3431. &pers_hdr_buf->raddr);
  3432. if (err)
  3433. goto bail;
  3434. /* Divide and store as N chunks, each of 1 page size */
  3435. hdr_bufs = kcalloc(num_pers_hdrs, sizeof(struct fastrpc_buf),
  3436. GFP_KERNEL);
  3437. if (!hdr_bufs) {
  3438. err = -ENOMEM;
  3439. goto bail;
  3440. }
  3441. spin_lock(&fl->hlock);
  3442. fl->pers_hdr_buf = pers_hdr_buf;
  3443. fl->num_pers_hdrs = num_pers_hdrs;
  3444. fl->hdr_bufs = hdr_bufs;
  3445. for (i = 0; i < num_pers_hdrs; i++) {
  3446. buf = &fl->hdr_bufs[i];
  3447. buf->fl = fl;
  3448. buf->virt = uint64_to_ptr(virtb + (i*PAGE_SIZE));
  3449. buf->phys = pers_hdr_buf->phys + (i*PAGE_SIZE);
  3450. buf->size = PAGE_SIZE;
  3451. buf->dma_attr = pers_hdr_buf->dma_attr;
  3452. buf->flags = pers_hdr_buf->flags;
  3453. buf->type = pers_hdr_buf->type;
  3454. buf->in_use = false;
  3455. }
  3456. spin_unlock(&fl->hlock);
  3457. bail:
  3458. if (err) {
  3459. ADSPRPC_ERR(
  3460. "failed to map len %zu, flags %d, user concurrency %u, num headers %u with err %d\n",
  3461. hdr_buf_alloc_len, ADSP_MMAP_PERSIST_HDR,
  3462. user_concurrency, num_pers_hdrs, err);
  3463. fl->pers_hdr_buf = NULL;
  3464. fl->hdr_bufs = NULL;
  3465. fl->num_pers_hdrs = 0;
  3466. if (!IS_ERR_OR_NULL(pers_hdr_buf))
  3467. fastrpc_buf_free(pers_hdr_buf, 0);
  3468. if (!IS_ERR_OR_NULL(hdr_bufs))
  3469. kfree(hdr_bufs);
  3470. }
  3471. return err;
  3472. }
  3473. int fastrpc_internal_invoke2(struct fastrpc_file *fl,
  3474. struct fastrpc_ioctl_invoke2 *inv2)
  3475. {
  3476. union {
  3477. struct fastrpc_ioctl_invoke_async inv;
  3478. struct fastrpc_ioctl_invoke_async_no_perf inv3;
  3479. struct fastrpc_ioctl_async_response async_res;
  3480. uint32_t user_concurrency;
  3481. struct fastrpc_ioctl_notif_rsp notif;
  3482. struct fastrpc_proc_sharedbuf_info buff_info;
  3483. struct fastrpc_proc_sess_info sess_info;
  3484. } p;
  3485. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  3486. uint32_t size = 0;
  3487. int err = 0, domain = fl->cid;
  3488. if (inv2->req == FASTRPC_INVOKE2_ASYNC ||
  3489. inv2->req == FASTRPC_INVOKE2_ASYNC_RESPONSE) {
  3490. VERIFY(err, domain == CDSP_DOMAIN_ID && fl->sctx != NULL);
  3491. if (err) {
  3492. err = -EBADR;
  3493. goto bail;
  3494. }
  3495. dsp_cap_ptr = &gcinfo[domain].dsp_cap_kernel;
  3496. VERIFY(err,
  3497. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP] == 1);
  3498. if (err) {
  3499. err = -EPROTONOSUPPORT;
  3500. goto bail;
  3501. }
  3502. }
  3503. switch (inv2->req) {
  3504. case FASTRPC_INVOKE2_ASYNC:
  3505. size = sizeof(struct fastrpc_ioctl_invoke_async);
  3506. VERIFY(err, size >= inv2->size);
  3507. if (err) {
  3508. err = -EBADE;
  3509. goto bail;
  3510. }
  3511. if (size > inv2->size) {
  3512. K_COPY_FROM_USER(err, fl->is_compat, &p.inv3, (void *)inv2->invparam,
  3513. sizeof(struct fastrpc_ioctl_invoke_async_no_perf));
  3514. if (err)
  3515. goto bail;
  3516. memcpy(&p.inv, &p.inv3, sizeof(struct fastrpc_ioctl_invoke_crc));
  3517. memcpy(&p.inv.job, &p.inv3.job, sizeof(p.inv.job));
  3518. } else {
  3519. K_COPY_FROM_USER(err, fl->is_compat, &p.inv, (void *)inv2->invparam, size);
  3520. if (err)
  3521. goto bail;
  3522. }
  3523. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode,
  3524. USER_MSG, &p.inv)));
  3525. if (err)
  3526. goto bail;
  3527. break;
  3528. case FASTRPC_INVOKE2_ASYNC_RESPONSE:
  3529. VERIFY(err,
  3530. sizeof(struct fastrpc_ioctl_async_response) >= inv2->size);
  3531. if (err) {
  3532. err = -EBADE;
  3533. goto bail;
  3534. }
  3535. err = fastrpc_get_async_response(&p.async_res,
  3536. (void *)inv2->invparam, fl);
  3537. break;
  3538. case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:
  3539. size = sizeof(uint32_t);
  3540. if (inv2->size != size) {
  3541. err = -EBADE;
  3542. goto bail;
  3543. }
  3544. K_COPY_FROM_USER(err, 0, &p.user_concurrency,
  3545. (void *)inv2->invparam, size);
  3546. if (err)
  3547. goto bail;
  3548. err = fastrpc_create_persistent_headers(fl,
  3549. p.user_concurrency);
  3550. break;
  3551. case FASTRPC_INVOKE2_STATUS_NOTIF:
  3552. VERIFY(err,
  3553. sizeof(struct fastrpc_ioctl_notif_rsp) >= inv2->size);
  3554. if (err) {
  3555. err = -EBADE;
  3556. goto bail;
  3557. }
  3558. err = fastrpc_get_notif_response(&p.notif,
  3559. (void *)inv2->invparam, fl);
  3560. break;
  3561. case FASTRPC_INVOKE2_PROC_SHAREDBUF_INFO:
  3562. VERIFY(err,
  3563. sizeof(struct fastrpc_proc_sharedbuf_info) >= inv2->size);
  3564. if (err) {
  3565. err = -EBADE;
  3566. goto bail;
  3567. }
  3568. K_COPY_FROM_USER(err, fl->is_compat, &p.buff_info,
  3569. (void *)inv2->invparam, inv2->size);
  3570. if (err)
  3571. goto bail;
  3572. fl->sharedbuf_info.buf_fd = p.buff_info.buf_fd;
  3573. fl->sharedbuf_info.buf_size = p.buff_info.buf_size;
  3574. break;
  3575. case FASTRPC_INVOKE2_SESS_INFO:
  3576. VERIFY(err,
  3577. sizeof(struct fastrpc_proc_sess_info) >= inv2->size);
  3578. if (err) {
  3579. err = -EBADE;
  3580. goto bail;
  3581. }
  3582. K_COPY_FROM_USER(err, fl->is_compat, &p.sess_info,
  3583. (void *)inv2->invparam, inv2->size);
  3584. if (err)
  3585. goto bail;
  3586. err = fastrpc_set_session_info(&p.sess_info,
  3587. (void *)inv2->invparam, fl);
  3588. break;
  3589. default:
  3590. err = -ENOTTY;
  3591. break;
  3592. }
  3593. bail:
  3594. return err;
  3595. }
  3596. static int fastrpc_get_spd_session(char *name, int *session, int *cid)
  3597. {
  3598. struct fastrpc_apps *me = &gfa;
  3599. int err = 0, i, j, match = 0;
  3600. for (i = 0; i < NUM_CHANNELS; i++) {
  3601. for (j = 0; j < NUM_SESSIONS; j++) {
  3602. if (!me->channel[i].spd[j].servloc_name)
  3603. continue;
  3604. if (!strcmp(name, me->channel[i].spd[j].servloc_name)) {
  3605. match = 1;
  3606. break;
  3607. }
  3608. }
  3609. if (match)
  3610. break;
  3611. }
  3612. VERIFY(err, i < NUM_CHANNELS && j < NUM_SESSIONS);
  3613. if (err) {
  3614. err = -EUSERS;
  3615. goto bail;
  3616. }
  3617. *cid = i;
  3618. *session = j;
  3619. bail:
  3620. return err;
  3621. }
  3622. static int fastrpc_mmap_remove_pdr(struct fastrpc_file *fl);
  3623. static int fastrpc_channel_open(struct fastrpc_file *fl, uint32_t flags);
  3624. static int fastrpc_mmap_remove_ssr(struct fastrpc_file *fl, int locked);
  3625. /*
  3626. * This function makes a call to create a thread group in the root
  3627. * process or static process on the remote subsystem.
  3628. * Examples:
  3629. * - guestOS daemons on all DSPs
  3630. * - sensors daemon on sensorsPD on SLPI/ADSP
  3631. */
  3632. static int fastrpc_init_attach_process(struct fastrpc_file *fl,
  3633. struct fastrpc_ioctl_init *init)
  3634. {
  3635. int err = 0, tgid = fl->tgid_frpc;
  3636. remote_arg_t ra[1];
  3637. struct fastrpc_ioctl_invoke_async ioctl;
  3638. if (fl->dev_minor == MINOR_NUM_DEV) {
  3639. err = -ECONNREFUSED;
  3640. ADSPRPC_ERR(
  3641. "untrusted app trying to attach to privileged DSP PD\n");
  3642. return err;
  3643. }
  3644. /*
  3645. * Prepare remote arguments for creating thread group
  3646. * in guestOS/staticPD on the remote subsystem.
  3647. * Send unique fastrpc id to dsp
  3648. */
  3649. ra[0].buf.pv = (void *)&tgid;
  3650. ra[0].buf.len = sizeof(tgid);
  3651. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  3652. ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 0);
  3653. ioctl.inv.pra = ra;
  3654. ioctl.fds = NULL;
  3655. ioctl.attrs = NULL;
  3656. ioctl.crc = NULL;
  3657. ioctl.perf_kernel = NULL;
  3658. ioctl.perf_dsp = NULL;
  3659. ioctl.job = NULL;
  3660. if (init->flags == FASTRPC_INIT_ATTACH)
  3661. fl->pd = FASTRPC_ROOT_PD;
  3662. else if (init->flags == FASTRPC_INIT_ATTACH_SENSORS)
  3663. /* Setting to 2 will route the message to sensorsPD */
  3664. fl->pd = FASTRPC_SENSORS_PD;
  3665. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  3666. if (err)
  3667. goto bail;
  3668. bail:
  3669. return err;
  3670. }
  3671. /*
  3672. * This function makes a call to spawn a dynamic process
  3673. * on the remote subsystem.
  3674. * Example: all compute offloads to CDSP
  3675. */
  3676. static int fastrpc_init_create_dynamic_process(struct fastrpc_file *fl,
  3677. struct fastrpc_ioctl_init_attrs *uproc)
  3678. {
  3679. int err = 0, memlen = 0, mflags = 0, locked = 0;
  3680. struct fastrpc_ioctl_invoke_async ioctl;
  3681. struct fastrpc_ioctl_init *init = &uproc->init;
  3682. /* First page for init-mem and second page for proc-attrs */
  3683. struct smq_phy_page pages[PAGESLEN_WITH_SHAREDBUF];
  3684. struct fastrpc_mmap *file = NULL;
  3685. struct fastrpc_buf *imem = NULL;
  3686. unsigned long imem_dma_attr = 0;
  3687. remote_arg_t ra[6];
  3688. int fds[6];
  3689. unsigned int gid = 0, one_mb = 1024*1024;
  3690. unsigned int dsp_userpd_memlen = 0;
  3691. struct fastrpc_buf *init_mem;
  3692. struct fastrpc_mmap *sharedbuf_map = NULL;
  3693. struct {
  3694. int pgid;
  3695. unsigned int namelen;
  3696. unsigned int filelen;
  3697. unsigned int pageslen;
  3698. int attrs;
  3699. int siglen;
  3700. } inbuf;
  3701. spin_lock(&fl->hlock);
  3702. if (fl->dsp_process_state) {
  3703. err = -EALREADY;
  3704. ADSPRPC_ERR("Already in create dynamic process\n");
  3705. spin_unlock(&fl->hlock);
  3706. return err;
  3707. }
  3708. fl->dsp_process_state = PROCESS_CREATE_IS_INPROGRESS;
  3709. if (init->memlen) {
  3710. if(init->memlen > INIT_MEMLEN_MAX_DYNAMIC || init->memlen < INIT_MEMLEN_MIN_DYNAMIC) {
  3711. ADSPRPC_ERR(
  3712. "init memory for process %d should be between %d and %d\n",
  3713. init->memlen, INIT_MEMLEN_MIN_DYNAMIC, INIT_MEMLEN_MAX_DYNAMIC);
  3714. err = -EINVAL;
  3715. spin_unlock(&fl->hlock);
  3716. goto bail;
  3717. }
  3718. dsp_userpd_memlen = init->memlen;
  3719. } else {
  3720. dsp_userpd_memlen = 3*one_mb;
  3721. }
  3722. spin_unlock(&fl->hlock);
  3723. inbuf.pgid = fl->tgid_frpc;
  3724. inbuf.namelen = strlen(current->comm) + 1;
  3725. inbuf.filelen = init->filelen;
  3726. fl->pd = FASTRPC_USER_PD;
  3727. if (uproc->attrs & FASTRPC_MODE_UNSIGNED_MODULE)
  3728. fl->is_unsigned_pd = true;
  3729. /* Check if file memory passed by userspace is valid */
  3730. VERIFY(err, access_ok((void __user *)init->file, init->filelen));
  3731. if (err)
  3732. goto bail;
  3733. if (init->filelen) {
  3734. /* Map the shell file buffer to remote subsystem */
  3735. mutex_lock(&fl->map_mutex);
  3736. err = fastrpc_mmap_create(fl, init->filefd, NULL, 0,
  3737. init->file, init->filelen, mflags, &file);
  3738. if (file)
  3739. file->is_filemap = true;
  3740. mutex_unlock(&fl->map_mutex);
  3741. if (err)
  3742. goto bail;
  3743. }
  3744. inbuf.pageslen = 1;
  3745. /* Disregard any system unsigned PD attribute from userspace */
  3746. uproc->attrs &= (~FASTRPC_MODE_SYSTEM_UNSIGNED_PD);
  3747. /* Untrusted apps are not allowed to offload to signedPD on DSP. */
  3748. if (fl->untrusted_process) {
  3749. VERIFY(err, fl->is_unsigned_pd);
  3750. if (err) {
  3751. err = -ECONNREFUSED;
  3752. ADSPRPC_ERR(
  3753. "untrusted app trying to offload to signed remote process\n");
  3754. goto bail;
  3755. }
  3756. } else {
  3757. /* Trusted apps will be launched as system unsigned PDs */
  3758. if (fl->is_unsigned_pd)
  3759. uproc->attrs |= FASTRPC_MODE_SYSTEM_UNSIGNED_PD;
  3760. }
  3761. /* Disregard any privilege bits from userspace */
  3762. uproc->attrs &= (~FASTRPC_MODE_PRIVILEGED);
  3763. /*
  3764. * Check if the primary or supplementary group(s) of the process is
  3765. * one of the 'privileged' fastrpc GIDs stored in the device-tree.
  3766. */
  3767. gid = sorted_lists_intersection(fl->gidlist.gids,
  3768. fl->gidlist.gidcount, gfa.gidlist.gids, gfa.gidlist.gidcount);
  3769. if (gid) {
  3770. ADSPRPC_INFO("PID %d, GID %u is a privileged process\n",
  3771. fl->tgid, gid);
  3772. uproc->attrs |= FASTRPC_MODE_PRIVILEGED;
  3773. }
  3774. /*
  3775. * Userspace client should try to allocate the initial memory donated
  3776. * to remote subsystem as only the kernel and DSP should have access
  3777. * to that memory.
  3778. */
  3779. VERIFY(err, !init->mem);
  3780. if (err) {
  3781. err = -EINVAL;
  3782. ADSPRPC_ERR("donated memory allocated in userspace\n");
  3783. goto bail;
  3784. }
  3785. /* Free any previous donated memory */
  3786. spin_lock(&fl->hlock);
  3787. locked = 1;
  3788. if (fl->init_mem) {
  3789. init_mem = fl->init_mem;
  3790. fl->init_mem = NULL;
  3791. spin_unlock(&fl->hlock);
  3792. locked = 0;
  3793. fastrpc_buf_free(init_mem, 0);
  3794. }
  3795. if (locked) {
  3796. spin_unlock(&fl->hlock);
  3797. locked = 0;
  3798. }
  3799. /* Allocate DMA buffer in kernel for donating to remote process
  3800. * Unsigned PD requires additional memory because of the
  3801. * additional static heap initialized within the process.
  3802. */
  3803. if (fl->is_unsigned_pd)
  3804. dsp_userpd_memlen = 5*one_mb;
  3805. memlen = ALIGN(max(dsp_userpd_memlen, init->filelen * 4), one_mb);
  3806. imem_dma_attr = DMA_ATTR_DELAYED_UNMAP | DMA_ATTR_NO_KERNEL_MAPPING;
  3807. err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0,
  3808. INITMEM_BUF, &imem);
  3809. if (err)
  3810. goto bail;
  3811. fl->init_mem = imem;
  3812. inbuf.pageslen = 1;
  3813. if ((fl->sharedbuf_info.buf_fd != -1) && fl->sharedbuf_info.buf_size) {
  3814. mutex_lock(&fl->map_mutex);
  3815. err = fastrpc_mmap_create(fl, fl->sharedbuf_info.buf_fd, NULL, 0,
  3816. 0, fl->sharedbuf_info.buf_size, mflags, &sharedbuf_map);
  3817. mutex_unlock(&fl->map_mutex);
  3818. if (err)
  3819. goto bail;
  3820. /* if shared buff is available send this as the second page and set pageslen as 2 */
  3821. inbuf.pageslen = PAGESLEN_WITH_SHAREDBUF;
  3822. }
  3823. /*
  3824. * Prepare remote arguments for dynamic process create
  3825. * call to remote subsystem.
  3826. */
  3827. ra[0].buf.pv = (void *)&inbuf;
  3828. ra[0].buf.len = sizeof(inbuf);
  3829. fds[0] = -1;
  3830. ra[1].buf.pv = (void *)current->comm;
  3831. ra[1].buf.len = inbuf.namelen;
  3832. fds[1] = -1;
  3833. ra[2].buf.pv = (void *)init->file;
  3834. ra[2].buf.len = inbuf.filelen;
  3835. fds[2] = init->filefd;
  3836. pages[0].addr = imem->phys;
  3837. pages[0].size = imem->size;
  3838. /* Update IOVA of second page shared with DSP */
  3839. if (inbuf.pageslen > 1) {
  3840. pages[1].addr = sharedbuf_map->phys;
  3841. pages[1].size = sharedbuf_map->size;
  3842. }
  3843. ra[3].buf.pv = (void *)pages;
  3844. ra[3].buf.len = (inbuf.pageslen) * sizeof(*pages);
  3845. fds[3] = -1;
  3846. inbuf.attrs = uproc->attrs;
  3847. ra[4].buf.pv = (void *)&(inbuf.attrs);
  3848. ra[4].buf.len = sizeof(inbuf.attrs);
  3849. fds[4] = -1;
  3850. inbuf.siglen = uproc->siglen;
  3851. ra[5].buf.pv = (void *)&(inbuf.siglen);
  3852. ra[5].buf.len = sizeof(inbuf.siglen);
  3853. fds[5] = -1;
  3854. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  3855. /*
  3856. * Choose appropriate remote method ID depending on whether the
  3857. * HLOS process has any attributes enabled (like unsignedPD,
  3858. * critical process, adaptive QoS, CRC checks etc).
  3859. */
  3860. ioctl.inv.sc = REMOTE_SCALARS_MAKE(6, 4, 0);
  3861. if (uproc->attrs)
  3862. ioctl.inv.sc = REMOTE_SCALARS_MAKE(7, 4, 0);
  3863. ioctl.inv.pra = ra;
  3864. ioctl.fds = fds;
  3865. ioctl.attrs = NULL;
  3866. ioctl.crc = NULL;
  3867. ioctl.perf_kernel = NULL;
  3868. ioctl.perf_dsp = NULL;
  3869. ioctl.job = NULL;
  3870. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  3871. if (err)
  3872. goto bail;
  3873. bail:
  3874. /*
  3875. * Shell is loaded into the donated memory on remote subsystem. So, the
  3876. * original file buffer can be DMA unmapped. In case of a failure also,
  3877. * the mapping needs to be removed.
  3878. */
  3879. if (file) {
  3880. mutex_lock(&fl->map_mutex);
  3881. fastrpc_mmap_free(file, 0);
  3882. mutex_unlock(&fl->map_mutex);
  3883. }
  3884. spin_lock(&fl->hlock);
  3885. locked = 1;
  3886. if (err) {
  3887. ADSPRPC_ERR("failed with err %d\n", err);
  3888. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  3889. if (!IS_ERR_OR_NULL(fl->init_mem)) {
  3890. init_mem = fl->init_mem;
  3891. fl->init_mem = NULL;
  3892. spin_unlock(&fl->hlock);
  3893. locked = 0;
  3894. fastrpc_buf_free(init_mem, 0);
  3895. }
  3896. } else {
  3897. fl->dsp_process_state = PROCESS_CREATE_SUCCESS;
  3898. }
  3899. if (locked) {
  3900. spin_unlock(&fl->hlock);
  3901. locked = 0;
  3902. }
  3903. return err;
  3904. }
  3905. /*
  3906. * This function makes a call to create a thread group in the static
  3907. * process on the remote subsystem.
  3908. * Example: audio daemon 'adsprpcd' on audioPD on ADSP
  3909. */
  3910. static int fastrpc_init_create_static_process(struct fastrpc_file *fl,
  3911. struct fastrpc_ioctl_init *init)
  3912. {
  3913. int err = 0, rh_hyp_done = 0;
  3914. struct fastrpc_apps *me = &gfa;
  3915. struct fastrpc_ioctl_invoke_async ioctl;
  3916. struct smq_phy_page pages[1];
  3917. struct fastrpc_mmap *mem = NULL;
  3918. char *proc_name = NULL;
  3919. remote_arg_t ra[3];
  3920. uint64_t phys = 0;
  3921. size_t size = 0;
  3922. int fds[3];
  3923. struct secure_vm *rhvm = &me->channel[fl->cid].rhvm;
  3924. struct {
  3925. int pgid;
  3926. unsigned int namelen;
  3927. unsigned int pageslen;
  3928. } inbuf;
  3929. unsigned long irq_flags = 0;
  3930. if (fl->dev_minor == MINOR_NUM_DEV) {
  3931. err = -ECONNREFUSED;
  3932. ADSPRPC_ERR(
  3933. "untrusted app trying to attach to audio PD\n");
  3934. return err;
  3935. }
  3936. VERIFY(err, init->memlen <= INIT_MEMLEN_MAX_STATIC);
  3937. if (err) {
  3938. ADSPRPC_ERR(
  3939. "init memory for static process %d is more than max allowed init len %d\n",
  3940. init->memlen, INIT_MEMLEN_MAX_STATIC);
  3941. err = -EFBIG;
  3942. goto bail;
  3943. }
  3944. if (!init->filelen)
  3945. goto bail;
  3946. proc_name = kzalloc(init->filelen + 1, GFP_KERNEL);
  3947. VERIFY(err, !IS_ERR_OR_NULL(proc_name));
  3948. if (err) {
  3949. err = -ENOMEM;
  3950. goto bail;
  3951. }
  3952. err = copy_from_user((void *)proc_name,
  3953. (void __user *)init->file, init->filelen);
  3954. if (err) {
  3955. err = -EFAULT;
  3956. goto bail;
  3957. }
  3958. fl->pd = FASTRPC_USER_PD;
  3959. inbuf.pgid = fl->tgid_frpc;
  3960. inbuf.namelen = init->filelen;
  3961. inbuf.pageslen = 0;
  3962. if (!strcmp(proc_name, "audiopd")) {
  3963. /*
  3964. * Remove any previous mappings in case process is trying
  3965. * to reconnect after a PD restart on remote subsystem.
  3966. */
  3967. err = fastrpc_mmap_remove_pdr(fl);
  3968. if (err)
  3969. goto bail;
  3970. } else {
  3971. ADSPRPC_ERR(
  3972. "Create static process is failed for proc_name %s",
  3973. proc_name);
  3974. goto bail;
  3975. }
  3976. if ((!me->staticpd_flags && !me->legacy_remote_heap)) {
  3977. inbuf.pageslen = 1;
  3978. if (!fastrpc_get_persistent_map(init->memlen, &mem)) {
  3979. mutex_lock(&fl->map_mutex);
  3980. err = fastrpc_mmap_create(fl, -1, NULL, 0, init->mem,
  3981. init->memlen, ADSP_MMAP_REMOTE_HEAP_ADDR, &mem);
  3982. if (mem)
  3983. mem->is_filemap = true;
  3984. mutex_unlock(&fl->map_mutex);
  3985. if (err || (!mem))
  3986. goto bail;
  3987. spin_lock_irqsave(&me->hlock, irq_flags);
  3988. mem->in_use = true;
  3989. spin_unlock_irqrestore(&me->hlock, irq_flags);
  3990. }
  3991. VERIFY(err, mem);
  3992. if (err)
  3993. goto bail;
  3994. phys = mem->phys;
  3995. size = mem->size;
  3996. /*
  3997. * If remote-heap VMIDs are defined in DTSI, then do
  3998. * hyp_assign from HLOS to those VMs (LPASS, ADSP).
  3999. */
  4000. if (rhvm->vmid && mem->refs == 1 && size) {
  4001. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  4002. struct qcom_scm_vmperm *dst_perms;
  4003. uint32_t i = 0;
  4004. VERIFY(err, NULL != (dst_perms = kcalloc(rhvm->vmcount,
  4005. sizeof(struct qcom_scm_vmperm), GFP_KERNEL)));
  4006. if (err)
  4007. goto bail;
  4008. for (i = 0; i < rhvm->vmcount; i++) {
  4009. dst_perms[i].vmid = rhvm->vmid[i];
  4010. dst_perms[i].perm = rhvm->vmperm[i];
  4011. }
  4012. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4013. &src_perms, dst_perms, rhvm->vmcount);
  4014. kfree(dst_perms);
  4015. if (err) {
  4016. ADSPRPC_ERR(
  4017. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  4018. err, phys, size);
  4019. err = -EADDRNOTAVAIL;
  4020. goto bail;
  4021. }
  4022. rh_hyp_done = 1;
  4023. }
  4024. me->staticpd_flags = 1;
  4025. mem->is_persistent = true;
  4026. }
  4027. /*
  4028. * Prepare remote arguments for static process create
  4029. * call to remote subsystem.
  4030. */
  4031. ra[0].buf.pv = (void *)&inbuf;
  4032. ra[0].buf.len = sizeof(inbuf);
  4033. fds[0] = -1;
  4034. ra[1].buf.pv = (void *)proc_name;
  4035. ra[1].buf.len = inbuf.namelen;
  4036. fds[1] = -1;
  4037. pages[0].addr = phys;
  4038. pages[0].size = size;
  4039. ra[2].buf.pv = (void *)pages;
  4040. ra[2].buf.len = sizeof(*pages);
  4041. fds[2] = -1;
  4042. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4043. ioctl.inv.sc = REMOTE_SCALARS_MAKE(8, 3, 0);
  4044. ioctl.inv.pra = ra;
  4045. ioctl.fds = NULL;
  4046. ioctl.attrs = NULL;
  4047. ioctl.crc = NULL;
  4048. ioctl.perf_kernel = NULL;
  4049. ioctl.perf_dsp = NULL;
  4050. ioctl.job = NULL;
  4051. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4052. if (err)
  4053. goto bail;
  4054. bail:
  4055. kfree(proc_name);
  4056. if (err) {
  4057. me->staticpd_flags = 0;
  4058. if (rh_hyp_done) {
  4059. int hyp_err = 0;
  4060. u64 src_perms = 0;
  4061. struct qcom_scm_vmperm dst_perms;
  4062. uint32_t i = 0;
  4063. for (i = 0; i < rhvm->vmcount; i++) {
  4064. src_perms |= BIT(rhvm->vmid[i]);
  4065. }
  4066. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4067. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4068. /* Assign memory back to HLOS in case of errors */
  4069. hyp_err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4070. &src_perms, &dst_perms, 1);
  4071. if (hyp_err)
  4072. ADSPRPC_WARN(
  4073. "rh hyp unassign failed with %d for phys 0x%llx of size %zu\n",
  4074. hyp_err, phys, size);
  4075. }
  4076. mutex_lock(&fl->map_mutex);
  4077. fastrpc_mmap_free(mem, 0);
  4078. mutex_unlock(&fl->map_mutex);
  4079. }
  4080. return err;
  4081. }
  4082. /*
  4083. * This function sets fastrpc service location name
  4084. * based on ioctl init flags.
  4085. */
  4086. static void fastrpc_set_servloc(struct fastrpc_file *fl,
  4087. struct fastrpc_ioctl_init *init)
  4088. {
  4089. char *proc_name = NULL;
  4090. int err = 0;
  4091. if (init->flags == FASTRPC_INIT_ATTACH_SENSORS) {
  4092. if (fl->cid == ADSP_DOMAIN_ID)
  4093. fl->servloc_name =
  4094. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME;
  4095. else if (fl->cid == SDSP_DOMAIN_ID)
  4096. fl->servloc_name =
  4097. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME;
  4098. } else if (init->flags == FASTRPC_INIT_CREATE_STATIC) {
  4099. if (!init->filelen)
  4100. goto bail;
  4101. proc_name = kzalloc(init->filelen + 1, GFP_KERNEL);
  4102. VERIFY(err, !IS_ERR_OR_NULL(proc_name));
  4103. if (err) {
  4104. err = -ENOMEM;
  4105. goto bail;
  4106. }
  4107. err = copy_from_user((void *)proc_name,
  4108. (void __user *)init->file, init->filelen);
  4109. if (err) {
  4110. err = -EFAULT;
  4111. goto bail;
  4112. }
  4113. if (!strcmp(proc_name, "audiopd"))
  4114. fl->servloc_name = AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME;
  4115. }
  4116. bail:
  4117. kfree(proc_name);
  4118. }
  4119. int fastrpc_init_process(struct fastrpc_file *fl,
  4120. struct fastrpc_ioctl_init_attrs *uproc)
  4121. {
  4122. int err = 0;
  4123. struct fastrpc_ioctl_init *init = &uproc->init;
  4124. int cid = fl->cid;
  4125. struct fastrpc_apps *me = &gfa;
  4126. struct fastrpc_channel_ctx *chan = NULL;
  4127. VERIFY(err, init->filelen < INIT_FILELEN_MAX
  4128. && init->memlen <= INIT_MEMLEN_MAX_DYNAMIC);
  4129. if (err) {
  4130. ADSPRPC_ERR(
  4131. "file size 0x%x or init memory 0x%x is more than max allowed file size 0x%x or init len 0x%x\n",
  4132. init->filelen, init->memlen,
  4133. INIT_FILELEN_MAX, INIT_MEMLEN_MAX_DYNAMIC);
  4134. err = -EFBIG;
  4135. goto bail;
  4136. }
  4137. VERIFY(err, VALID_FASTRPC_CID(cid));
  4138. if (err) {
  4139. err = -ECHRNG;
  4140. goto bail;
  4141. }
  4142. chan = &me->channel[cid];
  4143. if (chan->unsigned_support && fl->dev_minor == MINOR_NUM_DEV) {
  4144. /* Make sure third party applications */
  4145. /* can spawn only unsigned PD when */
  4146. /* channel configured as secure. */
  4147. if (chan->secure && !(fl->is_unsigned_pd)) {
  4148. err = -ECONNREFUSED;
  4149. goto bail;
  4150. }
  4151. }
  4152. if (fl->sharedcb == 1) {
  4153. // Only attach sensors pd use cases can share CB
  4154. VERIFY(err, init->flags == FASTRPC_INIT_ATTACH_SENSORS);
  4155. if (err) {
  4156. err = -EACCES;
  4157. goto bail;
  4158. }
  4159. }
  4160. fastrpc_set_servloc(fl, init);
  4161. err = fastrpc_set_tvm_remote_domain(fl, init);
  4162. if (err)
  4163. goto bail;
  4164. err = fastrpc_channel_open(fl, init->flags);
  4165. if (err)
  4166. goto bail;
  4167. fl->proc_flags = init->flags;
  4168. switch (init->flags) {
  4169. case FASTRPC_INIT_ATTACH:
  4170. case FASTRPC_INIT_ATTACH_SENSORS:
  4171. err = fastrpc_init_attach_process(fl, init);
  4172. break;
  4173. case FASTRPC_INIT_CREATE:
  4174. err = fastrpc_init_create_dynamic_process(fl, uproc);
  4175. break;
  4176. case FASTRPC_INIT_CREATE_STATIC:
  4177. err = fastrpc_init_create_static_process(fl, init);
  4178. break;
  4179. default:
  4180. err = -ENOTTY;
  4181. break;
  4182. }
  4183. if (err)
  4184. goto bail;
  4185. fl->dsp_proc_init = 1;
  4186. VERIFY(err, 0 == (err = fastrpc_device_create(fl)));
  4187. if (err)
  4188. goto bail;
  4189. bail:
  4190. return err;
  4191. }
  4192. static int fastrpc_send_cpuinfo_to_dsp(struct fastrpc_file *fl)
  4193. {
  4194. int err = 0;
  4195. uint64_t cpuinfo = 0;
  4196. struct fastrpc_apps *me = &gfa;
  4197. struct fastrpc_ioctl_invoke_async ioctl;
  4198. remote_arg_t ra[1];
  4199. int cid = -1;
  4200. if (!fl) {
  4201. err = -EBADF;
  4202. goto bail;
  4203. }
  4204. cid = fl->cid;
  4205. VERIFY(err, VALID_FASTRPC_CID(cid));
  4206. if (err) {
  4207. err = -ECHRNG;
  4208. ADSPRPC_ERR(
  4209. "invalid channel 0x%zx set for session\n",
  4210. cid);
  4211. goto bail;
  4212. }
  4213. cpuinfo = me->channel[cid].cpuinfo_todsp;
  4214. /* return success if already updated to remote processor */
  4215. if (me->channel[cid].cpuinfo_status)
  4216. return 0;
  4217. ra[0].buf.pv = (void *)&cpuinfo;
  4218. ra[0].buf.len = sizeof(cpuinfo);
  4219. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
  4220. ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
  4221. ioctl.inv.pra = ra;
  4222. ioctl.fds = NULL;
  4223. ioctl.attrs = NULL;
  4224. ioctl.crc = NULL;
  4225. ioctl.perf_kernel = NULL;
  4226. ioctl.perf_dsp = NULL;
  4227. ioctl.job = NULL;
  4228. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4229. if (!err)
  4230. me->channel[cid].cpuinfo_status = true;
  4231. bail:
  4232. return err;
  4233. }
  4234. int fastrpc_get_info_from_dsp(struct fastrpc_file *fl,
  4235. uint32_t *dsp_attr_buf,
  4236. uint32_t dsp_attr_buf_len,
  4237. uint32_t domain)
  4238. {
  4239. int err = 0;
  4240. struct fastrpc_ioctl_invoke_async ioctl;
  4241. remote_arg_t ra[2];
  4242. dsp_attr_buf[0] = 0; // Capability filled in userspace
  4243. // Fastrpc to modem not supported
  4244. if (domain == MDSP_DOMAIN_ID)
  4245. goto bail;
  4246. err = fastrpc_channel_open(fl, FASTRPC_INIT_NO_CREATE);
  4247. if (err)
  4248. goto bail;
  4249. ra[0].buf.pv = (void *)&dsp_attr_buf_len;
  4250. ra[0].buf.len = sizeof(dsp_attr_buf_len);
  4251. ra[1].buf.pv = (void *)(&dsp_attr_buf[1]);
  4252. ra[1].buf.len = dsp_attr_buf_len * sizeof(uint32_t);
  4253. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
  4254. ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 1);
  4255. ioctl.inv.pra = ra;
  4256. ioctl.fds = NULL;
  4257. ioctl.attrs = NULL;
  4258. ioctl.crc = NULL;
  4259. ioctl.perf_kernel = NULL;
  4260. ioctl.perf_dsp = NULL;
  4261. ioctl.job = NULL;
  4262. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4263. bail:
  4264. if (err)
  4265. ADSPRPC_ERR("could not obtain dsp information, err val %d\n",
  4266. err);
  4267. return err;
  4268. }
  4269. int fastrpc_get_info_from_kernel(
  4270. struct fastrpc_ioctl_capability *cap,
  4271. struct fastrpc_file *fl)
  4272. {
  4273. int err = 0;
  4274. uint32_t domain = cap->domain, attribute_ID = cap->attribute_ID;
  4275. uint32_t async_capability = 0;
  4276. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  4277. VERIFY(err, domain < NUM_CHANNELS);
  4278. if (err) {
  4279. err = -ECHRNG;
  4280. goto bail;
  4281. }
  4282. /*
  4283. * Check if number of attribute IDs obtained from userspace
  4284. * is less than the number of attribute IDs supported by
  4285. * kernel
  4286. */
  4287. if (attribute_ID >= FASTRPC_MAX_ATTRIBUTES) {
  4288. err = -EOVERFLOW;
  4289. goto bail;
  4290. }
  4291. dsp_cap_ptr = &gcinfo[domain].dsp_cap_kernel;
  4292. if (attribute_ID >= FASTRPC_MAX_DSP_ATTRIBUTES) {
  4293. // Driver capability, pass it to user
  4294. memcpy(&cap->capability,
  4295. &kernel_capabilities[attribute_ID -
  4296. FASTRPC_MAX_DSP_ATTRIBUTES],
  4297. sizeof(cap->capability));
  4298. } else if (!dsp_cap_ptr->is_cached) {
  4299. /*
  4300. * Information not on kernel, query device for information
  4301. * and cache on kernel
  4302. */
  4303. err = fastrpc_get_info_from_dsp(fl,
  4304. dsp_cap_ptr->dsp_attributes,
  4305. FASTRPC_MAX_DSP_ATTRIBUTES - 1,
  4306. domain);
  4307. if (err)
  4308. goto bail;
  4309. /* Async capability support depends on both kernel and DSP */
  4310. async_capability = IS_ASYNC_FASTRPC_AVAILABLE &&
  4311. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP];
  4312. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP]
  4313. = async_capability;
  4314. memcpy(&cap->capability,
  4315. &dsp_cap_ptr->dsp_attributes[attribute_ID],
  4316. sizeof(cap->capability));
  4317. dsp_cap_ptr->is_cached = 1;
  4318. } else {
  4319. // Information on Kernel, pass it to user
  4320. memcpy(&cap->capability,
  4321. &dsp_cap_ptr->dsp_attributes[attribute_ID],
  4322. sizeof(cap->capability));
  4323. }
  4324. bail:
  4325. return err;
  4326. }
  4327. static int fastrpc_release_current_dsp_process(struct fastrpc_file *fl)
  4328. {
  4329. int err = 0;
  4330. struct fastrpc_ioctl_invoke_async ioctl;
  4331. remote_arg_t ra[1];
  4332. int tgid = 0;
  4333. int cid = -1;
  4334. unsigned long irq_flags = 0;
  4335. if (!fl) {
  4336. err = -EBADF;
  4337. goto bail;
  4338. }
  4339. cid = fl->cid;
  4340. VERIFY(err, VALID_FASTRPC_CID(cid));
  4341. if (err) {
  4342. err = -ECHRNG;
  4343. goto bail;
  4344. }
  4345. VERIFY(err, fl->sctx != NULL);
  4346. if (err) {
  4347. err = -EBADR;
  4348. goto bail;
  4349. }
  4350. err = verify_transport_device(cid, fl->tvm_remote_domain);
  4351. if (err)
  4352. goto bail;
  4353. VERIFY(err, fl->apps->channel[cid].subsystemstate != SUBSYSTEM_RESTARTING);
  4354. if (err) {
  4355. wait_for_completion(&fl->shutdown);
  4356. err = -ECONNRESET;
  4357. goto bail;
  4358. }
  4359. /* Send unique fastrpc process ID to dsp */
  4360. tgid = fl->tgid_frpc;
  4361. ra[0].buf.pv = (void *)&tgid;
  4362. ra[0].buf.len = sizeof(tgid);
  4363. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4364. ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
  4365. ioctl.inv.pra = ra;
  4366. ioctl.fds = NULL;
  4367. ioctl.attrs = NULL;
  4368. ioctl.crc = NULL;
  4369. ioctl.perf_kernel = NULL;
  4370. ioctl.perf_dsp = NULL;
  4371. ioctl.job = NULL;
  4372. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4373. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_INIT;
  4374. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4375. /*
  4376. * Pass 2 for "kernel" arg to send kernel msg to DSP
  4377. * with non-zero msg PID for the DSP to directly use
  4378. * that info to kill the remote process.
  4379. */
  4380. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4381. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_NONZERO_PID, &ioctl)));
  4382. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4383. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_COMPLETE;
  4384. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4385. if (err && fl->dsp_proc_init)
  4386. ADSPRPC_ERR(
  4387. "releasing DSP process failed with %d (0x%x) for %s\n",
  4388. err, err, current->comm);
  4389. bail:
  4390. if (err && fl && fl->apps) {
  4391. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4392. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_ERROR;
  4393. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4394. }
  4395. return err;
  4396. }
  4397. static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
  4398. uint32_t flags, uintptr_t va, uint64_t phys,
  4399. size_t size, uintptr_t *raddr)
  4400. {
  4401. struct fastrpc_ioctl_invoke_async ioctl;
  4402. struct smq_phy_page page;
  4403. remote_arg_t ra[4];
  4404. int err = 0;
  4405. struct {
  4406. int pid;
  4407. int fd;
  4408. int offset;
  4409. uint32_t flags;
  4410. uint64_t vaddrin;
  4411. int num;
  4412. int data_len;
  4413. } inargs;
  4414. struct {
  4415. uint64_t vaddrout;
  4416. } routargs;
  4417. /* Send unique fastrpc process ID to dsp */
  4418. inargs.pid = fl->tgid_frpc;
  4419. inargs.fd = fd;
  4420. inargs.offset = offset;
  4421. inargs.vaddrin = (uintptr_t)va;
  4422. inargs.flags = flags;
  4423. inargs.num = sizeof(page);
  4424. inargs.data_len = 0;
  4425. ra[0].buf.pv = (void *)&inargs;
  4426. ra[0].buf.len = sizeof(inargs);
  4427. page.addr = phys;
  4428. page.size = size;
  4429. ra[1].buf.pv = (void *)&page;
  4430. ra[1].buf.len = sizeof(page);
  4431. ra[2].buf.pv = (void *)&page;
  4432. ra[2].buf.len = 0;
  4433. ra[3].buf.pv = (void *)&routargs;
  4434. ra[3].buf.len = sizeof(routargs);
  4435. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4436. ioctl.inv.sc = REMOTE_SCALARS_MAKE(10, 3, 1);
  4437. ioctl.inv.pra = ra;
  4438. ioctl.fds = NULL;
  4439. ioctl.attrs = NULL;
  4440. ioctl.crc = NULL;
  4441. ioctl.perf_kernel = NULL;
  4442. ioctl.perf_dsp = NULL;
  4443. ioctl.job = NULL;
  4444. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4445. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4446. if (err)
  4447. goto bail;
  4448. if (raddr)
  4449. *raddr = (uintptr_t)routargs.vaddrout;
  4450. bail:
  4451. return err;
  4452. }
  4453. static int fastrpc_mem_unmap_to_dsp(struct fastrpc_file *fl, int fd,
  4454. uint32_t flags, uintptr_t va,
  4455. uint64_t phys, size_t size)
  4456. {
  4457. struct fastrpc_ioctl_invoke_async ioctl;
  4458. remote_arg_t ra[1];
  4459. int err = 0;
  4460. struct {
  4461. int pid;
  4462. int fd;
  4463. uint64_t vaddrin;
  4464. uint64_t len;
  4465. } inargs;
  4466. /* Send unique fastrpc process ID to dsp */
  4467. inargs.pid = fl->tgid_frpc;
  4468. inargs.fd = fd;
  4469. inargs.vaddrin = (uint64_t)va;
  4470. inargs.len = (uint64_t)size;
  4471. ra[0].buf.pv = (void *)&inargs;
  4472. ra[0].buf.len = sizeof(inargs);
  4473. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4474. ioctl.inv.sc = REMOTE_SCALARS_MAKE(11, 1, 0);
  4475. ioctl.inv.pra = ra;
  4476. ioctl.fds = NULL;
  4477. ioctl.attrs = NULL;
  4478. ioctl.crc = NULL;
  4479. ioctl.perf_kernel = NULL;
  4480. ioctl.perf_dsp = NULL;
  4481. ioctl.job = NULL;
  4482. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4483. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4484. if (err)
  4485. goto bail;
  4486. bail:
  4487. return err;
  4488. }
  4489. static int fastrpc_unmap_on_dsp(struct fastrpc_file *fl,
  4490. uintptr_t raddr, uint64_t phys, size_t size, uint32_t flags)
  4491. {
  4492. struct fastrpc_ioctl_invoke_async ioctl;
  4493. remote_arg_t ra[1] = {};
  4494. int err = 0;
  4495. struct {
  4496. int pid;
  4497. uintptr_t vaddrout;
  4498. size_t size;
  4499. } inargs;
  4500. /* Send unique fastrpc process ID to dsp */
  4501. inargs.pid = fl->tgid_frpc;
  4502. inargs.size = size;
  4503. inargs.vaddrout = raddr;
  4504. ra[0].buf.pv = (void *)&inargs;
  4505. ra[0].buf.len = sizeof(inargs);
  4506. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4507. if (fl->apps->compat)
  4508. ioctl.inv.sc = REMOTE_SCALARS_MAKE(5, 1, 0);
  4509. else
  4510. ioctl.inv.sc = REMOTE_SCALARS_MAKE(3, 1, 0);
  4511. ioctl.inv.pra = ra;
  4512. ioctl.fds = NULL;
  4513. ioctl.attrs = NULL;
  4514. ioctl.crc = NULL;
  4515. ioctl.perf_kernel = NULL;
  4516. ioctl.perf_dsp = NULL;
  4517. ioctl.job = NULL;
  4518. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4519. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4520. if (err)
  4521. goto bail;
  4522. bail:
  4523. return err;
  4524. }
  4525. static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags,
  4526. uintptr_t va, uint64_t phys,
  4527. size_t size, int refs, uintptr_t *raddr)
  4528. {
  4529. struct fastrpc_ioctl_invoke_async ioctl;
  4530. struct fastrpc_apps *me = &gfa;
  4531. struct smq_phy_page page;
  4532. int num = 1;
  4533. remote_arg_t ra[3];
  4534. int err = 0;
  4535. struct {
  4536. int pid;
  4537. uint32_t flags;
  4538. uintptr_t vaddrin;
  4539. int num;
  4540. } inargs;
  4541. struct {
  4542. uintptr_t vaddrout;
  4543. } routargs;
  4544. int cid = -1;
  4545. if (!fl) {
  4546. err = -EBADF;
  4547. goto bail;
  4548. }
  4549. cid = fl->cid;
  4550. /* Send unique fastrpc process ID to dsp */
  4551. inargs.pid = fl->tgid_frpc;
  4552. inargs.vaddrin = (uintptr_t)va;
  4553. inargs.flags = flags;
  4554. inargs.num = fl->apps->compat ? num * sizeof(page) : num;
  4555. ra[0].buf.pv = (void *)&inargs;
  4556. ra[0].buf.len = sizeof(inargs);
  4557. page.addr = phys;
  4558. page.size = size;
  4559. ra[1].buf.pv = (void *)&page;
  4560. ra[1].buf.len = num * sizeof(page);
  4561. ra[2].buf.pv = (void *)&routargs;
  4562. ra[2].buf.len = sizeof(routargs);
  4563. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4564. if (fl->apps->compat)
  4565. ioctl.inv.sc = REMOTE_SCALARS_MAKE(4, 2, 1);
  4566. else
  4567. ioctl.inv.sc = REMOTE_SCALARS_MAKE(2, 2, 1);
  4568. ioctl.inv.pra = ra;
  4569. ioctl.fds = NULL;
  4570. ioctl.attrs = NULL;
  4571. ioctl.crc = NULL;
  4572. ioctl.perf_kernel = NULL;
  4573. ioctl.perf_dsp = NULL;
  4574. ioctl.job = NULL;
  4575. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4576. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4577. *raddr = (uintptr_t)routargs.vaddrout;
  4578. if (err)
  4579. goto bail;
  4580. if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4581. VERIFY(err, VALID_FASTRPC_CID(cid));
  4582. if (err) {
  4583. err = -ECHRNG;
  4584. ADSPRPC_ERR(
  4585. "invalid channel 0x%zx set for session\n",
  4586. cid);
  4587. goto bail;
  4588. }
  4589. }
  4590. if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR
  4591. && me->channel[cid].rhvm.vmid && refs == 1) {
  4592. struct secure_vm *rhvm = &me->channel[cid].rhvm;
  4593. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  4594. struct qcom_scm_vmperm *dst_perms;
  4595. uint32_t i = 0;
  4596. VERIFY(err, NULL != (dst_perms = kcalloc(rhvm->vmcount,
  4597. sizeof(struct qcom_scm_vmperm), GFP_KERNEL)));
  4598. if (err)
  4599. goto bail;
  4600. for (i = 0; i < rhvm->vmcount; i++) {
  4601. dst_perms[i].vmid = rhvm->vmid[i];
  4602. dst_perms[i].perm = rhvm->vmperm[i];
  4603. }
  4604. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4605. &src_perms, dst_perms, rhvm->vmcount);
  4606. kfree(dst_perms);
  4607. if (err) {
  4608. int unmap_err = 0;
  4609. ADSPRPC_ERR(
  4610. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  4611. err, phys, size);
  4612. err = -EADDRNOTAVAIL;
  4613. unmap_err = fastrpc_unmap_on_dsp(fl,
  4614. *raddr, phys, size, flags);
  4615. if (unmap_err) {
  4616. ADSPRPC_ERR(
  4617. "failed to unmap %d for phys 0x%llx, size %zd\n",
  4618. unmap_err, phys, size);
  4619. }
  4620. goto bail;
  4621. }
  4622. }
  4623. bail:
  4624. return err;
  4625. }
  4626. static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl, uint64_t phys,
  4627. size_t size, uint32_t flags, int locked)
  4628. {
  4629. int err = 0;
  4630. int tgid = 0;
  4631. struct fastrpc_apps *me = &gfa;
  4632. int cid = -1;
  4633. struct fastrpc_ioctl_invoke_async ioctl;
  4634. remote_arg_t ra[2];
  4635. struct {
  4636. uint8_t skey;
  4637. } routargs;
  4638. if (!fl) {
  4639. err = -EBADF;
  4640. goto bail;
  4641. }
  4642. cid = fl->cid;
  4643. VERIFY(err, VALID_FASTRPC_CID(cid));
  4644. if (err) {
  4645. err = -ECHRNG;
  4646. ADSPRPC_ERR(
  4647. "invalid channel 0x%zx set for session\n",
  4648. cid);
  4649. goto bail;
  4650. }
  4651. /* Send unique fastrpc process ID to dsp */
  4652. tgid = fl->tgid_frpc;
  4653. ra[0].buf.pv = (void *)&tgid;
  4654. ra[0].buf.len = sizeof(tgid);
  4655. ra[1].buf.pv = (void *)&routargs;
  4656. ra[1].buf.len = sizeof(routargs);
  4657. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4658. ioctl.inv.sc = REMOTE_SCALARS_MAKE(9, 1, 1);
  4659. ioctl.inv.pra = ra;
  4660. ioctl.fds = NULL;
  4661. ioctl.attrs = NULL;
  4662. ioctl.crc = NULL;
  4663. ioctl.perf_kernel = NULL;
  4664. ioctl.perf_dsp = NULL;
  4665. ioctl.job = NULL;
  4666. if (locked) {
  4667. mutex_unlock(&fl->map_mutex);
  4668. mutex_unlock(&me->channel[cid].smd_mutex);
  4669. }
  4670. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4671. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4672. if (locked) {
  4673. mutex_lock(&me->channel[cid].smd_mutex);
  4674. mutex_lock(&fl->map_mutex);
  4675. }
  4676. if (err)
  4677. goto bail;
  4678. bail:
  4679. return err;
  4680. }
  4681. static int fastrpc_munmap_rh(uint64_t phys, size_t size,
  4682. uint32_t flags)
  4683. {
  4684. int err = 0;
  4685. struct fastrpc_apps *me = &gfa;
  4686. struct secure_vm *rhvm = &me->channel[RH_CID].rhvm;
  4687. if ((rhvm->vmid)
  4688. && (me->channel[RH_CID].in_hib == 0)) {
  4689. u64 src_perms = 0;
  4690. struct qcom_scm_vmperm dst_perms = {0};
  4691. uint32_t i = 0;
  4692. for (i = 0; i < rhvm->vmcount; i++) {
  4693. src_perms |= BIT(rhvm->vmid[i]);
  4694. }
  4695. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4696. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4697. err = qcom_scm_assign_mem(phys,
  4698. (uint64_t)size, &src_perms, &dst_perms, 1);
  4699. if (err) {
  4700. ADSPRPC_ERR(
  4701. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  4702. err, phys, size);
  4703. err = -EADDRNOTAVAIL;
  4704. return err;
  4705. }
  4706. }
  4707. return err;
  4708. }
  4709. static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, uintptr_t raddr,
  4710. uint64_t phys, size_t size, uint32_t flags)
  4711. {
  4712. int err = 0;
  4713. VERIFY(err, 0 == (err = fastrpc_unmap_on_dsp(fl, raddr, phys,
  4714. size, flags)));
  4715. if (err)
  4716. goto bail;
  4717. if (flags == ADSP_MMAP_HEAP_ADDR) {
  4718. VERIFY(err, !(err = fastrpc_munmap_on_dsp_rh(fl, phys,
  4719. size, flags, 0)));
  4720. if (err)
  4721. goto bail;
  4722. } else if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4723. VERIFY(err, !(err = fastrpc_munmap_rh(phys,
  4724. size, flags)));
  4725. if (err)
  4726. goto bail;
  4727. }
  4728. bail:
  4729. return err;
  4730. }
  4731. static int fastrpc_mmap_remove_ssr(struct fastrpc_file *fl, int locked)
  4732. {
  4733. struct fastrpc_mmap *match = NULL, *map = NULL;
  4734. struct hlist_node *n = NULL;
  4735. int err = 0, ret = 0, lock = 0;
  4736. struct fastrpc_apps *me = &gfa;
  4737. struct qcom_dump_segment ramdump_segments_rh;
  4738. struct list_head head;
  4739. unsigned long irq_flags = 0;
  4740. INIT_LIST_HEAD(&head);
  4741. if (fl) {
  4742. VERIFY(err, fl->cid == RH_CID);
  4743. if (err) {
  4744. err = -EBADR;
  4745. goto bail;
  4746. }
  4747. }
  4748. spin_lock_irqsave(&me->hlock, irq_flags);
  4749. lock = 1;
  4750. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  4751. if (!lock) {
  4752. spin_lock_irqsave(&me->hlock, irq_flags);
  4753. lock = 1;
  4754. }
  4755. /* In hibernation suspend case fl is NULL, check !fl to cleanup */
  4756. if (!fl || (fl && map->servloc_name && fl->servloc_name
  4757. && !strcmp(map->servloc_name, fl->servloc_name))) {
  4758. match = map;
  4759. if (map->is_persistent && map->in_use) {
  4760. struct secure_vm *rhvm = &me->channel[RH_CID].rhvm;
  4761. uint64_t phys = map->phys;
  4762. size_t size = map->size;
  4763. if (lock) {
  4764. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4765. lock = 0;
  4766. }
  4767. //scm assign it back to HLOS
  4768. if (rhvm->vmid) {
  4769. u64 src_perms = 0;
  4770. struct qcom_scm_vmperm dst_perms = {0};
  4771. uint32_t i = 0;
  4772. for (i = 0; i < rhvm->vmcount; i++) {
  4773. src_perms |= BIT(rhvm->vmid[i]);
  4774. }
  4775. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4776. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4777. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4778. &src_perms, &dst_perms, 1);
  4779. }
  4780. if (err) {
  4781. ADSPRPC_ERR(
  4782. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  4783. err, phys, size);
  4784. err = -EADDRNOTAVAIL;
  4785. goto bail;
  4786. }
  4787. if (!lock) {
  4788. spin_lock_irqsave(&me->hlock, irq_flags);
  4789. lock = 1;
  4790. }
  4791. map->in_use = false;
  4792. /*
  4793. * decrementing refcount for persistent mappings
  4794. * as incrementing it in fastrpc_get_persistent_map
  4795. */
  4796. map->refs--;
  4797. }
  4798. if (!match->is_persistent)
  4799. hlist_del_init(&map->hn);
  4800. }
  4801. if (lock) {
  4802. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4803. lock = 0;
  4804. }
  4805. if (match) {
  4806. if (!match->is_persistent) {
  4807. if (match->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4808. err = fastrpc_munmap_rh(match->phys,
  4809. match->size, match->flags);
  4810. } else if (match->flags == ADSP_MMAP_HEAP_ADDR) {
  4811. if (fl)
  4812. err = fastrpc_munmap_on_dsp_rh(fl, match->phys,
  4813. match->size, match->flags, 0);
  4814. else {
  4815. pr_err("Cannot communicate with DSP, ADSP is down\n");
  4816. fastrpc_mmap_add(match);
  4817. }
  4818. }
  4819. }
  4820. memset(&ramdump_segments_rh, 0, sizeof(ramdump_segments_rh));
  4821. ramdump_segments_rh.da = match->phys;
  4822. ramdump_segments_rh.va = (void *)page_address((struct page *)match->va);
  4823. ramdump_segments_rh.size = match->size;
  4824. INIT_LIST_HEAD(&head);
  4825. list_add(&ramdump_segments_rh.node, &head);
  4826. if (me->dev && dump_enabled()) {
  4827. ret = qcom_elf_dump(&head, me->dev, ELF_CLASS);
  4828. if (ret < 0)
  4829. pr_err("adsprpc: %s: unable to dump heap (err %d)\n",
  4830. __func__, ret);
  4831. }
  4832. if (!match->is_persistent) {
  4833. if (!locked)
  4834. mutex_lock(&fl->map_mutex);
  4835. fastrpc_mmap_free(match, 0);
  4836. if (!locked)
  4837. mutex_unlock(&fl->map_mutex);
  4838. }
  4839. }
  4840. }
  4841. bail:
  4842. if (lock) {
  4843. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4844. lock = 0;
  4845. }
  4846. if (err && match) {
  4847. if (!locked)
  4848. mutex_lock(&fl->map_mutex);
  4849. fastrpc_mmap_add(match);
  4850. if (!locked)
  4851. mutex_unlock(&fl->map_mutex);
  4852. }
  4853. return err;
  4854. }
  4855. static int fastrpc_mmap_remove_pdr(struct fastrpc_file *fl)
  4856. {
  4857. struct fastrpc_apps *me = &gfa;
  4858. int session = 0, err = 0, cid = -1;
  4859. if (!fl) {
  4860. err = -EBADF;
  4861. goto bail;
  4862. }
  4863. err = fastrpc_get_spd_session(fl->servloc_name,
  4864. &session, &cid);
  4865. if (err)
  4866. goto bail;
  4867. VERIFY(err, cid == fl->cid);
  4868. if (err) {
  4869. err = -EBADR;
  4870. goto bail;
  4871. }
  4872. if (atomic_read(&me->channel[cid].spd[session].ispdup) == 0) {
  4873. err = -ENOTCONN;
  4874. goto bail;
  4875. }
  4876. if (me->channel[cid].spd[session].pdrcount !=
  4877. me->channel[cid].spd[session].prevpdrcount) {
  4878. err = fastrpc_mmap_remove_ssr(fl, 0);
  4879. if (err)
  4880. ADSPRPC_WARN("failed to unmap remote heap (err %d)\n",
  4881. err);
  4882. me->channel[cid].spd[session].prevpdrcount =
  4883. me->channel[cid].spd[session].pdrcount;
  4884. }
  4885. bail:
  4886. return err;
  4887. }
  4888. static inline void get_fastrpc_ioctl_mmap_64(
  4889. struct fastrpc_ioctl_mmap_64 *mmap64,
  4890. struct fastrpc_ioctl_mmap *immap)
  4891. {
  4892. immap->fd = mmap64->fd;
  4893. immap->flags = mmap64->flags;
  4894. immap->vaddrin = (uintptr_t)mmap64->vaddrin;
  4895. immap->size = mmap64->size;
  4896. }
  4897. static inline void put_fastrpc_ioctl_mmap_64(
  4898. struct fastrpc_ioctl_mmap_64 *mmap64,
  4899. struct fastrpc_ioctl_mmap *immap)
  4900. {
  4901. mmap64->vaddrout = (uint64_t)immap->vaddrout;
  4902. }
  4903. static inline void get_fastrpc_ioctl_munmap_64(
  4904. struct fastrpc_ioctl_munmap_64 *munmap64,
  4905. struct fastrpc_ioctl_munmap *imunmap)
  4906. {
  4907. imunmap->vaddrout = (uintptr_t)munmap64->vaddrout;
  4908. imunmap->size = munmap64->size;
  4909. }
  4910. int fastrpc_internal_munmap(struct fastrpc_file *fl,
  4911. struct fastrpc_ioctl_munmap *ud)
  4912. {
  4913. int err = 0;
  4914. struct fastrpc_mmap *map = NULL;
  4915. struct fastrpc_buf *rbuf = NULL, *free = NULL;
  4916. struct hlist_node *n;
  4917. VERIFY(err, fl->dsp_proc_init == 1);
  4918. if (err) {
  4919. ADSPRPC_ERR(
  4920. "user application %s trying to unmap without initialization\n",
  4921. current->comm);
  4922. err = -EHOSTDOWN;
  4923. return err;
  4924. }
  4925. mutex_lock(&fl->internal_map_mutex);
  4926. spin_lock(&fl->hlock);
  4927. hlist_for_each_entry_safe(rbuf, n, &fl->remote_bufs, hn_rem) {
  4928. if (rbuf->raddr && ((rbuf->flags == ADSP_MMAP_ADD_PAGES) ||
  4929. (rbuf->flags == ADSP_MMAP_ADD_PAGES_LLC))) {
  4930. if ((rbuf->raddr == ud->vaddrout) &&
  4931. (rbuf->size == ud->size)) {
  4932. free = rbuf;
  4933. break;
  4934. }
  4935. }
  4936. }
  4937. spin_unlock(&fl->hlock);
  4938. if (free) {
  4939. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, free->raddr,
  4940. free->phys, free->size, free->flags)));
  4941. if (err)
  4942. goto bail;
  4943. fastrpc_buf_free(rbuf, 0);
  4944. mutex_unlock(&fl->internal_map_mutex);
  4945. return err;
  4946. }
  4947. mutex_lock(&fl->map_mutex);
  4948. VERIFY(err, !(err = fastrpc_mmap_remove(fl, -1, ud->vaddrout,
  4949. ud->size, &map)));
  4950. mutex_unlock(&fl->map_mutex);
  4951. if (err)
  4952. goto bail;
  4953. VERIFY(err, map != NULL);
  4954. if (err) {
  4955. err = -EINVAL;
  4956. goto bail;
  4957. }
  4958. if (!map->is_persistent) {
  4959. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, map->raddr,
  4960. map->phys, map->size, map->flags)));
  4961. }
  4962. if (err)
  4963. goto bail;
  4964. mutex_lock(&fl->map_mutex);
  4965. fastrpc_mmap_free(map, 0);
  4966. mutex_unlock(&fl->map_mutex);
  4967. bail:
  4968. if (err && map) {
  4969. mutex_lock(&fl->map_mutex);
  4970. fastrpc_mmap_add(map);
  4971. mutex_unlock(&fl->map_mutex);
  4972. }
  4973. mutex_unlock(&fl->internal_map_mutex);
  4974. return err;
  4975. }
  4976. /*
  4977. * fastrpc_internal_munmap_fd can only be used for buffers
  4978. * mapped with persist attributes. This can only be called
  4979. * once for any persist buffer
  4980. */
  4981. static int fastrpc_internal_munmap_fd(struct fastrpc_file *fl,
  4982. struct fastrpc_ioctl_munmap_fd *ud)
  4983. {
  4984. int err = 0;
  4985. struct fastrpc_mmap *map = NULL;
  4986. VERIFY(err, (fl && ud));
  4987. if (err) {
  4988. err = -EINVAL;
  4989. return err;
  4990. }
  4991. VERIFY(err, fl->dsp_proc_init == 1);
  4992. if (err) {
  4993. ADSPRPC_ERR(
  4994. "user application %s trying to unmap without initialization\n",
  4995. current->comm);
  4996. err = -EHOSTDOWN;
  4997. return err;
  4998. }
  4999. mutex_lock(&fl->internal_map_mutex);
  5000. mutex_lock(&fl->map_mutex);
  5001. err = fastrpc_mmap_find(fl, ud->fd, NULL, ud->va, ud->len, 0, 0, &map);
  5002. if (err) {
  5003. ADSPRPC_ERR(
  5004. "mapping not found to unmap fd 0x%x, va 0x%llx, len 0x%x, err %d\n",
  5005. ud->fd, (unsigned long long)ud->va,
  5006. (unsigned int)ud->len, err);
  5007. mutex_unlock(&fl->map_mutex);
  5008. goto bail;
  5009. }
  5010. if (map && (map->attr & FASTRPC_ATTR_KEEP_MAP)) {
  5011. map->attr = map->attr & (~FASTRPC_ATTR_KEEP_MAP);
  5012. fastrpc_mmap_free(map, 0);
  5013. }
  5014. mutex_unlock(&fl->map_mutex);
  5015. bail:
  5016. mutex_unlock(&fl->internal_map_mutex);
  5017. return err;
  5018. }
  5019. int fastrpc_internal_mem_map(struct fastrpc_file *fl,
  5020. struct fastrpc_ioctl_mem_map *ud)
  5021. {
  5022. int err = 0;
  5023. struct fastrpc_mmap *map = NULL;
  5024. mutex_lock(&fl->internal_map_mutex);
  5025. VERIFY(err, fl->dsp_proc_init == 1);
  5026. if (err) {
  5027. pr_err("adsprpc: ERROR: %s: user application %s trying to map without initialization\n",
  5028. __func__, current->comm);
  5029. err = EBADR;
  5030. goto bail;
  5031. }
  5032. /* create SMMU mapping */
  5033. mutex_lock(&fl->map_mutex);
  5034. VERIFY(err, !(err = fastrpc_mmap_create(fl, ud->m.fd, NULL, ud->m.attrs,
  5035. ud->m.vaddrin, ud->m.length,
  5036. ud->m.flags, &map)));
  5037. mutex_unlock(&fl->map_mutex);
  5038. if (err)
  5039. goto bail;
  5040. if (map->raddr) {
  5041. err = -EEXIST;
  5042. goto bail;
  5043. }
  5044. /* create DSP mapping */
  5045. VERIFY(err, !(err = fastrpc_mem_map_to_dsp(fl, ud->m.fd, ud->m.offset,
  5046. ud->m.flags, map->va, map->phys, map->size, &map->raddr)));
  5047. if (err)
  5048. goto bail;
  5049. ud->m.vaddrout = map->raddr;
  5050. bail:
  5051. if (err) {
  5052. ADSPRPC_ERR("failed to map fd %d, len 0x%x, flags %d, map %pK, err %d\n",
  5053. ud->m.fd, ud->m.length, ud->m.flags, map, err);
  5054. if (map) {
  5055. mutex_lock(&fl->map_mutex);
  5056. fastrpc_mmap_free(map, 0);
  5057. mutex_unlock(&fl->map_mutex);
  5058. }
  5059. }
  5060. mutex_unlock(&fl->internal_map_mutex);
  5061. return err;
  5062. }
  5063. int fastrpc_internal_mem_unmap(struct fastrpc_file *fl,
  5064. struct fastrpc_ioctl_mem_unmap *ud)
  5065. {
  5066. int err = 0;
  5067. struct fastrpc_mmap *map = NULL;
  5068. size_t map_size = 0;
  5069. mutex_lock(&fl->internal_map_mutex);
  5070. VERIFY(err, fl->dsp_proc_init == 1);
  5071. if (err) {
  5072. pr_err("adsprpc: ERROR: %s: user application %s trying to map without initialization\n",
  5073. __func__, current->comm);
  5074. err = EBADR;
  5075. goto bail;
  5076. }
  5077. mutex_lock(&fl->map_mutex);
  5078. VERIFY(err, !(err = fastrpc_mmap_remove(fl, ud->um.fd,
  5079. (uintptr_t)ud->um.vaddr, ud->um.length, &map)));
  5080. mutex_unlock(&fl->map_mutex);
  5081. if (err)
  5082. goto bail;
  5083. VERIFY(err, map->flags == FASTRPC_MAP_FD ||
  5084. map->flags == FASTRPC_MAP_FD_DELAYED ||
  5085. map->flags == FASTRPC_MAP_STATIC);
  5086. if (err) {
  5087. err = -EBADMSG;
  5088. goto bail;
  5089. }
  5090. map_size = map->size;
  5091. /* remove mapping on DSP */
  5092. VERIFY(err, !(err = fastrpc_mem_unmap_to_dsp(fl, map->fd, map->flags,
  5093. map->raddr, map->phys, map->size)));
  5094. if (err)
  5095. goto bail;
  5096. /* remove SMMU mapping */
  5097. mutex_lock(&fl->map_mutex);
  5098. fastrpc_mmap_free(map, 0);
  5099. mutex_unlock(&fl->map_mutex);
  5100. map = NULL;
  5101. bail:
  5102. if (err) {
  5103. ADSPRPC_ERR(
  5104. "failed to unmap fd %d addr 0x%llx length %zu map size %zu err 0x%x\n",
  5105. ud->um.fd, ud->um.vaddr, ud->um.length, map_size, err);
  5106. /* Add back to map list in case of error to unmap on DSP */
  5107. if (map) {
  5108. mutex_lock(&fl->map_mutex);
  5109. fastrpc_mmap_add(map);
  5110. mutex_unlock(&fl->map_mutex);
  5111. }
  5112. }
  5113. mutex_unlock(&fl->internal_map_mutex);
  5114. return err;
  5115. }
  5116. int fastrpc_internal_mmap(struct fastrpc_file *fl,
  5117. struct fastrpc_ioctl_mmap *ud)
  5118. {
  5119. struct fastrpc_mmap *map = NULL;
  5120. struct fastrpc_buf *rbuf = NULL;
  5121. unsigned long dma_attr = 0;
  5122. uintptr_t raddr = 0;
  5123. int err = 0;
  5124. VERIFY(err, fl->dsp_proc_init == 1);
  5125. if (err) {
  5126. ADSPRPC_ERR(
  5127. "user application %s trying to map without initialization\n",
  5128. current->comm);
  5129. err = -EHOSTDOWN;
  5130. return err;
  5131. }
  5132. mutex_lock(&fl->internal_map_mutex);
  5133. /* Pages for unsigned PD's user-heap should be allocated in userspace */
  5134. if (((ud->flags == ADSP_MMAP_ADD_PAGES) ||
  5135. (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)) && !fl->is_unsigned_pd) {
  5136. if (ud->vaddrin) {
  5137. err = -EINVAL;
  5138. ADSPRPC_ERR(
  5139. "adding user allocated pages is not supported\n");
  5140. goto bail;
  5141. }
  5142. dma_attr = DMA_ATTR_DELAYED_UNMAP | DMA_ATTR_NO_KERNEL_MAPPING;
  5143. if (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)
  5144. dma_attr |= DMA_ATTR_SYS_CACHE_ONLY;
  5145. err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,
  5146. USERHEAP_BUF, &rbuf);
  5147. if (err)
  5148. goto bail;
  5149. err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,
  5150. rbuf->phys, rbuf->size, 0, &raddr);
  5151. if (err)
  5152. goto bail;
  5153. rbuf->raddr = raddr;
  5154. } else {
  5155. uintptr_t va_to_dsp;
  5156. if (fl->is_unsigned_pd && ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  5157. err = -EINVAL;
  5158. ADSPRPC_ERR(
  5159. "Secure memory allocation is not supported in unsigned PD");
  5160. goto bail;
  5161. }
  5162. mutex_lock(&fl->map_mutex);
  5163. VERIFY(err, !(err = fastrpc_mmap_create(fl, ud->fd, NULL, 0,
  5164. (uintptr_t)ud->vaddrin, ud->size,
  5165. ud->flags, &map)));
  5166. mutex_unlock(&fl->map_mutex);
  5167. if (err)
  5168. goto bail;
  5169. if (ud->flags == ADSP_MMAP_HEAP_ADDR ||
  5170. ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
  5171. va_to_dsp = 0;
  5172. else
  5173. va_to_dsp = (uintptr_t)map->va;
  5174. VERIFY(err, 0 == (err = fastrpc_mmap_on_dsp(fl, ud->flags,
  5175. va_to_dsp, map->phys, map->size, map->refs, &raddr)));
  5176. if (err)
  5177. goto bail;
  5178. map->raddr = raddr;
  5179. }
  5180. ud->vaddrout = raddr;
  5181. bail:
  5182. if (err) {
  5183. if (map) {
  5184. mutex_lock(&fl->map_mutex);
  5185. fastrpc_mmap_free(map, 0);
  5186. mutex_unlock(&fl->map_mutex);
  5187. }
  5188. if (!IS_ERR_OR_NULL(rbuf))
  5189. fastrpc_buf_free(rbuf, 0);
  5190. }
  5191. mutex_unlock(&fl->internal_map_mutex);
  5192. return err;
  5193. }
  5194. static void fastrpc_context_list_dtor(struct fastrpc_file *fl);
  5195. static int fastrpc_session_alloc_locked(struct fastrpc_channel_ctx *chan,
  5196. int secure, int sharedcb, int pd_type, struct fastrpc_session_ctx **session)
  5197. {
  5198. struct fastrpc_apps *me = &gfa;
  5199. uint64_t idx = 0;
  5200. int err = 0;
  5201. /*
  5202. * PD type can be either unused(DEFAULT_UNUSED) (or) if PD type
  5203. * is used, choose the context bank with matching PD type.
  5204. */
  5205. if (chan->sesscount) {
  5206. for (idx = 0; idx < chan->sesscount; ++idx) {
  5207. if (!chan->session[idx].used &&
  5208. chan->session[idx].smmu.secure == secure &&
  5209. chan->session[idx].smmu.sharedcb == sharedcb &&
  5210. (pd_type == DEFAULT_UNUSED ||
  5211. chan->session[idx].smmu.pd_type == pd_type)) {
  5212. chan->session[idx].used = 1;
  5213. break;
  5214. }
  5215. }
  5216. if (idx >= chan->sesscount) {
  5217. err = -EUSERS;
  5218. goto bail;
  5219. }
  5220. chan->session[idx].smmu.faults = 0;
  5221. } else {
  5222. VERIFY(err, me->dev != NULL);
  5223. if (err) {
  5224. err = -ENODEV;
  5225. goto bail;
  5226. }
  5227. chan->session[0].dev = me->dev;
  5228. chan->session[0].smmu.dev = me->dev;
  5229. }
  5230. *session = &chan->session[idx];
  5231. bail:
  5232. return err;
  5233. }
  5234. static void handle_remote_signal(uint64_t msg, int cid)
  5235. {
  5236. struct fastrpc_apps *me = &gfa;
  5237. uint32_t pid = msg >> 32;
  5238. uint32_t signal_id = msg & 0xffffffff;
  5239. struct fastrpc_file *fl = NULL;
  5240. struct hlist_node *n = NULL;
  5241. unsigned long irq_flags = 0;
  5242. DSPSIGNAL_VERBOSE("Received queue signal %llx: PID %u, signal %u\n", msg, pid, signal_id);
  5243. if (signal_id >= DSPSIGNAL_NUM_SIGNALS) {
  5244. ADSPRPC_ERR("Received bad signal %u for PID %u\n", signal_id, pid);
  5245. return;
  5246. }
  5247. spin_lock_irqsave(&me->hlock, irq_flags);
  5248. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  5249. /* Response from DSP contains unique fastrpc process id, use unique fastrpc process ID to compare */
  5250. if ((fl->tgid_frpc == pid) && (fl->cid == cid)) {
  5251. unsigned long fflags = 0;
  5252. spin_lock_irqsave(&fl->dspsignals_lock, fflags);
  5253. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE]) {
  5254. struct fastrpc_dspsignal *group =
  5255. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  5256. struct fastrpc_dspsignal *sig =
  5257. &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  5258. if ((sig->state == DSPSIGNAL_STATE_PENDING) ||
  5259. (sig->state == DSPSIGNAL_STATE_SIGNALED)) {
  5260. DSPSIGNAL_VERBOSE("Signaling signal %u for PID %u\n",
  5261. signal_id, pid);
  5262. trace_fastrpc_dspsignal("complete", signal_id, sig->state, 0);
  5263. complete(&sig->comp);
  5264. sig->state = DSPSIGNAL_STATE_SIGNALED;
  5265. } else if (sig->state == DSPSIGNAL_STATE_UNUSED) {
  5266. ADSPRPC_ERR("Received unknown signal %u for PID %u\n",
  5267. signal_id, pid);
  5268. }
  5269. } else {
  5270. ADSPRPC_ERR("Received unknown signal %u for PID %u\n",
  5271. signal_id, pid);
  5272. }
  5273. spin_unlock_irqrestore(&fl->dspsignals_lock, fflags);
  5274. break;
  5275. }
  5276. }
  5277. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5278. }
  5279. int fastrpc_handle_rpc_response(void *data, int len, int cid)
  5280. {
  5281. struct smq_invoke_rsp *rsp = (struct smq_invoke_rsp *)data;
  5282. struct smq_notif_rspv3 *notif = (struct smq_notif_rspv3 *)data;
  5283. struct smq_invoke_rspv2 *rspv2 = NULL;
  5284. struct smq_invoke_ctx *ctx = NULL;
  5285. struct fastrpc_apps *me = &gfa;
  5286. uint32_t index, rsp_flags = 0, early_wake_time = 0, ver = 0;
  5287. int err = 0, ignore_rsp_err = 0;
  5288. struct fastrpc_channel_ctx *chan = NULL;
  5289. unsigned long irq_flags = 0;
  5290. int64_t ns = 0;
  5291. uint64_t xo_time_in_us = 0, dspsig_msg = 0;
  5292. xo_time_in_us = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  5293. if (len == sizeof(uint64_t)) {
  5294. /*
  5295. * dspsignal message from the DSP
  5296. */
  5297. dspsig_msg = *((uint64_t *)data);
  5298. trace_fastrpc_transport_response(cid, dspsig_msg, 0, 0, 0);
  5299. handle_remote_signal(dspsig_msg, cid);
  5300. goto bail;
  5301. }
  5302. chan = &me->channel[cid];
  5303. VERIFY(err, (rsp && len >= sizeof(*rsp)));
  5304. if (err) {
  5305. err = -EINVAL;
  5306. goto bail;
  5307. }
  5308. if (notif->ctx == FASTRPC_NOTIF_CTX_RESERVED) {
  5309. VERIFY(err, (notif->type == STATUS_RESPONSE &&
  5310. len >= sizeof(*notif)));
  5311. if (err)
  5312. goto bail;
  5313. fastrpc_notif_find_process(cid, notif);
  5314. goto bail;
  5315. }
  5316. if (len >= sizeof(struct smq_invoke_rspv2))
  5317. rspv2 = (struct smq_invoke_rspv2 *)data;
  5318. if (rspv2) {
  5319. early_wake_time = rspv2->early_wake_time;
  5320. rsp_flags = rspv2->flags;
  5321. ver = rspv2->version;
  5322. }
  5323. trace_fastrpc_transport_response(cid, rsp->ctx,
  5324. rsp->retval, rsp_flags, early_wake_time);
  5325. ns = get_timestamp_in_ns();
  5326. fastrpc_update_rxmsg_buf(chan, rsp->ctx, rsp->retval,
  5327. rsp_flags, early_wake_time, ver, ns, xo_time_in_us);
  5328. index = (uint32_t)GET_TABLE_IDX_FROM_CTXID(rsp->ctx);
  5329. VERIFY(err, index < FASTRPC_CTX_MAX);
  5330. if (err)
  5331. goto bail;
  5332. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  5333. ctx = chan->ctxtable[index];
  5334. VERIFY(err, !IS_ERR_OR_NULL(ctx) &&
  5335. (ctx->ctxid == GET_CTXID_FROM_RSP_CTX(rsp->ctx)) &&
  5336. ctx->magic == FASTRPC_CTX_MAGIC);
  5337. if (err) {
  5338. /*
  5339. * Received an anticipatory COMPLETE_SIGNAL from DSP for a
  5340. * context after CPU successfully polling on memory and
  5341. * completed processing of context. Ignore the message.
  5342. * Also ignore response for a call which was already
  5343. * completed by update of poll memory and the context was
  5344. * removed from the table and possibly reused for another call.
  5345. */
  5346. ignore_rsp_err = ((rsp_flags == COMPLETE_SIGNAL) || !ctx ||
  5347. (ctx && (ctx->ctxid != GET_CTXID_FROM_RSP_CTX(rsp->ctx)))) ? 1 : 0;
  5348. goto bail_unlock;
  5349. }
  5350. if (rspv2) {
  5351. VERIFY(err, rspv2->version == FASTRPC_RSP_VERSION2);
  5352. if (err)
  5353. goto bail_unlock;
  5354. }
  5355. VERIFY(err, VALID_FASTRPC_CID(ctx->fl->cid));
  5356. if (err) {
  5357. err = -ECHRNG;
  5358. goto bail_unlock;
  5359. }
  5360. context_notify_user(ctx, rsp->retval, rsp_flags, early_wake_time);
  5361. bail_unlock:
  5362. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  5363. bail:
  5364. if (err) {
  5365. err = -ENOKEY;
  5366. if (!ignore_rsp_err)
  5367. ADSPRPC_ERR(
  5368. "invalid response data %pK, len %d from remote subsystem err %d\n",
  5369. data, len, err);
  5370. else {
  5371. err = 0;
  5372. me->duplicate_rsp_err_cnt++;
  5373. }
  5374. }
  5375. return err;
  5376. }
  5377. static int fastrpc_session_alloc_secure_memory(
  5378. struct fastrpc_channel_ctx *chan, int secure,
  5379. int sharedcb, int pd_type, struct fastrpc_session_ctx **session)
  5380. {
  5381. int err = 0;
  5382. struct fastrpc_apps *me = &gfa;
  5383. /*
  5384. * If PD type is configured for context banks,
  5385. * Use CPZ_USERPD, to allocate secure context bank type.
  5386. */
  5387. if (pd_type != DEFAULT_UNUSED && me->cb_pd_type)
  5388. pd_type = CPZ_USERPD;
  5389. mutex_lock(&chan->smd_mutex);
  5390. if (!*session)
  5391. err = fastrpc_session_alloc_locked(chan, secure, sharedcb, pd_type, session);
  5392. mutex_unlock(&chan->smd_mutex);
  5393. if (err == -EUSERS) {
  5394. ADSPRPC_WARN(
  5395. "max concurrent sessions limit (%d) already reached on %s err %d\n",
  5396. chan->sesscount, chan->subsys, err);
  5397. }
  5398. return err;
  5399. }
  5400. static void fastrpc_session_free(struct fastrpc_channel_ctx *chan,
  5401. struct fastrpc_session_ctx *session)
  5402. {
  5403. mutex_lock(&chan->smd_mutex);
  5404. session->used = 0;
  5405. mutex_unlock(&chan->smd_mutex);
  5406. }
  5407. static int fastrpc_file_free(struct fastrpc_file *fl)
  5408. {
  5409. struct hlist_node *n = NULL;
  5410. struct fastrpc_mmap *map = NULL, *lmap = NULL;
  5411. unsigned long flags;
  5412. int cid;
  5413. struct fastrpc_apps *me = &gfa;
  5414. bool is_driver_closed = false;
  5415. int err = 0;
  5416. unsigned long irq_flags = 0;
  5417. bool is_locked = false;
  5418. int i;
  5419. if (!fl)
  5420. return 0;
  5421. cid = fl->cid;
  5422. spin_lock_irqsave(&me->hlock, irq_flags);
  5423. if (fl->device) {
  5424. fl->device->dev_close = true;
  5425. if (fl->device->refs == 0) {
  5426. is_driver_closed = true;
  5427. hlist_del_init(&fl->device->hn);
  5428. }
  5429. }
  5430. fl->file_close = FASTRPC_PROCESS_EXIT_START;
  5431. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5432. (void)fastrpc_release_current_dsp_process(fl);
  5433. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5434. is_locked = true;
  5435. if (!fl->is_dma_invoke_pend)
  5436. goto skip_dmainvoke_wait;
  5437. is_locked = false;
  5438. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5439. wait_for_completion(&fl->dma_invoke);
  5440. skip_dmainvoke_wait:
  5441. if (!is_locked) {
  5442. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5443. is_locked = true;
  5444. }
  5445. if (!fl->is_ramdump_pend)
  5446. goto skip_dump_wait;
  5447. is_locked = false;
  5448. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5449. wait_for_completion(&fl->work);
  5450. skip_dump_wait:
  5451. if (!is_locked) {
  5452. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5453. is_locked = true;
  5454. }
  5455. hlist_del_init(&fl->hn);
  5456. fl->is_ramdump_pend = false;
  5457. fl->is_dma_invoke_pend = false;
  5458. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  5459. is_locked = false;
  5460. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5461. if (!fl->sctx) {
  5462. spin_lock_irqsave(&me->hlock, irq_flags);
  5463. /* Reset the tgid usage to false */
  5464. if (fl->tgid_frpc != -1)
  5465. frpc_tgid_usage_array[fl->tgid_frpc] = false;
  5466. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5467. kfree(fl);
  5468. return 0;
  5469. }
  5470. //Dummy wake up to exit Async worker thread
  5471. spin_lock_irqsave(&fl->aqlock, flags);
  5472. atomic_add(1, &fl->async_queue_job_count);
  5473. wake_up_interruptible(&fl->async_wait_queue);
  5474. spin_unlock_irqrestore(&fl->aqlock, flags);
  5475. // Dummy wake up to exit notification worker thread
  5476. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  5477. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  5478. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  5479. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  5480. if (!IS_ERR_OR_NULL(fl->init_mem))
  5481. fastrpc_buf_free(fl->init_mem, 0);
  5482. fastrpc_context_list_dtor(fl);
  5483. fastrpc_cached_buf_list_free(fl);
  5484. if (!IS_ERR_OR_NULL(fl->hdr_bufs))
  5485. kfree(fl->hdr_bufs);
  5486. if (!IS_ERR_OR_NULL(fl->pers_hdr_buf))
  5487. fastrpc_buf_free(fl->pers_hdr_buf, 0);
  5488. mutex_lock(&fl->internal_map_mutex);
  5489. mutex_lock(&fl->map_mutex);
  5490. do {
  5491. lmap = NULL;
  5492. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5493. hlist_del_init(&map->hn);
  5494. lmap = map;
  5495. break;
  5496. }
  5497. fastrpc_mmap_free(lmap, 1);
  5498. } while (lmap);
  5499. mutex_unlock(&fl->map_mutex);
  5500. mutex_unlock(&fl->internal_map_mutex);
  5501. fastrpc_pm_relax(fl, gcinfo[fl->cid].secure);
  5502. if (fl->device && is_driver_closed)
  5503. device_unregister(&fl->device->dev);
  5504. spin_lock_irqsave(&me->hlock, irq_flags);
  5505. /* Reset the tgid usage to false */
  5506. if (fl->tgid_frpc != -1)
  5507. frpc_tgid_usage_array[fl->tgid_frpc] = false;
  5508. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5509. VERIFY(err, VALID_FASTRPC_CID(cid));
  5510. if (!err && fl->sctx)
  5511. fastrpc_session_free(&fl->apps->channel[cid], fl->sctx);
  5512. if (!err && fl->secsctx)
  5513. fastrpc_session_free(&fl->apps->channel[cid], fl->secsctx);
  5514. for (i = 0; i < (DSPSIGNAL_NUM_SIGNALS / DSPSIGNAL_GROUP_SIZE); i++)
  5515. kfree(fl->signal_groups[i]);
  5516. mutex_destroy(&fl->signal_create_mutex);
  5517. fastrpc_remote_buf_list_free(fl);
  5518. mutex_destroy(&fl->map_mutex);
  5519. mutex_destroy(&fl->internal_map_mutex);
  5520. kfree(fl->dev_pm_qos_req);
  5521. kfree(fl->gidlist.gids);
  5522. kfree(fl);
  5523. return 0;
  5524. }
  5525. static int fastrpc_device_release(struct inode *inode, struct file *file)
  5526. {
  5527. struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
  5528. struct fastrpc_apps *me = &gfa;
  5529. unsigned int ii;
  5530. if (!fl)
  5531. return 0;
  5532. if (fl->qos_request && fl->dev_pm_qos_req) {
  5533. for (ii = 0; ii < me->lowest_capacity_core_count; ii++) {
  5534. if (!dev_pm_qos_request_active(&fl->dev_pm_qos_req[ii]))
  5535. continue;
  5536. dev_pm_qos_remove_request(&fl->dev_pm_qos_req[ii]);
  5537. }
  5538. }
  5539. debugfs_remove(fl->debugfs_file);
  5540. fastrpc_file_free(fl);
  5541. file->private_data = NULL;
  5542. return 0;
  5543. }
  5544. static ssize_t fastrpc_debugfs_read(struct file *filp, char __user *buffer,
  5545. size_t count, loff_t *position)
  5546. {
  5547. struct fastrpc_apps *me = &gfa;
  5548. struct fastrpc_file *fl = filp->private_data;
  5549. struct hlist_node *n;
  5550. struct fastrpc_buf *buf = NULL;
  5551. struct fastrpc_mmap *map = NULL;
  5552. struct fastrpc_mmap *gmaps = NULL;
  5553. struct smq_invoke_ctx *ictx = NULL;
  5554. struct fastrpc_channel_ctx *chan = NULL;
  5555. unsigned int len = 0;
  5556. int i, j, sess_used = 0, ret = 0;
  5557. char *fileinfo = NULL;
  5558. char single_line[] = "----------------";
  5559. char title[] = "=========================";
  5560. unsigned long irq_flags = 0;
  5561. fileinfo = kzalloc(DEBUGFS_SIZE, GFP_KERNEL);
  5562. if (!fileinfo) {
  5563. ret = -ENOMEM;
  5564. goto bail;
  5565. }
  5566. if (fl == NULL) {
  5567. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5568. "\n%s %s %s\n", title, " CHANNEL INFO ", title);
  5569. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5570. "%-7s|%-10s|%-15s|%-9s|%-13s\n",
  5571. "subsys", "sesscount", "subsystemstate",
  5572. "ssrcount", "session_used");
  5573. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5574. "-%s%s%s%s-\n", single_line, single_line,
  5575. single_line, single_line);
  5576. for (i = 0; i < NUM_CHANNELS; i++) {
  5577. sess_used = 0;
  5578. chan = &gcinfo[i];
  5579. len += scnprintf(fileinfo + len,
  5580. DEBUGFS_SIZE - len, "%-7s", chan->subsys);
  5581. len += scnprintf(fileinfo + len,
  5582. DEBUGFS_SIZE - len, "|%-10u",
  5583. chan->sesscount);
  5584. len += scnprintf(fileinfo + len,
  5585. DEBUGFS_SIZE - len, "|%-15d",
  5586. chan->subsystemstate);
  5587. len += scnprintf(fileinfo + len,
  5588. DEBUGFS_SIZE - len, "|%-9u",
  5589. chan->ssrcount);
  5590. for (j = 0; j < chan->sesscount; j++)
  5591. sess_used += chan->session[j].used;
  5592. len += scnprintf(fileinfo + len,
  5593. DEBUGFS_SIZE - len, "|%-13d\n", sess_used);
  5594. }
  5595. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5596. "\n%s%s%s\n", "=============",
  5597. " CMA HEAP ", "==============");
  5598. len += scnprintf(fileinfo + len,
  5599. DEBUGFS_SIZE - len, "%-20s|%-20s\n", "addr", "size");
  5600. len += scnprintf(fileinfo + len,
  5601. DEBUGFS_SIZE - len, "--%s%s---\n",
  5602. single_line, single_line);
  5603. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5604. "\n==========%s %s %s===========\n",
  5605. title, " GMAPS ", title);
  5606. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5607. "%-20s|%-20s|%-20s|%-20s\n",
  5608. "fd", "phys", "size", "va");
  5609. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5610. "%s%s%s%s%s\n", single_line, single_line,
  5611. single_line, single_line, single_line);
  5612. spin_lock_irqsave(&me->hlock, irq_flags);
  5613. hlist_for_each_entry_safe(gmaps, n, &me->maps, hn) {
  5614. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5615. "%-20d|0x%-18llX|0x%-18X|0x%-20lX\n\n",
  5616. gmaps->fd, gmaps->phys,
  5617. (uint32_t)gmaps->size,
  5618. gmaps->va);
  5619. }
  5620. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5621. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5622. "%-20s|%-20s|%-20s|%-20s\n",
  5623. "len", "refs", "raddr", "flags");
  5624. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5625. "%s%s%s%s%s\n", single_line, single_line,
  5626. single_line, single_line, single_line);
  5627. spin_lock_irqsave(&me->hlock, irq_flags);
  5628. hlist_for_each_entry_safe(gmaps, n, &me->maps, hn) {
  5629. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5630. "0x%-18X|%-20d|%-20lu|%-20u\n",
  5631. (uint32_t)gmaps->len, gmaps->refs,
  5632. gmaps->raddr, gmaps->flags);
  5633. }
  5634. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5635. } else {
  5636. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5637. "\n%s %13s %d\n", "cid", ":", fl->cid);
  5638. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5639. "%s %12s %d\n", "tgid", ":", fl->tgid);
  5640. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5641. "%s %14s %d\n", "tgid_frpc", ":", fl->tgid_frpc);
  5642. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5643. "%s %7s %d\n", "sessionid", ":", fl->sessionid);
  5644. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5645. "%s %8s %u\n", "ssrcount", ":", fl->ssrcount);
  5646. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5647. "%s %14s %d\n", "pd", ":", fl->pd);
  5648. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5649. "%s %9s %s\n", "servloc_name", ":", fl->servloc_name);
  5650. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5651. "%s %6s %d\n", "file_close", ":", fl->file_close);
  5652. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5653. "%s %9s %d\n", "profile", ":", fl->profile);
  5654. if (fl->sctx) {
  5655. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5656. "%s %3s %d\n", "smmu.coherent", ":",
  5657. fl->sctx->smmu.coherent);
  5658. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5659. "%s %4s %d\n", "smmu.enabled", ":",
  5660. fl->sctx->smmu.enabled);
  5661. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5662. "%s %9s %d\n", "smmu.cb", ":", fl->sctx->smmu.cb);
  5663. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5664. "%s %5s %d\n", "smmu.secure", ":",
  5665. fl->sctx->smmu.secure);
  5666. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5667. "%s %5s %d\n", "smmu.faults", ":",
  5668. fl->sctx->smmu.faults);
  5669. }
  5670. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5671. "\n=======%s %s %s======\n", title,
  5672. " LIST OF MAPS ", title);
  5673. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5674. "%-20s|%-20s|%-20s|%-20s\n", "va", "phys", "size", "flags");
  5675. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5676. "%s%s%s%s%s\n",
  5677. single_line, single_line, single_line,
  5678. single_line, single_line);
  5679. mutex_lock(&fl->map_mutex);
  5680. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5681. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5682. "0x%-20lX|0x%-20llX|0x%-20zu|0x%-17llX\n\n",
  5683. map->va, map->phys,
  5684. map->size, map->flags);
  5685. }
  5686. mutex_unlock(&fl->map_mutex);
  5687. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5688. "%-20s|%-20s|%-20s\n",
  5689. "len", "refs",
  5690. "raddr");
  5691. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5692. "%s%s%s%s%s\n",
  5693. single_line, single_line, single_line,
  5694. single_line, single_line);
  5695. mutex_lock(&fl->map_mutex);
  5696. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5697. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5698. "%-20zu|%-20d|0x%-20lX\n\n",
  5699. map->len, map->refs, map->raddr);
  5700. }
  5701. mutex_unlock(&fl->map_mutex);
  5702. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5703. "%-20s|%-20s\n", "secure", "attr");
  5704. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5705. "%s%s%s%s%s\n",
  5706. single_line, single_line, single_line,
  5707. single_line, single_line);
  5708. mutex_lock(&fl->map_mutex);
  5709. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5710. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5711. "%-20d|0x%-20lX\n\n",
  5712. map->secure, map->attr);
  5713. }
  5714. mutex_unlock(&fl->map_mutex);
  5715. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5716. "\n======%s %s %s======\n", title,
  5717. " LIST OF BUFS ", title);
  5718. spin_lock(&fl->hlock);
  5719. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5720. "%-19s|%-19s|%-19s|%-19s\n",
  5721. "virt", "phys", "size", "flags");
  5722. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5723. "%s%s%s%s%s\n", single_line, single_line,
  5724. single_line, single_line, single_line);
  5725. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  5726. len += scnprintf(fileinfo + len,
  5727. DEBUGFS_SIZE - len,
  5728. "0x%-17p|0x%-17llX|%-19zu|0x%-17llX\n",
  5729. buf->virt, (uint64_t)buf->phys, buf->size, buf->flags);
  5730. }
  5731. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5732. "\n======%s %s %s======\n", title,
  5733. " LIST OF REMOTE BUFS ", title);
  5734. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5735. "%-19s|%-19s|%-19s|%-19s\n",
  5736. "virt", "phys", "size", "flags");
  5737. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5738. "%s%s%s%s%s\n", single_line, single_line,
  5739. single_line, single_line, single_line);
  5740. hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {
  5741. len += scnprintf(fileinfo + len,
  5742. DEBUGFS_SIZE - len,
  5743. "0x%-17p|0x%-17llX|%-19zu|0x%-17llX\n",
  5744. buf->virt, (uint64_t)buf->phys, buf->size, buf->flags);
  5745. }
  5746. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5747. "\n%s %s %s\n", title,
  5748. " LIST OF PENDING SMQCONTEXTS ", title);
  5749. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5750. "%-20s|%-10s|%-10s|%-10s|%-20s\n",
  5751. "sc", "pid", "tgid", "used", "ctxid");
  5752. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5753. "%s%s%s%s%s\n", single_line, single_line,
  5754. single_line, single_line, single_line);
  5755. hlist_for_each_entry_safe(ictx, n, &fl->clst.pending, hn) {
  5756. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5757. "0x%-18X|%-10d|%-10d|%-10zu|0x%-20llX\n\n",
  5758. ictx->sc, ictx->pid, ictx->tgid,
  5759. ictx->used, ictx->ctxid);
  5760. }
  5761. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5762. "\n%s %s %s\n", title,
  5763. " LIST OF INTERRUPTED SMQCONTEXTS ", title);
  5764. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5765. "%-20s|%-10s|%-10s|%-10s|%-20s\n",
  5766. "sc", "pid", "tgid", "used", "ctxid");
  5767. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5768. "%s%s%s%s%s\n", single_line, single_line,
  5769. single_line, single_line, single_line);
  5770. hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {
  5771. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5772. "%-20u|%-20d|%-20d|%-20zu|0x%-20llX\n\n",
  5773. ictx->sc, ictx->pid, ictx->tgid,
  5774. ictx->used, ictx->ctxid);
  5775. }
  5776. spin_unlock(&fl->hlock);
  5777. }
  5778. if (len > DEBUGFS_SIZE)
  5779. len = DEBUGFS_SIZE;
  5780. ret = simple_read_from_buffer(buffer, count, position, fileinfo, len);
  5781. kfree(fileinfo);
  5782. bail:
  5783. return ret;
  5784. }
  5785. static const struct file_operations debugfs_fops = {
  5786. .open = simple_open,
  5787. .read = fastrpc_debugfs_read,
  5788. };
  5789. static int fastrpc_channel_open(struct fastrpc_file *fl, uint32_t flags)
  5790. {
  5791. struct fastrpc_apps *me = &gfa;
  5792. int cid = -1, err = 0;
  5793. VERIFY(err, fl && fl->sctx && fl->cid >= 0 && fl->cid < NUM_CHANNELS);
  5794. if (err) {
  5795. ADSPRPC_ERR("kernel session not initialized yet for %s\n",
  5796. current->comm);
  5797. err = -EBADR;
  5798. return err;
  5799. }
  5800. cid = fl->cid;
  5801. err = fastrpc_wait_for_transport_interrupt(cid, flags);
  5802. if (err)
  5803. goto bail;
  5804. err = verify_transport_device(cid, fl->tvm_remote_domain);
  5805. if (err)
  5806. goto bail;
  5807. mutex_lock(&me->channel[cid].smd_mutex);
  5808. if (me->channel[cid].ssrcount !=
  5809. me->channel[cid].prevssrcount) {
  5810. if (me->channel[cid].subsystemstate != SUBSYSTEM_UP) {
  5811. err = -ECONNREFUSED;
  5812. mutex_unlock(&me->channel[cid].smd_mutex);
  5813. goto bail;
  5814. }
  5815. }
  5816. fl->ssrcount = me->channel[cid].ssrcount;
  5817. if (cid == ADSP_DOMAIN_ID && me->channel[cid].ssrcount !=
  5818. me->channel[cid].prevssrcount) {
  5819. mutex_unlock(&me->channel[cid].smd_mutex);
  5820. mutex_lock(&fl->map_mutex);
  5821. err = fastrpc_mmap_remove_ssr(fl, 1);
  5822. mutex_unlock(&fl->map_mutex);
  5823. if (err)
  5824. ADSPRPC_WARN(
  5825. "failed to unmap remote heap for %s (err %d)\n",
  5826. me->channel[cid].subsys, err);
  5827. mutex_lock(&me->channel[cid].smd_mutex);
  5828. me->channel[cid].prevssrcount =
  5829. me->channel[cid].ssrcount;
  5830. }
  5831. me->channel[cid].in_hib = 0;
  5832. mutex_unlock(&me->channel[cid].smd_mutex);
  5833. bail:
  5834. return err;
  5835. }
  5836. static inline void fastrpc_register_wakeup_source(struct device *dev,
  5837. const char *client_name, struct wakeup_source **device_wake_source)
  5838. {
  5839. struct wakeup_source *wake_source = NULL;
  5840. wake_source = wakeup_source_register(dev, client_name);
  5841. if (IS_ERR_OR_NULL(wake_source)) {
  5842. ADSPRPC_ERR(
  5843. "wakeup_source_register failed for dev %s, client %s with err %ld\n",
  5844. dev_name(dev), client_name, PTR_ERR(wake_source));
  5845. return;
  5846. }
  5847. *device_wake_source = wake_source;
  5848. }
  5849. static int fastrpc_device_open(struct inode *inode, struct file *filp)
  5850. {
  5851. int err = 0;
  5852. struct fastrpc_file *fl = NULL;
  5853. struct fastrpc_apps *me = &gfa;
  5854. unsigned long irq_flags = 0;
  5855. /*
  5856. * Indicates the device node opened
  5857. * MINOR_NUM_DEV or MINOR_NUM_SECURE_DEV
  5858. */
  5859. int dev_minor = MINOR(inode->i_rdev);
  5860. VERIFY(err, ((dev_minor == MINOR_NUM_DEV) ||
  5861. (dev_minor == MINOR_NUM_SECURE_DEV)));
  5862. if (err) {
  5863. ADSPRPC_ERR("Invalid dev minor num %d\n",
  5864. dev_minor);
  5865. return err;
  5866. }
  5867. VERIFY(err, NULL != (fl = kzalloc(sizeof(*fl), GFP_KERNEL)));
  5868. if (err) {
  5869. err = -ENOMEM;
  5870. return err;
  5871. }
  5872. context_list_ctor(&fl->clst);
  5873. spin_lock_init(&fl->hlock);
  5874. spin_lock_init(&fl->aqlock);
  5875. spin_lock_init(&fl->proc_state_notif.nqlock);
  5876. INIT_HLIST_HEAD(&fl->maps);
  5877. INIT_HLIST_HEAD(&fl->cached_bufs);
  5878. fl->num_cached_buf = 0;
  5879. INIT_HLIST_HEAD(&fl->remote_bufs);
  5880. init_waitqueue_head(&fl->async_wait_queue);
  5881. init_waitqueue_head(&fl->proc_state_notif.notif_wait_queue);
  5882. INIT_HLIST_NODE(&fl->hn);
  5883. fl->sessionid = 0;
  5884. fl->tgid_open = current->tgid;
  5885. /* PD type is not known, when device is opened */
  5886. fl->pd_type = DEFAULT_UNUSED;
  5887. fl->apps = me;
  5888. fl->mode = FASTRPC_MODE_SERIAL;
  5889. fl->cid = -1;
  5890. fl->tgid_frpc = -1;
  5891. fl->tvm_remote_domain = -1;
  5892. fl->dev_minor = dev_minor;
  5893. fl->init_mem = NULL;
  5894. fl->qos_request = 0;
  5895. fl->dsp_proc_init = 0;
  5896. fl->is_ramdump_pend = false;
  5897. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  5898. fl->is_unsigned_pd = false;
  5899. fl->is_compat = false;
  5900. fl->exit_notif = false;
  5901. fl->exit_async = false;
  5902. fl->multi_session_support = false;
  5903. fl->set_session_info = false;
  5904. init_completion(&fl->work);
  5905. init_completion(&fl->dma_invoke);
  5906. fl->file_close = FASTRPC_PROCESS_DEFAULT_STATE;
  5907. filp->private_data = fl;
  5908. fl->sharedbuf_info.buf_fd = -1;
  5909. mutex_init(&fl->internal_map_mutex);
  5910. mutex_init(&fl->map_mutex);
  5911. spin_lock_irqsave(&me->hlock, irq_flags);
  5912. hlist_add_head(&fl->hn, &me->drivers);
  5913. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5914. if (me->lowest_capacity_core_count)
  5915. fl->dev_pm_qos_req = kzalloc((me->lowest_capacity_core_count) *
  5916. sizeof(struct dev_pm_qos_request),
  5917. GFP_KERNEL);
  5918. spin_lock_init(&fl->dspsignals_lock);
  5919. mutex_init(&fl->signal_create_mutex);
  5920. init_completion(&fl->shutdown);
  5921. return 0;
  5922. }
  5923. static int fastrpc_get_process_gids(struct gid_list *gidlist)
  5924. {
  5925. struct group_info *group_info = get_current_groups();
  5926. int i = 0, err = 0, num_gids = group_info->ngroups + 1;
  5927. unsigned int *gids = NULL;
  5928. gids = kcalloc(num_gids, sizeof(unsigned int), GFP_KERNEL);
  5929. if (!gids) {
  5930. err = -ENOMEM;
  5931. goto bail;
  5932. }
  5933. /* Get the real GID */
  5934. gids[0] = __kgid_val(current_gid());
  5935. /* Get the supplemental GIDs */
  5936. for (i = 1; i < num_gids; i++)
  5937. gids[i] = __kgid_val(group_info->gid[i - 1]);
  5938. sort(gids, num_gids, sizeof(*gids), uint_cmp_func, NULL);
  5939. gidlist->gids = gids;
  5940. gidlist->gidcount = num_gids;
  5941. bail:
  5942. if (err)
  5943. kfree(gids);
  5944. return err;
  5945. }
  5946. // Generate a unique process ID to DSP process
  5947. static int get_unique_hlos_process_id(void)
  5948. {
  5949. int tgid_frpc = -1, tgid_index = 1;
  5950. struct fastrpc_apps *me = &gfa;
  5951. unsigned long irq_flags = 0;
  5952. spin_lock_irqsave(&me->hlock, irq_flags);
  5953. for (tgid_index = 1; tgid_index < MAX_FRPC_TGID; tgid_index++) {
  5954. if (!frpc_tgid_usage_array[tgid_index]) {
  5955. tgid_frpc = tgid_index;
  5956. /* Set the tgid usage to false */
  5957. frpc_tgid_usage_array[tgid_index] = true;
  5958. break;
  5959. }
  5960. }
  5961. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5962. return tgid_frpc;
  5963. }
  5964. static int fastrpc_set_process_info(struct fastrpc_file *fl, uint32_t cid)
  5965. {
  5966. int err = 0, buf_size = 0;
  5967. char strpid[PID_SIZE];
  5968. char cur_comm[TASK_COMM_LEN];
  5969. memcpy(cur_comm, current->comm, TASK_COMM_LEN);
  5970. cur_comm[TASK_COMM_LEN-1] = '\0';
  5971. fl->tgid = current->tgid;
  5972. fl->tgid_frpc = get_unique_hlos_process_id();
  5973. VERIFY(err, fl->tgid_frpc != -1);
  5974. if (err) {
  5975. ADSPRPC_ERR("too many fastrpc clients, max %u allowed\n", MAX_FRPC_TGID);
  5976. err = -EUSERS;
  5977. goto bail;
  5978. }
  5979. ADSPRPC_INFO("HLOS pid %d, cid %d is mapped to unique sessions pid %d",
  5980. fl->tgid, cid, fl->tgid_frpc);
  5981. /*
  5982. * Third-party apps don't have permission to open the fastrpc device, so
  5983. * it is opened on their behalf by DSP HAL. This is detected by
  5984. * comparing current PID with the one stored during device open.
  5985. */
  5986. if (current->tgid != fl->tgid_open)
  5987. fl->untrusted_process = true;
  5988. snprintf(strpid, PID_SIZE, "%d", current->pid);
  5989. if (debugfs_root) {
  5990. VERIFY(err, VALID_FASTRPC_CID(cid));
  5991. if (err) {
  5992. err = -ECHRNG;
  5993. goto bail;
  5994. }
  5995. buf_size = strlen(cur_comm) + strlen("_") + strlen(strpid)
  5996. + strlen("_") + strlen(__TOSTR__(NUM_CHANNELS)) + 1;
  5997. spin_lock(&fl->hlock);
  5998. if (fl->debug_buf_alloced_attempted) {
  5999. spin_unlock(&fl->hlock);
  6000. return err;
  6001. }
  6002. fl->debug_buf_alloced_attempted = 1;
  6003. spin_unlock(&fl->hlock);
  6004. fl->debug_buf = kzalloc(buf_size, GFP_KERNEL);
  6005. if (!fl->debug_buf) {
  6006. err = -ENOMEM;
  6007. return err;
  6008. }
  6009. /* Use HLOS PID, unique fastrpc PID, CID in debugfs filename,
  6010. * for better ability to debug
  6011. */
  6012. snprintf(fl->debug_buf, buf_size, "%.10s%s%d%s%d%s%d",
  6013. cur_comm, "_", current->pid, "_", fl->tgid_frpc, "_", cid);
  6014. fl->debugfs_file = debugfs_create_file(fl->debug_buf, 0644,
  6015. debugfs_root, fl, &debugfs_fops);
  6016. if (IS_ERR_OR_NULL(fl->debugfs_file)) {
  6017. pr_warn("Error: %s: %s: failed to create debugfs file %s\n",
  6018. cur_comm, __func__, fl->debug_buf);
  6019. fl->debugfs_file = NULL;
  6020. }
  6021. kfree(fl->debug_buf);
  6022. fl->debug_buf = NULL;
  6023. }
  6024. bail:
  6025. return err;
  6026. }
  6027. int fastrpc_get_info(struct fastrpc_file *fl, uint32_t *info)
  6028. {
  6029. int err = 0;
  6030. uint32_t cid = *info;
  6031. struct fastrpc_apps *me = &gfa;
  6032. VERIFY(err, fl != NULL);
  6033. if (err) {
  6034. err = -EBADF;
  6035. goto bail;
  6036. }
  6037. spin_lock(&fl->hlock);
  6038. if (fl->set_session_info) {
  6039. spin_unlock(&fl->hlock);
  6040. ADSPRPC_ERR("Set session info invoked multiple times\n");
  6041. err = -EBADR;
  6042. goto bail;
  6043. }
  6044. // Set set_session_info to true
  6045. fl->set_session_info = true;
  6046. spin_unlock(&fl->hlock);
  6047. VERIFY(err, VALID_FASTRPC_CID(cid));
  6048. if (err) {
  6049. err = -ECHRNG;
  6050. goto bail;
  6051. }
  6052. fastrpc_get_process_gids(&fl->gidlist);
  6053. err = fastrpc_set_process_info(fl, cid);
  6054. if (err)
  6055. goto bail;
  6056. if (fl->cid == -1) {
  6057. struct fastrpc_channel_ctx *chan = NULL;
  6058. chan = &me->channel[cid];
  6059. /* Check to see if the device node is non-secure */
  6060. if (fl->dev_minor == MINOR_NUM_DEV) {
  6061. /*
  6062. * If an app is trying to offload to a secure remote
  6063. * channel by opening the non-secure device node, allow
  6064. * the access if the subsystem supports unsigned
  6065. * offload. Untrusted apps will be restricted from
  6066. * offloading to signed PD using DSP HAL.
  6067. */
  6068. if (chan->secure == SECURE_CHANNEL
  6069. && !chan->unsigned_support) {
  6070. ADSPRPC_ERR(
  6071. "cannot use domain %d with non-secure device\n",
  6072. cid);
  6073. err = -EACCES;
  6074. goto bail;
  6075. }
  6076. }
  6077. fl->cid = cid;
  6078. fl->ssrcount = fl->apps->channel[cid].ssrcount;
  6079. mutex_lock(&fl->apps->channel[cid].smd_mutex);
  6080. err = fastrpc_session_alloc_locked(&fl->apps->channel[cid],
  6081. 0, fl->sharedcb, fl->pd_type, &fl->sctx);
  6082. mutex_unlock(&fl->apps->channel[cid].smd_mutex);
  6083. if (err == -EUSERS) {
  6084. ADSPRPC_WARN(
  6085. "max concurrent sessions limit (%d) already reached on %s err %d\n",
  6086. chan->sesscount, chan->subsys, err);
  6087. }
  6088. if (err)
  6089. goto bail;
  6090. }
  6091. VERIFY(err, fl->sctx != NULL);
  6092. if (err) {
  6093. err = -EBADR;
  6094. goto bail;
  6095. }
  6096. *info = (fl->sctx->smmu.enabled ? 1 : 0);
  6097. bail:
  6098. return err;
  6099. }
  6100. static int fastrpc_manage_poll_mode(struct fastrpc_file *fl, uint32_t enable, uint32_t timeout)
  6101. {
  6102. int err = 0;
  6103. const unsigned int MAX_POLL_TIMEOUT_US = 10000;
  6104. if ((fl->cid != CDSP_DOMAIN_ID) || (fl->proc_flags != FASTRPC_INIT_CREATE)) {
  6105. err = -EPERM;
  6106. ADSPRPC_ERR("flags %d, cid %d, poll mode allowed only for dynamic CDSP process\n",
  6107. fl->proc_flags, fl->cid);
  6108. goto bail;
  6109. }
  6110. if (timeout > MAX_POLL_TIMEOUT_US) {
  6111. err = -EBADMSG;
  6112. ADSPRPC_ERR("poll timeout %u is greater than max allowed value %u\n",
  6113. timeout, MAX_POLL_TIMEOUT_US);
  6114. goto bail;
  6115. }
  6116. spin_lock(&fl->hlock);
  6117. if (enable) {
  6118. fl->poll_mode = true;
  6119. fl->poll_timeout = timeout;
  6120. } else {
  6121. fl->poll_mode = false;
  6122. fl->poll_timeout = 0;
  6123. }
  6124. spin_unlock(&fl->hlock);
  6125. ADSPRPC_INFO("updated poll mode to %d, timeout %u\n", enable, timeout);
  6126. bail:
  6127. return err;
  6128. }
  6129. int fastrpc_internal_control(struct fastrpc_file *fl,
  6130. struct fastrpc_ioctl_control *cp)
  6131. {
  6132. int err = 0;
  6133. unsigned int latency;
  6134. struct fastrpc_apps *me = &gfa;
  6135. unsigned int cpu;
  6136. unsigned long flags = 0;
  6137. VERIFY(err, !IS_ERR_OR_NULL(fl) && !IS_ERR_OR_NULL(fl->apps));
  6138. if (err) {
  6139. err = -EBADF;
  6140. goto bail;
  6141. }
  6142. VERIFY(err, !IS_ERR_OR_NULL(cp));
  6143. if (err) {
  6144. err = -EINVAL;
  6145. goto bail;
  6146. }
  6147. switch (cp->req) {
  6148. case FASTRPC_CONTROL_LATENCY:
  6149. latency = cp->lp.enable == FASTRPC_LATENCY_CTRL_ENB ?
  6150. fl->apps->latency : PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
  6151. VERIFY(err, latency != 0);
  6152. if (err) {
  6153. err = -EINVAL;
  6154. goto bail;
  6155. }
  6156. VERIFY(err, (me->lowest_capacity_core_count && fl->dev_pm_qos_req));
  6157. if (err) {
  6158. ADSPRPC_INFO("Skipping PM QoS latency voting, core count: %u\n",
  6159. me->lowest_capacity_core_count);
  6160. err = -EINVAL;
  6161. goto bail;
  6162. }
  6163. /*
  6164. * Add voting request for all possible cores corresponding to cluster
  6165. * id 0. If DT property 'qcom,single-core-latency-vote' is enabled
  6166. * then add voting request for only one core of cluster id 0.
  6167. */
  6168. for (cpu = 0; cpu < me->lowest_capacity_core_count; cpu++) {
  6169. if (!fl->qos_request) {
  6170. err = dev_pm_qos_add_request(
  6171. get_cpu_device(cpu),
  6172. &fl->dev_pm_qos_req[cpu],
  6173. DEV_PM_QOS_RESUME_LATENCY,
  6174. latency);
  6175. } else {
  6176. err = dev_pm_qos_update_request(
  6177. &fl->dev_pm_qos_req[cpu],
  6178. latency);
  6179. }
  6180. /* PM QoS request APIs return 0 or 1 on success */
  6181. if (err < 0) {
  6182. ADSPRPC_WARN("QoS with lat %u failed for CPU %d, err %d, req %d\n",
  6183. latency, cpu, err, fl->qos_request);
  6184. break;
  6185. }
  6186. }
  6187. if (err >= 0) {
  6188. fl->qos_request = 1;
  6189. err = 0;
  6190. }
  6191. /* Ensure CPU feature map updated to DSP for early WakeUp */
  6192. fastrpc_send_cpuinfo_to_dsp(fl);
  6193. break;
  6194. case FASTRPC_CONTROL_KALLOC:
  6195. cp->kalloc.kalloc_support = 1;
  6196. break;
  6197. case FASTRPC_CONTROL_WAKELOCK:
  6198. if (fl->dev_minor != MINOR_NUM_SECURE_DEV) {
  6199. ADSPRPC_ERR(
  6200. "PM voting not allowed for non-secure device node %d\n",
  6201. fl->dev_minor);
  6202. err = -EPERM;
  6203. goto bail;
  6204. }
  6205. fl->wake_enable = cp->wp.enable;
  6206. break;
  6207. case FASTRPC_CONTROL_PM:
  6208. if (!fl->wake_enable) {
  6209. /* Kernel PM voting not requested by this application */
  6210. err = -EACCES;
  6211. goto bail;
  6212. }
  6213. if (cp->pm.timeout > MAX_PM_TIMEOUT_MS)
  6214. fl->ws_timeout = MAX_PM_TIMEOUT_MS;
  6215. else
  6216. fl->ws_timeout = cp->pm.timeout;
  6217. VERIFY(err, VALID_FASTRPC_CID(fl->cid));
  6218. if (err) {
  6219. err = -ECHRNG;
  6220. goto bail;
  6221. }
  6222. fastrpc_pm_awake(fl, gcinfo[fl->cid].secure);
  6223. break;
  6224. case FASTRPC_CONTROL_DSPPROCESS_CLEAN:
  6225. (void)fastrpc_release_current_dsp_process(fl);
  6226. fastrpc_queue_pd_status(fl, fl->cid, FASTRPC_USER_PD_FORCE_KILL, fl->sessionid);
  6227. break;
  6228. case FASTRPC_CONTROL_RPC_POLL:
  6229. err = fastrpc_manage_poll_mode(fl, cp->lp.enable, cp->lp.latency);
  6230. if (err)
  6231. goto bail;
  6232. break;
  6233. case FASTRPC_CONTROL_SMMU:
  6234. fl->sharedcb = cp->smmu.sharedcb;
  6235. break;
  6236. case FASTRPC_CONTROL_ASYNC_WAKE:
  6237. fl->exit_async = true;
  6238. spin_lock_irqsave(&fl->aqlock, flags);
  6239. atomic_add(1, &fl->async_queue_job_count);
  6240. wake_up_interruptible(&fl->async_wait_queue);
  6241. spin_unlock_irqrestore(&fl->aqlock, flags);
  6242. break;
  6243. case FASTRPC_CONTROL_NOTIF_WAKE:
  6244. fl->exit_notif = true;
  6245. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  6246. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  6247. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  6248. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  6249. break;
  6250. default:
  6251. err = -EBADRQC;
  6252. break;
  6253. }
  6254. bail:
  6255. return err;
  6256. }
  6257. /* Wait for PD to be up before audio or sensors daemons try connecting */
  6258. static int fastrpc_check_pd_status(struct fastrpc_file *fl, char *sloc_name)
  6259. {
  6260. int err = 0, session = -1, cid = -1;
  6261. struct fastrpc_apps *me = &gfa;
  6262. if (fl->servloc_name && sloc_name
  6263. && !strcmp(fl->servloc_name, sloc_name)) {
  6264. err = fastrpc_get_spd_session(sloc_name, &session, &cid);
  6265. if (err || cid != fl->cid)
  6266. goto bail;
  6267. if ((!strcmp(fl->servloc_name,
  6268. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME)) &&
  6269. (me->channel[cid].spd[session].pdrcount !=
  6270. me->channel[cid].spd[session].prevpdrcount)) {
  6271. err = -ECONNRESET;
  6272. goto bail;
  6273. }
  6274. #if IS_ENABLED(CONFIG_QCOM_PDR_HELPERS)
  6275. if (!strcmp(fl->servloc_name,
  6276. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME) || !strcmp(fl->servloc_name,
  6277. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME) ||
  6278. !strcmp(fl->servloc_name,
  6279. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME)) {
  6280. err = wait_event_interruptible(
  6281. me->channel[cid].spd[session].wait_for_pdup,
  6282. atomic_read(&me->channel[cid].spd[session].ispdup));
  6283. goto bail;
  6284. }
  6285. #else
  6286. (void)me;
  6287. #endif
  6288. }
  6289. bail:
  6290. return err;
  6291. }
  6292. int fastrpc_setmode(unsigned long ioctl_param,
  6293. struct fastrpc_file *fl)
  6294. {
  6295. int err = 0;
  6296. switch ((uint32_t)ioctl_param) {
  6297. case FASTRPC_MODE_PARALLEL:
  6298. case FASTRPC_MODE_SERIAL:
  6299. fl->mode = (uint32_t)ioctl_param;
  6300. break;
  6301. case FASTRPC_MODE_PROFILE:
  6302. fl->profile = (uint32_t)ioctl_param;
  6303. break;
  6304. case FASTRPC_MODE_SESSION:
  6305. if (!fl->multi_session_support)
  6306. fl->sessionid = 1;
  6307. break;
  6308. default:
  6309. err = -ENOTTY;
  6310. break;
  6311. }
  6312. return err;
  6313. }
  6314. int fastrpc_control(struct fastrpc_ioctl_control *cp,
  6315. void *param, struct fastrpc_file *fl)
  6316. {
  6317. int err = 0;
  6318. K_COPY_FROM_USER(err, 0, cp, param,
  6319. sizeof(*cp));
  6320. if (err) {
  6321. err = -EFAULT;
  6322. goto bail;
  6323. }
  6324. VERIFY(err, 0 == (err = fastrpc_internal_control(fl, cp)));
  6325. if (err)
  6326. goto bail;
  6327. if (cp->req == FASTRPC_CONTROL_KALLOC) {
  6328. K_COPY_TO_USER(err, 0, param, cp, sizeof(*cp));
  6329. if (err) {
  6330. err = -EFAULT;
  6331. goto bail;
  6332. }
  6333. }
  6334. bail:
  6335. return err;
  6336. }
  6337. static int fastrpc_get_dsp_info(
  6338. struct fastrpc_ioctl_capability *cap,
  6339. void *param, struct fastrpc_file *fl)
  6340. {
  6341. int err = 0;
  6342. K_COPY_FROM_USER(err, 0, cap, param,
  6343. sizeof(struct fastrpc_ioctl_capability));
  6344. VERIFY(err, cap->domain < NUM_CHANNELS);
  6345. if (err) {
  6346. err = -ECHRNG;
  6347. goto bail;
  6348. }
  6349. cap->capability = 0;
  6350. err = fastrpc_get_info_from_kernel(cap, fl);
  6351. if (err)
  6352. goto bail;
  6353. K_COPY_TO_USER(err, 0, &((struct fastrpc_ioctl_capability *)
  6354. param)->capability, &cap->capability, sizeof(cap->capability));
  6355. bail:
  6356. return err;
  6357. }
  6358. int fastrpc_dspsignal_signal(struct fastrpc_file *fl,
  6359. struct fastrpc_ioctl_dspsignal_signal *sig)
  6360. {
  6361. int err = 0, cid = -1;
  6362. struct fastrpc_channel_ctx *channel_ctx = NULL;
  6363. uint64_t msg = 0;
  6364. // We don't check if the signal has even been allocated since we don't
  6365. // track outgoing signals in the driver. The userspace library does a
  6366. // basic sanity check and any security validation needs to be done by
  6367. // the recipient.
  6368. DSPSIGNAL_VERBOSE("Send signal PID %u, unique fastrpc pid %u signal %u\n",
  6369. (unsigned int)fl->tgid, (unsigned int)fl->tgid_frpc,
  6370. (unsigned int)sig->signal_id);
  6371. VERIFY(err, sig->signal_id < DSPSIGNAL_NUM_SIGNALS);
  6372. if (err) {
  6373. ADSPRPC_ERR("Sending bad signal %u for PID %u",
  6374. sig->signal_id, (unsigned int)fl->tgid);
  6375. err = -EBADR;
  6376. goto bail;
  6377. }
  6378. cid = fl->cid;
  6379. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6380. if (err) {
  6381. err = -EBADR;
  6382. goto bail;
  6383. }
  6384. channel_ctx = &fl->apps->channel[cid];
  6385. mutex_lock(&channel_ctx->smd_mutex);
  6386. if (fl->ssrcount != channel_ctx->ssrcount) {
  6387. err = -ECONNRESET;
  6388. mutex_unlock(&channel_ctx->smd_mutex);
  6389. goto bail;
  6390. }
  6391. /* Use unique fastrpc pid, to signal DSP process */
  6392. msg = (((uint64_t)fl->tgid_frpc) << 32) | ((uint64_t)sig->signal_id);
  6393. err = fastrpc_transport_send(cid, (void *)&msg, sizeof(msg), fl->tvm_remote_domain);
  6394. mutex_unlock(&channel_ctx->smd_mutex);
  6395. trace_fastrpc_dspsignal("signal", sig->signal_id, 0, 0);
  6396. bail:
  6397. return err;
  6398. }
  6399. int fastrpc_dspsignal_wait(struct fastrpc_file *fl,
  6400. struct fastrpc_ioctl_dspsignal_wait *wait)
  6401. {
  6402. int err = 0, cid = -1;
  6403. unsigned long timeout = usecs_to_jiffies(wait->timeout_usec);
  6404. uint32_t signal_id = wait->signal_id;
  6405. struct fastrpc_dspsignal *s = NULL;
  6406. long ret = 0;
  6407. unsigned long irq_flags = 0;
  6408. DSPSIGNAL_VERBOSE("Wait for signal %u\n", signal_id);
  6409. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6410. if (err) {
  6411. ADSPRPC_ERR("Waiting on bad signal %u", signal_id);
  6412. err = -EINVAL;
  6413. goto bail;
  6414. }
  6415. cid = fl->cid;
  6416. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6417. if (err) {
  6418. err = -EBADR;
  6419. goto bail;
  6420. }
  6421. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6422. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6423. struct fastrpc_dspsignal *group =
  6424. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6425. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6426. }
  6427. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6428. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6429. ADSPRPC_ERR("Unknown signal id %u\n", signal_id);
  6430. err = -ENOENT;
  6431. goto bail;
  6432. }
  6433. if (s->state != DSPSIGNAL_STATE_PENDING) {
  6434. if ((s->state == DSPSIGNAL_STATE_CANCELED) || (s->state == DSPSIGNAL_STATE_UNUSED))
  6435. err = -EINTR;
  6436. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6437. DSPSIGNAL_VERBOSE("Signal %u in state %u, complete wait immediately",
  6438. signal_id, s->state);
  6439. goto bail;
  6440. }
  6441. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6442. trace_fastrpc_dspsignal("wait", signal_id, s->state, wait->timeout_usec);
  6443. if (timeout != 0xffffffff)
  6444. ret = wait_for_completion_interruptible_timeout(&s->comp, timeout);
  6445. else
  6446. ret = wait_for_completion_interruptible(&s->comp);
  6447. trace_fastrpc_dspsignal("wakeup", signal_id, s->state, wait->timeout_usec);
  6448. if (ret == 0) {
  6449. DSPSIGNAL_VERBOSE("Wait for signal %u timed out\n", signal_id);
  6450. err = -ETIMEDOUT;
  6451. goto bail;
  6452. } else if (ret < 0) {
  6453. ADSPRPC_ERR("Wait for signal %u failed %d\n", signal_id, (int)ret);
  6454. err = ret;
  6455. goto bail;
  6456. }
  6457. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6458. if (s->state == DSPSIGNAL_STATE_SIGNALED) {
  6459. s->state = DSPSIGNAL_STATE_PENDING;
  6460. DSPSIGNAL_VERBOSE("Signal %u completed\n", signal_id);
  6461. } else if ((s->state == DSPSIGNAL_STATE_CANCELED) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6462. DSPSIGNAL_VERBOSE("Signal %u cancelled or destroyed\n", signal_id);
  6463. err = -EINTR;
  6464. }
  6465. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6466. bail:
  6467. return err;
  6468. }
  6469. int fastrpc_dspsignal_create(struct fastrpc_file *fl,
  6470. struct fastrpc_ioctl_dspsignal_create *create)
  6471. {
  6472. int err = 0, cid = -1;
  6473. uint32_t signal_id = create->signal_id;
  6474. struct fastrpc_dspsignal *group, *sig;
  6475. unsigned long irq_flags = 0;
  6476. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6477. if (err) {
  6478. err = -EINVAL;
  6479. goto bail;
  6480. }
  6481. cid = fl->cid;
  6482. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6483. if (err) {
  6484. err = -EBADR;
  6485. goto bail;
  6486. }
  6487. // Use a separate mutex for creating signals. This avoids holding on
  6488. // to a spinlock if we need to allocate a whole group of signals. The
  6489. // mutex ensures nobody else will allocate the same group.
  6490. mutex_lock(&fl->signal_create_mutex);
  6491. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6492. group = fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6493. if (group == NULL) {
  6494. int i;
  6495. // Release the spinlock while we allocate a new group but take
  6496. // it back before taking the group into use. No other code
  6497. // allocates groups so the mutex is sufficient.
  6498. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6499. VERIFY(err, (group = kzalloc(DSPSIGNAL_GROUP_SIZE * sizeof(*group),
  6500. GFP_KERNEL)) != NULL);
  6501. if (err) {
  6502. ADSPRPC_ERR("Unable to allocate signal group\n");
  6503. err = -ENOMEM;
  6504. mutex_unlock(&fl->signal_create_mutex);
  6505. goto bail;
  6506. }
  6507. for (i = 0; i < DSPSIGNAL_GROUP_SIZE; i++) {
  6508. sig = &group[i];
  6509. init_completion(&sig->comp);
  6510. sig->state = DSPSIGNAL_STATE_UNUSED;
  6511. }
  6512. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6513. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] = group;
  6514. }
  6515. sig = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6516. if (sig->state != DSPSIGNAL_STATE_UNUSED) {
  6517. err = -EBUSY;
  6518. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6519. mutex_unlock(&fl->signal_create_mutex);
  6520. ADSPRPC_ERR("Attempting to create signal %u already in use (state %u)\n",
  6521. signal_id, sig->state);
  6522. goto bail;
  6523. }
  6524. sig->state = DSPSIGNAL_STATE_PENDING;
  6525. reinit_completion(&sig->comp);
  6526. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6527. mutex_unlock(&fl->signal_create_mutex);
  6528. DSPSIGNAL_VERBOSE("Signal %u created\n", signal_id);
  6529. bail:
  6530. return err;
  6531. }
  6532. int fastrpc_dspsignal_destroy(struct fastrpc_file *fl,
  6533. struct fastrpc_ioctl_dspsignal_destroy *destroy)
  6534. {
  6535. int err = 0, cid = -1;
  6536. uint32_t signal_id = destroy->signal_id;
  6537. struct fastrpc_dspsignal *s = NULL;
  6538. unsigned long irq_flags = 0;
  6539. DSPSIGNAL_VERBOSE("Destroy signal %u\n", signal_id);
  6540. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6541. if (err) {
  6542. err = -EINVAL;
  6543. goto bail;
  6544. }
  6545. cid = fl->cid;
  6546. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6547. if (err) {
  6548. err = -EBADR;
  6549. goto bail;
  6550. }
  6551. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6552. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6553. struct fastrpc_dspsignal *group =
  6554. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6555. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6556. }
  6557. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6558. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6559. ADSPRPC_ERR("Attempting to destroy unused signal %u\n", signal_id);
  6560. err = -ENOENT;
  6561. goto bail;
  6562. }
  6563. s->state = DSPSIGNAL_STATE_UNUSED;
  6564. complete_all(&s->comp);
  6565. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6566. DSPSIGNAL_VERBOSE("Signal %u destroyed\n", signal_id);
  6567. bail:
  6568. return err;
  6569. }
  6570. int fastrpc_dspsignal_cancel_wait(struct fastrpc_file *fl,
  6571. struct fastrpc_ioctl_dspsignal_cancel_wait *cancel)
  6572. {
  6573. int err = 0, cid = -1;
  6574. uint32_t signal_id = cancel->signal_id;
  6575. struct fastrpc_dspsignal *s = NULL;
  6576. unsigned long irq_flags = 0;
  6577. DSPSIGNAL_VERBOSE("Cancel wait for signal %u\n", signal_id);
  6578. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6579. if (err) {
  6580. err = -EINVAL;
  6581. goto bail;
  6582. }
  6583. cid = fl->cid;
  6584. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6585. if (err) {
  6586. err = -EBADR;
  6587. goto bail;
  6588. }
  6589. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6590. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6591. struct fastrpc_dspsignal *group =
  6592. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6593. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6594. }
  6595. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6596. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6597. ADSPRPC_ERR("Attempting to cancel unused signal %u\n", signal_id);
  6598. err = -ENOENT;
  6599. goto bail;
  6600. }
  6601. if (s->state != DSPSIGNAL_STATE_CANCELED) {
  6602. s->state = DSPSIGNAL_STATE_CANCELED;
  6603. trace_fastrpc_dspsignal("cancel", signal_id, s->state, 0);
  6604. complete_all(&s->comp);
  6605. }
  6606. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6607. DSPSIGNAL_VERBOSE("Signal %u cancelled\n", signal_id);
  6608. bail:
  6609. return err;
  6610. }
  6611. static inline int fastrpc_mmap_device_ioctl(struct fastrpc_file *fl,
  6612. unsigned int ioctl_num, union fastrpc_ioctl_param *p,
  6613. void *param)
  6614. {
  6615. union {
  6616. struct fastrpc_ioctl_mmap mmap;
  6617. struct fastrpc_ioctl_munmap munmap;
  6618. } i;
  6619. int err = 0;
  6620. switch (ioctl_num) {
  6621. case FASTRPC_IOCTL_MEM_MAP:
  6622. K_COPY_FROM_USER(err, 0, &p->mem_map, param,
  6623. sizeof(p->mem_map));
  6624. if (err) {
  6625. err = -EFAULT;
  6626. goto bail;
  6627. }
  6628. VERIFY(err, 0 == (err = fastrpc_internal_mem_map(fl,
  6629. &p->mem_map)));
  6630. if (err)
  6631. goto bail;
  6632. K_COPY_TO_USER(err, 0, param, &p->mem_map, sizeof(p->mem_map));
  6633. if (err) {
  6634. err = -EFAULT;
  6635. goto bail;
  6636. }
  6637. break;
  6638. case FASTRPC_IOCTL_MEM_UNMAP:
  6639. K_COPY_FROM_USER(err, 0, &p->mem_unmap, param,
  6640. sizeof(p->mem_unmap));
  6641. if (err) {
  6642. err = -EFAULT;
  6643. goto bail;
  6644. }
  6645. VERIFY(err, 0 == (err = fastrpc_internal_mem_unmap(fl,
  6646. &p->mem_unmap)));
  6647. if (err)
  6648. goto bail;
  6649. K_COPY_TO_USER(err, 0, param, &p->mem_unmap,
  6650. sizeof(p->mem_unmap));
  6651. if (err) {
  6652. err = -EFAULT;
  6653. goto bail;
  6654. }
  6655. break;
  6656. case FASTRPC_IOCTL_MMAP:
  6657. K_COPY_FROM_USER(err, 0, &p->mmap, param,
  6658. sizeof(p->mmap));
  6659. if (err) {
  6660. err = -EFAULT;
  6661. goto bail;
  6662. }
  6663. VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &p->mmap)));
  6664. if (err)
  6665. goto bail;
  6666. K_COPY_TO_USER(err, 0, param, &p->mmap, sizeof(p->mmap));
  6667. if (err) {
  6668. err = -EFAULT;
  6669. goto bail;
  6670. }
  6671. break;
  6672. case FASTRPC_IOCTL_MUNMAP:
  6673. K_COPY_FROM_USER(err, 0, &p->munmap, param,
  6674. sizeof(p->munmap));
  6675. if (err) {
  6676. err = -EFAULT;
  6677. goto bail;
  6678. }
  6679. VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl,
  6680. &p->munmap)));
  6681. if (err)
  6682. goto bail;
  6683. break;
  6684. case FASTRPC_IOCTL_MMAP_64:
  6685. K_COPY_FROM_USER(err, 0, &p->mmap64, param,
  6686. sizeof(p->mmap64));
  6687. if (err) {
  6688. err = -EFAULT;
  6689. goto bail;
  6690. }
  6691. get_fastrpc_ioctl_mmap_64(&p->mmap64, &i.mmap);
  6692. VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &i.mmap)));
  6693. if (err)
  6694. goto bail;
  6695. put_fastrpc_ioctl_mmap_64(&p->mmap64, &i.mmap);
  6696. K_COPY_TO_USER(err, 0, param, &p->mmap64, sizeof(p->mmap64));
  6697. if (err) {
  6698. err = -EFAULT;
  6699. goto bail;
  6700. }
  6701. break;
  6702. case FASTRPC_IOCTL_MUNMAP_64:
  6703. K_COPY_FROM_USER(err, 0, &p->munmap64, param,
  6704. sizeof(p->munmap64));
  6705. if (err) {
  6706. err = -EFAULT;
  6707. goto bail;
  6708. }
  6709. get_fastrpc_ioctl_munmap_64(&p->munmap64, &i.munmap);
  6710. VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl,
  6711. &i.munmap)));
  6712. if (err)
  6713. goto bail;
  6714. break;
  6715. case FASTRPC_IOCTL_MUNMAP_FD:
  6716. K_COPY_FROM_USER(err, 0, &p->munmap_fd, param,
  6717. sizeof(p->munmap_fd));
  6718. if (err) {
  6719. err = -EFAULT;
  6720. goto bail;
  6721. }
  6722. VERIFY(err, 0 == (err = fastrpc_internal_munmap_fd(fl,
  6723. &p->munmap_fd)));
  6724. if (err)
  6725. goto bail;
  6726. break;
  6727. default:
  6728. err = -ENOTTY;
  6729. pr_info("bad ioctl: %d\n", ioctl_num);
  6730. break;
  6731. }
  6732. bail:
  6733. return err;
  6734. }
  6735. static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num,
  6736. unsigned long ioctl_param)
  6737. {
  6738. union fastrpc_ioctl_param p;
  6739. void *param = (char *)ioctl_param;
  6740. struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
  6741. int size = 0, err = 0;
  6742. uint32_t info;
  6743. p.inv.fds = NULL;
  6744. p.inv.attrs = NULL;
  6745. p.inv.crc = NULL;
  6746. p.inv.perf_kernel = NULL;
  6747. p.inv.perf_dsp = NULL;
  6748. p.inv.job = NULL;
  6749. spin_lock(&fl->hlock);
  6750. if (fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  6751. err = -ESHUTDOWN;
  6752. pr_warn("adsprpc: fastrpc_device_release is happening, So not sending any new requests to DSP\n");
  6753. spin_unlock(&fl->hlock);
  6754. goto bail;
  6755. }
  6756. spin_unlock(&fl->hlock);
  6757. switch (ioctl_num) {
  6758. case FASTRPC_IOCTL_INVOKE:
  6759. size = sizeof(struct fastrpc_ioctl_invoke);
  6760. fallthrough;
  6761. case FASTRPC_IOCTL_INVOKE_FD:
  6762. if (!size)
  6763. size = sizeof(struct fastrpc_ioctl_invoke_fd);
  6764. fallthrough;
  6765. case FASTRPC_IOCTL_INVOKE_ATTRS:
  6766. if (!size)
  6767. size = sizeof(struct fastrpc_ioctl_invoke_attrs);
  6768. fallthrough;
  6769. case FASTRPC_IOCTL_INVOKE_CRC:
  6770. if (!size)
  6771. size = sizeof(struct fastrpc_ioctl_invoke_crc);
  6772. fallthrough;
  6773. case FASTRPC_IOCTL_INVOKE_PERF:
  6774. if (!size)
  6775. size = sizeof(struct fastrpc_ioctl_invoke_perf);
  6776. trace_fastrpc_msg("invoke: begin");
  6777. K_COPY_FROM_USER(err, 0, &p.inv, param, size);
  6778. if (err) {
  6779. err = -EFAULT;
  6780. goto bail;
  6781. }
  6782. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode,
  6783. USER_MSG, &p.inv)));
  6784. trace_fastrpc_msg("invoke: end");
  6785. if (err)
  6786. goto bail;
  6787. break;
  6788. case FASTRPC_IOCTL_INVOKE2:
  6789. K_COPY_FROM_USER(err, 0, &p.inv2, param,
  6790. sizeof(struct fastrpc_ioctl_invoke2));
  6791. if (err) {
  6792. err = -EFAULT;
  6793. goto bail;
  6794. }
  6795. VERIFY(err, 0 == (err = fastrpc_internal_invoke2(fl, &p.inv2)));
  6796. if (err)
  6797. goto bail;
  6798. break;
  6799. case FASTRPC_IOCTL_SETMODE:
  6800. err = fastrpc_setmode(ioctl_param, fl);
  6801. break;
  6802. case FASTRPC_IOCTL_CONTROL:
  6803. err = fastrpc_control(&p.cp, param, fl);
  6804. break;
  6805. case FASTRPC_IOCTL_GETINFO:
  6806. K_COPY_FROM_USER(err, 0, &info, param, sizeof(info));
  6807. if (err) {
  6808. err = -EFAULT;
  6809. goto bail;
  6810. }
  6811. VERIFY(err, 0 == (err = fastrpc_get_info(fl, &info)));
  6812. if (err)
  6813. goto bail;
  6814. K_COPY_TO_USER(err, 0, param, &info, sizeof(info));
  6815. if (err) {
  6816. err = -EFAULT;
  6817. goto bail;
  6818. }
  6819. break;
  6820. case FASTRPC_IOCTL_INIT:
  6821. p.init.attrs = 0;
  6822. p.init.siglen = 0;
  6823. size = sizeof(struct fastrpc_ioctl_init);
  6824. fallthrough;
  6825. case FASTRPC_IOCTL_INIT_ATTRS:
  6826. if (!size)
  6827. size = sizeof(struct fastrpc_ioctl_init_attrs);
  6828. K_COPY_FROM_USER(err, 0, &p.init, param, size);
  6829. if (err) {
  6830. err = -EFAULT;
  6831. goto bail;
  6832. }
  6833. VERIFY(err, 0 == (err = fastrpc_init_process(fl, &p.init)));
  6834. if (err)
  6835. goto bail;
  6836. break;
  6837. case FASTRPC_IOCTL_GET_DSP_INFO:
  6838. err = fastrpc_get_dsp_info(&p.cap, param, fl);
  6839. break;
  6840. case FASTRPC_IOCTL_MEM_MAP:
  6841. fallthrough;
  6842. case FASTRPC_IOCTL_MEM_UNMAP:
  6843. fallthrough;
  6844. case FASTRPC_IOCTL_MMAP:
  6845. fallthrough;
  6846. case FASTRPC_IOCTL_MUNMAP:
  6847. fallthrough;
  6848. case FASTRPC_IOCTL_MMAP_64:
  6849. fallthrough;
  6850. case FASTRPC_IOCTL_MUNMAP_64:
  6851. fallthrough;
  6852. case FASTRPC_IOCTL_MUNMAP_FD:
  6853. err = fastrpc_mmap_device_ioctl(fl, ioctl_num, &p, param);
  6854. break;
  6855. case FASTRPC_IOCTL_DSPSIGNAL_SIGNAL:
  6856. K_COPY_FROM_USER(err, 0, &p.sig, param,
  6857. sizeof(struct fastrpc_ioctl_dspsignal_signal));
  6858. if (err) {
  6859. err = -EFAULT;
  6860. goto bail;
  6861. }
  6862. VERIFY(err, 0 == (err = fastrpc_dspsignal_signal(fl, &p.sig)));
  6863. if (err)
  6864. goto bail;
  6865. break;
  6866. case FASTRPC_IOCTL_DSPSIGNAL_WAIT:
  6867. K_COPY_FROM_USER(err, 0, &p.wait, param,
  6868. sizeof(struct fastrpc_ioctl_dspsignal_wait));
  6869. if (err) {
  6870. err = -EFAULT;
  6871. goto bail;
  6872. }
  6873. VERIFY(err, 0 == (err = fastrpc_dspsignal_wait(fl, &p.wait)));
  6874. if (err)
  6875. goto bail;
  6876. break;
  6877. case FASTRPC_IOCTL_DSPSIGNAL_CREATE:
  6878. K_COPY_FROM_USER(err, 0, &p.cre, param,
  6879. sizeof(struct fastrpc_ioctl_dspsignal_create));
  6880. if (err) {
  6881. err = -EFAULT;
  6882. goto bail;
  6883. }
  6884. VERIFY(err, 0 == (err = fastrpc_dspsignal_create(fl, &p.cre)));
  6885. if (err)
  6886. goto bail;
  6887. break;
  6888. case FASTRPC_IOCTL_DSPSIGNAL_DESTROY:
  6889. K_COPY_FROM_USER(err, 0, &p.des, param,
  6890. sizeof(struct fastrpc_ioctl_dspsignal_destroy));
  6891. if (err) {
  6892. err = -EFAULT;
  6893. goto bail;
  6894. }
  6895. VERIFY(err, 0 == (err = fastrpc_dspsignal_destroy(fl, &p.des)));
  6896. if (err)
  6897. goto bail;
  6898. break;
  6899. case FASTRPC_IOCTL_DSPSIGNAL_CANCEL_WAIT:
  6900. K_COPY_FROM_USER(err, 0, &p.canc, param,
  6901. sizeof(struct fastrpc_ioctl_dspsignal_cancel_wait));
  6902. if (err) {
  6903. err = -EFAULT;
  6904. goto bail;
  6905. }
  6906. VERIFY(err, 0 == (err = fastrpc_dspsignal_cancel_wait(fl, &p.canc)));
  6907. if (err)
  6908. goto bail;
  6909. break;
  6910. default:
  6911. err = -ENOTTY;
  6912. pr_info("bad ioctl: %d\n", ioctl_num);
  6913. break;
  6914. }
  6915. bail:
  6916. return err;
  6917. }
  6918. /*
  6919. * fastrpc_smq_ctx_detail : Store smq_invoke_ctx structure parameter.
  6920. * Input :
  6921. * structure smq_invoke_ctx
  6922. * void* mini_dump_buff
  6923. */
  6924. static void fastrpc_smq_ctx_detail(struct smq_invoke_ctx *smq_ctx, int cid, void *mini_dump_buff)
  6925. {
  6926. int i = 0;
  6927. remote_arg64_t *rpra = NULL;
  6928. struct fastrpc_mmap *map = NULL;
  6929. if (!smq_ctx)
  6930. return;
  6931. if (smq_ctx->buf && smq_ctx->buf->virt)
  6932. rpra = smq_ctx->buf->virt;
  6933. for (i = 0; rpra &&
  6934. i < (REMOTE_SCALARS_INBUFS(smq_ctx->sc) + REMOTE_SCALARS_OUTBUFS(smq_ctx->sc));
  6935. ++i) {
  6936. map = smq_ctx->maps[i];
  6937. if (map) {
  6938. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  6939. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  6940. smq_invoke_ctx_params,
  6941. smq_ctx->pid, smq_ctx->tgid, smq_ctx->handle,
  6942. smq_ctx->sc, smq_ctx->fl, smq_ctx->fds,
  6943. smq_ctx->magic);
  6944. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  6945. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  6946. fastrpc_mmap_params,
  6947. map->fd, map->flags, map->buf,
  6948. map->phys, map->size, map->va,
  6949. map->raddr, map->len, map->refs,
  6950. map->secure);
  6951. } else {
  6952. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  6953. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  6954. smq_invoke_ctx_params, smq_ctx->pid, smq_ctx->tgid,
  6955. smq_ctx->handle, smq_ctx->sc, smq_ctx->fl, smq_ctx->fds,
  6956. smq_ctx->magic);
  6957. }
  6958. break;
  6959. }
  6960. }
  6961. /*
  6962. * fastrpc_print_fastrpcbuf : Print fastrpc_buf structure parameter.
  6963. * Input :
  6964. * structure fastrpc_buf
  6965. * void* buffer
  6966. */
  6967. static void fastrpc_print_fastrpcbuf(struct fastrpc_buf *buf, void *buffer)
  6968. {
  6969. if (!buf || !buffer)
  6970. return;
  6971. scnprintf(buffer + strlen(buffer),
  6972. MINI_DUMP_DBG_SIZE - strlen(buffer),
  6973. fastrpc_buf_params, buf->fl, buf->phys,
  6974. buf->virt, buf->size, buf->dma_attr, buf->raddr,
  6975. buf->flags, buf->type, buf->in_use);
  6976. }
  6977. /*
  6978. * fastrpc_print_debug_data : Print debug structure variable in CMA memory.
  6979. * Input cid: Channel id
  6980. */
  6981. static void fastrpc_print_debug_data(int cid)
  6982. {
  6983. unsigned int i = 0, count = 0, gmsg_log_iter = 3, err = 0, len = 0;
  6984. unsigned int tx_index = 0, rx_index = 0;
  6985. unsigned long flags = 0;
  6986. char *gmsg_log_tx = NULL;
  6987. char *gmsg_log_rx = NULL;
  6988. void *mini_dump_buff = NULL;
  6989. struct fastrpc_apps *me = &gfa;
  6990. struct smq_invoke_rspv2 *rsp = NULL;
  6991. struct fastrpc_file *fl = NULL;
  6992. struct fastrpc_channel_ctx *chan = NULL;
  6993. struct hlist_node *n = NULL;
  6994. struct smq_invoke_ctx *ictx = NULL;
  6995. struct fastrpc_tx_msg *tx_msg = NULL;
  6996. struct fastrpc_buf *buf = NULL;
  6997. struct fastrpc_mmap *map = NULL;
  6998. unsigned long irq_flags = 0;
  6999. VERIFY(err, NULL != (gmsg_log_tx = kzalloc(MD_GMSG_BUFFER, GFP_KERNEL)));
  7000. if (err) {
  7001. err = -ENOMEM;
  7002. goto free_buf;
  7003. }
  7004. VERIFY(err, NULL != (gmsg_log_rx = kzalloc(MD_GMSG_BUFFER, GFP_KERNEL)));
  7005. if (err) {
  7006. err = -ENOMEM;
  7007. goto free_buf;
  7008. }
  7009. chan = &me->channel[cid];
  7010. if ((!chan) || (!chan->buf))
  7011. goto free_buf;
  7012. mini_dump_buff = chan->buf->virt;
  7013. if (!mini_dump_buff)
  7014. goto free_buf;
  7015. if (chan) {
  7016. tx_index = chan->gmsg_log.tx_index;
  7017. rx_index = chan->gmsg_log.rx_index;
  7018. }
  7019. spin_lock_irqsave(&me->hlock, irq_flags);
  7020. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  7021. if (fl->cid == cid && fl->is_ramdump_pend) {
  7022. scnprintf(mini_dump_buff +
  7023. strlen(mini_dump_buff),
  7024. MINI_DUMP_DBG_SIZE -
  7025. strlen(mini_dump_buff),
  7026. "\nfastrpc_file : %p\n", fl);
  7027. scnprintf(mini_dump_buff +
  7028. strlen(mini_dump_buff),
  7029. MINI_DUMP_DBG_SIZE -
  7030. strlen(mini_dump_buff),
  7031. fastrpc_file_params, fl->tgid,
  7032. fl->cid, fl->ssrcount, fl->pd,
  7033. fl->profile, fl->mode,
  7034. fl->tgid_open, fl->num_cached_buf,
  7035. fl->num_pers_hdrs, fl->sessionid,
  7036. fl->servloc_name, fl->file_close,
  7037. fl->dsp_proc_init, fl->apps,
  7038. fl->qos_request, fl->dev_minor,
  7039. fl->debug_buf,
  7040. fl->debug_buf_alloced_attempted,
  7041. fl->wake_enable,
  7042. fl->ws_timeout,
  7043. fl->untrusted_process);
  7044. scnprintf(mini_dump_buff +
  7045. strlen(mini_dump_buff),
  7046. MINI_DUMP_DBG_SIZE -
  7047. strlen(mini_dump_buff),
  7048. "\nSession Maps\n");
  7049. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  7050. scnprintf(mini_dump_buff +
  7051. strlen(mini_dump_buff),
  7052. MINI_DUMP_DBG_SIZE -
  7053. strlen(mini_dump_buff),
  7054. fastrpc_mmap_params,
  7055. map->fd,
  7056. map->flags, map->buf,
  7057. map->phys, map->size,
  7058. map->va, map->raddr,
  7059. map->len, map->refs,
  7060. map->secure);
  7061. }
  7062. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7063. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7064. "\ncached_bufs\n");
  7065. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  7066. fastrpc_print_fastrpcbuf(buf, mini_dump_buff);
  7067. }
  7068. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7069. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7070. "\ninit_mem: %p\n", fl->init_mem);
  7071. fastrpc_print_fastrpcbuf(fl->init_mem, mini_dump_buff);
  7072. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7073. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7074. "\npers_hdr_buf: %p\n", fl->pers_hdr_buf);
  7075. fastrpc_print_fastrpcbuf(fl->pers_hdr_buf, mini_dump_buff);
  7076. snprintf(mini_dump_buff + strlen(mini_dump_buff),
  7077. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7078. "\nhdr_bufs: %p\n", fl->hdr_bufs);
  7079. fastrpc_print_fastrpcbuf(fl->hdr_bufs, mini_dump_buff);
  7080. if (fl->debugfs_file) {
  7081. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7082. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7083. "\nfl->debugfs_file.d_iname : %s\n",
  7084. fl->debugfs_file->d_iname);
  7085. }
  7086. if (fl->sctx) {
  7087. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7088. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7089. "\nfl->sctx->smmu.cb : %d\n",
  7090. fl->sctx->smmu.cb);
  7091. }
  7092. if (fl->secsctx) {
  7093. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7094. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7095. "\nfl->secsctx->smmu.cb : %d\n",
  7096. fl->secsctx->smmu.cb);
  7097. }
  7098. spin_lock(&fl->hlock);
  7099. scnprintf(mini_dump_buff +
  7100. strlen(mini_dump_buff),
  7101. MINI_DUMP_DBG_SIZE -
  7102. strlen(mini_dump_buff),
  7103. "\nPending Ctx:\n");
  7104. hlist_for_each_entry_safe(ictx, n, &fl->clst.pending, hn) {
  7105. fastrpc_smq_ctx_detail(ictx,
  7106. cid, mini_dump_buff);
  7107. }
  7108. scnprintf(mini_dump_buff +
  7109. strlen(mini_dump_buff),
  7110. MINI_DUMP_DBG_SIZE -
  7111. strlen(mini_dump_buff),
  7112. "\nInterrupted Ctx:\n");
  7113. hlist_for_each_entry_safe(ictx, n,
  7114. &fl->clst.interrupted,
  7115. hn) {
  7116. fastrpc_smq_ctx_detail(ictx,
  7117. cid, mini_dump_buff);
  7118. }
  7119. spin_unlock(&fl->hlock);
  7120. }
  7121. }
  7122. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7123. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  7124. if (rx_index) {
  7125. for (i = rx_index, count = 0, len = 0 ; i > 0 &&
  7126. count <= gmsg_log_iter; i--, count++) {
  7127. rsp = &chan->gmsg_log.rx_msgs[i].rsp;
  7128. len += scnprintf(gmsg_log_rx + len, MD_GMSG_BUFFER - len,
  7129. "ctx: 0x%x, retval: %d, flags: %d, early_wake_time: %d, version: %d\n",
  7130. rsp->ctx, rsp->retval, rsp->flags,
  7131. rsp->early_wake_time, rsp->version);
  7132. }
  7133. }
  7134. if (tx_index) {
  7135. for (i = tx_index, count = 0, len = 0;
  7136. i > 0 && count <= gmsg_log_iter;
  7137. i--, count++) {
  7138. tx_msg = &chan->gmsg_log.tx_msgs[i];
  7139. len += scnprintf(gmsg_log_tx + len, MD_GMSG_BUFFER - len,
  7140. "pid: %d, tid: %d, ctx: 0x%x, handle: 0x%x, sc: 0x%x, addr: 0x%x, size:%d\n",
  7141. tx_msg->msg.pid,
  7142. tx_msg->msg.tid,
  7143. tx_msg->msg.invoke.header.ctx,
  7144. tx_msg->msg.invoke.header.handle,
  7145. tx_msg->msg.invoke.header.sc,
  7146. tx_msg->msg.invoke.page.addr,
  7147. tx_msg->msg.invoke.page.size);
  7148. }
  7149. }
  7150. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  7151. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7152. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7153. "gmsg_log_tx:\n%s\n", gmsg_log_tx);
  7154. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7155. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7156. "gmsg_log_rx:\n %s\n", gmsg_log_rx);
  7157. if (chan && chan->buf)
  7158. chan->buf->size = strlen(mini_dump_buff);
  7159. free_buf:
  7160. kfree(gmsg_log_tx);
  7161. kfree(gmsg_log_rx);
  7162. }
  7163. void fastrpc_restart_drivers(int cid)
  7164. {
  7165. struct fastrpc_apps *me = &gfa;
  7166. fastrpc_notify_drivers(me, cid);
  7167. me->channel[cid].ssrcount++;
  7168. }
  7169. static int fastrpc_restart_notifier_cb(struct notifier_block *nb,
  7170. unsigned long code,
  7171. void *data)
  7172. {
  7173. struct fastrpc_apps *me = &gfa;
  7174. struct fastrpc_channel_ctx *ctx;
  7175. struct fastrpc_file *fl;
  7176. struct hlist_node *n;
  7177. int cid = -1;
  7178. struct timespec64 startT = {0};
  7179. unsigned long irq_flags = 0;
  7180. ctx = container_of(nb, struct fastrpc_channel_ctx, nb);
  7181. cid = ctx - &me->channel[0];
  7182. switch (code) {
  7183. case QCOM_SSR_BEFORE_SHUTDOWN:
  7184. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7185. "QCOM_SSR_BEFORE_SHUTDOWN", "fastrpc_restart_notifier-enter");
  7186. pr_info("adsprpc: %s: %s subsystem is restarting\n",
  7187. __func__, gcinfo[cid].subsys);
  7188. mutex_lock(&me->channel[cid].smd_mutex);
  7189. ctx->ssrcount++;
  7190. ctx->subsystemstate = SUBSYSTEM_RESTARTING;
  7191. mutex_unlock(&me->channel[cid].smd_mutex);
  7192. if (cid == RH_CID)
  7193. me->staticpd_flags = 0;
  7194. break;
  7195. case QCOM_SSR_AFTER_SHUTDOWN:
  7196. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7197. "QCOM_SSR_AFTER_SHUTDOWN", "fastrpc_restart_notifier-enter");
  7198. spin_lock_irqsave(&me->hlock, irq_flags);
  7199. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  7200. if (fl->cid != cid)
  7201. continue;
  7202. complete(&fl->shutdown);
  7203. }
  7204. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7205. ctx->subsystemstate = SUBSYSTEM_DOWN;
  7206. pr_info("adsprpc: %s: received RAMDUMP notification for %s\n",
  7207. __func__, gcinfo[cid].subsys);
  7208. break;
  7209. case QCOM_SSR_BEFORE_POWERUP:
  7210. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7211. "QCOM_SSR_BEFORE_POWERUP", "fastrpc_restart_notifier-enter");
  7212. pr_info("adsprpc: %s: subsystem %s is about to start\n",
  7213. __func__, gcinfo[cid].subsys);
  7214. if (cid == CDSP_DOMAIN_ID && dump_enabled() &&
  7215. ctx->ssrcount)
  7216. fastrpc_update_ramdump_status(cid);
  7217. fastrpc_notify_drivers(me, cid);
  7218. /* Skip ram dump collection in first boot */
  7219. if (cid == CDSP_DOMAIN_ID && dump_enabled() &&
  7220. ctx->ssrcount) {
  7221. mutex_lock(&me->channel[cid].smd_mutex);
  7222. fastrpc_print_debug_data(cid);
  7223. mutex_unlock(&me->channel[cid].smd_mutex);
  7224. ktime_get_real_ts64(&startT);
  7225. fastrpc_ramdump_collection(cid);
  7226. pr_info("adsprpc: %s: fastrpc ramdump finished in %lu (us)\n",
  7227. __func__, getnstimediff(&startT));
  7228. }
  7229. break;
  7230. case QCOM_SSR_AFTER_POWERUP:
  7231. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7232. "QCOM_SSR_AFTER_POWERUP", "fastrpc_restart_notifier-enter");
  7233. pr_info("adsprpc: %s: %s subsystem is up\n",
  7234. __func__, gcinfo[cid].subsys);
  7235. ctx->subsystemstate = SUBSYSTEM_UP;
  7236. break;
  7237. default:
  7238. break;
  7239. }
  7240. fastrpc_rproc_trace_events(dev_name(me->dev), "fastrpc_restart_notifier", "exit");
  7241. return NOTIFY_DONE;
  7242. }
  7243. static void fastrpc_pdr_cb(int state, char *service_path, void *priv)
  7244. {
  7245. struct fastrpc_apps *me = &gfa;
  7246. struct fastrpc_static_pd *spd;
  7247. int err = 0;
  7248. spd = priv;
  7249. VERIFY(err, spd);
  7250. if (err)
  7251. goto bail;
  7252. switch (state) {
  7253. case SERVREG_SERVICE_STATE_DOWN:
  7254. pr_info("adsprpc: %s: %s (%s) is down for PDR on %s\n",
  7255. __func__, spd->spdname,
  7256. spd->servloc_name,
  7257. gcinfo[spd->cid].subsys);
  7258. mutex_lock(&me->channel[spd->cid].smd_mutex);
  7259. spd->pdrcount++;
  7260. atomic_set(&spd->ispdup, 0);
  7261. mutex_unlock(&me->channel[spd->cid].smd_mutex);
  7262. if (!strcmp(spd->servloc_name,
  7263. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME))
  7264. me->staticpd_flags = 0;
  7265. fastrpc_notify_pdr_drivers(me, spd->servloc_name);
  7266. break;
  7267. case SERVREG_SERVICE_STATE_UP:
  7268. pr_info("adsprpc: %s: %s (%s) is up for PDR on %s\n",
  7269. __func__, spd->spdname,
  7270. spd->servloc_name,
  7271. gcinfo[spd->cid].subsys);
  7272. atomic_set(&spd->ispdup, 1);
  7273. wake_up_interruptible(&spd->wait_for_pdup);
  7274. break;
  7275. default:
  7276. break;
  7277. }
  7278. bail:
  7279. if (err) {
  7280. pr_err("adsprpc: %s: failed for path %s, state %d, spd %pK\n",
  7281. __func__, service_path, state, spd);
  7282. }
  7283. }
  7284. static const struct file_operations fops = {
  7285. .open = fastrpc_device_open,
  7286. .release = fastrpc_device_release,
  7287. .unlocked_ioctl = fastrpc_device_ioctl,
  7288. /* Only DSP service 64-bit app will interface with fastrpc TVM driver.
  7289. * There is not need to support 32-bit fastrpc driver on TVM.
  7290. */
  7291. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  7292. .compat_ioctl = NULL,
  7293. #else
  7294. .compat_ioctl = compat_fastrpc_device_ioctl,
  7295. #endif
  7296. };
  7297. static const struct of_device_id fastrpc_match_table[] = {
  7298. { .compatible = "qcom,msm-fastrpc-adsp", },
  7299. { .compatible = "qcom,msm-fastrpc-compute", },
  7300. { .compatible = "qcom,msm-fastrpc-compute-cb", },
  7301. { .compatible = "qcom,msm-adsprpc-mem-region", },
  7302. {}
  7303. };
  7304. static int fastrpc_cb_probe(struct device *dev)
  7305. {
  7306. struct fastrpc_channel_ctx *chan = NULL;
  7307. struct fastrpc_session_ctx *sess = NULL;
  7308. struct of_phandle_args iommuspec;
  7309. struct fastrpc_apps *me = &gfa;
  7310. struct fastrpc_buf *buf = NULL;
  7311. struct gen_pool *gen_pool = NULL;
  7312. struct iommu_domain *domain = NULL;
  7313. const char *name;
  7314. int err = 0, cid = -1, i = 0;
  7315. u32 sharedcb_count = 0, j = 0;
  7316. uint32_t dma_addr_pool[2] = {0, 0};
  7317. VERIFY(err, NULL != (name = of_get_property(dev->of_node,
  7318. "label", NULL)));
  7319. if (err) {
  7320. err = -EINVAL;
  7321. goto bail;
  7322. }
  7323. for (i = 0; i < NUM_CHANNELS; i++) {
  7324. if (!gcinfo[i].name)
  7325. continue;
  7326. if (!strcmp(name, gcinfo[i].name))
  7327. break;
  7328. }
  7329. VERIFY(err, i < NUM_CHANNELS);
  7330. if (err) {
  7331. err = -ECHRNG;
  7332. goto bail;
  7333. }
  7334. cid = i;
  7335. chan = &gcinfo[i];
  7336. VERIFY(err, chan->sesscount < NUM_SESSIONS);
  7337. if (err) {
  7338. err = -EINVAL;
  7339. goto bail;
  7340. }
  7341. err = of_parse_phandle_with_args(dev->of_node, "iommus",
  7342. "#iommu-cells", 0, &iommuspec);
  7343. if (err) {
  7344. pr_err("Error: adsprpc: %s: parsing iommu arguments failed for %s with err %d\n",
  7345. __func__, dev_name(dev), err);
  7346. goto bail;
  7347. }
  7348. sess = &chan->session[chan->sesscount];
  7349. sess->used = 0;
  7350. sess->smmu.coherent = of_property_read_bool(dev->of_node,
  7351. "dma-coherent");
  7352. sess->smmu.secure = of_property_read_bool(dev->of_node,
  7353. "qcom,secure-context-bank");
  7354. sess->smmu.cb = iommuspec.args[0] & 0xf;
  7355. sess->smmu.dev = dev;
  7356. sess->smmu.dev_name = dev_name(dev);
  7357. sess->smmu.enabled = 1;
  7358. if (!sess->smmu.dev->dma_parms)
  7359. sess->smmu.dev->dma_parms = devm_kzalloc(sess->smmu.dev,
  7360. sizeof(*sess->smmu.dev->dma_parms), GFP_KERNEL);
  7361. dma_set_max_seg_size(sess->smmu.dev, DMA_BIT_MASK(32));
  7362. dma_set_seg_boundary(sess->smmu.dev, (unsigned long)DMA_BIT_MASK(64));
  7363. of_property_read_u32_array(dev->of_node, "qcom,iommu-dma-addr-pool",
  7364. dma_addr_pool, 2);
  7365. me->max_size_limit = (dma_addr_pool[1] == 0 ? 0x78000000 :
  7366. dma_addr_pool[1]);
  7367. if (of_get_property(dev->of_node, "pd-type", NULL) != NULL) {
  7368. err = of_property_read_u32(dev->of_node, "pd-type",
  7369. &(sess->smmu.pd_type));
  7370. /* Set cb_pd_type, if the process type is set for context banks */
  7371. me->cb_pd_type = true;
  7372. if (err)
  7373. goto bail;
  7374. }
  7375. if (of_get_property(dev->of_node, "shared-cb", NULL) != NULL) {
  7376. sess->smmu.sharedcb = 1;
  7377. // Set share_securecb, if the secure context bank is shared
  7378. if (sess->smmu.secure)
  7379. me->share_securecb = 1;
  7380. err = of_property_read_u32(dev->of_node, "shared-cb",
  7381. &sharedcb_count);
  7382. if (err)
  7383. goto bail;
  7384. if (sharedcb_count > 0) {
  7385. struct fastrpc_session_ctx *dup_sess;
  7386. for (j = 1; j < sharedcb_count &&
  7387. chan->sesscount < NUM_SESSIONS; j++) {
  7388. chan->sesscount++;
  7389. dup_sess = &chan->session[chan->sesscount];
  7390. memcpy(dup_sess, sess,
  7391. sizeof(struct fastrpc_session_ctx));
  7392. }
  7393. }
  7394. }
  7395. if (of_get_property(dev->of_node, "qrtr-gen-pool", NULL) != NULL) {
  7396. u32 frpc_gen_addr_pool[2] = {0, 0};
  7397. struct sg_table sgt;
  7398. err = of_property_read_u32_array(dev->of_node, "frpc-gen-addr-pool",
  7399. frpc_gen_addr_pool, 2);
  7400. if (err) {
  7401. pr_err("Error: adsprpc: %s: parsing frpc-gen-addr-pool arguments failed for %s with err %d\n",
  7402. __func__, dev_name(dev), err);
  7403. goto bail;
  7404. }
  7405. sess->smmu.genpool_iova = frpc_gen_addr_pool[0];
  7406. sess->smmu.genpool_size = frpc_gen_addr_pool[1];
  7407. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  7408. if (err) {
  7409. err = -ENOMEM;
  7410. ADSPRPC_ERR(
  7411. "allocation failed for size 0x%zx\n", sizeof(*buf));
  7412. goto bail;
  7413. }
  7414. INIT_HLIST_NODE(&buf->hn);
  7415. buf->virt = NULL;
  7416. buf->phys = 0;
  7417. buf->size = frpc_gen_addr_pool[1];
  7418. buf->dma_attr = DMA_ATTR_DELAYED_UNMAP;
  7419. /* Allocate memory for adding to genpool */
  7420. buf->virt = dma_alloc_attrs(sess->smmu.dev, buf->size,
  7421. (dma_addr_t *)&buf->phys,
  7422. GFP_KERNEL, buf->dma_attr);
  7423. if (IS_ERR_OR_NULL(buf->virt)) {
  7424. ADSPRPC_ERR(
  7425. "dma_alloc_attrs failed for size 0x%zx, returned %pK\n",
  7426. buf->size, buf->virt);
  7427. err = -ENOBUFS;
  7428. goto dma_alloc_bail;
  7429. }
  7430. err = dma_get_sgtable_attrs(sess->smmu.dev, &sgt, buf->virt,
  7431. buf->phys, buf->size, buf->dma_attr);
  7432. if (err) {
  7433. ADSPRPC_ERR("dma_get_sgtable_attrs failed with err %d", err);
  7434. goto iommu_map_bail;
  7435. }
  7436. domain = iommu_get_domain_for_dev(sess->smmu.dev);
  7437. if (!domain) {
  7438. ADSPRPC_ERR("iommu_get_domain_for_dev failed ");
  7439. goto iommu_map_bail;
  7440. }
  7441. /* Map the allocated memory with fixed IOVA and is shared to remote subsystem */
  7442. err = iommu_map_sg(domain, frpc_gen_addr_pool[0], sgt.sgl,
  7443. sgt.nents, IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE);
  7444. if (err < 0) {
  7445. ADSPRPC_ERR("iommu_map_sg failed with err %d", err);
  7446. goto iommu_map_bail;
  7447. }
  7448. /* Create genpool using SMMU device */
  7449. gen_pool = devm_gen_pool_create(sess->smmu.dev, 0,
  7450. NUMA_NO_NODE, NULL);
  7451. if (IS_ERR(gen_pool)) {
  7452. err = PTR_ERR(gen_pool);
  7453. ADSPRPC_ERR("devm_gen_pool_create failed with err %d", err);
  7454. goto genpool_create_bail;
  7455. }
  7456. /* Add allocated memory to genpool */
  7457. err = gen_pool_add_virt(gen_pool, (unsigned long)buf->virt,
  7458. buf->phys, buf->size, NUMA_NO_NODE);
  7459. if (err) {
  7460. ADSPRPC_ERR("gen_pool_add_virt failed with err %d", err);
  7461. goto genpool_add_bail;
  7462. }
  7463. sess->smmu.frpc_genpool = gen_pool;
  7464. sess->smmu.frpc_genpool_buf = buf;
  7465. }
  7466. chan->sesscount++;
  7467. if (debugfs_root && !debugfs_global_file) {
  7468. debugfs_global_file = debugfs_create_file("global", 0644,
  7469. debugfs_root, NULL, &debugfs_fops);
  7470. if (IS_ERR_OR_NULL(debugfs_global_file)) {
  7471. pr_warn("Error: %s: %s: failed to create debugfs global file\n",
  7472. current->comm, __func__);
  7473. debugfs_global_file = NULL;
  7474. }
  7475. }
  7476. bail:
  7477. return err;
  7478. genpool_add_bail:
  7479. gen_pool_destroy(gen_pool);
  7480. genpool_create_bail:
  7481. iommu_unmap(domain, sess->smmu.genpool_iova,
  7482. sess->smmu.genpool_size);
  7483. iommu_map_bail:
  7484. dma_free_attrs(sess->smmu.dev, buf->size, buf->virt,
  7485. buf->phys, buf->dma_attr);
  7486. dma_alloc_bail:
  7487. kfree(buf);
  7488. return err;
  7489. }
  7490. static void init_secure_vmid_list(struct device *dev, char *prop_name,
  7491. struct secure_vm *destvm)
  7492. {
  7493. int err = 0;
  7494. u32 len = 0, i = 0;
  7495. u32 *rhvmlist = NULL;
  7496. u32 *rhvmpermlist = NULL;
  7497. if (!of_find_property(dev->of_node, prop_name, &len))
  7498. goto bail;
  7499. if (len == 0)
  7500. goto bail;
  7501. len /= sizeof(u32);
  7502. VERIFY(err, NULL != (rhvmlist = kcalloc(len, sizeof(u32), GFP_KERNEL)));
  7503. if (err)
  7504. goto bail;
  7505. VERIFY(err, NULL != (rhvmpermlist = kcalloc(len, sizeof(u32),
  7506. GFP_KERNEL)));
  7507. if (err)
  7508. goto bail;
  7509. for (i = 0; i < len; i++) {
  7510. err = of_property_read_u32_index(dev->of_node, prop_name, i,
  7511. &rhvmlist[i]);
  7512. if (err) {
  7513. pr_err("Error: adsprpc: %s: failed to read VMID\n",
  7514. __func__);
  7515. goto bail;
  7516. }
  7517. ADSPRPC_INFO("secure VMID = %d\n",
  7518. rhvmlist[i]);
  7519. rhvmpermlist[i] = QCOM_SCM_PERM_RWX;
  7520. }
  7521. destvm->vmid = rhvmlist;
  7522. destvm->vmperm = rhvmpermlist;
  7523. destvm->vmcount = len;
  7524. bail:
  7525. if (err) {
  7526. kfree(rhvmlist);
  7527. kfree(rhvmpermlist);
  7528. }
  7529. }
  7530. static void fastrpc_init_privileged_gids(struct device *dev, char *prop_name,
  7531. struct gid_list *gidlist)
  7532. {
  7533. int err = 0;
  7534. u32 len = 0, i = 0;
  7535. u32 *gids = NULL;
  7536. if (!of_find_property(dev->of_node, prop_name, &len))
  7537. goto bail;
  7538. if (len == 0)
  7539. goto bail;
  7540. len /= sizeof(u32);
  7541. gids = kcalloc(len, sizeof(u32), GFP_KERNEL);
  7542. if (!gids) {
  7543. err = ENOMEM;
  7544. goto bail;
  7545. }
  7546. for (i = 0; i < len; i++) {
  7547. err = of_property_read_u32_index(dev->of_node, prop_name,
  7548. i, &gids[i]);
  7549. if (err) {
  7550. pr_err("Error: adsprpc: %s: failed to read GID %u\n",
  7551. __func__, i);
  7552. goto bail;
  7553. }
  7554. pr_info("adsprpc: %s: privileged GID: %u\n", __func__, gids[i]);
  7555. }
  7556. sort(gids, len, sizeof(*gids), uint_cmp_func, NULL);
  7557. gidlist->gids = gids;
  7558. gidlist->gidcount = len;
  7559. bail:
  7560. if (err)
  7561. kfree(gids);
  7562. }
  7563. static void configure_secure_channels(uint32_t secure_domains)
  7564. {
  7565. struct fastrpc_apps *me = &gfa;
  7566. int ii = 0;
  7567. /*
  7568. * secure_domains contains the bitmask of the secure channels
  7569. * Bit 0 - ADSP
  7570. * Bit 1 - MDSP
  7571. * Bit 2 - SLPI
  7572. * Bit 3 - CDSP
  7573. */
  7574. for (ii = ADSP_DOMAIN_ID; ii <= CDSP_DOMAIN_ID; ++ii) {
  7575. int secure = (secure_domains >> ii) & 0x01;
  7576. me->channel[ii].secure = secure;
  7577. ADSPRPC_INFO("domain %d configured as secure %d\n", ii, secure);
  7578. }
  7579. }
  7580. /*
  7581. * This function is used to create the service locator required for
  7582. * registering for remote process restart (PDR) notifications if that
  7583. * PDR property has been enabled in the fastrpc node on the DTSI.
  7584. */
  7585. static int fastrpc_setup_service_locator(struct device *dev,
  7586. const char *propname,
  7587. char *client_name, char *service_name,
  7588. char *service_path)
  7589. {
  7590. int err = 0, session = -1, cid = -1;
  7591. struct fastrpc_apps *me = &gfa;
  7592. struct pdr_handle *handle = NULL;
  7593. struct pdr_service *service = NULL;
  7594. if (of_property_read_bool(dev->of_node, propname)) {
  7595. err = fastrpc_get_spd_session(client_name, &session, &cid);
  7596. if (err)
  7597. goto bail;
  7598. /* Register the service locator's callback function */
  7599. handle = pdr_handle_alloc(fastrpc_pdr_cb, &me->channel[cid].spd[session]);
  7600. if (IS_ERR_OR_NULL(handle)) {
  7601. err = PTR_ERR(handle);
  7602. goto bail;
  7603. }
  7604. me->channel[cid].spd[session].pdrhandle = handle;
  7605. service = pdr_add_lookup(handle, service_name, service_path);
  7606. if (IS_ERR_OR_NULL(service)) {
  7607. err = PTR_ERR(service);
  7608. goto bail;
  7609. }
  7610. pr_info("adsprpc: %s: pdr_add_lookup enabled for %s (%s, %s), DTSI (%s)\n",
  7611. __func__, service_name, client_name, service_path, propname);
  7612. }
  7613. bail:
  7614. if (err) {
  7615. pr_warn("adsprpc: %s: failed for %s (%s, %s), DTSI (%s) with err %d\n",
  7616. __func__, service_name, client_name, service_path, propname, err);
  7617. }
  7618. return err;
  7619. }
  7620. /*
  7621. * fastrpc_nsp_status_show() - Updates the buffer with remote nsp status
  7622. * by reading the fastrpc node.
  7623. * @dev : pointer to device node.
  7624. * @attr: pointer to device attribute.
  7625. * @buf : Output parameter to be updated with remote nsp status.
  7626. * Return : bytes written to buffer.
  7627. */
  7628. static ssize_t fastrpc_nsp_status_show(struct device *dev,
  7629. struct device_attribute *attr, char *buf)
  7630. {
  7631. struct fastrpc_apps *me = &gfa;
  7632. /*
  7633. * Default remote DSP status: 0
  7634. * driver possibly not probed yet or not the main device.
  7635. */
  7636. if (!dev || !dev->driver ||
  7637. !of_device_is_compatible(dev->of_node, "qcom,msm-fastrpc-compute")) {
  7638. ADSPRPC_ERR("Driver not probed yet or not the main device\n");
  7639. return 0;
  7640. }
  7641. return scnprintf(buf, PAGE_SIZE, "%d",
  7642. me->fastrpc_nsp_status);
  7643. }
  7644. /* Remote nsp status attribute declaration as read only */
  7645. static DEVICE_ATTR_RO(fastrpc_nsp_status);
  7646. /* Declaring attribute for remote dsp */
  7647. static struct attribute *msm_remote_dsp_attrs[] = {
  7648. &dev_attr_fastrpc_nsp_status.attr,
  7649. NULL
  7650. };
  7651. /* Defining remote dsp attributes in attributes group */
  7652. static struct attribute_group msm_remote_dsp_attr_group = {
  7653. .attrs = msm_remote_dsp_attrs,
  7654. };
  7655. static int fastrpc_probe(struct platform_device *pdev)
  7656. {
  7657. int err = 0;
  7658. struct fastrpc_apps *me = &gfa;
  7659. struct device *dev = &pdev->dev;
  7660. int ret = 0;
  7661. uint32_t secure_domains = 0;
  7662. if (of_device_is_compatible(dev->of_node,
  7663. "qcom,msm-fastrpc-compute")) {
  7664. err = sysfs_create_group(&pdev->dev.kobj, &msm_remote_dsp_attr_group);
  7665. if (err) {
  7666. ADSPRPC_ERR(
  7667. "Initialization of sysfs create group failed with %d\n",
  7668. err);
  7669. goto bail;
  7670. }
  7671. init_secure_vmid_list(dev, "qcom,adsp-remoteheap-vmid",
  7672. &gcinfo[0].rhvm);
  7673. fastrpc_init_privileged_gids(dev, "qcom,fastrpc-gids",
  7674. &me->gidlist);
  7675. /*
  7676. * Check if latency voting for only one core
  7677. * is enabled for the platform
  7678. */
  7679. me->single_core_latency_vote = of_property_read_bool(dev->of_node,
  7680. "qcom,single-core-latency-vote");
  7681. if (me->single_core_latency_vote)
  7682. me->lowest_capacity_core_count = 1;
  7683. of_property_read_u32(dev->of_node, "qcom,rpc-latency-us",
  7684. &me->latency);
  7685. of_property_read_u32(dev->of_node, "qcom,max-sessions",
  7686. &me->max_sess_per_proc);
  7687. if (of_get_property(dev->of_node,
  7688. "qcom,secure-domains", NULL) != NULL) {
  7689. VERIFY(err, !of_property_read_u32(dev->of_node,
  7690. "qcom,secure-domains",
  7691. &secure_domains));
  7692. if (!err)
  7693. configure_secure_channels(secure_domains);
  7694. else
  7695. pr_info("adsprpc: unable to read the domain configuration from dts\n");
  7696. }
  7697. }
  7698. if (of_device_is_compatible(dev->of_node,
  7699. "qcom,msm-fastrpc-compute-cb"))
  7700. return fastrpc_cb_probe(dev);
  7701. if (of_device_is_compatible(dev->of_node,
  7702. "qcom,msm-adsprpc-mem-region")) {
  7703. me->dev = dev;
  7704. ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0);
  7705. if (ret) {
  7706. pr_warn("adsprpc: Error: %s: initialization of memory region adsp_mem failed with %d\n",
  7707. __func__, ret);
  7708. }
  7709. goto bail;
  7710. }
  7711. me->legacy_remote_heap = of_property_read_bool(dev->of_node,
  7712. "qcom,fastrpc-legacy-remote-heap");
  7713. err = fastrpc_setup_service_locator(dev, AUDIO_PDR_ADSP_DTSI_PROPERTY_NAME,
  7714. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME,
  7715. AUDIO_PDR_ADSP_SERVICE_NAME, ADSP_AUDIOPD_NAME);
  7716. if (err)
  7717. goto bail;
  7718. err = fastrpc_setup_service_locator(dev, SENSORS_PDR_ADSP_DTSI_PROPERTY_NAME,
  7719. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME,
  7720. SENSORS_PDR_ADSP_SERVICE_NAME, ADSP_SENSORPD_NAME);
  7721. if (err)
  7722. goto bail;
  7723. err = fastrpc_setup_service_locator(dev, SENSORS_PDR_SLPI_DTSI_PROPERTY_NAME,
  7724. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME,
  7725. SENSORS_PDR_SLPI_SERVICE_NAME, SLPI_SENSORPD_NAME);
  7726. if (err)
  7727. goto bail;
  7728. err = of_platform_populate(pdev->dev.of_node,
  7729. fastrpc_match_table,
  7730. NULL, &pdev->dev);
  7731. if (err)
  7732. goto bail;
  7733. bail:
  7734. return err;
  7735. }
  7736. /*
  7737. * Function to free fastrpc genpool buffer
  7738. */
  7739. static void fastrpc_genpool_free(struct fastrpc_session_ctx *sess)
  7740. {
  7741. struct fastrpc_buf *buf = NULL;
  7742. struct iommu_domain *domain = NULL;
  7743. if (!sess)
  7744. goto bail;
  7745. buf = sess->smmu.frpc_genpool_buf;
  7746. if (sess->smmu.frpc_genpool) {
  7747. gen_pool_destroy(sess->smmu.frpc_genpool);
  7748. sess->smmu.frpc_genpool = NULL;
  7749. }
  7750. if (buf && sess->smmu.dev) {
  7751. domain = iommu_get_domain_for_dev(sess->smmu.dev);
  7752. iommu_unmap(domain, sess->smmu.genpool_iova,
  7753. sess->smmu.genpool_size);
  7754. if (buf->phys)
  7755. dma_free_attrs(sess->smmu.dev, buf->size, buf->virt,
  7756. buf->phys, buf->dma_attr);
  7757. kfree(buf);
  7758. sess->smmu.frpc_genpool_buf = NULL;
  7759. }
  7760. bail:
  7761. return;
  7762. }
  7763. static void fastrpc_deinit(void)
  7764. {
  7765. struct fastrpc_channel_ctx *chan = gcinfo;
  7766. struct fastrpc_apps *me = &gfa;
  7767. int i, j;
  7768. for (i = 0; i < NUM_CHANNELS; i++, chan++) {
  7769. for (j = 0; j < NUM_SESSIONS; j++) {
  7770. struct fastrpc_session_ctx *sess = &chan->session[j];
  7771. fastrpc_genpool_free(sess);
  7772. if (sess->smmu.dev)
  7773. sess->smmu.dev = NULL;
  7774. }
  7775. kfree(chan->rhvm.vmid);
  7776. kfree(chan->rhvm.vmperm);
  7777. fastrpc_transport_session_deinit(i);
  7778. mutex_destroy(&chan->smd_mutex);
  7779. }
  7780. if (me->transport_initialized)
  7781. fastrpc_transport_deinit();
  7782. me->transport_initialized = 0;
  7783. mutex_destroy(&me->mut_uid);
  7784. }
  7785. #ifdef CONFIG_HIBERNATION
  7786. static bool hibernation;
  7787. static int fastrpc_hibernation_notifier(struct notifier_block *nb,
  7788. unsigned long event, void *dummy)
  7789. {
  7790. if (event == PM_HIBERNATION_PREPARE)
  7791. hibernation = true;
  7792. else if (event == PM_POST_HIBERNATION)
  7793. hibernation = false;
  7794. return NOTIFY_OK;
  7795. }
  7796. static struct notifier_block fastrpc_notif_block = {
  7797. .notifier_call = fastrpc_hibernation_notifier,
  7798. };
  7799. #endif
  7800. #ifdef CONFIG_PM_SLEEP
  7801. static int fastrpc_hibernation_suspend(struct device *dev)
  7802. {
  7803. int err = 0;
  7804. if (of_device_is_compatible(dev->of_node,
  7805. "qcom,msm-fastrpc-compute")) {
  7806. err = fastrpc_mmap_remove_ssr(NULL, 0);
  7807. if (err)
  7808. ADSPRPC_WARN("failed to unmap remote heap (err %d)\n",
  7809. err);
  7810. }
  7811. return err;
  7812. }
  7813. static int fastrpc_restore(struct device *dev)
  7814. {
  7815. struct fastrpc_apps *me = &gfa;
  7816. int cid;
  7817. pr_info("adsprpc: restore enter\n");
  7818. for (cid = 0; cid < NUM_CHANNELS; cid++)
  7819. me->channel[cid].in_hib = 1;
  7820. pr_info("adsprpc: restore exit\n");
  7821. return 0;
  7822. }
  7823. static const struct dev_pm_ops fastrpc_pm = {
  7824. .freeze = fastrpc_hibernation_suspend,
  7825. .restore = fastrpc_restore,
  7826. };
  7827. #endif
  7828. static struct platform_driver fastrpc_driver = {
  7829. .probe = fastrpc_probe,
  7830. .driver = {
  7831. .name = "fastrpc",
  7832. .of_match_table = fastrpc_match_table,
  7833. .suppress_bind_attrs = true,
  7834. #ifdef CONFIG_PM_SLEEP
  7835. .pm = &fastrpc_pm,
  7836. #endif
  7837. },
  7838. };
  7839. union fastrpc_dev_param {
  7840. struct fastrpc_dev_map_dma *map;
  7841. struct fastrpc_dev_unmap_dma *unmap;
  7842. struct fastrpc_dev_get_hlos_pid *hpid;
  7843. };
  7844. long fastrpc_dev_map_dma(struct fastrpc_device *dev, unsigned long invoke_param)
  7845. {
  7846. int err = 0;
  7847. union fastrpc_dev_param p;
  7848. struct fastrpc_file *fl = NULL;
  7849. struct fastrpc_mmap *map = NULL;
  7850. struct fastrpc_apps *me = &gfa;
  7851. uintptr_t raddr = 0;
  7852. unsigned long irq_flags = 0;
  7853. p.map = (struct fastrpc_dev_map_dma *)invoke_param;
  7854. spin_lock_irqsave(&me->hlock, irq_flags);
  7855. /* Verify if fastrpc device is closed*/
  7856. VERIFY(err, dev && !dev->dev_close);
  7857. if (err) {
  7858. err = -ESRCH;
  7859. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7860. return err;
  7861. }
  7862. fl = dev->fl;
  7863. /* Verify if fastrpc file is not NULL*/
  7864. if (!fl) {
  7865. err = -EBADF;
  7866. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7867. return err;
  7868. }
  7869. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7870. mutex_lock(&fl->internal_map_mutex);
  7871. spin_lock_irqsave(&me->hlock, irq_flags);
  7872. /* Verify if fastrpc file is being closed, holding device lock*/
  7873. if (fl->file_close) {
  7874. err = -ESRCH;
  7875. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7876. goto bail;
  7877. }
  7878. fl->is_dma_invoke_pend = true;
  7879. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7880. mutex_lock(&fl->map_mutex);
  7881. /* Map DMA buffer on SMMU device*/
  7882. err = fastrpc_mmap_create(fl, -1, p.map->buf,
  7883. p.map->attrs, 0, p.map->size,
  7884. ADSP_MMAP_DMA_BUFFER, &map);
  7885. mutex_unlock(&fl->map_mutex);
  7886. if (err)
  7887. goto bail;
  7888. /* Map DMA buffer on DSP*/
  7889. VERIFY(err, 0 == (err = fastrpc_mmap_on_dsp(fl,
  7890. map->flags, 0, map->phys, map->size, map->refs, &raddr)));
  7891. if (err)
  7892. goto bail;
  7893. map->raddr = raddr;
  7894. p.map->v_dsp_addr = raddr;
  7895. bail:
  7896. if (err && map) {
  7897. mutex_lock(&fl->map_mutex);
  7898. fastrpc_mmap_free(map, 0);
  7899. mutex_unlock(&fl->map_mutex);
  7900. }
  7901. if (fl) {
  7902. spin_lock_irqsave(&me->hlock, irq_flags);
  7903. if (fl->file_close && fl->is_dma_invoke_pend)
  7904. complete(&fl->dma_invoke);
  7905. fl->is_dma_invoke_pend = false;
  7906. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7907. }
  7908. mutex_unlock(&fl->internal_map_mutex);
  7909. return err;
  7910. }
  7911. long fastrpc_dev_unmap_dma(struct fastrpc_device *dev, unsigned long invoke_param)
  7912. {
  7913. int err = 0;
  7914. union fastrpc_dev_param p;
  7915. struct fastrpc_file *fl = NULL;
  7916. struct fastrpc_mmap *map = NULL;
  7917. struct fastrpc_apps *me = &gfa;
  7918. unsigned long irq_flags = 0;
  7919. p.unmap = (struct fastrpc_dev_unmap_dma *)invoke_param;
  7920. spin_lock_irqsave(&me->hlock, irq_flags);
  7921. /* Verify if fastrpc device is closed*/
  7922. VERIFY(err, dev && !dev->dev_close);
  7923. if (err) {
  7924. err = -ESRCH;
  7925. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7926. return err;
  7927. }
  7928. fl = dev->fl;
  7929. /* Verify if fastrpc file is not NULL*/
  7930. if (!fl) {
  7931. err = -EBADF;
  7932. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7933. return err;
  7934. }
  7935. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7936. mutex_lock(&fl->internal_map_mutex);
  7937. spin_lock_irqsave(&me->hlock, irq_flags);
  7938. /* Verify if fastrpc file is being closed, holding device lock*/
  7939. if (fl->file_close) {
  7940. err = -ESRCH;
  7941. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7942. goto bail;
  7943. }
  7944. fl->is_dma_invoke_pend = true;
  7945. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7946. mutex_lock(&fl->map_mutex);
  7947. if (!fastrpc_mmap_find(fl, -1, p.unmap->buf, 0, 0, ADSP_MMAP_DMA_BUFFER, 0, &map)) {
  7948. mutex_unlock(&fl->map_mutex);
  7949. if (err)
  7950. goto bail;
  7951. /* Un-map DMA buffer on DSP*/
  7952. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, map->raddr,
  7953. map->phys, map->size, map->flags)));
  7954. if (err)
  7955. goto bail;
  7956. mutex_lock(&fl->map_mutex);
  7957. fastrpc_mmap_free(map, 0);
  7958. }
  7959. mutex_unlock(&fl->map_mutex);
  7960. bail:
  7961. if (fl) {
  7962. spin_lock_irqsave(&me->hlock, irq_flags);
  7963. if (fl->file_close && fl->is_dma_invoke_pend)
  7964. complete(&fl->dma_invoke);
  7965. fl->is_dma_invoke_pend = false;
  7966. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7967. }
  7968. mutex_unlock(&fl->internal_map_mutex);
  7969. return err;
  7970. }
  7971. long fastrpc_dev_get_hlos_pid(struct fastrpc_device *dev, unsigned long invoke_param)
  7972. {
  7973. int err = 0;
  7974. union fastrpc_dev_param p;
  7975. struct fastrpc_file *fl = NULL;
  7976. struct fastrpc_apps *me = &gfa;
  7977. unsigned long irq_flags = 0;
  7978. p.hpid = (struct fastrpc_dev_get_hlos_pid *)invoke_param;
  7979. spin_lock_irqsave(&me->hlock, irq_flags);
  7980. /* Verify if fastrpc device is closed*/
  7981. VERIFY(err, dev && !dev->dev_close);
  7982. if (err) {
  7983. err = -ESRCH;
  7984. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7985. return err;
  7986. }
  7987. fl = dev->fl;
  7988. /* Verify if fastrpc file is not NULL*/
  7989. if (!fl) {
  7990. err = -EBADF;
  7991. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7992. return err;
  7993. }
  7994. p.hpid->hlos_pid = fl->tgid;
  7995. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7996. return err;
  7997. }
  7998. long fastrpc_driver_invoke(struct fastrpc_device *dev, unsigned int invoke_num,
  7999. unsigned long invoke_param)
  8000. {
  8001. int err = 0;
  8002. switch (invoke_num) {
  8003. case FASTRPC_DEV_MAP_DMA:
  8004. err = fastrpc_dev_map_dma(dev, invoke_param);
  8005. break;
  8006. case FASTRPC_DEV_UNMAP_DMA:
  8007. err = fastrpc_dev_unmap_dma(dev, invoke_param);
  8008. break;
  8009. case FASTRPC_DEV_GET_HLOS_PID:
  8010. err = fastrpc_dev_get_hlos_pid(dev, invoke_param);
  8011. break;
  8012. default:
  8013. err = -ENOTTY;
  8014. break;
  8015. }
  8016. return err;
  8017. }
  8018. EXPORT_SYMBOL(fastrpc_driver_invoke);
  8019. static struct device fastrpc_bus = {
  8020. .init_name = "fastrpc"
  8021. };
  8022. static int fastrpc_bus_match(struct device *dev, struct device_driver *driver)
  8023. {
  8024. struct fastrpc_apps *me = &gfa;
  8025. struct fastrpc_driver *frpc_driver = to_fastrpc_driver(driver);
  8026. struct fastrpc_device *frpc_device = to_fastrpc_device(dev);
  8027. unsigned long irq_flags = 0;
  8028. if (frpc_device->handle == frpc_driver->handle) {
  8029. spin_lock_irqsave(&me->hlock, irq_flags);
  8030. /* If device is being closed, fail the match */
  8031. if (frpc_device->dev_close) {
  8032. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8033. return 0;
  8034. }
  8035. frpc_device->refs++;
  8036. frpc_driver->device = dev;
  8037. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8038. return 1;
  8039. }
  8040. return 0;
  8041. }
  8042. static int fastrpc_bus_probe(struct device *dev)
  8043. {
  8044. struct fastrpc_device *frpc_dev = to_fastrpc_device(dev);
  8045. struct fastrpc_driver *frpc_drv = to_fastrpc_driver(dev->driver);
  8046. if (frpc_drv && frpc_drv->probe)
  8047. return frpc_drv->probe(frpc_dev);
  8048. return 0;
  8049. }
  8050. static void fastrpc_bus_remove(struct device *dev)
  8051. {
  8052. struct fastrpc_driver *frpc_drv = to_fastrpc_driver(dev->driver);
  8053. if (frpc_drv && frpc_drv->callback)
  8054. frpc_drv->callback(to_fastrpc_device(dev), FASTRPC_PROC_DOWN);
  8055. }
  8056. static struct bus_type fastrpc_bus_type = {
  8057. .name = "fastrpc",
  8058. .match = fastrpc_bus_match,
  8059. .probe = fastrpc_bus_probe,
  8060. .remove = fastrpc_bus_remove,
  8061. };
  8062. static void fastrpc_dev_release(struct device *dev)
  8063. {
  8064. kfree(to_fastrpc_device(dev));
  8065. }
  8066. static int fastrpc_device_create(struct fastrpc_file *fl)
  8067. {
  8068. int err = 0;
  8069. struct fastrpc_device *frpc_dev;
  8070. struct fastrpc_apps *me = &gfa;
  8071. unsigned long irq_flags = 0;
  8072. frpc_dev = kzalloc(sizeof(*frpc_dev), GFP_KERNEL);
  8073. if (!frpc_dev) {
  8074. err = -ENOMEM;
  8075. goto bail;
  8076. }
  8077. frpc_dev->dev.parent = &fastrpc_bus;
  8078. frpc_dev->dev.bus = &fastrpc_bus_type;
  8079. /* Use HLOS PID, unique fastrpc process ID and CID to create device file,
  8080. * Else names would conflict for multiple sessions
  8081. * And also for better ability to debug
  8082. */
  8083. dev_set_name(&frpc_dev->dev, "%s-%d-%d-%d",
  8084. dev_name(frpc_dev->dev.parent), fl->tgid, fl->tgid_frpc, fl->cid);
  8085. frpc_dev->dev.release = fastrpc_dev_release;
  8086. frpc_dev->fl = fl;
  8087. /* Use unique fastrpc tgid as handle */
  8088. frpc_dev->handle = fl->tgid_frpc;
  8089. err = device_register(&frpc_dev->dev);
  8090. if (err) {
  8091. put_device(&frpc_dev->dev);
  8092. ADSPRPC_ERR(
  8093. "fastrpc device register failed for process %d unique fastrpc tgid %d session %d with error %d\n",
  8094. fl->tgid, fl->tgid_frpc, fl->sessionid, err);
  8095. goto bail;
  8096. }
  8097. fl->device = frpc_dev;
  8098. spin_lock_irqsave(&me->hlock, irq_flags);
  8099. hlist_add_head(&frpc_dev->hn, &me->frpc_devices);
  8100. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8101. bail:
  8102. return err;
  8103. }
  8104. void fastrpc_driver_unregister(struct fastrpc_driver *frpc_driver)
  8105. {
  8106. struct fastrpc_apps *me = &gfa;
  8107. struct device *dev = NULL;
  8108. struct fastrpc_device *frpc_dev = NULL;
  8109. bool is_device_closed = false;
  8110. unsigned long irq_flags = 0;
  8111. spin_lock_irqsave(&me->hlock, irq_flags);
  8112. dev = frpc_driver->device;
  8113. if (dev) {
  8114. frpc_dev = to_fastrpc_device(dev);
  8115. if (frpc_dev->refs > 0)
  8116. frpc_dev->refs--;
  8117. else
  8118. ADSPRPC_ERR("Fastrpc device for driver %s is already freed\n",
  8119. frpc_driver->driver.name);
  8120. if (frpc_dev->dev_close) {
  8121. hlist_del_init(&frpc_dev->hn);
  8122. is_device_closed = true;
  8123. }
  8124. }
  8125. hlist_del_init(&frpc_driver->hn);
  8126. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8127. if (is_device_closed) {
  8128. ADSPRPC_INFO("un-registering fastrpc device with handle %d\n",
  8129. frpc_dev->handle);
  8130. device_unregister(dev);
  8131. }
  8132. driver_unregister(&frpc_driver->driver);
  8133. ADSPRPC_INFO("Un-registering fastrpc driver %s with handle %d\n",
  8134. frpc_driver->driver.name, frpc_driver->handle);
  8135. }
  8136. EXPORT_SYMBOL(fastrpc_driver_unregister);
  8137. int fastrpc_driver_register(struct fastrpc_driver *frpc_driver)
  8138. {
  8139. int err = 0;
  8140. struct fastrpc_apps *me = &gfa;
  8141. unsigned long irq_flags = 0;
  8142. frpc_driver->driver.bus = &fastrpc_bus_type;
  8143. frpc_driver->driver.owner = THIS_MODULE;
  8144. err = driver_register(&frpc_driver->driver);
  8145. if (err) {
  8146. ADSPRPC_ERR("fastrpc driver %s failed to register with error %d\n",
  8147. frpc_driver->driver.name, err);
  8148. goto bail;
  8149. }
  8150. ADSPRPC_INFO("fastrpc driver %s registered with handle %d\n",
  8151. frpc_driver->driver.name, frpc_driver->handle);
  8152. spin_lock_irqsave(&me->hlock, irq_flags);
  8153. hlist_add_head(&frpc_driver->hn, &me->frpc_drivers);
  8154. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8155. bail:
  8156. return err;
  8157. }
  8158. EXPORT_SYMBOL(fastrpc_driver_register);
  8159. static int __init fastrpc_device_init(void)
  8160. {
  8161. struct fastrpc_apps *me = &gfa;
  8162. int err = 0, i;
  8163. uintptr_t attr = 0;
  8164. dma_addr_t region_phys = 0;
  8165. void *region_vaddr = NULL;
  8166. struct fastrpc_buf *buf = NULL;
  8167. debugfs_root = debugfs_create_dir("adsprpc", NULL);
  8168. if (IS_ERR_OR_NULL(debugfs_root)) {
  8169. pr_warn("Error: %s: %s: failed to create debugfs root dir\n",
  8170. current->comm, __func__);
  8171. debugfs_remove_recursive(debugfs_root);
  8172. debugfs_root = NULL;
  8173. }
  8174. memset(me, 0, sizeof(*me));
  8175. fastrpc_init(me);
  8176. fastrpc_get_nsp_status(me);
  8177. me->dev = NULL;
  8178. me->legacy_remote_heap = false;
  8179. err = bus_register(&fastrpc_bus_type);
  8180. if (err) {
  8181. ADSPRPC_ERR("fastrpc bus register failed with err %d\n",
  8182. err);
  8183. goto bus_register_bail;
  8184. }
  8185. err = device_register(&fastrpc_bus);
  8186. if (err) {
  8187. ADSPRPC_ERR("fastrpc bus device register failed with err %d\n",
  8188. err);
  8189. goto bus_device_register_bail;
  8190. }
  8191. me->fastrpc_bus_register = true;
  8192. fastrpc_lowest_capacity_corecount(me);
  8193. VERIFY(err, 0 == platform_driver_register(&fastrpc_driver));
  8194. if (err)
  8195. goto register_bail;
  8196. VERIFY(err, 0 == alloc_chrdev_region(&me->dev_no, 0, NUM_CHANNELS,
  8197. DEVICE_NAME));
  8198. if (err)
  8199. goto alloc_chrdev_bail;
  8200. cdev_init(&me->cdev, &fops);
  8201. me->cdev.owner = THIS_MODULE;
  8202. VERIFY(err, 0 == cdev_add(&me->cdev, MKDEV(MAJOR(me->dev_no), 0),
  8203. NUM_DEVICES));
  8204. if (err)
  8205. goto cdev_init_bail;
  8206. me->class = class_create(THIS_MODULE, "fastrpc");
  8207. VERIFY(err, !IS_ERR(me->class));
  8208. if (err)
  8209. goto class_create_bail;
  8210. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  8211. me->compat = 1;
  8212. #else
  8213. me->compat = (fops.compat_ioctl == NULL) ? 0 : 1;
  8214. #endif
  8215. /*
  8216. * Create devices and register with sysfs
  8217. * Create first device with minor number 0
  8218. */
  8219. me->non_secure_dev = device_create(me->class, NULL,
  8220. MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV),
  8221. NULL, DEVICE_NAME);
  8222. VERIFY(err, !IS_ERR_OR_NULL(me->non_secure_dev));
  8223. if (err) {
  8224. err = -ENODEV;
  8225. goto device_create_bail;
  8226. }
  8227. /* Create secure device with minor number for secure device */
  8228. me->secure_dev = device_create(me->class, NULL,
  8229. MKDEV(MAJOR(me->dev_no), MINOR_NUM_SECURE_DEV),
  8230. NULL, DEVICE_NAME_SECURE);
  8231. VERIFY(err, !IS_ERR_OR_NULL(me->secure_dev));
  8232. if (err)
  8233. goto device_create_bail;
  8234. for (i = 0; i < NUM_CHANNELS; i++) {
  8235. me->jobid[i] = 1;
  8236. me->channel[i].dev = me->secure_dev;
  8237. me->channel[i].ssrcount = 0;
  8238. me->channel[i].in_hib = 0;
  8239. me->channel[i].prevssrcount = 0;
  8240. me->channel[i].subsystemstate = SUBSYSTEM_UP;
  8241. me->channel[i].rh_dump_dev = NULL;
  8242. me->channel[i].nb.notifier_call = fastrpc_restart_notifier_cb;
  8243. me->channel[i].handle = qcom_register_ssr_notifier(
  8244. gcinfo[i].subsys,
  8245. &me->channel[i].nb);
  8246. if (i == CDSP_DOMAIN_ID) {
  8247. me->channel[i].dev = me->non_secure_dev;
  8248. #if !IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  8249. /*
  8250. * Allocate CMA memory for mini dump.
  8251. * Ignore error as CMA node may not be available on all targets.
  8252. */
  8253. err = fastrpc_alloc_cma_memory(&region_phys,
  8254. &region_vaddr,
  8255. MINI_DUMP_DBG_SIZE,
  8256. (unsigned long)attr);
  8257. #endif
  8258. if (err) {
  8259. ADSPRPC_WARN("CMA alloc failed err 0x%x\n", err);
  8260. err = 0;
  8261. }
  8262. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  8263. if (err) {
  8264. err = -ENOMEM;
  8265. ADSPRPC_WARN("kzalloc failed err 0x%x\n", err);
  8266. err = 0;
  8267. } else {
  8268. INIT_HLIST_NODE(&buf->hn);
  8269. buf->virt = region_vaddr;
  8270. buf->phys = (uintptr_t)region_phys;
  8271. buf->size = MINI_DUMP_DBG_SIZE;
  8272. buf->dma_attr = attr;
  8273. buf->raddr = 0;
  8274. me->channel[i].buf = buf;
  8275. }
  8276. }
  8277. if (IS_ERR_OR_NULL(me->channel[i].handle))
  8278. pr_warn("adsprpc: %s: SSR notifier register failed for %s with err %d\n",
  8279. __func__, gcinfo[i].subsys,
  8280. PTR_ERR(me->channel[i].handle));
  8281. else
  8282. pr_info("adsprpc: %s: SSR notifier registered for %s\n",
  8283. __func__, gcinfo[i].subsys);
  8284. }
  8285. err = fastrpc_transport_init();
  8286. if (err)
  8287. goto device_create_bail;
  8288. me->transport_initialized = 1;
  8289. #ifdef CONFIG_HIBERNATION
  8290. err = register_pm_notifier(&fastrpc_notif_block);
  8291. if (err)
  8292. goto device_create_bail;
  8293. #endif
  8294. fastrpc_register_wakeup_source(me->non_secure_dev,
  8295. FASTRPC_NON_SECURE_WAKE_SOURCE_CLIENT_NAME,
  8296. &me->wake_source);
  8297. fastrpc_register_wakeup_source(me->secure_dev,
  8298. FASTRPC_SECURE_WAKE_SOURCE_CLIENT_NAME,
  8299. &me->wake_source_secure);
  8300. return 0;
  8301. device_create_bail:
  8302. for (i = 0; i < NUM_CHANNELS; i++) {
  8303. if (me->channel[i].handle)
  8304. qcom_unregister_ssr_notifier(me->channel[i].handle,
  8305. &me->channel[i].nb);
  8306. }
  8307. if (!IS_ERR_OR_NULL(me->non_secure_dev))
  8308. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8309. MINOR_NUM_DEV));
  8310. if (!IS_ERR_OR_NULL(me->secure_dev))
  8311. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8312. MINOR_NUM_SECURE_DEV));
  8313. class_destroy(me->class);
  8314. class_create_bail:
  8315. cdev_del(&me->cdev);
  8316. cdev_init_bail:
  8317. unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
  8318. alloc_chrdev_bail:
  8319. platform_driver_unregister(&fastrpc_driver);
  8320. register_bail:
  8321. device_unregister(&fastrpc_bus);
  8322. bus_device_register_bail:
  8323. bus_unregister(&fastrpc_bus_type);
  8324. bus_register_bail:
  8325. fastrpc_deinit();
  8326. return err;
  8327. }
  8328. static void __exit fastrpc_device_exit(void)
  8329. {
  8330. struct fastrpc_apps *me = &gfa;
  8331. int i;
  8332. fastrpc_file_list_dtor(me);
  8333. fastrpc_deinit();
  8334. wakeup_source_unregister(me->wake_source);
  8335. wakeup_source_unregister(me->wake_source_secure);
  8336. for (i = 0; i < NUM_CHANNELS; i++) {
  8337. if (i == CDSP_DOMAIN_ID)
  8338. kfree(me->channel[i].buf);
  8339. if (!gcinfo[i].name)
  8340. continue;
  8341. qcom_unregister_ssr_notifier(me->channel[i].handle,
  8342. &me->channel[i].nb);
  8343. }
  8344. /* Destroy the secure and non secure devices */
  8345. device_destroy(me->class, MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV));
  8346. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8347. MINOR_NUM_SECURE_DEV));
  8348. of_reserved_mem_device_release(me->dev);
  8349. class_destroy(me->class);
  8350. cdev_del(&me->cdev);
  8351. unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
  8352. if (me->transport_initialized)
  8353. fastrpc_transport_deinit();
  8354. me->transport_initialized = 0;
  8355. if (me->fastrpc_bus_register) {
  8356. bus_unregister(&fastrpc_bus_type);
  8357. device_unregister(&fastrpc_bus);
  8358. }
  8359. kfree(me->gidlist.gids);
  8360. debugfs_remove_recursive(debugfs_root);
  8361. }
  8362. module_init(fastrpc_device_init);
  8363. module_exit(fastrpc_device_exit);
  8364. MODULE_LICENSE("GPL v2");