adsprpc.c 243 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2022-2023, Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. /* Uncomment this block to log an error on every VERIFY failure */
  7. /*
  8. * #ifndef VERIFY_PRINT_ERROR
  9. * #define VERIFY_PRINT_ERROR
  10. * #endif
  11. */
  12. #include <linux/dma-buf.h>
  13. #include <linux/dma-mapping.h>
  14. #include <linux/qcom-dma-mapping.h>
  15. #include <linux/slab.h>
  16. #include <linux/completion.h>
  17. #include <linux/pagemap.h>
  18. #include <linux/mm.h>
  19. #include <linux/wait.h>
  20. #include <linux/sched.h>
  21. #include <linux/module.h>
  22. #include <linux/list.h>
  23. #include <linux/arch_topology.h>
  24. #include <linux/hash.h>
  25. #include <linux/msm_ion.h>
  26. #include <linux/qcom_scm.h>
  27. #include <linux/ipc_logging.h>
  28. #include <linux/remoteproc/qcom_rproc.h>
  29. #include <linux/scatterlist.h>
  30. #include <linux/uaccess.h>
  31. #include <linux/device.h>
  32. #include <linux/of.h>
  33. #include <linux/of_address.h>
  34. #include <linux/of_platform.h>
  35. #include <linux/dma-map-ops.h>
  36. #include <linux/cma.h>
  37. #include <linux/sort.h>
  38. #include <linux/cred.h>
  39. #include <linux/msm_dma_iommu_mapping.h>
  40. #include "adsprpc_compat.h"
  41. #include "adsprpc_shared.h"
  42. #include <soc/qcom/qcom_ramdump.h>
  43. #include <soc/qcom/minidump.h>
  44. #include <soc/qcom/secure_buffer.h>
  45. #include <linux/delay.h>
  46. #include <linux/debugfs.h>
  47. #include <linux/pm_qos.h>
  48. #include <linux/stat.h>
  49. #include <linux/preempt.h>
  50. #include <linux/of_reserved_mem.h>
  51. #include <linux/soc/qcom/pdr.h>
  52. #include <linux/soc/qcom/qmi.h>
  53. #include <linux/mem-buf.h>
  54. #include <linux/iommu.h>
  55. #include <asm/arch_timer.h>
  56. #include <linux/genalloc.h>
  57. #include <soc/qcom/socinfo.h>
  58. #ifdef CONFIG_HIBERNATION
  59. #include <linux/suspend.h>
  60. #include <linux/notifier.h>
  61. #endif
  62. #define CREATE_TRACE_POINTS
  63. #include "fastrpc_trace.h"
  64. #ifdef CONFIG_MSM_ADSPRPC_TRUSTED
  65. #include "../include/linux/fastrpc.h"
  66. #else
  67. #include "fastrpc.h"
  68. #endif
  69. #define TZ_PIL_PROTECT_MEM_SUBSYS_ID 0x0C
  70. #define TZ_PIL_CLEAR_PROTECT_MEM_SUBSYS_ID 0x0D
  71. #define TZ_PIL_AUTH_QDSP6_PROC 1
  72. #define FASTRPC_ENOSUCH 39
  73. #define VMID_SSC_Q6 5
  74. #define VMID_ADSP_Q6 6
  75. #define DEBUGFS_SIZE 3072
  76. #define PID_SIZE 10
  77. #define AUDIO_PDR_ADSP_DTSI_PROPERTY_NAME "qcom,fastrpc-adsp-audio-pdr"
  78. #define AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME "audio_pdr_adsprpc"
  79. #define AUDIO_PDR_ADSP_SERVICE_NAME "avs/audio"
  80. #define ADSP_AUDIOPD_NAME "msm/adsp/audio_pd"
  81. #define SENSORS_PDR_ADSP_DTSI_PROPERTY_NAME "qcom,fastrpc-adsp-sensors-pdr"
  82. #define SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME "sensors_pdr_adsprpc"
  83. #define SENSORS_PDR_ADSP_SERVICE_NAME "tms/servreg"
  84. #define ADSP_SENSORPD_NAME "msm/adsp/sensor_pd"
  85. #define SENSORS_PDR_SLPI_DTSI_PROPERTY_NAME "qcom,fastrpc-slpi-sensors-pdr"
  86. #define SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME "sensors_pdr_sdsprpc"
  87. #define SENSORS_PDR_SLPI_SERVICE_NAME SENSORS_PDR_ADSP_SERVICE_NAME
  88. #define SLPI_SENSORPD_NAME "msm/slpi/sensor_pd"
  89. #define FASTRPC_SECURE_WAKE_SOURCE_CLIENT_NAME "adsprpc-secure"
  90. #define FASTRPC_NON_SECURE_WAKE_SOURCE_CLIENT_NAME "adsprpc-non_secure"
  91. #define RPC_TIMEOUT (5 * HZ)
  92. #define BALIGN 128
  93. #define M_FDLIST (16)
  94. #define M_CRCLIST (64)
  95. #define M_KERNEL_PERF_LIST (PERF_KEY_MAX)
  96. #define M_DSP_PERF_LIST (12)
  97. #define FASTRPC_CTX_MAGIC (0xbeeddeed)
  98. /* Process status notifications from DSP will be sent with this unique context */
  99. #define FASTRPC_NOTIF_CTX_RESERVED 0xABCDABCD
  100. #define FASTRPC_CTX_JOB_TYPE_POS (4)
  101. #define FASTRPC_CTX_TABLE_IDX_POS (6)
  102. #define FASTRPC_CTX_JOBID_POS (16)
  103. #define FASTRPC_CTX_TABLE_IDX_MASK \
  104. ((FASTRPC_CTX_MAX - 1) << FASTRPC_CTX_TABLE_IDX_POS)
  105. #define FASTRPC_ASYNC_JOB_MASK (1)
  106. #define GET_TABLE_IDX_FROM_CTXID(ctxid) \
  107. ((ctxid & FASTRPC_CTX_TABLE_IDX_MASK) >> FASTRPC_CTX_TABLE_IDX_POS)
  108. /* Reserve few entries in context table for critical kernel and static RPC
  109. * calls to avoid user invocations from exhausting all entries.
  110. */
  111. #define NUM_KERNEL_AND_STATIC_ONLY_CONTEXTS (70)
  112. /* Maximum number of pending contexts per remote session */
  113. #define MAX_PENDING_CTX_PER_SESSION (64)
  114. #define NUM_DEVICES 2 /* adsprpc-smd, adsprpc-smd-secure */
  115. #define MINOR_NUM_DEV 0
  116. #define MINOR_NUM_SECURE_DEV 1
  117. #define NON_SECURE_CHANNEL 0
  118. #define SECURE_CHANNEL 1
  119. #define IS_CACHE_ALIGNED(x) (((x) & ((L1_CACHE_BYTES)-1)) == 0)
  120. #ifndef ION_FLAG_CACHED
  121. #define ION_FLAG_CACHED (1)
  122. #endif
  123. #ifndef topology_cluster_id
  124. #define topology_cluster_id(cpu) topology_physical_package_id(cpu)
  125. #endif
  126. /*
  127. * ctxid of every message is OR-ed with fastrpc_remote_pd_type before
  128. * it is sent to DSP. So mask 2 LSBs to retrieve actual context
  129. */
  130. #define CONTEXT_PD_CHECK (3)
  131. #define GET_CTXID_FROM_RSP_CTX(rsp_ctx) (rsp_ctx & ~CONTEXT_PD_CHECK)
  132. #define RH_CID ADSP_DOMAIN_ID
  133. #define FASTRPC_STATIC_HANDLE_PROCESS_GROUP (1)
  134. #define FASTRPC_STATIC_HANDLE_DSP_UTILITIES (2)
  135. #define FASTRPC_STATIC_HANDLE_LISTENER (3)
  136. #define FASTRPC_STATIC_HANDLE_MAX (20)
  137. #define FASTRPC_LATENCY_CTRL_ENB (1)
  138. /* Maximum PM timeout that can be voted through fastrpc */
  139. #define MAX_PM_TIMEOUT_MS 50
  140. /* timeout in us for busy polling after early response from remote processor */
  141. #define FASTRPC_POLL_TIME (4000)
  142. /* timeout in us for polling until memory barrier */
  143. #define FASTRPC_POLL_TIME_MEM_UPDATE (500)
  144. /* timeout in us for polling completion signal after user early hint */
  145. #define FASTRPC_USER_EARLY_HINT_TIMEOUT (500)
  146. /* Early wake up poll completion number received from remote processor */
  147. #define FASTRPC_EARLY_WAKEUP_POLL (0xabbccdde)
  148. /* Poll response number from remote processor for call completion */
  149. #define FASTRPC_POLL_RESPONSE (0xdecaf)
  150. /* latency in us, early wake up signal used below this value */
  151. #define FASTRPC_EARLY_WAKEUP_LATENCY (200)
  152. /* response version number */
  153. #define FASTRPC_RSP_VERSION2 (2)
  154. /* CPU feature information to DSP */
  155. #define FASTRPC_CPUINFO_DEFAULT (0)
  156. #define FASTRPC_CPUINFO_EARLY_WAKEUP (1)
  157. #define INIT_FILELEN_MAX (2*1024*1024)
  158. #define INIT_MEMLEN_MAX_STATIC (8*1024*1024)
  159. #define INIT_MEMLEN_MAX_DYNAMIC (200*1024*1024)
  160. #define INIT_MEMLEN_MIN_DYNAMIC (3*1024*1024)
  161. #define MAX_CACHE_BUF_SIZE (8*1024*1024)
  162. /* Maximum buffers cached in cached buffer list */
  163. #define MAX_CACHED_BUFS (32)
  164. /* Max no. of persistent headers pre-allocated per process */
  165. #define MAX_PERSISTENT_HEADERS (25)
  166. /* Max value of unique fastrpc tgid */
  167. #define MAX_FRPC_TGID 256
  168. #define PERF_CAPABILITY_SUPPORT (1 << 1)
  169. #define KERNEL_ERROR_CODE_V1_SUPPORT 1
  170. #define USERSPACE_ALLOCATION_SUPPORT 1
  171. #define DSPSIGNAL_SUPPORT 1
  172. #define MD_GMSG_BUFFER (1000)
  173. #define MINI_DUMP_DBG_SIZE (200*1024)
  174. /* Max number of region supported */
  175. #define MAX_UNIQUE_ID 5
  176. /* Convert the 19.2MHz clock count to micro-seconds */
  177. #define CONVERT_CNT_TO_US(CNT) (CNT * 10ull / 192ull)
  178. #define FASTRPC_USER_PD_FORCE_KILL 2
  179. /*
  180. * No. of pages shared with DSP during process init
  181. * First page for init-mem and second page for proc-attrs
  182. */
  183. #define PAGESLEN_WITH_SHAREDBUF 2
  184. /* Unique index flag used for mini dump */
  185. static int md_unique_index_flag[MAX_UNIQUE_ID] = { 0, 0, 0, 0, 0 };
  186. /* Array to keep track unique tgid_frpc usage */
  187. static bool frpc_tgid_usage_array[MAX_FRPC_TGID] = {0};
  188. /* Fastrpc remote process attributes */
  189. enum fastrpc_proc_attr {
  190. /* Macro for Debug attr */
  191. FASTRPC_MODE_DEBUG = 1 << 0,
  192. /* Macro for Ptrace */
  193. FASTRPC_MODE_PTRACE = 1 << 1,
  194. /* Macro for CRC Check */
  195. FASTRPC_MODE_CRC = 1 << 2,
  196. /* Macro for Unsigned PD */
  197. FASTRPC_MODE_UNSIGNED_MODULE = 1 << 3,
  198. /* Macro for Adaptive QoS */
  199. FASTRPC_MODE_ADAPTIVE_QOS = 1 << 4,
  200. /* Macro for System Process */
  201. FASTRPC_MODE_SYSTEM_PROCESS = 1 << 5,
  202. /* Macro for Prvileged Process */
  203. FASTRPC_MODE_PRIVILEGED = (1 << 6),
  204. /* Macro for system unsigned PD */
  205. FASTRPC_MODE_SYSTEM_UNSIGNED_PD = 1 << 17,
  206. };
  207. /* FastRPC remote subsystem state*/
  208. enum fastrpc_remote_subsys_state {
  209. SUBSYSTEM_RESTARTING = 0,
  210. SUBSYSTEM_DOWN,
  211. SUBSYSTEM_UP,
  212. };
  213. #define PERF_END ((void)0)
  214. #define PERF(enb, cnt, ff) \
  215. {\
  216. struct timespec64 startT = {0};\
  217. uint64_t *counter = cnt;\
  218. if (enb && counter) {\
  219. ktime_get_real_ts64(&startT);\
  220. } \
  221. ff ;\
  222. if (enb && counter) {\
  223. *counter += getnstimediff(&startT);\
  224. } \
  225. }
  226. #define GET_COUNTER(perf_ptr, offset) \
  227. (perf_ptr != NULL ?\
  228. (((offset >= 0) && (offset < PERF_KEY_MAX)) ?\
  229. (uint64_t *)(perf_ptr + offset)\
  230. : (uint64_t *)NULL) : (uint64_t *)NULL)
  231. /* Macro for comparing local client and PD names with those from callback */
  232. #define COMPARE_SERVICE_LOCATOR_NAMES(cb_client, local_client, \
  233. cb_pdname, local_pdname) \
  234. ((!strcmp(cb_client, local_client)) \
  235. && (!strcmp(cb_pdname, local_pdname)))
  236. #define IS_ASYNC_FASTRPC_AVAILABLE (1)
  237. /* Use the second definition to enable additional dspsignal debug logging */
  238. #define DSPSIGNAL_VERBOSE(x, ...)
  239. /*#define DSPSIGNAL_VERBOSE ADSPRPC_INFO*/
  240. MODULE_IMPORT_NS(DMA_BUF);
  241. static struct dentry *debugfs_root;
  242. static struct dentry *debugfs_global_file;
  243. static inline uint64_t buf_page_start(uint64_t buf)
  244. {
  245. uint64_t start = (uint64_t) buf & PAGE_MASK;
  246. return start;
  247. }
  248. static inline uint64_t buf_page_offset(uint64_t buf)
  249. {
  250. uint64_t offset = (uint64_t) buf & (PAGE_SIZE - 1);
  251. return offset;
  252. }
  253. static inline uint64_t buf_num_pages(uint64_t buf, size_t len)
  254. {
  255. uint64_t start = buf_page_start(buf) >> PAGE_SHIFT;
  256. uint64_t end = (((uint64_t) buf + len - 1) & PAGE_MASK) >> PAGE_SHIFT;
  257. uint64_t nPages = end - start + 1;
  258. return nPages;
  259. }
  260. static inline uint64_t buf_page_size(uint32_t size)
  261. {
  262. uint64_t sz = (size + (PAGE_SIZE - 1)) & PAGE_MASK;
  263. return sz > PAGE_SIZE ? sz : PAGE_SIZE;
  264. }
  265. static inline void *uint64_to_ptr(uint64_t addr)
  266. {
  267. void *ptr = (void *)((uintptr_t)addr);
  268. return ptr;
  269. }
  270. static inline uint64_t ptr_to_uint64(void *ptr)
  271. {
  272. uint64_t addr = (uint64_t)((uintptr_t)ptr);
  273. return addr;
  274. }
  275. static struct fastrpc_apps gfa;
  276. static struct fastrpc_channel_ctx gcinfo[NUM_CHANNELS] = {
  277. {
  278. .name = "adsprpc-smd",
  279. .subsys = "lpass",
  280. .spd = {
  281. {
  282. .servloc_name =
  283. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME,
  284. .spdname = ADSP_AUDIOPD_NAME,
  285. .cid = ADSP_DOMAIN_ID,
  286. },
  287. {
  288. .servloc_name =
  289. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME,
  290. .spdname = ADSP_SENSORPD_NAME,
  291. .cid = ADSP_DOMAIN_ID,
  292. }
  293. },
  294. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  295. .cpuinfo_status = false,
  296. },
  297. {
  298. .name = "mdsprpc-smd",
  299. .subsys = "mpss",
  300. .spd = {
  301. {
  302. .cid = MDSP_DOMAIN_ID,
  303. }
  304. },
  305. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  306. .cpuinfo_status = false,
  307. },
  308. {
  309. .name = "sdsprpc-smd",
  310. .subsys = "dsps",
  311. .spd = {
  312. {
  313. .servloc_name =
  314. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME,
  315. .spdname = SLPI_SENSORPD_NAME,
  316. .cid = SDSP_DOMAIN_ID,
  317. }
  318. },
  319. .cpuinfo_todsp = FASTRPC_CPUINFO_DEFAULT,
  320. .cpuinfo_status = false,
  321. },
  322. {
  323. .name = "cdsprpc-smd",
  324. .subsys = "cdsp",
  325. .spd = {
  326. {
  327. .cid = CDSP_DOMAIN_ID,
  328. }
  329. },
  330. .cpuinfo_todsp = FASTRPC_CPUINFO_EARLY_WAKEUP,
  331. .cpuinfo_status = false,
  332. },
  333. };
  334. static uint32_t kernel_capabilities[FASTRPC_MAX_ATTRIBUTES -
  335. FASTRPC_MAX_DSP_ATTRIBUTES] = {
  336. PERF_CAPABILITY_SUPPORT,
  337. /* PERF_LOGGING_V2_SUPPORT feature is supported, unsupported = 0 */
  338. KERNEL_ERROR_CODE_V1_SUPPORT,
  339. /* Fastrpc Driver error code changes present */
  340. USERSPACE_ALLOCATION_SUPPORT,
  341. /* Userspace allocation allowed for DSP memory request*/
  342. DSPSIGNAL_SUPPORT
  343. /* Lightweight driver-based signaling */
  344. };
  345. static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type);
  346. static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
  347. uint32_t flags, uintptr_t va, uint64_t phys,
  348. size_t size, uintptr_t *raddr);
  349. static inline void fastrpc_update_rxmsg_buf(struct fastrpc_channel_ctx *chan,
  350. uint64_t ctx, int retval, uint32_t rsp_flags,
  351. uint32_t early_wake_time, uint32_t ver, int64_t ns, uint64_t xo_time_in_us);
  352. /**
  353. * fastrpc_device_create - Create device for the fastrpc process file
  354. * @fl : Fastrpc process file
  355. * Returns: 0 on Success
  356. */
  357. static int fastrpc_device_create(struct fastrpc_file *fl);
  358. static inline int64_t getnstimediff(struct timespec64 *start)
  359. {
  360. int64_t ns;
  361. struct timespec64 ts, b;
  362. ktime_get_real_ts64(&ts);
  363. b = timespec64_sub(ts, *start);
  364. ns = timespec64_to_ns(&b);
  365. return ns;
  366. }
  367. /**
  368. * get_timestamp_in_ns - Gets time of day in nanoseconds
  369. *
  370. * Returns: Timestamp in nanoseconds
  371. */
  372. static inline int64_t get_timestamp_in_ns(void)
  373. {
  374. int64_t ns = 0;
  375. struct timespec64 ts;
  376. ktime_get_real_ts64(&ts);
  377. ns = timespec64_to_ns(&ts);
  378. return ns;
  379. }
  380. static inline int poll_for_remote_response(struct smq_invoke_ctx *ctx, uint32_t timeout)
  381. {
  382. int err = -EIO;
  383. uint32_t sc = ctx->sc, ii = 0, jj = 0;
  384. struct smq_invoke_buf *list;
  385. struct smq_phy_page *pages;
  386. uint64_t *fdlist = NULL;
  387. uint32_t *crclist = NULL, *poll = NULL;
  388. unsigned int inbufs, outbufs, handles;
  389. /* calculate poll memory location */
  390. inbufs = REMOTE_SCALARS_INBUFS(sc);
  391. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  392. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  393. list = smq_invoke_buf_start(ctx->rpra, sc);
  394. pages = smq_phy_page_start(sc, list);
  395. fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
  396. crclist = (uint32_t *)(fdlist + M_FDLIST);
  397. poll = (uint32_t *)(crclist + M_CRCLIST);
  398. /* poll on memory for DSP response. Return failure on timeout */
  399. for (ii = 0, jj = 0; ii < timeout; ii++, jj++) {
  400. if (*poll == FASTRPC_EARLY_WAKEUP_POLL) {
  401. /* Remote processor sent early response */
  402. err = 0;
  403. break;
  404. } else if (*poll == FASTRPC_POLL_RESPONSE) {
  405. /* Remote processor sent poll response to complete the call */
  406. err = 0;
  407. ctx->is_work_done = true;
  408. ctx->retval = 0;
  409. /* Update DSP response history */
  410. fastrpc_update_rxmsg_buf(&gfa.channel[ctx->fl->cid],
  411. ctx->msg.invoke.header.ctx, 0, POLL_MODE, 0,
  412. FASTRPC_RSP_VERSION2, get_timestamp_in_ns(),
  413. CONVERT_CNT_TO_US(__arch_counter_get_cntvct()));
  414. break;
  415. }
  416. if (jj == FASTRPC_POLL_TIME_MEM_UPDATE) {
  417. /* Wait for DSP to finish updating poll memory */
  418. rmb();
  419. jj = 0;
  420. }
  421. udelay(1);
  422. }
  423. return err;
  424. }
  425. enum interrupted_state {
  426. DEFAULT_STATE = 0,
  427. INTERRUPTED_STATE = 1,
  428. RESTORED_STATE = 2,
  429. };
  430. /**
  431. * fastrpc_update_txmsg_buf - Update history of sent glink messages
  432. * @msg : Pointer to RPC message to remote subsystem
  433. * @transport_send_err : Error from transport
  434. * @ns : Timestamp (in ns) of sent message
  435. * @xo_time_in_us : XO Timestamp (in us) of sent message
  436. * @ctx : invoke ctx
  437. * @interrupted : 0/1/2 (default/interrupted/restored)
  438. *
  439. * Returns none
  440. */
  441. static inline void fastrpc_update_txmsg_buf(struct smq_msg *msg,
  442. int transport_send_err, int64_t ns, uint64_t xo_time_in_us,
  443. struct smq_invoke_ctx *ctx, enum interrupted_state interrupted)
  444. {
  445. unsigned long flags = 0;
  446. unsigned int tx_index = 0;
  447. struct fastrpc_tx_msg *tx_msg = NULL;
  448. struct fastrpc_channel_ctx *chan = NULL;
  449. struct fastrpc_file *fl = ctx->fl;
  450. int err = 0, cid = -1;
  451. if (!fl) {
  452. err = -EBADF;
  453. goto bail;
  454. }
  455. cid = fl->cid;
  456. VERIFY(err, VALID_FASTRPC_CID(cid));
  457. if (err) {
  458. err = -ECHRNG;
  459. goto bail;
  460. }
  461. chan = &fl->apps->channel[cid];
  462. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  463. if (interrupted){
  464. if (ctx->tx_index >= 0 && ctx->tx_index < GLINK_MSG_HISTORY_LEN) {
  465. tx_msg = &chan->gmsg_log.tx_msgs[ctx->tx_index];
  466. if (tx_msg->msg.invoke.header.ctx == ctx->msg.invoke.header.ctx) {
  467. tx_msg->xo_time_in_us_interrupted = ctx->xo_time_in_us_interrupted;
  468. tx_msg->xo_time_in_us_restored = ctx->xo_time_in_us_restored;
  469. }
  470. }
  471. } else {
  472. tx_index = chan->gmsg_log.tx_index;
  473. ctx->tx_index = tx_index;
  474. tx_msg = &chan->gmsg_log.tx_msgs[tx_index];
  475. memcpy(&tx_msg->msg, msg, sizeof(struct smq_msg));
  476. tx_msg->transport_send_err = transport_send_err;
  477. tx_msg->ns = ns;
  478. tx_msg->xo_time_in_us = xo_time_in_us;
  479. tx_index++;
  480. chan->gmsg_log.tx_index =
  481. (tx_index > (GLINK_MSG_HISTORY_LEN - 1)) ? 0 : tx_index;
  482. }
  483. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  484. bail:
  485. if (err)
  486. ADSPRPC_ERR("adsprpc: %s: unable to update txmsg buf (err %d) for ctx: 0x%x\n",
  487. __func__, err, ctx->msg.invoke.header.ctx);
  488. }
  489. /**
  490. * fastrpc_update_rxmsg_buf - Update history of received glink responses
  491. * @chan : Channel context
  492. * @ctx : Context of received response from DSP
  493. * @retval : Return value for RPC call
  494. * @rsp_flags : Response type
  495. * @early_wake_time : Poll time for early wakeup
  496. * @ver : Version of response
  497. * @ns : Timestamp (in ns) of response
  498. * @xo_time_in_us : XO Timestamp (in us) of response
  499. *
  500. * Returns none
  501. */
  502. static inline void fastrpc_update_rxmsg_buf(struct fastrpc_channel_ctx *chan,
  503. uint64_t ctx, int retval, uint32_t rsp_flags,
  504. uint32_t early_wake_time, uint32_t ver, int64_t ns, uint64_t xo_time_in_us)
  505. {
  506. unsigned long flags = 0;
  507. unsigned int rx_index = 0;
  508. struct fastrpc_rx_msg *rx_msg = NULL;
  509. struct smq_invoke_rspv2 *rsp = NULL;
  510. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  511. rx_index = chan->gmsg_log.rx_index;
  512. rx_msg = &chan->gmsg_log.rx_msgs[rx_index];
  513. rsp = &rx_msg->rsp;
  514. rsp->ctx = ctx;
  515. rsp->retval = retval;
  516. rsp->flags = rsp_flags;
  517. rsp->early_wake_time = early_wake_time;
  518. rsp->version = ver;
  519. rx_msg->ns = ns;
  520. rx_msg->xo_time_in_us = xo_time_in_us;
  521. rx_index++;
  522. chan->gmsg_log.rx_index =
  523. (rx_index > (GLINK_MSG_HISTORY_LEN - 1)) ? 0 : rx_index;
  524. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  525. }
  526. static inline int get_unique_index(void)
  527. {
  528. int index = -1;
  529. mutex_lock(&gfa.mut_uid);
  530. for (index = 0; index < MAX_UNIQUE_ID; index++) {
  531. if (md_unique_index_flag[index] == 0) {
  532. md_unique_index_flag[index] = 1;
  533. mutex_unlock(&gfa.mut_uid);
  534. return index;
  535. }
  536. }
  537. mutex_unlock(&gfa.mut_uid);
  538. return index;
  539. }
  540. static inline void reset_unique_index(int index)
  541. {
  542. mutex_lock(&gfa.mut_uid);
  543. if (index > -1 && index < MAX_UNIQUE_ID)
  544. md_unique_index_flag[index] = 0;
  545. mutex_unlock(&gfa.mut_uid);
  546. }
  547. /**
  548. * fastrpc_minidump_add_region - Add mini dump region
  549. * @fastrpc_mmap : Input structure mmap
  550. *
  551. * Returns int
  552. */
  553. static int fastrpc_minidump_add_region(struct fastrpc_mmap *map)
  554. {
  555. int err = 0, ret_val = 0, md_index = 0;
  556. struct md_region md_entry;
  557. md_index = get_unique_index();
  558. if (md_index > -1 && md_index < MAX_UNIQUE_ID) {
  559. scnprintf(md_entry.name, MAX_NAME_LENGTH, "FRPC_%d", md_index);
  560. md_entry.virt_addr = map->va;
  561. md_entry.phys_addr = map->phys;
  562. md_entry.size = map->size;
  563. ret_val = msm_minidump_add_region(&md_entry);
  564. if (ret_val < 0) {
  565. ADSPRPC_ERR(
  566. "Failed to add/update CMA to Minidump for phys: 0x%llx, size: %zu, md_index %d, md_entry.name %s\n",
  567. map->phys,
  568. map->size, md_index,
  569. md_entry.name);
  570. reset_unique_index(md_index);
  571. err = ret_val;
  572. } else {
  573. map->frpc_md_index = md_index;
  574. }
  575. } else {
  576. pr_warn("failed to generate valid unique id for mini dump : %d\n", md_index);
  577. }
  578. return err;
  579. }
  580. /**
  581. * fastrpc_minidump_remove_region - Remove mini dump region if added
  582. * @fastrpc_mmap : Input structure mmap
  583. *
  584. * Returns int
  585. */
  586. static int fastrpc_minidump_remove_region(struct fastrpc_mmap *map)
  587. {
  588. int err = -EINVAL;
  589. struct md_region md_entry;
  590. if (map->frpc_md_index > -1 && map->frpc_md_index < MAX_UNIQUE_ID) {
  591. scnprintf(md_entry.name, MAX_NAME_LENGTH, "FRPC_%d",
  592. map->frpc_md_index);
  593. md_entry.virt_addr = map->va;
  594. md_entry.phys_addr = map->phys;
  595. md_entry.size = map->size;
  596. err = msm_minidump_remove_region(&md_entry);
  597. if (err < 0) {
  598. ADSPRPC_ERR(
  599. "Failed to remove CMA from Minidump for phys: 0x%llx, size: %zu index = %d\n",
  600. map->phys, map->size, map->frpc_md_index);
  601. } else {
  602. reset_unique_index(map->frpc_md_index);
  603. map->frpc_md_index = -1;
  604. }
  605. } else {
  606. ADSPRPC_WARN("mini-dump enabled with invalid unique id: %d\n", map->frpc_md_index);
  607. }
  608. return err;
  609. }
  610. static void fastrpc_buf_free(struct fastrpc_buf *buf, int cache)
  611. {
  612. struct fastrpc_file *fl = buf == NULL ? NULL : buf->fl;
  613. int vmid, err = 0, cid = -1;
  614. if (!fl)
  615. return;
  616. if (buf->in_use) {
  617. /* Don't free persistent header buf. Just mark as available */
  618. spin_lock(&fl->hlock);
  619. buf->in_use = false;
  620. spin_unlock(&fl->hlock);
  621. return;
  622. }
  623. if (cache && buf->size < MAX_CACHE_BUF_SIZE) {
  624. spin_lock(&fl->hlock);
  625. if (fl->num_cached_buf > MAX_CACHED_BUFS) {
  626. spin_unlock(&fl->hlock);
  627. goto skip_buf_cache;
  628. }
  629. hlist_add_head(&buf->hn, &fl->cached_bufs);
  630. fl->num_cached_buf++;
  631. buf->type = -1;
  632. spin_unlock(&fl->hlock);
  633. return;
  634. }
  635. skip_buf_cache:
  636. if (buf->type == USERHEAP_BUF) {
  637. spin_lock(&fl->hlock);
  638. hlist_del_init(&buf->hn_rem);
  639. spin_unlock(&fl->hlock);
  640. buf->raddr = 0;
  641. }
  642. if (!IS_ERR_OR_NULL(buf->virt)) {
  643. VERIFY(err, fl->sctx != NULL);
  644. if (err)
  645. goto bail;
  646. if (fl->sctx->smmu.cb)
  647. buf->phys &= ~((uint64_t)fl->sctx->smmu.cb << 32);
  648. cid = fl->cid;
  649. VERIFY(err, VALID_FASTRPC_CID(cid));
  650. if (err) {
  651. err = -ECHRNG;
  652. ADSPRPC_ERR(
  653. "invalid channel 0x%zx set for session\n",
  654. cid);
  655. goto bail;
  656. }
  657. vmid = fl->apps->channel[cid].vmid;
  658. if ((vmid) && (fl->apps->channel[cid].in_hib == 0)) {
  659. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS)| BIT(vmid);
  660. struct qcom_scm_vmperm dest_perms = {0};
  661. int hyp_err = 0;
  662. dest_perms.vmid = QCOM_SCM_VMID_HLOS;
  663. dest_perms.perm = QCOM_SCM_PERM_RWX;
  664. hyp_err = qcom_scm_assign_mem(buf->phys,
  665. buf_page_size(buf->size),
  666. &src_perms, &dest_perms, 1);
  667. if (hyp_err) {
  668. ADSPRPC_ERR(
  669. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  670. hyp_err, buf->phys, buf->size);
  671. }
  672. }
  673. trace_fastrpc_dma_free(cid, buf->phys, buf->size);
  674. dma_free_attrs(fl->sctx->smmu.dev, buf->size, buf->virt,
  675. buf->phys, buf->dma_attr);
  676. }
  677. bail:
  678. kfree(buf);
  679. }
  680. static void fastrpc_cached_buf_list_free(struct fastrpc_file *fl)
  681. {
  682. struct fastrpc_buf *buf, *free;
  683. do {
  684. struct hlist_node *n;
  685. free = NULL;
  686. spin_lock(&fl->hlock);
  687. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  688. hlist_del_init(&buf->hn);
  689. fl->num_cached_buf--;
  690. free = buf;
  691. break;
  692. }
  693. spin_unlock(&fl->hlock);
  694. if (free)
  695. fastrpc_buf_free(free, 0);
  696. } while (free);
  697. }
  698. static void fastrpc_remote_buf_list_free(struct fastrpc_file *fl)
  699. {
  700. struct fastrpc_buf *buf, *free;
  701. do {
  702. struct hlist_node *n;
  703. free = NULL;
  704. spin_lock(&fl->hlock);
  705. hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {
  706. free = buf;
  707. break;
  708. }
  709. spin_unlock(&fl->hlock);
  710. if (free)
  711. fastrpc_buf_free(free, 0);
  712. } while (free);
  713. }
  714. static void fastrpc_mmap_add(struct fastrpc_mmap *map)
  715. {
  716. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  717. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  718. struct fastrpc_apps *me = &gfa;
  719. unsigned long irq_flags = 0;
  720. spin_lock_irqsave(&me->hlock, irq_flags);
  721. hlist_add_head(&map->hn, &me->maps);
  722. spin_unlock_irqrestore(&me->hlock, irq_flags);
  723. } else {
  724. struct fastrpc_file *fl = map->fl;
  725. hlist_add_head(&map->hn, &fl->maps);
  726. }
  727. }
  728. static int fastrpc_mmap_find(struct fastrpc_file *fl, int fd,
  729. struct dma_buf *buf, uintptr_t va, size_t len, int mflags, int refs,
  730. struct fastrpc_mmap **ppmap)
  731. {
  732. struct fastrpc_apps *me = &gfa;
  733. struct fastrpc_mmap *match = NULL, *map = NULL;
  734. struct hlist_node *n;
  735. unsigned long irq_flags = 0;
  736. if ((va + len) < va)
  737. return -EFAULT;
  738. if (mflags == ADSP_MMAP_HEAP_ADDR ||
  739. mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  740. spin_lock_irqsave(&me->hlock, irq_flags);
  741. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  742. if (va >= map->va &&
  743. va + len <= map->va + map->len &&
  744. map->fd == fd) {
  745. if (refs) {
  746. if (map->refs + 1 == INT_MAX) {
  747. spin_unlock_irqrestore(&me->hlock, irq_flags);
  748. return -ETOOMANYREFS;
  749. }
  750. map->refs++;
  751. }
  752. match = map;
  753. break;
  754. }
  755. }
  756. spin_unlock_irqrestore(&me->hlock, irq_flags);
  757. } else if (mflags == ADSP_MMAP_DMA_BUFFER) {
  758. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  759. if (map->buf == buf) {
  760. if (refs) {
  761. if (map->refs + 1 == INT_MAX)
  762. return -ETOOMANYREFS;
  763. map->refs++;
  764. }
  765. match = map;
  766. break;
  767. }
  768. }
  769. } else {
  770. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  771. if (va >= map->va &&
  772. va + len <= map->va + map->len &&
  773. map->fd == fd) {
  774. if (refs) {
  775. if (map->refs + 1 == INT_MAX)
  776. return -ETOOMANYREFS;
  777. map->refs++;
  778. }
  779. match = map;
  780. break;
  781. }
  782. }
  783. }
  784. if (match) {
  785. *ppmap = match;
  786. return 0;
  787. }
  788. return -ENXIO;
  789. }
  790. static int fastrpc_alloc_cma_memory(dma_addr_t *region_phys, void **vaddr,
  791. size_t size, unsigned long dma_attr)
  792. {
  793. int err = 0;
  794. struct fastrpc_apps *me = &gfa;
  795. if (me->dev == NULL) {
  796. ADSPRPC_ERR(
  797. "failed to allocate CMA memory, device adsprpc-mem is not initialized\n");
  798. return -ENODEV;
  799. }
  800. VERIFY(err, size > 0 && size < me->max_size_limit);
  801. if (err) {
  802. err = -EFAULT;
  803. pr_err("adsprpc: %s: invalid allocation size 0x%zx\n",
  804. __func__, size);
  805. return err;
  806. }
  807. *vaddr = dma_alloc_attrs(me->dev, size, region_phys,
  808. GFP_KERNEL, dma_attr);
  809. if (IS_ERR_OR_NULL(*vaddr)) {
  810. ADSPRPC_ERR(
  811. "dma_alloc_attrs failed for device %s size 0x%zx dma_attr %lu, returned %ld\n",
  812. dev_name(me->dev), size, dma_attr, PTR_ERR(*vaddr));
  813. return -ENOBUFS;
  814. }
  815. return 0;
  816. }
  817. static int fastrpc_mmap_remove(struct fastrpc_file *fl, int fd, uintptr_t va,
  818. size_t len, struct fastrpc_mmap **ppmap)
  819. {
  820. struct fastrpc_mmap *match = NULL, *map;
  821. struct hlist_node *n;
  822. struct fastrpc_apps *me = &gfa;
  823. unsigned long irq_flags = 0;
  824. /*
  825. * Search for a mapping by matching fd, remote address and length.
  826. * For backward compatibility, search for a mapping by matching is
  827. * limited to remote address and length when passed fd < 0.
  828. */
  829. spin_lock_irqsave(&me->hlock, irq_flags);
  830. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  831. if ((fd < 0 || map->fd == fd) && map->raddr == va &&
  832. map->raddr + map->len == va + len &&
  833. map->refs == 1 && !map->is_persistent &&
  834. /* Skip unmap if it is fastrpc shell memory */
  835. !map->is_filemap) {
  836. match = map;
  837. hlist_del_init(&map->hn);
  838. break;
  839. }
  840. }
  841. spin_unlock_irqrestore(&me->hlock, irq_flags);
  842. if (match) {
  843. *ppmap = match;
  844. return 0;
  845. }
  846. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  847. if ((fd < 0 || map->fd == fd) && map->raddr == va &&
  848. map->raddr + map->len == va + len &&
  849. map->refs == 1 &&
  850. /* Remove map only if it isn't being used in any pending RPC calls */
  851. !map->ctx_refs &&
  852. /* Skip unmap if it is fastrpc shell memory */
  853. !map->is_filemap) {
  854. match = map;
  855. hlist_del_init(&map->hn);
  856. break;
  857. }
  858. }
  859. if (match) {
  860. *ppmap = match;
  861. return 0;
  862. }
  863. return -ETOOMANYREFS;
  864. }
  865. static void fastrpc_mmap_free(struct fastrpc_mmap *map, uint32_t flags)
  866. {
  867. struct fastrpc_apps *me = &gfa;
  868. struct fastrpc_file *fl;
  869. int vmid, cid = -1, err = 0;
  870. struct fastrpc_session_ctx *sess;
  871. unsigned long irq_flags = 0;
  872. if (!map)
  873. return;
  874. fl = map->fl;
  875. if (fl && !(map->flags == ADSP_MMAP_HEAP_ADDR ||
  876. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)) {
  877. cid = fl->cid;
  878. VERIFY(err, VALID_FASTRPC_CID(cid));
  879. if (err) {
  880. err = -ECHRNG;
  881. pr_err("adsprpc: ERROR:%s, Invalid channel id: %d, err:%d\n",
  882. __func__, cid, err);
  883. return;
  884. }
  885. }
  886. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  887. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  888. spin_lock_irqsave(&me->hlock, irq_flags);
  889. map->refs--;
  890. if (!map->refs && !map->is_persistent && !map->ctx_refs)
  891. hlist_del_init(&map->hn);
  892. spin_unlock_irqrestore(&me->hlock, irq_flags);
  893. if (map->refs > 0) {
  894. ADSPRPC_WARN(
  895. "multiple references for remote heap size %zu va 0x%lx ref count is %d\n",
  896. map->size, map->va, map->refs);
  897. return;
  898. }
  899. spin_lock_irqsave(&me->hlock, irq_flags);
  900. if (map->is_persistent && map->in_use)
  901. map->in_use = false;
  902. spin_unlock_irqrestore(&me->hlock, irq_flags);
  903. } else {
  904. map->refs--;
  905. if (!map->refs && !map->ctx_refs)
  906. hlist_del_init(&map->hn);
  907. if (map->refs > 0 && !flags)
  908. return;
  909. }
  910. if (map->flags == ADSP_MMAP_HEAP_ADDR ||
  911. map->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  912. if (me->dev == NULL) {
  913. ADSPRPC_ERR(
  914. "failed to free remote heap allocation, device is not initialized\n");
  915. return;
  916. }
  917. if (msm_minidump_enabled() && !map->is_persistent)
  918. err = fastrpc_minidump_remove_region(map);
  919. if (map->phys && !map->is_persistent) {
  920. trace_fastrpc_dma_free(-1, map->phys, map->size);
  921. dma_free_attrs(me->dev, map->size, (void *)map->va,
  922. (dma_addr_t)map->phys, (unsigned long)map->attr);
  923. }
  924. } else if (map->flags == FASTRPC_MAP_FD_NOMAP) {
  925. trace_fastrpc_dma_unmap(cid, map->phys, map->size);
  926. if (!IS_ERR_OR_NULL(map->table))
  927. dma_buf_unmap_attachment(map->attach, map->table,
  928. DMA_BIDIRECTIONAL);
  929. if (!IS_ERR_OR_NULL(map->attach))
  930. dma_buf_detach(map->buf, map->attach);
  931. if (!IS_ERR_OR_NULL(map->buf))
  932. dma_buf_put(map->buf);
  933. } else {
  934. if (!fl)
  935. goto bail;
  936. if (map->secure)
  937. sess = fl->secsctx;
  938. else
  939. sess = fl->sctx;
  940. vmid = fl->apps->channel[cid].vmid;
  941. if (vmid && map->phys && (me->channel[cid].in_hib == 0)) {
  942. int hyp_err = 0;
  943. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS) | BIT(vmid);
  944. struct qcom_scm_vmperm dst_perms = {0};
  945. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  946. dst_perms.perm = QCOM_SCM_PERM_RWX;
  947. hyp_err = qcom_scm_assign_mem(map->phys,
  948. buf_page_size(map->size),
  949. &src_perms, &dst_perms, 1);
  950. if (hyp_err) {
  951. ADSPRPC_ERR(
  952. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  953. hyp_err, map->phys, map->size);
  954. }
  955. }
  956. trace_fastrpc_dma_unmap(cid, map->phys, map->size);
  957. if (!IS_ERR_OR_NULL(map->table))
  958. dma_buf_unmap_attachment(map->attach, map->table,
  959. DMA_BIDIRECTIONAL);
  960. if (!IS_ERR_OR_NULL(map->attach))
  961. dma_buf_detach(map->buf, map->attach);
  962. if (!IS_ERR_OR_NULL(map->buf))
  963. dma_buf_put(map->buf);
  964. }
  965. if (fl) {
  966. spin_lock(&fl->hlock);
  967. if ((map->flags == ADSP_MMAP_ADD_PAGES) || (map->flags == ADSP_MMAP_ADD_PAGES_LLC))
  968. fl->mem_snap.heap_bufs_size -= map->size;
  969. else
  970. fl->mem_snap.nonheap_bufs_size -= map->size;
  971. spin_unlock(&fl->hlock);
  972. }
  973. bail:
  974. if (!map->is_persistent)
  975. kfree(map);
  976. }
  977. static int fastrpc_session_alloc_secure_memory(
  978. struct fastrpc_channel_ctx *chan, int secure,
  979. int sharedcb, int pd_type, struct fastrpc_session_ctx **session);
  980. static inline bool fastrpc_get_persistent_map(size_t len, struct fastrpc_mmap **pers_map)
  981. {
  982. struct fastrpc_apps *me = &gfa;
  983. struct fastrpc_mmap *map = NULL;
  984. struct hlist_node *n = NULL;
  985. bool found = false;
  986. unsigned long irq_flags = 0;
  987. spin_lock_irqsave(&me->hlock, irq_flags);
  988. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  989. if (len == map->len &&
  990. map->is_persistent && !map->in_use) {
  991. *pers_map = map;
  992. map->in_use = true;
  993. /*
  994. * Incrementing map reference count when getting
  995. * the map to avoid negative reference count when
  996. * freeing the map.
  997. */
  998. map->refs++;
  999. found = true;
  1000. break;
  1001. }
  1002. }
  1003. spin_unlock_irqrestore(&me->hlock, irq_flags);
  1004. return found;
  1005. }
  1006. static int fastrpc_mmap_create_remote_heap(struct fastrpc_file *fl,
  1007. struct fastrpc_mmap *map, size_t len, int mflags)
  1008. {
  1009. int err = 0;
  1010. struct fastrpc_apps *me = &gfa;
  1011. dma_addr_t region_phys = 0;
  1012. void *region_vaddr = NULL;
  1013. map->apps = me;
  1014. map->fl = NULL;
  1015. map->attr |= DMA_ATTR_NO_KERNEL_MAPPING;
  1016. err = fastrpc_alloc_cma_memory(&region_phys, &region_vaddr,
  1017. len, (unsigned long) map->attr);
  1018. if (err)
  1019. goto bail;
  1020. trace_fastrpc_dma_alloc(fl->cid, (uint64_t)region_phys, len,
  1021. (unsigned long)map->attr, mflags);
  1022. map->phys = (uintptr_t)region_phys;
  1023. map->size = len;
  1024. map->va = (uintptr_t)region_vaddr;
  1025. map->servloc_name = fl->servloc_name;
  1026. bail:
  1027. return err;
  1028. }
  1029. static int get_buffer_attr(struct dma_buf *buf, bool *exclusive_access, bool *hlos_access)
  1030. {
  1031. const int *vmids_list = NULL, *perms = NULL;
  1032. int err = 0, vmids_list_len = 0;
  1033. *exclusive_access = false;
  1034. *hlos_access = false;
  1035. err = mem_buf_dma_buf_get_vmperm(buf, &vmids_list, &perms, &vmids_list_len);
  1036. if (err)
  1037. goto bail;
  1038. /*
  1039. * If one VM has access to buffer and is the current VM,
  1040. * then VM has exclusive access to buffer
  1041. */
  1042. if (vmids_list_len == 1 && vmids_list[0] == mem_buf_current_vmid())
  1043. *exclusive_access = true;
  1044. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1045. /*
  1046. * PVM (HLOS) can share buffers with TVM. In that case,
  1047. * it is expected to relinquish its ownership to those buffers
  1048. * before sharing. But if the PVM still retains access, then
  1049. * these buffers cannot be used by TVM.
  1050. */
  1051. for (int ii = 0; ii < vmids_list_len; ii++) {
  1052. if (vmids_list[ii] == VMID_HLOS) {
  1053. *hlos_access = true;
  1054. break;
  1055. }
  1056. }
  1057. #endif
  1058. bail:
  1059. return err;
  1060. }
  1061. static int set_buffer_secure_type(struct fastrpc_mmap *map)
  1062. {
  1063. int err = 0;
  1064. bool hlos_access = false, exclusive_access = false;
  1065. VERIFY(err, 0 == (err = get_buffer_attr(map->buf, &exclusive_access, &hlos_access)));
  1066. if (err) {
  1067. ADSPRPC_ERR("failed to obtain buffer attributes for fd %d ret %d\n", map->fd, err);
  1068. err = -EBADFD;
  1069. goto bail;
  1070. }
  1071. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  1072. if (hlos_access) {
  1073. ADSPRPC_ERR("Sharing HLOS buffer (fd %d) not allowed on TVM\n", map->fd);
  1074. err = -EACCES;
  1075. goto bail;
  1076. }
  1077. #endif
  1078. /*
  1079. * Secure buffers would always be owned by multiple VMs.
  1080. * If current VM is the exclusive owner of a buffer, it is considered non-secure.
  1081. * In PVM:
  1082. * - CPZ buffers are secure
  1083. * - All other buffers are non-secure
  1084. * In TVM:
  1085. * - Since it is a secure environment by default, there are no explicit "secure" buffers
  1086. * - All buffers are marked "non-secure"
  1087. */
  1088. map->secure = (exclusive_access) ? 0 : 1;
  1089. bail:
  1090. return err;
  1091. }
  1092. static int fastrpc_mmap_create(struct fastrpc_file *fl, int fd, struct dma_buf *buf,
  1093. unsigned int attr, uintptr_t va, size_t len, int mflags,
  1094. struct fastrpc_mmap **ppmap)
  1095. {
  1096. struct fastrpc_apps *me = &gfa;
  1097. struct fastrpc_session_ctx *sess;
  1098. struct fastrpc_apps *apps = NULL;
  1099. int cid = -1;
  1100. struct fastrpc_channel_ctx *chan = NULL;
  1101. struct fastrpc_mmap *map = NULL;
  1102. int err = 0, vmid, sgl_index = 0;
  1103. struct scatterlist *sgl = NULL;
  1104. bool dma_attach_fail = false;
  1105. size_t tot_bufs_size = 0;
  1106. if (!fl) {
  1107. err = -EBADF;
  1108. goto bail;
  1109. }
  1110. apps = fl->apps;
  1111. cid = fl->cid;
  1112. VERIFY(err, VALID_FASTRPC_CID(cid));
  1113. if (err) {
  1114. err = -ECHRNG;
  1115. goto bail;
  1116. }
  1117. chan = &apps->channel[cid];
  1118. if (!fastrpc_mmap_find(fl, fd, NULL, va, len, mflags, 1, ppmap))
  1119. return 0;
  1120. map = kzalloc(sizeof(*map), GFP_KERNEL);
  1121. VERIFY(err, !IS_ERR_OR_NULL(map));
  1122. if (err) {
  1123. err = -ENOMEM;
  1124. goto bail;
  1125. }
  1126. INIT_HLIST_NODE(&map->hn);
  1127. map->flags = mflags;
  1128. map->refs = 1;
  1129. map->fl = fl;
  1130. map->fd = fd;
  1131. map->attr = attr;
  1132. map->buf = buf;
  1133. map->frpc_md_index = -1;
  1134. map->is_filemap = false;
  1135. ktime_get_real_ts64(&map->map_start_time);
  1136. if (mflags == ADSP_MMAP_HEAP_ADDR ||
  1137. mflags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  1138. VERIFY(err, 0 == (err = fastrpc_mmap_create_remote_heap(fl, map,
  1139. len, mflags)));
  1140. if (err)
  1141. goto bail;
  1142. if (msm_minidump_enabled()) {
  1143. err = fastrpc_minidump_add_region(map);
  1144. if (err)
  1145. goto bail;
  1146. }
  1147. } else if (mflags == FASTRPC_MAP_FD_NOMAP) {
  1148. VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));
  1149. if (err) {
  1150. ADSPRPC_ERR("dma_buf_get failed for fd %d ret %ld\n",
  1151. fd, PTR_ERR(map->buf));
  1152. err = -EBADFD;
  1153. goto bail;
  1154. }
  1155. err = set_buffer_secure_type(map);
  1156. if (err)
  1157. goto bail;
  1158. map->va = 0;
  1159. map->phys = 0;
  1160. VERIFY(err, !IS_ERR_OR_NULL(map->attach =
  1161. dma_buf_attach(map->buf, me->dev)));
  1162. if (err) {
  1163. ADSPRPC_ERR(
  1164. "dma_buf_attach for fd %d for len 0x%zx failed to map buffer on SMMU device %s ret %ld\n",
  1165. fd, len, dev_name(me->dev), PTR_ERR(map->attach));
  1166. dma_attach_fail = true;
  1167. err = -EFAULT;
  1168. goto bail;
  1169. }
  1170. map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  1171. VERIFY(err, !IS_ERR_OR_NULL(map->table =
  1172. dma_buf_map_attachment(map->attach,
  1173. DMA_BIDIRECTIONAL)));
  1174. if (err) {
  1175. ADSPRPC_ERR(
  1176. "dma_buf_map_attachment for fd %d for len 0x%zx failed on device %s ret %ld\n",
  1177. fd, len, dev_name(me->dev), PTR_ERR(map->table));
  1178. err = -EFAULT;
  1179. goto bail;
  1180. }
  1181. VERIFY(err, map->table->nents == 1);
  1182. if (err) {
  1183. ADSPRPC_ERR(
  1184. "multiple scatter-gather entries (%u) present for NOMAP fd %d\n",
  1185. map->table->nents, fd);
  1186. err = -EFAULT;
  1187. goto bail;
  1188. }
  1189. map->phys = sg_dma_address(map->table->sgl);
  1190. map->size = len;
  1191. map->flags = FASTRPC_MAP_FD_DELAYED;
  1192. trace_fastrpc_dma_map(cid, fd, map->phys, map->size,
  1193. len, map->attach->dma_map_attrs, mflags);
  1194. } else {
  1195. if (map->attr && (map->attr & FASTRPC_ATTR_KEEP_MAP)) {
  1196. ADSPRPC_INFO("buffer mapped with persist attr 0x%x\n",
  1197. (unsigned int)map->attr);
  1198. map->refs = 2;
  1199. }
  1200. if (mflags == ADSP_MMAP_DMA_BUFFER) {
  1201. VERIFY(err, !IS_ERR_OR_NULL(map->buf));
  1202. if (err) {
  1203. ADSPRPC_ERR("Invalid DMA buffer address %pK\n",
  1204. map->buf);
  1205. err = -EFAULT;
  1206. goto bail;
  1207. }
  1208. /* Increment DMA buffer ref count,
  1209. * so that client cannot unmap DMA buffer, before freeing buffer
  1210. */
  1211. get_dma_buf(map->buf);
  1212. } else {
  1213. VERIFY(err, !IS_ERR_OR_NULL(map->buf = dma_buf_get(fd)));
  1214. if (err) {
  1215. ADSPRPC_ERR("dma_buf_get failed for fd %d ret %ld\n",
  1216. fd, PTR_ERR(map->buf));
  1217. err = -EBADFD;
  1218. goto bail;
  1219. }
  1220. }
  1221. err = set_buffer_secure_type(map);
  1222. if (err)
  1223. goto bail;
  1224. if (map->secure) {
  1225. if (!fl->secsctx)
  1226. err = fastrpc_session_alloc_secure_memory(chan, 1,
  1227. me->share_securecb, fl->pd_type, &fl->secsctx);
  1228. if (err) {
  1229. ADSPRPC_ERR(
  1230. "fastrpc_session_alloc_secure_memory failed for fd %d ret %d\n",
  1231. fd, err);
  1232. err = -ENOSR;
  1233. goto bail;
  1234. }
  1235. }
  1236. if (map->secure)
  1237. sess = fl->secsctx;
  1238. else
  1239. sess = fl->sctx;
  1240. VERIFY(err, !IS_ERR_OR_NULL(sess));
  1241. if (err) {
  1242. ADSPRPC_ERR(
  1243. "session is invalid for fd %d, secure flag %d\n",
  1244. fd, map->secure);
  1245. err = -EBADR;
  1246. goto bail;
  1247. }
  1248. VERIFY(err, !IS_ERR_OR_NULL(map->attach =
  1249. dma_buf_attach(map->buf, sess->smmu.dev)));
  1250. if (err) {
  1251. ADSPRPC_ERR(
  1252. "dma_buf_attach for fd %d failed for len 0x%zx to map buffer on SMMU device %s ret %ld\n",
  1253. fd, len, dev_name(sess->smmu.dev),
  1254. PTR_ERR(map->attach));
  1255. dma_attach_fail = true;
  1256. err = -EFAULT;
  1257. goto bail;
  1258. }
  1259. map->attach->dma_map_attrs |= DMA_ATTR_DELAYED_UNMAP;
  1260. /*
  1261. * Skip CPU sync if IO Cohernecy is not supported
  1262. */
  1263. if (!sess->smmu.coherent)
  1264. map->attach->dma_map_attrs |= DMA_ATTR_SKIP_CPU_SYNC;
  1265. VERIFY(err, !IS_ERR_OR_NULL(map->table =
  1266. dma_buf_map_attachment(map->attach,
  1267. DMA_BIDIRECTIONAL)));
  1268. if (err) {
  1269. ADSPRPC_ERR(
  1270. "dma_buf_map_attachment for fd %d failed for len 0x%zx on device %s ret %ld\n",
  1271. fd, len, dev_name(sess->smmu.dev),
  1272. PTR_ERR(map->table));
  1273. err = -EFAULT;
  1274. goto bail;
  1275. }
  1276. if (!sess->smmu.enabled) {
  1277. VERIFY(err, map->table->nents == 1);
  1278. if (err) {
  1279. ADSPRPC_ERR(
  1280. "multiple scatter-gather entries (%u) present for fd %d mapped on SMMU disabled device\n",
  1281. map->table->nents, fd);
  1282. err = -EFAULT;
  1283. goto bail;
  1284. }
  1285. }
  1286. map->phys = sg_dma_address(map->table->sgl);
  1287. if (sess->smmu.cb) {
  1288. map->phys += ((uint64_t)sess->smmu.cb << 32);
  1289. for_each_sg(map->table->sgl, sgl, map->table->nents,
  1290. sgl_index)
  1291. map->size += sg_dma_len(sgl);
  1292. } else {
  1293. map->size = buf_page_size(len);
  1294. }
  1295. trace_fastrpc_dma_map(cid, fd, map->phys, map->size,
  1296. len, map->attach->dma_map_attrs, mflags);
  1297. VERIFY(err, map->size >= len && map->size < me->max_size_limit);
  1298. if (err) {
  1299. err = -EFAULT;
  1300. pr_err("adsprpc: %s: invalid map size 0x%zx len 0x%zx\n",
  1301. __func__, map->size, len);
  1302. goto bail;
  1303. }
  1304. vmid = fl->apps->channel[cid].vmid;
  1305. if (vmid) {
  1306. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  1307. struct qcom_scm_vmperm dst_perms[2] = {0};
  1308. dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
  1309. dst_perms[0].perm = QCOM_SCM_PERM_RW;
  1310. dst_perms[1].vmid = vmid;
  1311. dst_perms[1].perm = QCOM_SCM_PERM_RWX;
  1312. err = qcom_scm_assign_mem(map->phys,
  1313. buf_page_size(map->size),
  1314. &src_perms, dst_perms, 2);
  1315. if (err) {
  1316. ADSPRPC_ERR(
  1317. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  1318. err, map->phys, map->size);
  1319. err = -EADDRNOTAVAIL;
  1320. goto bail;
  1321. }
  1322. }
  1323. map->va = va;
  1324. }
  1325. map->len = len;
  1326. spin_lock(&fl->hlock);
  1327. if ((mflags == ADSP_MMAP_ADD_PAGES) || (mflags == ADSP_MMAP_ADD_PAGES_LLC))
  1328. fl->mem_snap.heap_bufs_size += map->size;
  1329. else
  1330. fl->mem_snap.nonheap_bufs_size += map->size;
  1331. spin_unlock(&fl->hlock);
  1332. fastrpc_mmap_add(map);
  1333. *ppmap = map;
  1334. bail:
  1335. if (dma_attach_fail && fl) {
  1336. tot_bufs_size = fl->mem_snap.heap_bufs_size
  1337. + fl->mem_snap.nonheap_bufs_size;
  1338. ADSPRPC_INFO("Heapbufs size: %zu, non-heapbufs size: %zu, total size: %zu\n",
  1339. fl->mem_snap.heap_bufs_size, fl->mem_snap.nonheap_bufs_size,
  1340. tot_bufs_size);
  1341. }
  1342. if (map)
  1343. ktime_get_real_ts64(&map->map_end_time);
  1344. if (err && map)
  1345. fastrpc_mmap_free(map, 0);
  1346. return err;
  1347. }
  1348. static inline bool fastrpc_get_cached_buf(struct fastrpc_file *fl,
  1349. size_t size, int buf_type, struct fastrpc_buf **obuf)
  1350. {
  1351. bool found = false;
  1352. struct fastrpc_buf *buf = NULL, *fr = NULL;
  1353. struct hlist_node *n = NULL;
  1354. if (buf_type == USERHEAP_BUF)
  1355. goto bail;
  1356. /* find the smallest buffer that fits in the cache */
  1357. spin_lock(&fl->hlock);
  1358. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  1359. if (buf->size >= size && (!fr || fr->size > buf->size))
  1360. fr = buf;
  1361. }
  1362. if (fr) {
  1363. hlist_del_init(&fr->hn);
  1364. fl->num_cached_buf--;
  1365. }
  1366. spin_unlock(&fl->hlock);
  1367. if (fr) {
  1368. fr->type = buf_type;
  1369. *obuf = fr;
  1370. found = true;
  1371. }
  1372. bail:
  1373. return found;
  1374. }
  1375. static inline bool fastrpc_get_persistent_buf(struct fastrpc_file *fl,
  1376. size_t size, int buf_type, struct fastrpc_buf **obuf)
  1377. {
  1378. unsigned int i = 0;
  1379. bool found = false;
  1380. struct fastrpc_buf *buf = NULL;
  1381. spin_lock(&fl->hlock);
  1382. if (!fl->num_pers_hdrs)
  1383. goto bail;
  1384. /*
  1385. * Persistent header buffer can be used only if
  1386. * metadata length is less than 1 page size.
  1387. */
  1388. if (buf_type != METADATA_BUF || size > PAGE_SIZE)
  1389. goto bail;
  1390. for (i = 0; i < fl->num_pers_hdrs; i++) {
  1391. buf = &fl->hdr_bufs[i];
  1392. /* If buffer not in use, then assign it for requested alloc */
  1393. if (!buf->in_use) {
  1394. buf->in_use = true;
  1395. *obuf = buf;
  1396. found = true;
  1397. break;
  1398. }
  1399. }
  1400. bail:
  1401. spin_unlock(&fl->hlock);
  1402. return found;
  1403. }
  1404. static int fastrpc_buf_alloc(struct fastrpc_file *fl, size_t size,
  1405. unsigned long dma_attr, uint32_t rflags,
  1406. int buf_type, struct fastrpc_buf **obuf)
  1407. {
  1408. int err = 0, vmid;
  1409. struct fastrpc_apps *me = &gfa;
  1410. struct fastrpc_buf *buf = NULL;
  1411. int cid = -1;
  1412. VERIFY(err, fl && fl->sctx != NULL);
  1413. if (err) {
  1414. err = -EBADR;
  1415. goto bail;
  1416. }
  1417. cid = fl->cid;
  1418. VERIFY(err, VALID_FASTRPC_CID(cid));
  1419. if (err) {
  1420. err = -ECHRNG;
  1421. goto bail;
  1422. }
  1423. VERIFY(err, size > 0 && size < me->max_size_limit);
  1424. if (err) {
  1425. err = -EFAULT;
  1426. pr_err("adsprpc: %s: invalid allocation size 0x%zx\n",
  1427. __func__, size);
  1428. goto bail;
  1429. }
  1430. VERIFY(err, size > 0 && fl->sctx->smmu.dev);
  1431. if (err) {
  1432. err = (fl->sctx->smmu.dev == NULL) ? -ENODEV : err;
  1433. goto bail;
  1434. }
  1435. if (fastrpc_get_persistent_buf(fl, size, buf_type, obuf))
  1436. return err;
  1437. if (fastrpc_get_cached_buf(fl, size, buf_type, obuf))
  1438. return err;
  1439. /* If unable to get persistent or cached buf, allocate new buffer */
  1440. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  1441. if (err) {
  1442. err = -ENOMEM;
  1443. goto bail;
  1444. }
  1445. INIT_HLIST_NODE(&buf->hn);
  1446. buf->fl = fl;
  1447. buf->virt = NULL;
  1448. buf->phys = 0;
  1449. buf->size = size;
  1450. buf->dma_attr = dma_attr;
  1451. buf->flags = rflags;
  1452. buf->raddr = 0;
  1453. buf->type = buf_type;
  1454. ktime_get_real_ts64(&buf->buf_start_time);
  1455. buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
  1456. (dma_addr_t *)&buf->phys,
  1457. GFP_KERNEL, buf->dma_attr);
  1458. if (IS_ERR_OR_NULL(buf->virt)) {
  1459. /* free cache and retry */
  1460. fastrpc_cached_buf_list_free(fl);
  1461. buf->virt = dma_alloc_attrs(fl->sctx->smmu.dev, buf->size,
  1462. (dma_addr_t *)&buf->phys, GFP_KERNEL,
  1463. buf->dma_attr);
  1464. VERIFY(err, !IS_ERR_OR_NULL(buf->virt));
  1465. }
  1466. if (err) {
  1467. ADSPRPC_ERR(
  1468. "dma_alloc_attrs failed for size 0x%zx, returned %pK\n",
  1469. size, buf->virt);
  1470. err = -ENOBUFS;
  1471. goto bail;
  1472. }
  1473. if (fl->sctx->smmu.cb)
  1474. buf->phys += ((uint64_t)fl->sctx->smmu.cb << 32);
  1475. trace_fastrpc_dma_alloc(cid, buf->phys, size,
  1476. dma_attr, (int)rflags);
  1477. vmid = fl->apps->channel[cid].vmid;
  1478. if (vmid) {
  1479. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  1480. struct qcom_scm_vmperm dst_perms[2] = {0};
  1481. dst_perms[0].vmid = QCOM_SCM_VMID_HLOS;
  1482. dst_perms[0].perm = QCOM_SCM_PERM_RW;
  1483. dst_perms[1].vmid = vmid;
  1484. dst_perms[1].perm = QCOM_SCM_PERM_RWX;
  1485. err = qcom_scm_assign_mem(buf->phys, buf_page_size(size),
  1486. &src_perms, dst_perms, 2);
  1487. if (err) {
  1488. ADSPRPC_DEBUG(
  1489. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  1490. err, buf->phys, size);
  1491. err = -EADDRNOTAVAIL;
  1492. goto bail;
  1493. }
  1494. }
  1495. if (buf_type == USERHEAP_BUF) {
  1496. INIT_HLIST_NODE(&buf->hn_rem);
  1497. spin_lock(&fl->hlock);
  1498. hlist_add_head(&buf->hn_rem, &fl->remote_bufs);
  1499. spin_unlock(&fl->hlock);
  1500. }
  1501. *obuf = buf;
  1502. bail:
  1503. if (buf)
  1504. ktime_get_real_ts64(&buf->buf_end_time);
  1505. if (err && buf)
  1506. fastrpc_buf_free(buf, 0);
  1507. return err;
  1508. }
  1509. static int context_restore_interrupted(struct fastrpc_file *fl,
  1510. struct fastrpc_ioctl_invoke_async *inv,
  1511. struct smq_invoke_ctx **po)
  1512. {
  1513. int err = 0;
  1514. struct smq_invoke_ctx *ctx = NULL, *ictx = NULL;
  1515. struct hlist_node *n;
  1516. struct fastrpc_ioctl_invoke *invoke = &inv->inv;
  1517. spin_lock(&fl->hlock);
  1518. hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {
  1519. if (ictx->pid == current->pid) {
  1520. if (invoke->sc != ictx->sc || ictx->fl != fl) {
  1521. err = -EINVAL;
  1522. ictx->sc_interrupted = invoke->sc;
  1523. ictx->fl_interrupted = fl;
  1524. ictx->handle_interrupted = invoke->handle;
  1525. ADSPRPC_ERR(
  1526. "interrupted sc (0x%x) or fl (%pK) does not match with invoke sc (0x%x) or fl (%pK)\n",
  1527. ictx->sc, ictx->fl, invoke->sc, fl);
  1528. } else {
  1529. ictx->xo_time_in_us_restored = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1530. fastrpc_update_txmsg_buf(NULL, 0, 0, 0, ictx, RESTORED_STATE);
  1531. ADSPRPC_DEBUG(
  1532. "restored sc (0x%x) of fl (%pK), interrupt ts 0x%llx, restore ts 0x%llx \n",
  1533. ictx->sc, ictx->fl, ictx->xo_time_in_us_interrupted, ictx->xo_time_in_us_restored);
  1534. ctx = ictx;
  1535. hlist_del_init(&ctx->hn);
  1536. hlist_add_head(&ctx->hn, &fl->clst.pending);
  1537. }
  1538. break;
  1539. }
  1540. }
  1541. spin_unlock(&fl->hlock);
  1542. if (ctx)
  1543. *po = ctx;
  1544. return err;
  1545. }
  1546. static unsigned int sorted_lists_intersection(unsigned int *listA,
  1547. unsigned int lenA, unsigned int *listB, unsigned int lenB)
  1548. {
  1549. unsigned int i = 0, j = 0;
  1550. while (i < lenA && j < lenB) {
  1551. if (listA[i] < listB[j])
  1552. i++;
  1553. else if (listA[i] > listB[j])
  1554. j++;
  1555. else
  1556. return listA[i];
  1557. }
  1558. return 0;
  1559. }
  1560. #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
  1561. static int uint_cmp_func(const void *p1, const void *p2)
  1562. {
  1563. unsigned int a1 = *((unsigned int *)p1);
  1564. unsigned int a2 = *((unsigned int *)p2);
  1565. return CMP(a1, a2);
  1566. }
  1567. static int overlap_ptr_cmp(const void *a, const void *b)
  1568. {
  1569. struct overlap *pa = *((struct overlap **)a);
  1570. struct overlap *pb = *((struct overlap **)b);
  1571. /* sort with lowest starting buffer first */
  1572. int st = CMP(pa->start, pb->start);
  1573. /* sort with highest ending buffer first */
  1574. int ed = CMP(pb->end, pa->end);
  1575. return st == 0 ? ed : st;
  1576. }
  1577. static int context_build_overlap(struct smq_invoke_ctx *ctx)
  1578. {
  1579. int i, err = 0;
  1580. remote_arg_t *lpra = ctx->lpra;
  1581. int inbufs = REMOTE_SCALARS_INBUFS(ctx->sc);
  1582. int outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc);
  1583. int nbufs = inbufs + outbufs;
  1584. struct overlap max;
  1585. for (i = 0; i < nbufs; ++i) {
  1586. ctx->overs[i].start = (uintptr_t)lpra[i].buf.pv;
  1587. ctx->overs[i].end = ctx->overs[i].start + lpra[i].buf.len;
  1588. if (lpra[i].buf.len) {
  1589. VERIFY(err, ctx->overs[i].end > ctx->overs[i].start);
  1590. if (err) {
  1591. err = -EFAULT;
  1592. ADSPRPC_ERR(
  1593. "Invalid address 0x%llx and size %zu\n",
  1594. (uintptr_t)lpra[i].buf.pv,
  1595. lpra[i].buf.len);
  1596. goto bail;
  1597. }
  1598. }
  1599. ctx->overs[i].raix = i;
  1600. ctx->overps[i] = &ctx->overs[i];
  1601. }
  1602. sort(ctx->overps, nbufs, sizeof(*ctx->overps), overlap_ptr_cmp, NULL);
  1603. max.start = 0;
  1604. max.end = 0;
  1605. for (i = 0; i < nbufs; ++i) {
  1606. if (ctx->overps[i]->start < max.end) {
  1607. ctx->overps[i]->mstart = max.end;
  1608. ctx->overps[i]->mend = ctx->overps[i]->end;
  1609. ctx->overps[i]->offset = max.end -
  1610. ctx->overps[i]->start;
  1611. if (ctx->overps[i]->end > max.end) {
  1612. max.end = ctx->overps[i]->end;
  1613. } else {
  1614. if ((max.raix < inbufs &&
  1615. ctx->overps[i]->raix + 1 > inbufs) ||
  1616. (ctx->overps[i]->raix < inbufs &&
  1617. max.raix + 1 > inbufs))
  1618. ctx->overps[i]->do_cmo = 1;
  1619. ctx->overps[i]->mend = 0;
  1620. ctx->overps[i]->mstart = 0;
  1621. }
  1622. } else {
  1623. ctx->overps[i]->mend = ctx->overps[i]->end;
  1624. ctx->overps[i]->mstart = ctx->overps[i]->start;
  1625. ctx->overps[i]->offset = 0;
  1626. max = *ctx->overps[i];
  1627. }
  1628. }
  1629. bail:
  1630. return err;
  1631. }
  1632. #define K_COPY_FROM_USER(err, kernel, dst, src, size) \
  1633. do {\
  1634. if (!(kernel))\
  1635. err = copy_from_user((dst),\
  1636. (void const __user *)(src),\
  1637. (size));\
  1638. else\
  1639. memmove((dst), (src), (size));\
  1640. } while (0)
  1641. #define K_COPY_TO_USER(err, kernel, dst, src, size) \
  1642. do {\
  1643. if (!(kernel))\
  1644. err = copy_to_user((void __user *)(dst),\
  1645. (src), (size));\
  1646. else\
  1647. memmove((dst), (src), (size));\
  1648. } while (0)
  1649. static void context_free(struct smq_invoke_ctx *ctx);
  1650. static int context_alloc(struct fastrpc_file *fl, uint32_t kernel,
  1651. struct fastrpc_ioctl_invoke_async *invokefd,
  1652. struct smq_invoke_ctx **po)
  1653. {
  1654. struct fastrpc_apps *me = &gfa;
  1655. int err = 0, bufs, ii, size = 0, cid = fl->cid;
  1656. struct smq_invoke_ctx *ctx = NULL;
  1657. struct fastrpc_ctx_lst *clst = &fl->clst;
  1658. struct fastrpc_ioctl_invoke *invoke = &invokefd->inv;
  1659. struct fastrpc_channel_ctx *chan = NULL;
  1660. unsigned long irq_flags = 0;
  1661. uint32_t is_kernel_memory = 0;
  1662. spin_lock(&fl->hlock);
  1663. if (fl->clst.num_active_ctxs > MAX_PENDING_CTX_PER_SESSION &&
  1664. !(kernel || invoke->handle < FASTRPC_STATIC_HANDLE_MAX)) {
  1665. err = -EDQUOT;
  1666. spin_unlock(&fl->hlock);
  1667. goto bail;
  1668. }
  1669. spin_unlock(&fl->hlock);
  1670. bufs = REMOTE_SCALARS_LENGTH(invoke->sc);
  1671. size = bufs * sizeof(*ctx->lpra) + bufs * sizeof(*ctx->maps) +
  1672. sizeof(*ctx->fds) * (bufs) +
  1673. sizeof(*ctx->attrs) * (bufs) +
  1674. sizeof(*ctx->overs) * (bufs) +
  1675. sizeof(*ctx->overps) * (bufs);
  1676. VERIFY(err, NULL != (ctx = kzalloc(sizeof(*ctx) + size, GFP_KERNEL)));
  1677. if (err) {
  1678. err = -ENOMEM;
  1679. goto bail;
  1680. }
  1681. INIT_HLIST_NODE(&ctx->hn);
  1682. INIT_LIST_HEAD(&ctx->asyncn);
  1683. hlist_add_fake(&ctx->hn);
  1684. ctx->fl = fl;
  1685. ctx->maps = (struct fastrpc_mmap **)(&ctx[1]);
  1686. ctx->lpra = (remote_arg_t *)(&ctx->maps[bufs]);
  1687. ctx->fds = (int *)(&ctx->lpra[bufs]);
  1688. ctx->attrs = (unsigned int *)(&ctx->fds[bufs]);
  1689. ctx->overs = (struct overlap *)(&ctx->attrs[bufs]);
  1690. ctx->overps = (struct overlap **)(&ctx->overs[bufs]);
  1691. /* If user message, do not use copy_from_user to copy buffers for
  1692. * compat driver,as memory is already copied to kernel memory
  1693. * for compat driver
  1694. */
  1695. is_kernel_memory = ((kernel == USER_MSG) ? (fl->is_compat) : kernel);
  1696. K_COPY_FROM_USER(err, is_kernel_memory, (void *)ctx->lpra, invoke->pra,
  1697. bufs * sizeof(*ctx->lpra));
  1698. if (err) {
  1699. ADSPRPC_ERR(
  1700. "copy from user failed with %d for remote arguments list\n",
  1701. err);
  1702. err = -EFAULT;
  1703. goto bail;
  1704. }
  1705. if (invokefd->fds) {
  1706. K_COPY_FROM_USER(err, kernel, ctx->fds, invokefd->fds,
  1707. bufs * sizeof(*ctx->fds));
  1708. if (err) {
  1709. ADSPRPC_ERR(
  1710. "copy from user failed with %d for fd list\n",
  1711. err);
  1712. err = -EFAULT;
  1713. goto bail;
  1714. }
  1715. } else {
  1716. ctx->fds = NULL;
  1717. }
  1718. if (invokefd->attrs) {
  1719. K_COPY_FROM_USER(err, kernel, ctx->attrs, invokefd->attrs,
  1720. bufs * sizeof(*ctx->attrs));
  1721. if (err) {
  1722. ADSPRPC_ERR(
  1723. "copy from user failed with %d for attribute list\n",
  1724. err);
  1725. err = -EFAULT;
  1726. goto bail;
  1727. }
  1728. }
  1729. ctx->crc = (uint32_t *)invokefd->crc;
  1730. ctx->perf_dsp = (uint64_t *)invokefd->perf_dsp;
  1731. ctx->perf_kernel = (uint64_t *)invokefd->perf_kernel;
  1732. ctx->handle = invoke->handle;
  1733. ctx->sc = invoke->sc;
  1734. if (bufs) {
  1735. VERIFY(err, 0 == (err = context_build_overlap(ctx)));
  1736. if (err)
  1737. goto bail;
  1738. }
  1739. ctx->retval = -1;
  1740. ctx->pid = current->pid;
  1741. /* Store HLOS PID in context, it is not being sent to DSP */
  1742. ctx->tgid = fl->tgid;
  1743. init_completion(&ctx->work);
  1744. ctx->magic = FASTRPC_CTX_MAGIC;
  1745. ctx->rsp_flags = NORMAL_RESPONSE;
  1746. ctx->is_work_done = false;
  1747. ctx->copybuf = NULL;
  1748. ctx->is_early_wakeup = false;
  1749. ctx->is_job_sent_to_remote_ss = false;
  1750. if (ctx->fl->profile) {
  1751. ctx->perf = kzalloc(sizeof(*(ctx->perf)), GFP_KERNEL);
  1752. VERIFY(err, !IS_ERR_OR_NULL(ctx->perf));
  1753. if (err) {
  1754. kfree(ctx->perf);
  1755. err = -ENOMEM;
  1756. goto bail;
  1757. }
  1758. memset(ctx->perf, 0, sizeof(*(ctx->perf)));
  1759. /* Use HLOS PID, as perf tid is not being sent to DSP and is used to log in traces */
  1760. ctx->perf->tid = fl->tgid;
  1761. }
  1762. if (invokefd->job) {
  1763. K_COPY_FROM_USER(err, kernel, &ctx->asyncjob, invokefd->job,
  1764. sizeof(ctx->asyncjob));
  1765. if (err)
  1766. goto bail;
  1767. }
  1768. VERIFY(err, VALID_FASTRPC_CID(cid));
  1769. if (err) {
  1770. err = -ECHRNG;
  1771. goto bail;
  1772. }
  1773. chan = &me->channel[cid];
  1774. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  1775. me->jobid[cid]++;
  1776. for (ii = ((kernel || ctx->handle < FASTRPC_STATIC_HANDLE_MAX)
  1777. ? 0 : NUM_KERNEL_AND_STATIC_ONLY_CONTEXTS);
  1778. ii < FASTRPC_CTX_MAX; ii++) {
  1779. if (!chan->ctxtable[ii]) {
  1780. chan->ctxtable[ii] = ctx;
  1781. ctx->ctxid = (me->jobid[cid] << FASTRPC_CTX_JOBID_POS)
  1782. | (ii << FASTRPC_CTX_TABLE_IDX_POS)
  1783. | ((ctx->asyncjob.isasyncjob &&
  1784. FASTRPC_ASYNC_JOB_MASK) << FASTRPC_CTX_JOB_TYPE_POS);
  1785. break;
  1786. }
  1787. }
  1788. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  1789. VERIFY(err, ii < FASTRPC_CTX_MAX);
  1790. if (err) {
  1791. ADSPRPC_ERR(
  1792. "adsprpc: out of context table entries for handle 0x%x, sc 0x%x\n",
  1793. ctx->handle, ctx->sc);
  1794. err = -ENOKEY;
  1795. goto bail;
  1796. }
  1797. ctx->xo_time_in_us_created = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1798. spin_lock(&fl->hlock);
  1799. hlist_add_head(&ctx->hn, &clst->pending);
  1800. clst->num_active_ctxs++;
  1801. spin_unlock(&fl->hlock);
  1802. trace_fastrpc_context_alloc((uint64_t)ctx,
  1803. ctx->ctxid | fl->pd, ctx->handle, ctx->sc);
  1804. *po = ctx;
  1805. bail:
  1806. if (ctx && err)
  1807. context_free(ctx);
  1808. return err;
  1809. }
  1810. static void context_save_interrupted(struct smq_invoke_ctx *ctx)
  1811. {
  1812. struct fastrpc_ctx_lst *clst = &ctx->fl->clst;
  1813. ctx->xo_time_in_us_interrupted = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  1814. fastrpc_update_txmsg_buf(NULL, 0, 0, 0, ctx, INTERRUPTED_STATE);
  1815. spin_lock(&ctx->fl->hlock);
  1816. hlist_del_init(&ctx->hn);
  1817. hlist_add_head(&ctx->hn, &clst->interrupted);
  1818. spin_unlock(&ctx->fl->hlock);
  1819. }
  1820. static void context_free(struct smq_invoke_ctx *ctx)
  1821. {
  1822. uint32_t i = 0;
  1823. struct fastrpc_apps *me = &gfa;
  1824. int nbufs = REMOTE_SCALARS_INBUFS(ctx->sc) +
  1825. REMOTE_SCALARS_OUTBUFS(ctx->sc);
  1826. int cid = ctx->fl->cid;
  1827. struct fastrpc_channel_ctx *chan = NULL;
  1828. unsigned long irq_flags = 0;
  1829. int err = 0;
  1830. VERIFY(err, VALID_FASTRPC_CID(cid));
  1831. if (err) {
  1832. ADSPRPC_ERR(
  1833. "invalid channel 0x%zx set for session\n",
  1834. cid);
  1835. return;
  1836. }
  1837. chan = &me->channel[cid];
  1838. i = (uint32_t)GET_TABLE_IDX_FROM_CTXID(ctx->ctxid);
  1839. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  1840. if (i < FASTRPC_CTX_MAX && chan->ctxtable[i] == ctx) {
  1841. chan->ctxtable[i] = NULL;
  1842. } else {
  1843. for (i = 0; i < FASTRPC_CTX_MAX; i++) {
  1844. if (chan->ctxtable[i] == ctx) {
  1845. chan->ctxtable[i] = NULL;
  1846. break;
  1847. }
  1848. }
  1849. }
  1850. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  1851. spin_lock(&ctx->fl->hlock);
  1852. if (!hlist_unhashed(&ctx->hn)) {
  1853. hlist_del_init(&ctx->hn);
  1854. ctx->fl->clst.num_active_ctxs--;
  1855. }
  1856. spin_unlock(&ctx->fl->hlock);
  1857. mutex_lock(&ctx->fl->map_mutex);
  1858. for (i = 0; i < nbufs; ++i) {
  1859. /*
  1860. * Decrement ctx refs count before mmap free,
  1861. * indicate remote call no longer using it
  1862. */
  1863. if (ctx->maps[i] && ctx->maps[i]->ctx_refs)
  1864. ctx->maps[i]->ctx_refs--;
  1865. fastrpc_mmap_free(ctx->maps[i], 0);
  1866. }
  1867. mutex_unlock(&ctx->fl->map_mutex);
  1868. fastrpc_buf_free(ctx->buf, 1);
  1869. if (ctx->copybuf != ctx->buf)
  1870. fastrpc_buf_free(ctx->copybuf, 1);
  1871. kfree(ctx->lrpra);
  1872. ctx->lrpra = NULL;
  1873. ctx->magic = 0;
  1874. ctx->ctxid = 0;
  1875. if (ctx->fl->profile)
  1876. kfree(ctx->perf);
  1877. trace_fastrpc_context_free((uint64_t)ctx,
  1878. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  1879. kfree(ctx);
  1880. }
  1881. static void fastrpc_queue_completed_async_job(struct smq_invoke_ctx *ctx)
  1882. {
  1883. struct fastrpc_file *fl = ctx->fl;
  1884. unsigned long flags;
  1885. spin_lock_irqsave(&fl->aqlock, flags);
  1886. if (ctx->is_early_wakeup)
  1887. goto bail;
  1888. list_add_tail(&ctx->asyncn, &fl->clst.async_queue);
  1889. atomic_add(1, &fl->async_queue_job_count);
  1890. ctx->is_early_wakeup = true;
  1891. wake_up_interruptible(&fl->async_wait_queue);
  1892. bail:
  1893. spin_unlock_irqrestore(&fl->aqlock, flags);
  1894. }
  1895. static void fastrpc_queue_pd_status(struct fastrpc_file *fl, int domain, int status, int sessionid)
  1896. {
  1897. struct smq_notif_rsp *notif_rsp = NULL;
  1898. unsigned long flags;
  1899. int err = 0;
  1900. VERIFY(err, NULL != (notif_rsp = kzalloc(sizeof(*notif_rsp), GFP_ATOMIC)));
  1901. if (err) {
  1902. ADSPRPC_ERR(
  1903. "allocation failed for size 0x%zx\n",
  1904. sizeof(*notif_rsp));
  1905. return;
  1906. }
  1907. notif_rsp->status = status;
  1908. notif_rsp->domain = domain;
  1909. notif_rsp->session = sessionid;
  1910. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  1911. list_add_tail(&notif_rsp->notifn, &fl->clst.notif_queue);
  1912. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  1913. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  1914. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  1915. }
  1916. static void fastrpc_notif_find_process(int domain, struct smq_notif_rspv3 *notif)
  1917. {
  1918. struct fastrpc_apps *me = &gfa;
  1919. struct fastrpc_file *fl = NULL;
  1920. struct hlist_node *n;
  1921. bool is_process_found = false;
  1922. unsigned long irq_flags = 0;
  1923. spin_lock_irqsave(&me->hlock, irq_flags);
  1924. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  1925. if (fl->tgid_frpc == notif->pid) {
  1926. is_process_found = true;
  1927. break;
  1928. }
  1929. }
  1930. spin_unlock_irqrestore(&me->hlock, irq_flags);
  1931. if (!is_process_found)
  1932. return;
  1933. fastrpc_queue_pd_status(fl, domain, notif->status, fl->sessionid);
  1934. }
  1935. static void context_notify_user(struct smq_invoke_ctx *ctx,
  1936. int retval, uint32_t rsp_flags, uint32_t early_wake_time)
  1937. {
  1938. fastrpc_pm_awake(ctx->fl, gcinfo[ctx->fl->cid].secure);
  1939. ctx->retval = retval;
  1940. ctx->rsp_flags = (enum fastrpc_response_flags)rsp_flags;
  1941. trace_fastrpc_context_complete(ctx->fl->cid, (uint64_t)ctx, retval,
  1942. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  1943. switch (rsp_flags) {
  1944. case NORMAL_RESPONSE:
  1945. fallthrough;
  1946. case COMPLETE_SIGNAL:
  1947. /* normal and complete response with return value */
  1948. ctx->is_work_done = true;
  1949. if (ctx->asyncjob.isasyncjob)
  1950. fastrpc_queue_completed_async_job(ctx);
  1951. trace_fastrpc_msg("wakeup_task: begin");
  1952. complete(&ctx->work);
  1953. trace_fastrpc_msg("wakeup_task: end");
  1954. break;
  1955. case USER_EARLY_SIGNAL:
  1956. /* user hint of approximate time of completion */
  1957. ctx->early_wake_time = early_wake_time;
  1958. if (ctx->asyncjob.isasyncjob)
  1959. break;
  1960. fallthrough;
  1961. case EARLY_RESPONSE:
  1962. /* rpc framework early response with return value */
  1963. if (ctx->asyncjob.isasyncjob)
  1964. fastrpc_queue_completed_async_job(ctx);
  1965. else {
  1966. trace_fastrpc_msg("wakeup_task: begin");
  1967. complete(&ctx->work);
  1968. trace_fastrpc_msg("wakeup_task: end");
  1969. }
  1970. break;
  1971. default:
  1972. break;
  1973. }
  1974. }
  1975. static void fastrpc_notify_users(struct fastrpc_file *me)
  1976. {
  1977. struct smq_invoke_ctx *ictx;
  1978. struct hlist_node *n;
  1979. unsigned long irq_flags = 0;
  1980. spin_lock_irqsave(&me->hlock, irq_flags);
  1981. hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
  1982. ictx->is_work_done = true;
  1983. ictx->retval = -ECONNRESET;
  1984. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  1985. ictx->retval, ictx->msg.invoke.header.ctx,
  1986. ictx->handle, ictx->sc);
  1987. if (ictx->asyncjob.isasyncjob && ictx->is_job_sent_to_remote_ss)
  1988. fastrpc_queue_completed_async_job(ictx);
  1989. else
  1990. complete(&ictx->work);
  1991. }
  1992. hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
  1993. ictx->is_work_done = true;
  1994. ictx->retval = -ECONNRESET;
  1995. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  1996. ictx->retval, ictx->msg.invoke.header.ctx,
  1997. ictx->handle, ictx->sc);
  1998. complete(&ictx->work);
  1999. }
  2000. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2001. }
  2002. static void fastrpc_notify_users_staticpd_pdr(struct fastrpc_file *me)
  2003. {
  2004. struct smq_invoke_ctx *ictx;
  2005. struct hlist_node *n;
  2006. unsigned long irq_flags = 0;
  2007. spin_lock_irqsave(&me->hlock, irq_flags);
  2008. hlist_for_each_entry_safe(ictx, n, &me->clst.pending, hn) {
  2009. if (ictx->msg.pid) {
  2010. ictx->is_work_done = true;
  2011. ictx->retval = -ECONNRESET;
  2012. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2013. ictx->retval, ictx->msg.invoke.header.ctx,
  2014. ictx->handle, ictx->sc);
  2015. if (ictx->asyncjob.isasyncjob && ictx->is_job_sent_to_remote_ss)
  2016. fastrpc_queue_completed_async_job(ictx);
  2017. else
  2018. complete(&ictx->work);
  2019. }
  2020. }
  2021. hlist_for_each_entry_safe(ictx, n, &me->clst.interrupted, hn) {
  2022. if (ictx->msg.pid) {
  2023. ictx->is_work_done = true;
  2024. ictx->retval = -ECONNRESET;
  2025. trace_fastrpc_context_complete(me->cid, (uint64_t)ictx,
  2026. ictx->retval, ictx->msg.invoke.header.ctx,
  2027. ictx->handle, ictx->sc);
  2028. complete(&ictx->work);
  2029. }
  2030. }
  2031. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2032. }
  2033. static void fastrpc_update_ramdump_status(int cid)
  2034. {
  2035. struct fastrpc_file *fl = NULL;
  2036. struct hlist_node *n = NULL;
  2037. struct fastrpc_apps *me = &gfa;
  2038. struct fastrpc_channel_ctx *chan = &me->channel[cid];
  2039. unsigned long irq_flags = 0;
  2040. spin_lock_irqsave(&me->hlock, irq_flags);
  2041. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2042. if (fl->cid == cid && fl->init_mem &&
  2043. fl->file_close < FASTRPC_PROCESS_DSP_EXIT_COMPLETE &&
  2044. fl->dsp_proc_init) {
  2045. hlist_add_head(&fl->init_mem->hn_init, &chan->initmems);
  2046. fl->is_ramdump_pend = true;
  2047. }
  2048. }
  2049. if (chan->buf)
  2050. hlist_add_head(&chan->buf->hn_init, &chan->initmems);
  2051. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2052. }
  2053. static void fastrpc_ramdump_collection(int cid)
  2054. {
  2055. struct fastrpc_file *fl = NULL;
  2056. struct hlist_node *n = NULL;
  2057. struct fastrpc_apps *me = &gfa;
  2058. struct fastrpc_channel_ctx *chan = &me->channel[cid];
  2059. struct qcom_dump_segment ramdump_entry;
  2060. struct fastrpc_buf *buf = NULL;
  2061. int ret = 0;
  2062. unsigned long irq_flags = 0;
  2063. struct list_head head;
  2064. hlist_for_each_entry_safe(buf, n, &chan->initmems, hn_init) {
  2065. fl = buf->fl;
  2066. memset(&ramdump_entry, 0, sizeof(ramdump_entry));
  2067. ramdump_entry.da = buf->phys;
  2068. ramdump_entry.va = (void *)buf->virt;
  2069. ramdump_entry.size = buf->size;
  2070. INIT_LIST_HEAD(&head);
  2071. list_add(&ramdump_entry.node, &head);
  2072. if (fl && fl->sctx && fl->sctx->smmu.dev)
  2073. ret = qcom_elf_dump(&head, fl->sctx->smmu.dev, ELF_CLASS);
  2074. else {
  2075. if (me->dev != NULL)
  2076. ret = qcom_elf_dump(&head, me->dev, ELF_CLASS);
  2077. }
  2078. if (ret < 0)
  2079. ADSPRPC_ERR("adsprpc: %s: unable to dump PD memory (err %d)\n",
  2080. __func__, ret);
  2081. hlist_del_init(&buf->hn_init);
  2082. if (fl) {
  2083. spin_lock_irqsave(&me->hlock, irq_flags);
  2084. if (fl->file_close)
  2085. complete(&fl->work);
  2086. fl->is_ramdump_pend = false;
  2087. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2088. }
  2089. }
  2090. }
  2091. static void fastrpc_notify_drivers(struct fastrpc_apps *me, int cid)
  2092. {
  2093. struct fastrpc_file *fl;
  2094. struct hlist_node *n;
  2095. unsigned long irq_flags = 0;
  2096. spin_lock_irqsave(&me->hlock, irq_flags);
  2097. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2098. if (fl->cid == cid) {
  2099. fastrpc_queue_pd_status(fl, cid, FASTRPC_DSP_SSR, 0);
  2100. fastrpc_notify_users(fl);
  2101. }
  2102. }
  2103. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2104. }
  2105. static void fastrpc_notify_pdr_drivers(struct fastrpc_apps *me,
  2106. char *servloc_name)
  2107. {
  2108. struct fastrpc_file *fl;
  2109. struct hlist_node *n;
  2110. unsigned long irq_flags = 0;
  2111. spin_lock_irqsave(&me->hlock, irq_flags);
  2112. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2113. if (fl->servloc_name && !strcmp(servloc_name, fl->servloc_name))
  2114. fastrpc_notify_users_staticpd_pdr(fl);
  2115. }
  2116. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2117. }
  2118. static void context_list_ctor(struct fastrpc_ctx_lst *me)
  2119. {
  2120. INIT_HLIST_HEAD(&me->interrupted);
  2121. INIT_HLIST_HEAD(&me->pending);
  2122. me->num_active_ctxs = 0;
  2123. INIT_LIST_HEAD(&me->async_queue);
  2124. INIT_LIST_HEAD(&me->notif_queue);
  2125. }
  2126. static void fastrpc_context_list_dtor(struct fastrpc_file *fl)
  2127. {
  2128. struct fastrpc_ctx_lst *clst = &fl->clst;
  2129. struct smq_invoke_ctx *ictx = NULL, *ctxfree;
  2130. struct hlist_node *n;
  2131. do {
  2132. ctxfree = NULL;
  2133. spin_lock(&fl->hlock);
  2134. hlist_for_each_entry_safe(ictx, n, &clst->interrupted, hn) {
  2135. hlist_del_init(&ictx->hn);
  2136. clst->num_active_ctxs--;
  2137. ctxfree = ictx;
  2138. break;
  2139. }
  2140. spin_unlock(&fl->hlock);
  2141. if (ctxfree)
  2142. context_free(ctxfree);
  2143. } while (ctxfree);
  2144. do {
  2145. ctxfree = NULL;
  2146. spin_lock(&fl->hlock);
  2147. hlist_for_each_entry_safe(ictx, n, &clst->pending, hn) {
  2148. hlist_del_init(&ictx->hn);
  2149. clst->num_active_ctxs--;
  2150. ctxfree = ictx;
  2151. break;
  2152. }
  2153. spin_unlock(&fl->hlock);
  2154. if (ctxfree)
  2155. context_free(ctxfree);
  2156. } while (ctxfree);
  2157. }
  2158. static int fastrpc_file_free(struct fastrpc_file *fl);
  2159. static void fastrpc_file_list_dtor(struct fastrpc_apps *me)
  2160. {
  2161. struct fastrpc_file *fl, *free;
  2162. struct hlist_node *n;
  2163. unsigned long irq_flags = 0;
  2164. do {
  2165. free = NULL;
  2166. spin_lock_irqsave(&me->hlock, irq_flags);
  2167. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  2168. hlist_del_init(&fl->hn);
  2169. free = fl;
  2170. break;
  2171. }
  2172. spin_unlock_irqrestore(&me->hlock, irq_flags);
  2173. if (free)
  2174. fastrpc_file_free(free);
  2175. } while (free);
  2176. }
  2177. static int get_args(uint32_t kernel, struct smq_invoke_ctx *ctx)
  2178. {
  2179. remote_arg64_t *rpra, *lrpra;
  2180. remote_arg_t *lpra = ctx->lpra;
  2181. struct smq_invoke_buf *list;
  2182. struct smq_phy_page *pages, *ipage;
  2183. uint32_t sc = ctx->sc;
  2184. int inbufs = REMOTE_SCALARS_INBUFS(sc);
  2185. int outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2186. int handles, bufs = inbufs + outbufs;
  2187. uintptr_t args = 0;
  2188. size_t rlen = 0, copylen = 0, metalen = 0, lrpralen = 0, templen = 0;
  2189. size_t totallen = 0; //header and non ion copy buf len
  2190. int i, oix;
  2191. int err = 0, j = 0;
  2192. int mflags = 0;
  2193. uint64_t *fdlist = NULL;
  2194. uint32_t *crclist = NULL;
  2195. uint32_t early_hint;
  2196. uint64_t *perf_counter = NULL;
  2197. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  2198. if (ctx->fl->profile)
  2199. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  2200. /* calculate size of the metadata */
  2201. rpra = NULL;
  2202. lrpra = NULL;
  2203. list = smq_invoke_buf_start(rpra, sc);
  2204. pages = smq_phy_page_start(sc, list);
  2205. ipage = pages;
  2206. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_MAP),
  2207. for (i = 0; i < bufs; ++i) {
  2208. uintptr_t buf = (uintptr_t)lpra[i].buf.pv;
  2209. size_t len = lpra[i].buf.len;
  2210. mutex_lock(&ctx->fl->map_mutex);
  2211. if (ctx->fds && (ctx->fds[i] != -1))
  2212. err = fastrpc_mmap_create(ctx->fl, ctx->fds[i], NULL,
  2213. ctx->attrs[i], buf, len,
  2214. mflags, &ctx->maps[i]);
  2215. /*
  2216. * Increment ctx refs count for in/out buffer if map created,
  2217. * indicate map under use in remote call
  2218. */
  2219. if (ctx->maps[i])
  2220. ctx->maps[i]->ctx_refs++;
  2221. mutex_unlock(&ctx->fl->map_mutex);
  2222. if (err)
  2223. goto bail;
  2224. ipage += 1;
  2225. }
  2226. PERF_END);
  2227. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  2228. mutex_lock(&ctx->fl->map_mutex);
  2229. for (i = bufs; i < bufs + handles; i++) {
  2230. int dmaflags = 0;
  2231. if (ctx->attrs && (ctx->attrs[i] & FASTRPC_ATTR_NOMAP))
  2232. dmaflags = FASTRPC_MAP_FD_NOMAP;
  2233. VERIFY(err, VALID_FASTRPC_CID(ctx->fl->cid));
  2234. if (err) {
  2235. err = -ECHRNG;
  2236. mutex_unlock(&ctx->fl->map_mutex);
  2237. goto bail;
  2238. }
  2239. dsp_cap_ptr = &gcinfo[ctx->fl->cid].dsp_cap_kernel;
  2240. // Skip cpu mapping if DMA_HANDLE_REVERSE_RPC_CAP is true.
  2241. if (!dsp_cap_ptr->dsp_attributes[DMA_HANDLE_REVERSE_RPC_CAP] &&
  2242. ctx->fds && (ctx->fds[i] != -1))
  2243. err = fastrpc_mmap_create(ctx->fl, ctx->fds[i], NULL,
  2244. FASTRPC_ATTR_NOVA, 0, 0, dmaflags,
  2245. &ctx->maps[i]);
  2246. if (err) {
  2247. for (j = bufs; j < i; j++) {
  2248. /*
  2249. * Due to error decrement ctx refs count before mmap free
  2250. * for each in/out handle, if map created
  2251. */
  2252. if (ctx->maps[j] && ctx->maps[j]->ctx_refs)
  2253. ctx->maps[j]->ctx_refs--;
  2254. fastrpc_mmap_free(ctx->maps[j], 0);
  2255. }
  2256. mutex_unlock(&ctx->fl->map_mutex);
  2257. goto bail;
  2258. } else if (ctx->maps[i]) {
  2259. /*
  2260. * Increment ctx refs count for in/out handle if map created
  2261. * and no error, indicate map under use in remote call
  2262. */
  2263. ctx->maps[i]->ctx_refs++;
  2264. }
  2265. ipage += 1;
  2266. }
  2267. mutex_unlock(&ctx->fl->map_mutex);
  2268. /* metalen includes meta data, fds, crc, dsp perf and early wakeup hint */
  2269. metalen = totallen = (size_t)&ipage[0] + (sizeof(uint64_t) * M_FDLIST) +
  2270. (sizeof(uint32_t) * M_CRCLIST) + (sizeof(uint64_t) * M_DSP_PERF_LIST) +
  2271. sizeof(early_hint);
  2272. if (metalen) {
  2273. err = fastrpc_buf_alloc(ctx->fl, metalen, 0, 0,
  2274. METADATA_BUF, &ctx->buf);
  2275. if (err)
  2276. goto bail;
  2277. VERIFY(err, !IS_ERR_OR_NULL(ctx->buf->virt));
  2278. if (err)
  2279. goto bail;
  2280. memset(ctx->buf->virt, 0, ctx->buf->size);
  2281. }
  2282. ctx->used = metalen;
  2283. /* allocate new local rpra buffer */
  2284. lrpralen = (size_t)&list[0];
  2285. if (lrpralen) {
  2286. lrpra = kzalloc(lrpralen, GFP_KERNEL);
  2287. VERIFY(err, !IS_ERR_OR_NULL(lrpra));
  2288. if (err) {
  2289. err = -ENOMEM;
  2290. goto bail;
  2291. }
  2292. }
  2293. ctx->lrpra = lrpra;
  2294. /* calculate len required for copying */
  2295. for (oix = 0; oix < inbufs + outbufs; ++oix) {
  2296. int i = ctx->overps[oix]->raix;
  2297. uintptr_t mstart, mend;
  2298. size_t len = lpra[i].buf.len;
  2299. if (!len)
  2300. continue;
  2301. if (ctx->maps[i])
  2302. continue;
  2303. if (ctx->overps[oix]->offset == 0)
  2304. copylen = ALIGN(copylen, BALIGN);
  2305. mstart = ctx->overps[oix]->mstart;
  2306. mend = ctx->overps[oix]->mend;
  2307. templen = mend - mstart;
  2308. VERIFY(err, ((templen <= LONG_MAX) && (copylen <= (LONG_MAX - templen))));
  2309. if (err) {
  2310. err = -EFAULT;
  2311. goto bail;
  2312. }
  2313. if (templen > DEBUG_PRINT_SIZE_LIMIT)
  2314. ADSPRPC_WARN(
  2315. "user passed non ion buffer size %zu, mend 0x%llx mstart 0x%llx, sc 0x%x handle 0x%x\n",
  2316. templen, mend, mstart, sc, ctx->handle);
  2317. copylen += templen;
  2318. }
  2319. totallen = ALIGN(totallen, BALIGN) + copylen;
  2320. /* allocate non -ion copy buffer */
  2321. /* Checking if copylen can be accomodated in metalen*/
  2322. /*if not allocating new buffer */
  2323. if (totallen <= (size_t)buf_page_size(metalen)) {
  2324. args = (uintptr_t)ctx->buf->virt + metalen;
  2325. ctx->copybuf = ctx->buf;
  2326. rlen = totallen - metalen;
  2327. } else if (copylen) {
  2328. err = fastrpc_buf_alloc(ctx->fl, copylen, 0, 0, COPYDATA_BUF,
  2329. &ctx->copybuf);
  2330. if (err)
  2331. goto bail;
  2332. memset(ctx->copybuf->virt, 0, copylen);
  2333. args = (uintptr_t)ctx->copybuf->virt;
  2334. rlen = copylen;
  2335. totallen = copylen;
  2336. }
  2337. /* copy metadata */
  2338. rpra = ctx->buf->virt;
  2339. ctx->rpra = rpra;
  2340. list = smq_invoke_buf_start(rpra, sc);
  2341. pages = smq_phy_page_start(sc, list);
  2342. ipage = pages;
  2343. for (i = 0; i < bufs + handles; ++i) {
  2344. if (lpra[i].buf.len)
  2345. list[i].num = 1;
  2346. else
  2347. list[i].num = 0;
  2348. list[i].pgidx = ipage - pages;
  2349. ipage++;
  2350. }
  2351. /* map ion buffers */
  2352. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_MAP),
  2353. for (i = 0; rpra && i < inbufs + outbufs; ++i) {
  2354. struct fastrpc_mmap *map = ctx->maps[i];
  2355. uint64_t buf = ptr_to_uint64(lpra[i].buf.pv);
  2356. size_t len = lpra[i].buf.len;
  2357. rpra[i].buf.pv = 0;
  2358. rpra[i].buf.len = len;
  2359. if (!len)
  2360. continue;
  2361. if (map) {
  2362. struct vm_area_struct *vma;
  2363. uintptr_t offset;
  2364. uint64_t num = buf_num_pages(buf, len);
  2365. int idx = list[i].pgidx;
  2366. if (map->attr & FASTRPC_ATTR_NOVA) {
  2367. offset = 0;
  2368. } else {
  2369. down_read(&current->mm->mmap_lock);
  2370. VERIFY(err, NULL != (vma = find_vma(current->mm,
  2371. map->va)));
  2372. if (err) {
  2373. up_read(&current->mm->mmap_lock);
  2374. goto bail;
  2375. }
  2376. offset = buf_page_start(buf) - vma->vm_start;
  2377. up_read(&current->mm->mmap_lock);
  2378. VERIFY(err, offset + len <= (uintptr_t)map->size);
  2379. if (err) {
  2380. ADSPRPC_ERR(
  2381. "buffer address is invalid for the fd passed for %d address 0x%llx and size %zu\n",
  2382. i, (uintptr_t)lpra[i].buf.pv,
  2383. lpra[i].buf.len);
  2384. err = -EFAULT;
  2385. goto bail;
  2386. }
  2387. }
  2388. pages[idx].addr = map->phys + offset;
  2389. pages[idx].size = num << PAGE_SHIFT;
  2390. }
  2391. rpra[i].buf.pv = buf;
  2392. }
  2393. PERF_END);
  2394. for (i = bufs; i < bufs + handles; ++i) {
  2395. struct fastrpc_mmap *map = ctx->maps[i];
  2396. if (map) {
  2397. pages[i].addr = map->phys;
  2398. pages[i].size = map->size;
  2399. }
  2400. }
  2401. fdlist = (uint64_t *)&pages[bufs + handles];
  2402. crclist = (uint32_t *)&fdlist[M_FDLIST];
  2403. /* reset fds, crc and early wakeup hint memory */
  2404. /* remote process updates these values before responding */
  2405. memset(fdlist, 0, sizeof(uint64_t)*M_FDLIST + sizeof(uint32_t)*M_CRCLIST +
  2406. (sizeof(uint64_t) * M_DSP_PERF_LIST) + sizeof(early_hint));
  2407. /* copy non ion buffers */
  2408. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_COPY),
  2409. for (oix = 0; rpra && oix < inbufs + outbufs; ++oix) {
  2410. int i = ctx->overps[oix]->raix;
  2411. struct fastrpc_mmap *map = ctx->maps[i];
  2412. size_t mlen;
  2413. uint64_t buf;
  2414. size_t len = lpra[i].buf.len;
  2415. if (!len)
  2416. continue;
  2417. if (map)
  2418. continue;
  2419. if (ctx->overps[oix]->offset == 0) {
  2420. rlen -= ALIGN(args, BALIGN) - args;
  2421. args = ALIGN(args, BALIGN);
  2422. }
  2423. mlen = ctx->overps[oix]->mend - ctx->overps[oix]->mstart;
  2424. VERIFY(err, rlen >= mlen);
  2425. if (err) {
  2426. err = -EFAULT;
  2427. goto bail;
  2428. }
  2429. rpra[i].buf.pv =
  2430. (args - ctx->overps[oix]->offset);
  2431. pages[list[i].pgidx].addr = ctx->copybuf->phys -
  2432. ctx->overps[oix]->offset +
  2433. (totallen - rlen);
  2434. pages[list[i].pgidx].addr =
  2435. buf_page_start(pages[list[i].pgidx].addr);
  2436. buf = rpra[i].buf.pv;
  2437. pages[list[i].pgidx].size = buf_num_pages(buf, len) * PAGE_SIZE;
  2438. if (i < inbufs) {
  2439. K_COPY_FROM_USER(err, kernel, uint64_to_ptr(buf),
  2440. lpra[i].buf.pv, len);
  2441. if (err) {
  2442. ADSPRPC_ERR(
  2443. "copy from user failed with %d for dst 0x%llx, src %pK, size 0x%zx, arg %d\n",
  2444. err, buf, lpra[i].buf.pv, len, i+1);
  2445. err = -EFAULT;
  2446. goto bail;
  2447. }
  2448. }
  2449. if (len > DEBUG_PRINT_SIZE_LIMIT)
  2450. ADSPRPC_DEBUG(
  2451. "copied non ion buffer sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu\n",
  2452. sc, rpra[i].buf.pv,
  2453. ctx->overps[oix]->mend,
  2454. ctx->overps[oix]->mstart, len);
  2455. args = args + mlen;
  2456. rlen -= mlen;
  2457. }
  2458. PERF_END);
  2459. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_FLUSH),
  2460. for (oix = 0; oix < inbufs + outbufs; ++oix) {
  2461. int i = ctx->overps[oix]->raix;
  2462. struct fastrpc_mmap *map = ctx->maps[i];
  2463. if (i+1 > inbufs) // Avoiding flush for outbufs
  2464. continue;
  2465. if (ctx->fl->sctx && ctx->fl->sctx->smmu.coherent)
  2466. continue;
  2467. if (map && (map->attr & FASTRPC_ATTR_FORCE_NOFLUSH))
  2468. continue;
  2469. if (rpra && rpra[i].buf.len && (ctx->overps[oix]->mstart ||
  2470. ctx->overps[oix]->do_cmo == 1)) {
  2471. if (map && map->buf) {
  2472. if (((buf_page_size(ctx->overps[oix]->mend -
  2473. ctx->overps[oix]->mstart)) == map->size) ||
  2474. ctx->overps[oix]->do_cmo) {
  2475. dma_buf_begin_cpu_access(map->buf,
  2476. DMA_TO_DEVICE);
  2477. dma_buf_end_cpu_access(map->buf,
  2478. DMA_TO_DEVICE);
  2479. ADSPRPC_DEBUG(
  2480. "sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2481. sc, rpra[i].buf.pv,
  2482. ctx->overps[oix]->mend,
  2483. ctx->overps[oix]->mstart,
  2484. rpra[i].buf.len, map->size);
  2485. } else {
  2486. uintptr_t offset;
  2487. uint64_t flush_len;
  2488. struct vm_area_struct *vma;
  2489. down_read(&current->mm->mmap_lock);
  2490. VERIFY(err, NULL != (vma = find_vma(
  2491. current->mm, rpra[i].buf.pv)));
  2492. if (err) {
  2493. up_read(&current->mm->mmap_lock);
  2494. goto bail;
  2495. }
  2496. if (ctx->overps[oix]->do_cmo) {
  2497. offset = rpra[i].buf.pv -
  2498. vma->vm_start;
  2499. flush_len = rpra[i].buf.len;
  2500. } else {
  2501. offset =
  2502. ctx->overps[oix]->mstart
  2503. - vma->vm_start;
  2504. flush_len =
  2505. ctx->overps[oix]->mend -
  2506. ctx->overps[oix]->mstart;
  2507. }
  2508. up_read(&current->mm->mmap_lock);
  2509. dma_buf_begin_cpu_access_partial(
  2510. map->buf, DMA_TO_DEVICE, offset,
  2511. flush_len);
  2512. dma_buf_end_cpu_access_partial(
  2513. map->buf, DMA_TO_DEVICE, offset,
  2514. flush_len);
  2515. ADSPRPC_DEBUG(
  2516. "sc 0x%x vm_start 0x%llx pv 0x%llx, offset 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2517. sc, vma->vm_start,
  2518. rpra[i].buf.pv, offset,
  2519. ctx->overps[oix]->mend,
  2520. ctx->overps[oix]->mstart,
  2521. rpra[i].buf.len, map->size);
  2522. }
  2523. }
  2524. }
  2525. }
  2526. PERF_END);
  2527. for (i = bufs; ctx->fds && rpra && i < bufs + handles; i++) {
  2528. rpra[i].dma.fd = ctx->fds[i];
  2529. rpra[i].dma.len = (uint32_t)lpra[i].buf.len;
  2530. rpra[i].dma.offset =
  2531. (uint32_t)(uintptr_t)lpra[i].buf.pv;
  2532. }
  2533. /* Copy rpra to local buffer */
  2534. if (ctx->lrpra && rpra && lrpralen > 0)
  2535. memcpy(ctx->lrpra, rpra, lrpralen);
  2536. bail:
  2537. return err;
  2538. }
  2539. static int put_args(uint32_t kernel, struct smq_invoke_ctx *ctx,
  2540. remote_arg_t *upra)
  2541. {
  2542. uint32_t sc = ctx->sc;
  2543. struct smq_invoke_buf *list;
  2544. struct smq_phy_page *pages;
  2545. struct fastrpc_mmap *mmap;
  2546. uint64_t *fdlist;
  2547. uint32_t *crclist = NULL, *poll = NULL;
  2548. uint64_t *perf_dsp_list = NULL;
  2549. remote_arg64_t *rpra = ctx->lrpra;
  2550. int i, inbufs, outbufs, handles;
  2551. int err = 0, perfErr = 0;
  2552. inbufs = REMOTE_SCALARS_INBUFS(sc);
  2553. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2554. handles = REMOTE_SCALARS_INHANDLES(sc) + REMOTE_SCALARS_OUTHANDLES(sc);
  2555. list = smq_invoke_buf_start(ctx->rpra, sc);
  2556. pages = smq_phy_page_start(sc, list);
  2557. fdlist = (uint64_t *)(pages + inbufs + outbufs + handles);
  2558. crclist = (uint32_t *)(fdlist + M_FDLIST);
  2559. poll = (uint32_t *)(crclist + M_CRCLIST);
  2560. perf_dsp_list = (uint64_t *)(poll + 1);
  2561. for (i = inbufs; i < inbufs + outbufs; ++i) {
  2562. if (!ctx->maps[i]) {
  2563. K_COPY_TO_USER(err, kernel,
  2564. ctx->lpra[i].buf.pv,
  2565. uint64_to_ptr(rpra[i].buf.pv),
  2566. rpra[i].buf.len);
  2567. if (err) {
  2568. ADSPRPC_ERR(
  2569. "Invalid size 0x%llx for output argument %d ret %ld\n",
  2570. rpra[i].buf.len, i+1, err);
  2571. err = -EFAULT;
  2572. goto bail;
  2573. }
  2574. } else {
  2575. mutex_lock(&ctx->fl->map_mutex);
  2576. /*
  2577. * Decrement ctx refs count before mmap free,
  2578. * indicate remote call no longer using it
  2579. */
  2580. if (ctx->maps[i]->ctx_refs)
  2581. ctx->maps[i]->ctx_refs--;
  2582. fastrpc_mmap_free(ctx->maps[i], 0);
  2583. mutex_unlock(&ctx->fl->map_mutex);
  2584. ctx->maps[i] = NULL;
  2585. }
  2586. }
  2587. mutex_lock(&ctx->fl->map_mutex);
  2588. for (i = 0; i < M_FDLIST; i++) {
  2589. if (!fdlist[i])
  2590. break;
  2591. if (!fastrpc_mmap_find(ctx->fl, (int)fdlist[i], NULL, 0, 0,
  2592. 0, 0, &mmap)) {
  2593. /*
  2594. * Decrement ctx refs count before mmap free,
  2595. * indicate remote call no longer using it
  2596. */
  2597. if (mmap && mmap->ctx_refs)
  2598. mmap->ctx_refs--;
  2599. fastrpc_mmap_free(mmap, 0);
  2600. }
  2601. }
  2602. mutex_unlock(&ctx->fl->map_mutex);
  2603. if (ctx->crc && crclist && rpra)
  2604. K_COPY_TO_USER(err, kernel, ctx->crc,
  2605. crclist, M_CRCLIST*sizeof(uint32_t));
  2606. if (ctx->perf_dsp && perf_dsp_list) {
  2607. K_COPY_TO_USER(perfErr, kernel, ctx->perf_dsp,
  2608. perf_dsp_list, M_DSP_PERF_LIST*sizeof(uint64_t));
  2609. if (perfErr)
  2610. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  2611. }
  2612. bail:
  2613. return err;
  2614. }
  2615. static void inv_args(struct smq_invoke_ctx *ctx)
  2616. {
  2617. int i, inbufs, outbufs;
  2618. uint32_t sc = ctx->sc;
  2619. remote_arg64_t *rpra = ctx->lrpra;
  2620. int err = 0;
  2621. inbufs = REMOTE_SCALARS_INBUFS(sc);
  2622. outbufs = REMOTE_SCALARS_OUTBUFS(sc);
  2623. for (i = 0; i < inbufs + outbufs; ++i) {
  2624. int over = ctx->overps[i]->raix;
  2625. struct fastrpc_mmap *map = ctx->maps[over];
  2626. if ((over + 1 <= inbufs))
  2627. continue;
  2628. if (!rpra[over].buf.len)
  2629. continue;
  2630. if (ctx->fl && ctx->fl->sctx && ctx->fl->sctx->smmu.coherent)
  2631. continue;
  2632. if (map && (map->attr & FASTRPC_ATTR_FORCE_NOINVALIDATE))
  2633. continue;
  2634. if (buf_page_start(ptr_to_uint64((void *)rpra)) ==
  2635. buf_page_start(rpra[over].buf.pv)) {
  2636. continue;
  2637. }
  2638. if (ctx->overps[i]->mstart || ctx->overps[i]->do_cmo == 1) {
  2639. if (map && map->buf) {
  2640. if (((buf_page_size(ctx->overps[i]->mend -
  2641. ctx->overps[i]->mstart)) == map->size) ||
  2642. ctx->overps[i]->do_cmo) {
  2643. dma_buf_begin_cpu_access(map->buf,
  2644. DMA_FROM_DEVICE);
  2645. dma_buf_end_cpu_access(map->buf,
  2646. DMA_TO_DEVICE);
  2647. ADSPRPC_DEBUG(
  2648. "sc 0x%x pv 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2649. sc, rpra[over].buf.pv,
  2650. ctx->overps[i]->mend,
  2651. ctx->overps[i]->mstart,
  2652. rpra[over].buf.len, map->size);
  2653. } else {
  2654. uintptr_t offset;
  2655. uint64_t inv_len;
  2656. struct vm_area_struct *vma;
  2657. down_read(&current->mm->mmap_lock);
  2658. VERIFY(err, NULL != (vma = find_vma(
  2659. current->mm,
  2660. rpra[over].buf.pv)));
  2661. if (err) {
  2662. up_read(&current->mm->mmap_lock);
  2663. goto bail;
  2664. }
  2665. if (ctx->overps[i]->do_cmo) {
  2666. offset = rpra[over].buf.pv -
  2667. vma->vm_start;
  2668. inv_len = rpra[over].buf.len;
  2669. } else {
  2670. offset =
  2671. ctx->overps[i]->mstart -
  2672. vma->vm_start;
  2673. inv_len =
  2674. ctx->overps[i]->mend -
  2675. ctx->overps[i]->mstart;
  2676. }
  2677. up_read(&current->mm->mmap_lock);
  2678. dma_buf_begin_cpu_access_partial(
  2679. map->buf, DMA_FROM_DEVICE, offset,
  2680. inv_len);
  2681. dma_buf_end_cpu_access_partial(map->buf,
  2682. DMA_TO_DEVICE, offset,
  2683. inv_len);
  2684. ADSPRPC_DEBUG(
  2685. "sc 0x%x vm_start 0x%llx pv 0x%llx, offset 0x%llx, mend 0x%llx mstart 0x%llx, len %zu size %zu\n",
  2686. sc, vma->vm_start,
  2687. rpra[over].buf.pv,
  2688. offset, ctx->overps[i]->mend,
  2689. ctx->overps[i]->mstart,
  2690. rpra[over].buf.len, map->size);
  2691. }
  2692. }
  2693. }
  2694. }
  2695. bail:
  2696. return;
  2697. }
  2698. static int fastrpc_invoke_send(struct smq_invoke_ctx *ctx,
  2699. uint32_t kernel, uint32_t handle)
  2700. {
  2701. struct smq_msg *msg = &ctx->msg;
  2702. struct smq_msg msg_temp;
  2703. struct smq_invoke_ctx ctx_temp;
  2704. struct fastrpc_file *fl = ctx->fl;
  2705. struct fastrpc_channel_ctx *channel_ctx = NULL;
  2706. int err = 0, cid = -1;
  2707. uint32_t sc = ctx->sc;
  2708. int64_t ns = 0;
  2709. uint64_t xo_time_in_us = 0;
  2710. int isasync = (ctx->asyncjob.isasyncjob ? true : false);
  2711. unsigned long irq_flags = 0;
  2712. uint32_t index = 0;
  2713. if (!fl) {
  2714. err = -EBADF;
  2715. goto bail;
  2716. }
  2717. cid = fl->cid;
  2718. VERIFY(err, VALID_FASTRPC_CID(cid));
  2719. if (err) {
  2720. err = -ECHRNG;
  2721. goto bail;
  2722. }
  2723. channel_ctx = &fl->apps->channel[cid];
  2724. mutex_lock(&channel_ctx->smd_mutex);
  2725. /* Send unique fastrpc process ID to dsp */
  2726. msg->pid = fl->tgid_frpc;
  2727. msg->tid = current->pid;
  2728. if (kernel == KERNEL_MSG_WITH_ZERO_PID)
  2729. msg->pid = 0;
  2730. msg->invoke.header.ctx = ctx->ctxid | fl->pd;
  2731. msg->invoke.header.handle = handle;
  2732. msg->invoke.header.sc = sc;
  2733. msg->invoke.page.addr = ctx->buf ? ctx->buf->phys : 0;
  2734. msg->invoke.page.size = buf_page_size(ctx->used);
  2735. if (fl->ssrcount != channel_ctx->ssrcount) {
  2736. err = -ECONNRESET;
  2737. mutex_unlock(&channel_ctx->smd_mutex);
  2738. goto bail;
  2739. }
  2740. mutex_unlock(&channel_ctx->smd_mutex);
  2741. xo_time_in_us = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  2742. if (isasync) {
  2743. /*
  2744. * After message is sent to DSP, async response thread could immediately
  2745. * get the response and free context, which will result in a use-after-free
  2746. * in this function. So use a local variable for message and context.
  2747. */
  2748. memcpy(&msg_temp, msg, sizeof(struct smq_msg));
  2749. msg = &msg_temp;
  2750. memcpy(&ctx_temp, ctx, sizeof(struct smq_invoke_ctx));
  2751. index = (uint32_t)GET_TABLE_IDX_FROM_CTXID(ctx->ctxid);
  2752. }
  2753. err = fastrpc_transport_send(cid, (void *)msg, sizeof(*msg), fl->tvm_remote_domain);
  2754. if (isasync) {
  2755. if (!err) {
  2756. /*
  2757. * Validate the ctx as this could have been already
  2758. * freed by async response.
  2759. */
  2760. spin_lock_irqsave(&channel_ctx->ctxlock, irq_flags);
  2761. if (index < FASTRPC_CTX_MAX && channel_ctx->ctxtable[index] == ctx)
  2762. ctx->is_job_sent_to_remote_ss = true;
  2763. spin_unlock_irqrestore(&channel_ctx->ctxlock, irq_flags);
  2764. }
  2765. ctx = &ctx_temp;
  2766. }
  2767. trace_fastrpc_transport_send(cid, (uint64_t)ctx, msg->invoke.header.ctx,
  2768. handle, sc, msg->invoke.page.addr, msg->invoke.page.size);
  2769. ns = get_timestamp_in_ns();
  2770. fastrpc_update_txmsg_buf(msg, err, ns, xo_time_in_us, ctx, DEFAULT_STATE);
  2771. bail:
  2772. return err;
  2773. }
  2774. /* fastrpc_get_nsp_status() - Reads the property string from soc_info
  2775. * denoted for nsp part, and updates the nsp device avialbility status
  2776. * if the nsp is not defective.
  2777. * @me : pointer to fastrpc_apps.
  2778. */
  2779. static void fastrpc_get_nsp_status(struct fastrpc_apps *me)
  2780. {
  2781. if (socinfo_get_part_info(PART_NSP)) {
  2782. me->fastrpc_nsp_status = 0;
  2783. ADSPRPC_ERR(
  2784. "nsp part defective with status:%x\n", me->fastrpc_nsp_status);
  2785. } else {
  2786. me->fastrpc_nsp_status = 1;
  2787. ADSPRPC_INFO("nsp available with status: %x\n", me->fastrpc_nsp_status);
  2788. }
  2789. }
  2790. /*
  2791. * Counts number of cores corresponding
  2792. * to cluster id 0. If a core is defective or unavailable, skip counting
  2793. * that core.
  2794. * @me : pointer to fastrpc_apps.
  2795. */
  2796. static void fastrpc_lowest_capacity_corecount(struct fastrpc_apps *me)
  2797. {
  2798. unsigned int cpu = 0;
  2799. cpu = cpumask_first(cpu_possible_mask);
  2800. for_each_cpu(cpu, cpu_possible_mask) {
  2801. if (topology_cluster_id(cpu) == 0)
  2802. me->lowest_capacity_core_count++;
  2803. }
  2804. ADSPRPC_INFO("lowest capacity core count: %u\n",
  2805. me->lowest_capacity_core_count);
  2806. }
  2807. static void fastrpc_init(struct fastrpc_apps *me)
  2808. {
  2809. int i, jj;
  2810. INIT_HLIST_HEAD(&me->drivers);
  2811. INIT_HLIST_HEAD(&me->maps);
  2812. spin_lock_init(&me->hlock);
  2813. me->channel = &gcinfo[0];
  2814. mutex_init(&me->mut_uid);
  2815. me->max_sess_per_proc = DEFAULT_MAX_SESS_PER_PROC;
  2816. for (i = 0; i < NUM_CHANNELS; i++) {
  2817. init_completion(&me->channel[i].work);
  2818. init_completion(&me->channel[i].workport);
  2819. me->channel[i].sesscount = 0;
  2820. /* All channels are secure by default except CDSP */
  2821. me->channel[i].secure = SECURE_CHANNEL;
  2822. me->channel[i].unsigned_support = false;
  2823. mutex_init(&me->channel[i].smd_mutex);
  2824. fastrpc_transport_session_init(i, me->channel[i].subsys);
  2825. spin_lock_init(&me->channel[i].ctxlock);
  2826. spin_lock_init(&me->channel[i].gmsg_log.lock);
  2827. INIT_HLIST_HEAD(&me->channel[i].initmems);
  2828. for (jj = 0; jj < NUM_SESSIONS; jj++)
  2829. init_waitqueue_head(&me->channel[i].spd[jj].wait_for_pdup);
  2830. }
  2831. /* Set CDSP channel to non secure */
  2832. me->channel[CDSP_DOMAIN_ID].secure = NON_SECURE_CHANNEL;
  2833. me->channel[CDSP_DOMAIN_ID].unsigned_support = true;
  2834. }
  2835. static inline void fastrpc_pm_awake(struct fastrpc_file *fl, int channel_type)
  2836. {
  2837. struct fastrpc_apps *me = &gfa;
  2838. struct wakeup_source *wake_source = NULL;
  2839. if (!fl->wake_enable)
  2840. return;
  2841. /*
  2842. * Vote with PM to abort any suspend in progress and
  2843. * keep system awake for specified timeout
  2844. */
  2845. if (channel_type == SECURE_CHANNEL)
  2846. wake_source = me->wake_source_secure;
  2847. else if (channel_type == NON_SECURE_CHANNEL)
  2848. wake_source = me->wake_source;
  2849. if (wake_source)
  2850. pm_wakeup_ws_event(wake_source, fl->ws_timeout, true);
  2851. }
  2852. static inline void fastrpc_pm_relax(struct fastrpc_file *fl, int channel_type)
  2853. {
  2854. struct fastrpc_apps *me = &gfa;
  2855. struct wakeup_source *wake_source = NULL;
  2856. if (!fl->wake_enable)
  2857. return;
  2858. if (channel_type == SECURE_CHANNEL)
  2859. wake_source = me->wake_source_secure;
  2860. else if (channel_type == NON_SECURE_CHANNEL)
  2861. wake_source = me->wake_source;
  2862. ADSPRPC_INFO("done for tgid %d\n", fl->tgid);
  2863. if (wake_source)
  2864. __pm_relax(wake_source);
  2865. }
  2866. static inline int fastrpc_wait_for_response(struct smq_invoke_ctx *ctx,
  2867. uint32_t kernel)
  2868. {
  2869. int interrupted = 0;
  2870. if (kernel)
  2871. wait_for_completion(&ctx->work);
  2872. else
  2873. interrupted = wait_for_completion_interruptible(&ctx->work);
  2874. return interrupted;
  2875. }
  2876. static void fastrpc_wait_for_completion(struct smq_invoke_ctx *ctx,
  2877. int *ptr_interrupted, uint32_t kernel, uint32_t async,
  2878. bool *ptr_isworkdone)
  2879. {
  2880. int interrupted = 0, err = 0;
  2881. int jj;
  2882. bool wait_resp;
  2883. uint32_t wTimeout = FASTRPC_USER_EARLY_HINT_TIMEOUT;
  2884. uint32_t wakeTime = 0;
  2885. unsigned long flags;
  2886. if (!ctx) {
  2887. /* This failure is not expected */
  2888. err = *ptr_interrupted = EFAULT;
  2889. *ptr_isworkdone = false;
  2890. ADSPRPC_ERR("ctx is NULL, cannot wait for response err %d\n",
  2891. err);
  2892. return;
  2893. }
  2894. wakeTime = ctx->early_wake_time;
  2895. do {
  2896. switch (ctx->rsp_flags) {
  2897. /* try polling on completion with timeout */
  2898. case USER_EARLY_SIGNAL:
  2899. /* try wait if completion time is less than timeout */
  2900. /* disable preempt to avoid context switch latency */
  2901. preempt_disable();
  2902. jj = 0;
  2903. wait_resp = false;
  2904. for (; wakeTime < wTimeout && jj < wTimeout; jj++) {
  2905. wait_resp = try_wait_for_completion(&ctx->work);
  2906. if (wait_resp)
  2907. break;
  2908. udelay(1);
  2909. }
  2910. preempt_enable();
  2911. if (async) {
  2912. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2913. if (!ctx->is_work_done) {
  2914. ctx->is_early_wakeup = false;
  2915. *ptr_isworkdone = false;
  2916. } else
  2917. *ptr_isworkdone = true;
  2918. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2919. goto bail;
  2920. } else if (!wait_resp) {
  2921. interrupted = fastrpc_wait_for_response(ctx,
  2922. kernel);
  2923. *ptr_interrupted = interrupted;
  2924. if (interrupted || ctx->is_work_done)
  2925. goto bail;
  2926. }
  2927. break;
  2928. /* busy poll on memory for actual job done */
  2929. case EARLY_RESPONSE:
  2930. trace_fastrpc_msg("early_response: poll_begin");
  2931. err = poll_for_remote_response(ctx, FASTRPC_POLL_TIME);
  2932. /* Mark job done if poll on memory successful */
  2933. /* Wait for completion if poll on memory timoeut */
  2934. if (!err) {
  2935. ctx->is_work_done = true;
  2936. *ptr_isworkdone = true;
  2937. goto bail;
  2938. }
  2939. trace_fastrpc_msg("early_response: poll_timeout");
  2940. ADSPRPC_INFO("early rsp poll timeout (%u us) for handle 0x%x, sc 0x%x\n",
  2941. FASTRPC_POLL_TIME, ctx->handle, ctx->sc);
  2942. if (async) {
  2943. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2944. if (!ctx->is_work_done) {
  2945. ctx->is_early_wakeup = false;
  2946. *ptr_isworkdone = false;
  2947. } else
  2948. *ptr_isworkdone = true;
  2949. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2950. goto bail;
  2951. } else if (!ctx->is_work_done) {
  2952. interrupted = fastrpc_wait_for_response(ctx,
  2953. kernel);
  2954. *ptr_interrupted = interrupted;
  2955. if (interrupted || ctx->is_work_done)
  2956. goto bail;
  2957. }
  2958. break;
  2959. case COMPLETE_SIGNAL:
  2960. case NORMAL_RESPONSE:
  2961. if (!async) {
  2962. interrupted = fastrpc_wait_for_response(ctx,
  2963. kernel);
  2964. *ptr_interrupted = interrupted;
  2965. if (interrupted || ctx->is_work_done)
  2966. goto bail;
  2967. } else {
  2968. spin_lock_irqsave(&ctx->fl->aqlock, flags);
  2969. if (!ctx->is_work_done) {
  2970. ctx->is_early_wakeup = false;
  2971. *ptr_isworkdone = false;
  2972. } else
  2973. *ptr_isworkdone = true;
  2974. spin_unlock_irqrestore(&ctx->fl->aqlock, flags);
  2975. goto bail;
  2976. }
  2977. break;
  2978. case POLL_MODE:
  2979. trace_fastrpc_msg("poll_mode: begin");
  2980. err = poll_for_remote_response(ctx, ctx->fl->poll_timeout);
  2981. /* If polling timed out, move to normal response state */
  2982. if (err) {
  2983. trace_fastrpc_msg("poll_mode: timeout");
  2984. ADSPRPC_INFO("poll mode timeout (%u us) for handle 0x%x, sc 0x%x\n",
  2985. ctx->fl->poll_timeout, ctx->handle, ctx->sc);
  2986. ctx->rsp_flags = NORMAL_RESPONSE;
  2987. } else {
  2988. *ptr_interrupted = 0;
  2989. *ptr_isworkdone = true;
  2990. }
  2991. break;
  2992. default:
  2993. *ptr_interrupted = EBADR;
  2994. *ptr_isworkdone = false;
  2995. ADSPRPC_ERR(
  2996. "unsupported response flags 0x%x for handle 0x%x, sc 0x%x\n",
  2997. ctx->rsp_flags, ctx->handle, ctx->sc);
  2998. goto bail;
  2999. } /* end of switch */
  3000. } while (!ctx->is_work_done);
  3001. bail:
  3002. return;
  3003. }
  3004. static void fastrpc_update_invoke_count(uint32_t handle, uint64_t *perf_counter,
  3005. struct timespec64 *invoket)
  3006. {
  3007. /* update invoke count for dynamic handles */
  3008. if (handle != FASTRPC_STATIC_HANDLE_LISTENER) {
  3009. uint64_t *count = GET_COUNTER(perf_counter, PERF_INVOKE);
  3010. if (count)
  3011. *count += getnstimediff(invoket);
  3012. }
  3013. if (handle > FASTRPC_STATIC_HANDLE_MAX) {
  3014. uint64_t *count = GET_COUNTER(perf_counter, PERF_COUNT);
  3015. if (count)
  3016. *count += 1;
  3017. }
  3018. }
  3019. static int fastrpc_check_pd_status(struct fastrpc_file *fl, char *sloc_name);
  3020. int fastrpc_internal_invoke(struct fastrpc_file *fl, uint32_t mode,
  3021. uint32_t kernel,
  3022. struct fastrpc_ioctl_invoke_async *inv)
  3023. {
  3024. struct smq_invoke_ctx *ctx = NULL;
  3025. struct fastrpc_ioctl_invoke *invoke = &inv->inv;
  3026. int err = 0, interrupted = 0, cid = -1, perfErr = 0;
  3027. struct timespec64 invoket = {0};
  3028. uint64_t *perf_counter = NULL;
  3029. bool isasyncinvoke = false, isworkdone = false;
  3030. cid = fl->cid;
  3031. VERIFY(err, VALID_FASTRPC_CID(cid) &&
  3032. fl->sctx != NULL);
  3033. if (err) {
  3034. ADSPRPC_ERR("kernel session not initialized yet for %s\n",
  3035. current->comm);
  3036. err = -EBADR;
  3037. goto bail;
  3038. }
  3039. if (fl->profile)
  3040. ktime_get_real_ts64(&invoket);
  3041. if (!kernel) {
  3042. VERIFY(err, invoke->handle !=
  3043. FASTRPC_STATIC_HANDLE_PROCESS_GROUP);
  3044. VERIFY(err, invoke->handle !=
  3045. FASTRPC_STATIC_HANDLE_DSP_UTILITIES);
  3046. if (err) {
  3047. err = -EINVAL;
  3048. ADSPRPC_ERR(
  3049. "user application trying to send a kernel RPC message to channel %d, handle 0x%x\n",
  3050. cid, invoke->handle);
  3051. goto bail;
  3052. }
  3053. }
  3054. if (!kernel) {
  3055. VERIFY(err, 0 == (err = context_restore_interrupted(fl,
  3056. inv, &ctx)));
  3057. if (err)
  3058. goto bail;
  3059. if (fl->sctx->smmu.faults)
  3060. err = -FASTRPC_ENOSUCH;
  3061. if (err)
  3062. goto bail;
  3063. if (ctx) {
  3064. trace_fastrpc_context_restore(cid, (uint64_t)ctx,
  3065. ctx->msg.invoke.header.ctx,
  3066. ctx->handle, ctx->sc);
  3067. goto wait;
  3068. }
  3069. }
  3070. trace_fastrpc_msg("context_alloc: begin");
  3071. VERIFY(err, 0 == (err = context_alloc(fl, kernel, inv, &ctx)));
  3072. trace_fastrpc_msg("context_alloc: end");
  3073. if (err)
  3074. goto bail;
  3075. if (fl->servloc_name) {
  3076. err = fastrpc_check_pd_status(fl,
  3077. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME);
  3078. err |= fastrpc_check_pd_status(fl,
  3079. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME);
  3080. err |= fastrpc_check_pd_status(fl,
  3081. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME);
  3082. if (err)
  3083. goto bail;
  3084. }
  3085. isasyncinvoke = (ctx->asyncjob.isasyncjob ? true : false);
  3086. if (fl->profile)
  3087. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  3088. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_GETARGS),
  3089. VERIFY(err, 0 == (err = get_args(kernel, ctx)));
  3090. PERF_END);
  3091. trace_fastrpc_msg("get_args: end");
  3092. if (err)
  3093. goto bail;
  3094. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3095. inv_args(ctx);
  3096. PERF_END);
  3097. trace_fastrpc_msg("inv_args_1: end");
  3098. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_LINK),
  3099. VERIFY(err, 0 == (err = fastrpc_invoke_send(ctx,
  3100. kernel, invoke->handle)));
  3101. PERF_END);
  3102. trace_fastrpc_msg("invoke_send: end");
  3103. if (err)
  3104. goto bail;
  3105. if (isasyncinvoke)
  3106. goto invoke_end;
  3107. wait:
  3108. /* Poll mode allowed only for non-static handle calls to dynamic CDSP process */
  3109. if (fl->poll_mode && (invoke->handle > FASTRPC_STATIC_HANDLE_MAX)
  3110. && (cid == CDSP_DOMAIN_ID)
  3111. && (fl->proc_flags == FASTRPC_INIT_CREATE))
  3112. ctx->rsp_flags = POLL_MODE;
  3113. fastrpc_wait_for_completion(ctx, &interrupted, kernel, 0, &isworkdone);
  3114. trace_fastrpc_msg("wait_for_completion: end");
  3115. VERIFY(err, 0 == (err = interrupted));
  3116. if (err)
  3117. goto bail;
  3118. if (!ctx->is_work_done) {
  3119. err = -ETIMEDOUT;
  3120. ADSPRPC_ERR(
  3121. "WorkDone state is invalid for handle 0x%x, sc 0x%x\n",
  3122. invoke->handle, ctx->sc);
  3123. goto bail;
  3124. }
  3125. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3126. inv_args(ctx);
  3127. PERF_END);
  3128. trace_fastrpc_msg("inv_args_2: end");
  3129. PERF(fl->profile, GET_COUNTER(perf_counter, PERF_PUTARGS),
  3130. VERIFY(err, 0 == (err = put_args(kernel, ctx, invoke->pra)));
  3131. PERF_END);
  3132. trace_fastrpc_msg("put_args: end");
  3133. if (err)
  3134. goto bail;
  3135. VERIFY(err, 0 == (err = ctx->retval));
  3136. if (err)
  3137. goto bail;
  3138. bail:
  3139. if (ctx && interrupted == -ERESTARTSYS) {
  3140. trace_fastrpc_context_interrupt(cid, (uint64_t)ctx,
  3141. ctx->msg.invoke.header.ctx, ctx->handle, ctx->sc);
  3142. context_save_interrupted(ctx);
  3143. } else if (ctx) {
  3144. if (fl->profile && !interrupted)
  3145. fastrpc_update_invoke_count(invoke->handle,
  3146. perf_counter, &invoket);
  3147. if (fl->profile && ctx->perf && ctx->handle > FASTRPC_STATIC_HANDLE_MAX) {
  3148. trace_fastrpc_perf_counters(ctx->handle, ctx->sc,
  3149. ctx->perf->count, ctx->perf->flush, ctx->perf->map,
  3150. ctx->perf->copy, ctx->perf->link, ctx->perf->getargs,
  3151. ctx->perf->putargs, ctx->perf->invargs,
  3152. ctx->perf->invoke, ctx->perf->tid);
  3153. if (ctx->perf_kernel) {
  3154. K_COPY_TO_USER(perfErr, kernel, ctx->perf_kernel,
  3155. ctx->perf, M_KERNEL_PERF_LIST*sizeof(uint64_t));
  3156. if (perfErr)
  3157. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  3158. }
  3159. }
  3160. context_free(ctx);
  3161. trace_fastrpc_msg("context_free: end");
  3162. }
  3163. if (!kernel) {
  3164. if (VALID_FASTRPC_CID(cid)
  3165. && (fl->ssrcount != fl->apps->channel[cid].ssrcount))
  3166. err = -ECONNRESET;
  3167. }
  3168. invoke_end:
  3169. if (fl->profile && !interrupted && isasyncinvoke)
  3170. fastrpc_update_invoke_count(invoke->handle, perf_counter,
  3171. &invoket);
  3172. return err;
  3173. }
  3174. static int fastrpc_wait_on_async_queue(
  3175. struct fastrpc_ioctl_async_response *async_res,
  3176. struct fastrpc_file *fl)
  3177. {
  3178. int err = 0, ierr = 0, interrupted = 0, perfErr = 0;
  3179. struct smq_invoke_ctx *ctx = NULL, *ictx = NULL, *n = NULL;
  3180. unsigned long flags;
  3181. uint64_t *perf_counter = NULL;
  3182. bool isworkdone = false;
  3183. read_async_job:
  3184. interrupted = wait_event_interruptible(fl->async_wait_queue,
  3185. atomic_read(&fl->async_queue_job_count));
  3186. if (!fl || fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  3187. err = -EBADF;
  3188. goto bail;
  3189. }
  3190. if (fl->exit_async) {
  3191. err = -EFAULT;
  3192. goto bail;
  3193. }
  3194. VERIFY(err, 0 == (err = interrupted));
  3195. if (err)
  3196. goto bail;
  3197. spin_lock_irqsave(&fl->aqlock, flags);
  3198. list_for_each_entry_safe(ictx, n, &fl->clst.async_queue, asyncn) {
  3199. list_del_init(&ictx->asyncn);
  3200. atomic_sub(1, &fl->async_queue_job_count);
  3201. ctx = ictx;
  3202. break;
  3203. }
  3204. spin_unlock_irqrestore(&fl->aqlock, flags);
  3205. if (ctx) {
  3206. if (fl->profile)
  3207. perf_counter = (uint64_t *)ctx->perf + PERF_COUNT;
  3208. fastrpc_wait_for_completion(ctx, &interrupted, 0, 1,
  3209. &isworkdone);
  3210. if (!isworkdone) {//In valid workdone state
  3211. ADSPRPC_DEBUG(
  3212. "Async early wake response did not reach on time for thread %d handle 0x%x, sc 0x%x\n",
  3213. ctx->pid, ctx->handle, ctx->sc);
  3214. goto read_async_job;
  3215. }
  3216. async_res->jobid = ctx->asyncjob.jobid;
  3217. async_res->result = ctx->retval;
  3218. async_res->handle = ctx->handle;
  3219. async_res->sc = ctx->sc;
  3220. async_res->perf_dsp = (uint64_t *)ctx->perf_dsp;
  3221. async_res->perf_kernel = (uint64_t *)ctx->perf_kernel;
  3222. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_INVARGS),
  3223. inv_args(ctx);
  3224. PERF_END);
  3225. if (ctx->retval != 0)
  3226. goto bail;
  3227. PERF(ctx->fl->profile, GET_COUNTER(perf_counter, PERF_PUTARGS),
  3228. VERIFY(ierr, 0 == (ierr = put_args(0, ctx, NULL)));
  3229. PERF_END);
  3230. if (ierr)
  3231. goto bail;
  3232. } else { // Go back to wait if ctx is invalid
  3233. ADSPRPC_ERR("Invalid async job wake up\n");
  3234. goto read_async_job;
  3235. }
  3236. bail:
  3237. if (ierr)
  3238. async_res->result = ierr;
  3239. if (ctx) {
  3240. if (fl->profile && ctx->perf && ctx->handle > FASTRPC_STATIC_HANDLE_MAX) {
  3241. trace_fastrpc_perf_counters(ctx->handle, ctx->sc,
  3242. ctx->perf->count, ctx->perf->flush, ctx->perf->map,
  3243. ctx->perf->copy, ctx->perf->link, ctx->perf->getargs,
  3244. ctx->perf->putargs, ctx->perf->invargs,
  3245. ctx->perf->invoke, ctx->perf->tid);
  3246. if (ctx->perf_kernel) {
  3247. K_COPY_TO_USER(perfErr, 0, ctx->perf_kernel,
  3248. ctx->perf, M_KERNEL_PERF_LIST*sizeof(uint64_t));
  3249. if (perfErr)
  3250. ADSPRPC_WARN("failed to copy perf data err %d\n", perfErr);
  3251. }
  3252. }
  3253. context_free(ctx);
  3254. }
  3255. return err;
  3256. }
  3257. static int fastrpc_wait_on_notif_queue(
  3258. struct fastrpc_ioctl_notif_rsp *notif_rsp,
  3259. struct fastrpc_file *fl)
  3260. {
  3261. int err = 0, interrupted = 0;
  3262. unsigned long flags;
  3263. struct smq_notif_rsp *notif = NULL, *inotif = NULL, *n = NULL;
  3264. read_notif_status:
  3265. interrupted = wait_event_interruptible(fl->proc_state_notif.notif_wait_queue,
  3266. atomic_read(&fl->proc_state_notif.notif_queue_count));
  3267. if (!fl) {
  3268. err = -EBADF;
  3269. goto bail;
  3270. }
  3271. if (fl->exit_notif) {
  3272. err = -EFAULT;
  3273. goto bail;
  3274. }
  3275. VERIFY(err, 0 == (err = interrupted));
  3276. if (err)
  3277. goto bail;
  3278. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  3279. list_for_each_entry_safe(inotif, n, &fl->clst.notif_queue, notifn) {
  3280. list_del_init(&inotif->notifn);
  3281. atomic_sub(1, &fl->proc_state_notif.notif_queue_count);
  3282. notif = inotif;
  3283. break;
  3284. }
  3285. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  3286. if (notif) {
  3287. notif_rsp->status = notif->status;
  3288. notif_rsp->domain = notif->domain;
  3289. notif_rsp->session = notif->session;
  3290. } else {// Go back to wait if ctx is invalid
  3291. ADSPRPC_ERR("Invalid status notification response\n");
  3292. goto read_notif_status;
  3293. }
  3294. bail:
  3295. kfree(notif);
  3296. return err;
  3297. }
  3298. static int fastrpc_get_async_response(
  3299. struct fastrpc_ioctl_async_response *async_res,
  3300. void *param, struct fastrpc_file *fl)
  3301. {
  3302. int err = 0;
  3303. err = fastrpc_wait_on_async_queue(async_res, fl);
  3304. if (err)
  3305. goto bail;
  3306. K_COPY_TO_USER(err, 0, param, async_res,
  3307. sizeof(struct fastrpc_ioctl_async_response));
  3308. bail:
  3309. return err;
  3310. }
  3311. static int fastrpc_get_notif_response(
  3312. struct fastrpc_ioctl_notif_rsp *notif,
  3313. void *param, struct fastrpc_file *fl)
  3314. {
  3315. int err = 0;
  3316. err = fastrpc_wait_on_notif_queue(notif, fl);
  3317. if (err)
  3318. goto bail;
  3319. K_COPY_TO_USER(err, 0, param, notif,
  3320. sizeof(struct fastrpc_ioctl_notif_rsp));
  3321. bail:
  3322. return err;
  3323. }
  3324. static int fastrpc_set_session_info(
  3325. struct fastrpc_proc_sess_info *sess_info,
  3326. void *param, struct fastrpc_file *fl)
  3327. {
  3328. int err = 0;
  3329. struct fastrpc_apps *me = &gfa;
  3330. if (fl->set_session_info) {
  3331. ADSPRPC_ERR("Set session info invoked multiple times\n");
  3332. err = -EBADR;
  3333. goto bail;
  3334. }
  3335. /*
  3336. * Third-party apps don't have permission to open the fastrpc device, so
  3337. * it is opened on their behalf by DSP HAL. This is detected by
  3338. * comparing current PID with the one stored during device open.
  3339. */
  3340. if (current->tgid != fl->tgid_open)
  3341. fl->untrusted_process = true;
  3342. VERIFY(err, sess_info->pd_type > DEFAULT_UNUSED &&
  3343. sess_info->pd_type < MAX_PD_TYPE);
  3344. if (err) {
  3345. ADSPRPC_ERR(
  3346. "Session PD type %u is invalid for the process\n",
  3347. sess_info->pd_type);
  3348. err = -EBADR;
  3349. goto bail;
  3350. }
  3351. if (fl->untrusted_process && sess_info->pd_type != USERPD) {
  3352. ADSPRPC_ERR(
  3353. "Session PD type %u not allowed for untrusted process\n",
  3354. sess_info->pd_type);
  3355. err = -EBADR;
  3356. goto bail;
  3357. }
  3358. /*
  3359. * If PD type is not configured for context banks,
  3360. * ignore PD type passed by the user, leave pd_type set to DEFAULT_UNUSED(0)
  3361. */
  3362. if (me->cb_pd_type)
  3363. fl->pd_type = sess_info->pd_type;
  3364. // Processes attaching to Sensor Static PD, share context bank.
  3365. if (sess_info->pd_type == SENSORS_STATICPD)
  3366. fl->sharedcb = 1;
  3367. if (sess_info->session_id >= me->max_sess_per_proc) {
  3368. ADSPRPC_ERR(
  3369. "Session ID %u cannot be beyond %u\n",
  3370. sess_info->session_id, me->max_sess_per_proc);
  3371. err = -EBADR;
  3372. goto bail;
  3373. }
  3374. fl->sessionid = sess_info->session_id;
  3375. // Set multi_session_support, to disable old way of setting session_id
  3376. fl->multi_session_support = true;
  3377. VERIFY(err, 0 == (err = fastrpc_get_info(fl, &(sess_info->domain_id))));
  3378. if (err)
  3379. goto bail;
  3380. K_COPY_TO_USER(err, 0, param, sess_info,
  3381. sizeof(struct fastrpc_proc_sess_info));
  3382. bail:
  3383. return err;
  3384. }
  3385. static int fastrpc_create_persistent_headers(struct fastrpc_file *fl,
  3386. uint32_t user_concurrency)
  3387. {
  3388. int err = 0, i = 0;
  3389. uint64_t virtb = 0;
  3390. struct fastrpc_buf *pers_hdr_buf = NULL, *hdr_bufs = NULL, *buf = NULL;
  3391. unsigned int num_pers_hdrs = 0;
  3392. size_t hdr_buf_alloc_len = 0;
  3393. if (fl->pers_hdr_buf || !user_concurrency)
  3394. goto bail;
  3395. /*
  3396. * Pre-allocate memory for persistent header buffers based
  3397. * on concurrency info passed by user. Upper limit enforced.
  3398. */
  3399. num_pers_hdrs = (user_concurrency > MAX_PERSISTENT_HEADERS) ?
  3400. MAX_PERSISTENT_HEADERS : user_concurrency;
  3401. hdr_buf_alloc_len = num_pers_hdrs*PAGE_SIZE;
  3402. err = fastrpc_buf_alloc(fl, hdr_buf_alloc_len, 0, 0,
  3403. METADATA_BUF, &pers_hdr_buf);
  3404. if (err)
  3405. goto bail;
  3406. virtb = ptr_to_uint64(pers_hdr_buf->virt);
  3407. /* Map entire buffer on remote subsystem in single RPC call */
  3408. err = fastrpc_mem_map_to_dsp(fl, -1, 0, ADSP_MMAP_PERSIST_HDR, 0,
  3409. pers_hdr_buf->phys, pers_hdr_buf->size,
  3410. &pers_hdr_buf->raddr);
  3411. if (err)
  3412. goto bail;
  3413. /* Divide and store as N chunks, each of 1 page size */
  3414. hdr_bufs = kcalloc(num_pers_hdrs, sizeof(struct fastrpc_buf),
  3415. GFP_KERNEL);
  3416. if (!hdr_bufs) {
  3417. err = -ENOMEM;
  3418. goto bail;
  3419. }
  3420. spin_lock(&fl->hlock);
  3421. fl->pers_hdr_buf = pers_hdr_buf;
  3422. fl->num_pers_hdrs = num_pers_hdrs;
  3423. fl->hdr_bufs = hdr_bufs;
  3424. for (i = 0; i < num_pers_hdrs; i++) {
  3425. buf = &fl->hdr_bufs[i];
  3426. buf->fl = fl;
  3427. buf->virt = uint64_to_ptr(virtb + (i*PAGE_SIZE));
  3428. buf->phys = pers_hdr_buf->phys + (i*PAGE_SIZE);
  3429. buf->size = PAGE_SIZE;
  3430. buf->dma_attr = pers_hdr_buf->dma_attr;
  3431. buf->flags = pers_hdr_buf->flags;
  3432. buf->type = pers_hdr_buf->type;
  3433. buf->in_use = false;
  3434. }
  3435. spin_unlock(&fl->hlock);
  3436. bail:
  3437. if (err) {
  3438. ADSPRPC_ERR(
  3439. "failed to map len %zu, flags %d, user concurrency %u, num headers %u with err %d\n",
  3440. hdr_buf_alloc_len, ADSP_MMAP_PERSIST_HDR,
  3441. user_concurrency, num_pers_hdrs, err);
  3442. fl->pers_hdr_buf = NULL;
  3443. fl->hdr_bufs = NULL;
  3444. fl->num_pers_hdrs = 0;
  3445. if (!IS_ERR_OR_NULL(pers_hdr_buf))
  3446. fastrpc_buf_free(pers_hdr_buf, 0);
  3447. if (!IS_ERR_OR_NULL(hdr_bufs))
  3448. kfree(hdr_bufs);
  3449. }
  3450. return err;
  3451. }
  3452. int fastrpc_internal_invoke2(struct fastrpc_file *fl,
  3453. struct fastrpc_ioctl_invoke2 *inv2)
  3454. {
  3455. union {
  3456. struct fastrpc_ioctl_invoke_async inv;
  3457. struct fastrpc_ioctl_invoke_async_no_perf inv3;
  3458. struct fastrpc_ioctl_async_response async_res;
  3459. uint32_t user_concurrency;
  3460. struct fastrpc_ioctl_notif_rsp notif;
  3461. struct fastrpc_proc_sharedbuf_info buff_info;
  3462. struct fastrpc_proc_sess_info sess_info;
  3463. } p;
  3464. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  3465. uint32_t size = 0;
  3466. int err = 0, domain = fl->cid;
  3467. if (inv2->req == FASTRPC_INVOKE2_ASYNC ||
  3468. inv2->req == FASTRPC_INVOKE2_ASYNC_RESPONSE) {
  3469. VERIFY(err, domain == CDSP_DOMAIN_ID && fl->sctx != NULL);
  3470. if (err) {
  3471. err = -EBADR;
  3472. goto bail;
  3473. }
  3474. dsp_cap_ptr = &gcinfo[domain].dsp_cap_kernel;
  3475. VERIFY(err,
  3476. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP] == 1);
  3477. if (err) {
  3478. err = -EPROTONOSUPPORT;
  3479. goto bail;
  3480. }
  3481. }
  3482. switch (inv2->req) {
  3483. case FASTRPC_INVOKE2_ASYNC:
  3484. size = sizeof(struct fastrpc_ioctl_invoke_async);
  3485. VERIFY(err, size >= inv2->size);
  3486. if (err) {
  3487. err = -EBADE;
  3488. goto bail;
  3489. }
  3490. if (size > inv2->size) {
  3491. K_COPY_FROM_USER(err, fl->is_compat, &p.inv3, (void *)inv2->invparam,
  3492. sizeof(struct fastrpc_ioctl_invoke_async_no_perf));
  3493. if (err)
  3494. goto bail;
  3495. memcpy(&p.inv, &p.inv3, sizeof(struct fastrpc_ioctl_invoke_crc));
  3496. memcpy(&p.inv.job, &p.inv3.job, sizeof(p.inv.job));
  3497. } else {
  3498. K_COPY_FROM_USER(err, fl->is_compat, &p.inv, (void *)inv2->invparam, size);
  3499. if (err)
  3500. goto bail;
  3501. }
  3502. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode,
  3503. USER_MSG, &p.inv)));
  3504. if (err)
  3505. goto bail;
  3506. break;
  3507. case FASTRPC_INVOKE2_ASYNC_RESPONSE:
  3508. VERIFY(err,
  3509. sizeof(struct fastrpc_ioctl_async_response) >= inv2->size);
  3510. if (err) {
  3511. err = -EBADE;
  3512. goto bail;
  3513. }
  3514. err = fastrpc_get_async_response(&p.async_res,
  3515. (void *)inv2->invparam, fl);
  3516. break;
  3517. case FASTRPC_INVOKE2_KERNEL_OPTIMIZATIONS:
  3518. size = sizeof(uint32_t);
  3519. if (inv2->size != size) {
  3520. err = -EBADE;
  3521. goto bail;
  3522. }
  3523. K_COPY_FROM_USER(err, 0, &p.user_concurrency,
  3524. (void *)inv2->invparam, size);
  3525. if (err)
  3526. goto bail;
  3527. err = fastrpc_create_persistent_headers(fl,
  3528. p.user_concurrency);
  3529. break;
  3530. case FASTRPC_INVOKE2_STATUS_NOTIF:
  3531. VERIFY(err,
  3532. sizeof(struct fastrpc_ioctl_notif_rsp) >= inv2->size);
  3533. if (err) {
  3534. err = -EBADE;
  3535. goto bail;
  3536. }
  3537. err = fastrpc_get_notif_response(&p.notif,
  3538. (void *)inv2->invparam, fl);
  3539. break;
  3540. case FASTRPC_INVOKE2_PROC_SHAREDBUF_INFO:
  3541. VERIFY(err,
  3542. sizeof(struct fastrpc_proc_sharedbuf_info) >= inv2->size);
  3543. if (err) {
  3544. err = -EBADE;
  3545. goto bail;
  3546. }
  3547. K_COPY_FROM_USER(err, fl->is_compat, &p.buff_info,
  3548. (void *)inv2->invparam, inv2->size);
  3549. if (err)
  3550. goto bail;
  3551. fl->sharedbuf_info.buf_fd = p.buff_info.buf_fd;
  3552. fl->sharedbuf_info.buf_size = p.buff_info.buf_size;
  3553. break;
  3554. case FASTRPC_INVOKE2_SESS_INFO:
  3555. VERIFY(err,
  3556. sizeof(struct fastrpc_proc_sess_info) >= inv2->size);
  3557. if (err) {
  3558. err = -EBADE;
  3559. goto bail;
  3560. }
  3561. K_COPY_FROM_USER(err, fl->is_compat, &p.sess_info,
  3562. (void *)inv2->invparam, inv2->size);
  3563. if (err)
  3564. goto bail;
  3565. err = fastrpc_set_session_info(&p.sess_info,
  3566. (void *)inv2->invparam, fl);
  3567. break;
  3568. default:
  3569. err = -ENOTTY;
  3570. break;
  3571. }
  3572. bail:
  3573. return err;
  3574. }
  3575. static int fastrpc_get_spd_session(char *name, int *session, int *cid)
  3576. {
  3577. struct fastrpc_apps *me = &gfa;
  3578. int err = 0, i, j, match = 0;
  3579. for (i = 0; i < NUM_CHANNELS; i++) {
  3580. for (j = 0; j < NUM_SESSIONS; j++) {
  3581. if (!me->channel[i].spd[j].servloc_name)
  3582. continue;
  3583. if (!strcmp(name, me->channel[i].spd[j].servloc_name)) {
  3584. match = 1;
  3585. break;
  3586. }
  3587. }
  3588. if (match)
  3589. break;
  3590. }
  3591. VERIFY(err, i < NUM_CHANNELS && j < NUM_SESSIONS);
  3592. if (err) {
  3593. err = -EUSERS;
  3594. goto bail;
  3595. }
  3596. *cid = i;
  3597. *session = j;
  3598. bail:
  3599. return err;
  3600. }
  3601. static int fastrpc_mmap_remove_pdr(struct fastrpc_file *fl);
  3602. static int fastrpc_channel_open(struct fastrpc_file *fl, uint32_t flags);
  3603. static int fastrpc_mmap_remove_ssr(struct fastrpc_file *fl, int locked);
  3604. /*
  3605. * This function makes a call to create a thread group in the root
  3606. * process or static process on the remote subsystem.
  3607. * Examples:
  3608. * - guestOS daemons on all DSPs
  3609. * - sensors daemon on sensorsPD on SLPI/ADSP
  3610. */
  3611. static int fastrpc_init_attach_process(struct fastrpc_file *fl,
  3612. struct fastrpc_ioctl_init *init)
  3613. {
  3614. int err = 0, tgid = fl->tgid_frpc;
  3615. remote_arg_t ra[1];
  3616. struct fastrpc_ioctl_invoke_async ioctl;
  3617. if (fl->dev_minor == MINOR_NUM_DEV) {
  3618. err = -ECONNREFUSED;
  3619. ADSPRPC_ERR(
  3620. "untrusted app trying to attach to privileged DSP PD\n");
  3621. return err;
  3622. }
  3623. /*
  3624. * Prepare remote arguments for creating thread group
  3625. * in guestOS/staticPD on the remote subsystem.
  3626. * Send unique fastrpc id to dsp
  3627. */
  3628. ra[0].buf.pv = (void *)&tgid;
  3629. ra[0].buf.len = sizeof(tgid);
  3630. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  3631. ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 0);
  3632. ioctl.inv.pra = ra;
  3633. ioctl.fds = NULL;
  3634. ioctl.attrs = NULL;
  3635. ioctl.crc = NULL;
  3636. ioctl.perf_kernel = NULL;
  3637. ioctl.perf_dsp = NULL;
  3638. ioctl.job = NULL;
  3639. if (init->flags == FASTRPC_INIT_ATTACH)
  3640. fl->pd = FASTRPC_ROOT_PD;
  3641. else if (init->flags == FASTRPC_INIT_ATTACH_SENSORS)
  3642. /* Setting to 2 will route the message to sensorsPD */
  3643. fl->pd = FASTRPC_SENSORS_PD;
  3644. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  3645. if (err)
  3646. goto bail;
  3647. bail:
  3648. return err;
  3649. }
  3650. /*
  3651. * This function makes a call to spawn a dynamic process
  3652. * on the remote subsystem.
  3653. * Example: all compute offloads to CDSP
  3654. */
  3655. static int fastrpc_init_create_dynamic_process(struct fastrpc_file *fl,
  3656. struct fastrpc_ioctl_init_attrs *uproc)
  3657. {
  3658. int err = 0, memlen = 0, mflags = 0, locked = 0;
  3659. struct fastrpc_ioctl_invoke_async ioctl;
  3660. struct fastrpc_ioctl_init *init = &uproc->init;
  3661. /* First page for init-mem and second page for proc-attrs */
  3662. struct smq_phy_page pages[PAGESLEN_WITH_SHAREDBUF];
  3663. struct fastrpc_mmap *file = NULL;
  3664. struct fastrpc_buf *imem = NULL;
  3665. unsigned long imem_dma_attr = 0, irq_flags = 0;
  3666. remote_arg_t ra[6];
  3667. int fds[6];
  3668. struct fastrpc_apps *me = &gfa;
  3669. struct hlist_node *n = NULL;
  3670. struct fastrpc_file *fl_curr = NULL;
  3671. unsigned int gid = 0, one_mb = 1024*1024;
  3672. unsigned int dsp_userpd_memlen = 0;
  3673. struct fastrpc_buf *init_mem;
  3674. struct fastrpc_mmap *sharedbuf_map = NULL;
  3675. struct {
  3676. int pgid;
  3677. unsigned int namelen;
  3678. unsigned int filelen;
  3679. unsigned int pageslen;
  3680. int attrs;
  3681. int siglen;
  3682. } inbuf;
  3683. spin_lock(&fl->hlock);
  3684. if (fl->dsp_process_state) {
  3685. err = -EALREADY;
  3686. ADSPRPC_ERR("Already in create dynamic process\n");
  3687. spin_unlock(&fl->hlock);
  3688. return err;
  3689. }
  3690. fl->dsp_process_state = PROCESS_CREATE_IS_INPROGRESS;
  3691. if (init->memlen) {
  3692. if(init->memlen > INIT_MEMLEN_MAX_DYNAMIC || init->memlen < INIT_MEMLEN_MIN_DYNAMIC) {
  3693. ADSPRPC_ERR(
  3694. "init memory for process %d should be between %d and %d\n",
  3695. init->memlen, INIT_MEMLEN_MIN_DYNAMIC, INIT_MEMLEN_MAX_DYNAMIC);
  3696. err = -EINVAL;
  3697. spin_unlock(&fl->hlock);
  3698. goto bail;
  3699. }
  3700. dsp_userpd_memlen = init->memlen;
  3701. } else {
  3702. dsp_userpd_memlen = 3*one_mb;
  3703. }
  3704. spin_unlock(&fl->hlock);
  3705. inbuf.pgid = fl->tgid_frpc;
  3706. inbuf.namelen = strlen(current->comm) + 1;
  3707. inbuf.filelen = init->filelen;
  3708. fl->pd = FASTRPC_USER_PD;
  3709. if (uproc->attrs & FASTRPC_MODE_UNSIGNED_MODULE)
  3710. fl->is_unsigned_pd = true;
  3711. /* Validate that any existing sessions of process are of same pd type */
  3712. spin_lock_irqsave(&me->hlock, irq_flags);
  3713. hlist_for_each_entry_safe(fl_curr, n, &me->drivers, hn) {
  3714. if ((fl != fl_curr) && (fl->tgid == fl_curr->tgid) && (fl->cid == fl_curr->cid)) {
  3715. err = (fl->is_unsigned_pd != fl_curr->is_unsigned_pd) ? -ECONNREFUSED : 0;
  3716. break;
  3717. }
  3718. }
  3719. spin_unlock_irqrestore(&me->hlock, irq_flags);
  3720. if (err) {
  3721. ADSPRPC_ERR("existing session pd type %u not same as requested pd type %u \n",
  3722. fl_curr->is_unsigned_pd, fl->is_unsigned_pd);
  3723. goto bail;
  3724. }
  3725. /* Check if file memory passed by userspace is valid */
  3726. VERIFY(err, access_ok((void __user *)init->file, init->filelen));
  3727. if (err)
  3728. goto bail;
  3729. if (init->filelen) {
  3730. /* Map the shell file buffer to remote subsystem */
  3731. mutex_lock(&fl->map_mutex);
  3732. err = fastrpc_mmap_create(fl, init->filefd, NULL, 0,
  3733. init->file, init->filelen, mflags, &file);
  3734. if (file)
  3735. file->is_filemap = true;
  3736. mutex_unlock(&fl->map_mutex);
  3737. if (err)
  3738. goto bail;
  3739. }
  3740. inbuf.pageslen = 1;
  3741. /* Disregard any system unsigned PD attribute from userspace */
  3742. uproc->attrs &= (~FASTRPC_MODE_SYSTEM_UNSIGNED_PD);
  3743. /* Untrusted apps are not allowed to offload to signedPD on DSP. */
  3744. if (fl->untrusted_process) {
  3745. VERIFY(err, fl->is_unsigned_pd);
  3746. if (err) {
  3747. err = -ECONNREFUSED;
  3748. ADSPRPC_ERR(
  3749. "untrusted app trying to offload to signed remote process\n");
  3750. goto bail;
  3751. }
  3752. } else {
  3753. /* Trusted apps will be launched as system unsigned PDs */
  3754. if (fl->is_unsigned_pd)
  3755. uproc->attrs |= FASTRPC_MODE_SYSTEM_UNSIGNED_PD;
  3756. }
  3757. /* Disregard any privilege bits from userspace */
  3758. uproc->attrs &= (~FASTRPC_MODE_PRIVILEGED);
  3759. /*
  3760. * Check if the primary or supplementary group(s) of the process is
  3761. * one of the 'privileged' fastrpc GIDs stored in the device-tree.
  3762. */
  3763. gid = sorted_lists_intersection(fl->gidlist.gids,
  3764. fl->gidlist.gidcount, gfa.gidlist.gids, gfa.gidlist.gidcount);
  3765. if (gid) {
  3766. ADSPRPC_INFO("PID %d, GID %u is a privileged process\n",
  3767. fl->tgid, gid);
  3768. uproc->attrs |= FASTRPC_MODE_PRIVILEGED;
  3769. }
  3770. /*
  3771. * Userspace client should try to allocate the initial memory donated
  3772. * to remote subsystem as only the kernel and DSP should have access
  3773. * to that memory.
  3774. */
  3775. VERIFY(err, !init->mem);
  3776. if (err) {
  3777. err = -EINVAL;
  3778. ADSPRPC_ERR("donated memory allocated in userspace\n");
  3779. goto bail;
  3780. }
  3781. /* Free any previous donated memory */
  3782. spin_lock(&fl->hlock);
  3783. locked = 1;
  3784. if (fl->init_mem) {
  3785. init_mem = fl->init_mem;
  3786. fl->init_mem = NULL;
  3787. spin_unlock(&fl->hlock);
  3788. locked = 0;
  3789. fastrpc_buf_free(init_mem, 0);
  3790. }
  3791. if (locked) {
  3792. spin_unlock(&fl->hlock);
  3793. locked = 0;
  3794. }
  3795. /* Allocate DMA buffer in kernel for donating to remote process
  3796. * Unsigned PD requires additional memory because of the
  3797. * additional static heap initialized within the process.
  3798. */
  3799. if (fl->is_unsigned_pd)
  3800. dsp_userpd_memlen = 5*one_mb;
  3801. memlen = ALIGN(max(dsp_userpd_memlen, init->filelen * 4), one_mb);
  3802. imem_dma_attr = DMA_ATTR_DELAYED_UNMAP | DMA_ATTR_NO_KERNEL_MAPPING;
  3803. err = fastrpc_buf_alloc(fl, memlen, imem_dma_attr, 0,
  3804. INITMEM_BUF, &imem);
  3805. if (err)
  3806. goto bail;
  3807. fl->init_mem = imem;
  3808. inbuf.pageslen = 1;
  3809. if ((fl->sharedbuf_info.buf_fd != -1) && fl->sharedbuf_info.buf_size) {
  3810. mutex_lock(&fl->map_mutex);
  3811. err = fastrpc_mmap_create(fl, fl->sharedbuf_info.buf_fd, NULL, 0,
  3812. 0, fl->sharedbuf_info.buf_size, mflags, &sharedbuf_map);
  3813. mutex_unlock(&fl->map_mutex);
  3814. if (err)
  3815. goto bail;
  3816. /* if shared buff is available send this as the second page and set pageslen as 2 */
  3817. inbuf.pageslen = PAGESLEN_WITH_SHAREDBUF;
  3818. }
  3819. /*
  3820. * Prepare remote arguments for dynamic process create
  3821. * call to remote subsystem.
  3822. */
  3823. ra[0].buf.pv = (void *)&inbuf;
  3824. ra[0].buf.len = sizeof(inbuf);
  3825. fds[0] = -1;
  3826. ra[1].buf.pv = (void *)current->comm;
  3827. ra[1].buf.len = inbuf.namelen;
  3828. fds[1] = -1;
  3829. ra[2].buf.pv = (void *)init->file;
  3830. ra[2].buf.len = inbuf.filelen;
  3831. fds[2] = init->filefd;
  3832. pages[0].addr = imem->phys;
  3833. pages[0].size = imem->size;
  3834. /* Update IOVA of second page shared with DSP */
  3835. if (inbuf.pageslen > 1) {
  3836. pages[1].addr = sharedbuf_map->phys;
  3837. pages[1].size = sharedbuf_map->size;
  3838. }
  3839. ra[3].buf.pv = (void *)pages;
  3840. ra[3].buf.len = (inbuf.pageslen) * sizeof(*pages);
  3841. fds[3] = -1;
  3842. inbuf.attrs = uproc->attrs;
  3843. ra[4].buf.pv = (void *)&(inbuf.attrs);
  3844. ra[4].buf.len = sizeof(inbuf.attrs);
  3845. fds[4] = -1;
  3846. inbuf.siglen = uproc->siglen;
  3847. ra[5].buf.pv = (void *)&(inbuf.siglen);
  3848. ra[5].buf.len = sizeof(inbuf.siglen);
  3849. fds[5] = -1;
  3850. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  3851. /*
  3852. * Choose appropriate remote method ID depending on whether the
  3853. * HLOS process has any attributes enabled (like unsignedPD,
  3854. * critical process, adaptive QoS, CRC checks etc).
  3855. */
  3856. ioctl.inv.sc = REMOTE_SCALARS_MAKE(6, 4, 0);
  3857. if (uproc->attrs)
  3858. ioctl.inv.sc = REMOTE_SCALARS_MAKE(7, 4, 0);
  3859. ioctl.inv.pra = ra;
  3860. ioctl.fds = fds;
  3861. ioctl.attrs = NULL;
  3862. ioctl.crc = NULL;
  3863. ioctl.perf_kernel = NULL;
  3864. ioctl.perf_dsp = NULL;
  3865. ioctl.job = NULL;
  3866. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  3867. if (err)
  3868. goto bail;
  3869. bail:
  3870. /*
  3871. * Shell is loaded into the donated memory on remote subsystem. So, the
  3872. * original file buffer can be DMA unmapped. In case of a failure also,
  3873. * the mapping needs to be removed.
  3874. */
  3875. if (file) {
  3876. mutex_lock(&fl->map_mutex);
  3877. fastrpc_mmap_free(file, 0);
  3878. mutex_unlock(&fl->map_mutex);
  3879. }
  3880. spin_lock(&fl->hlock);
  3881. locked = 1;
  3882. if (err) {
  3883. ADSPRPC_ERR("failed with err %d\n", err);
  3884. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  3885. if (!IS_ERR_OR_NULL(fl->init_mem)) {
  3886. init_mem = fl->init_mem;
  3887. fl->init_mem = NULL;
  3888. spin_unlock(&fl->hlock);
  3889. locked = 0;
  3890. fastrpc_buf_free(init_mem, 0);
  3891. }
  3892. } else {
  3893. fl->dsp_process_state = PROCESS_CREATE_SUCCESS;
  3894. }
  3895. if (locked) {
  3896. spin_unlock(&fl->hlock);
  3897. locked = 0;
  3898. }
  3899. return err;
  3900. }
  3901. /*
  3902. * This function makes a call to create a thread group in the static
  3903. * process on the remote subsystem.
  3904. * Example: audio daemon 'adsprpcd' on audioPD on ADSP
  3905. */
  3906. static int fastrpc_init_create_static_process(struct fastrpc_file *fl,
  3907. struct fastrpc_ioctl_init *init)
  3908. {
  3909. int err = 0, rh_hyp_done = 0;
  3910. struct fastrpc_apps *me = &gfa;
  3911. struct fastrpc_ioctl_invoke_async ioctl;
  3912. struct smq_phy_page pages[1];
  3913. struct fastrpc_mmap *mem = NULL;
  3914. char *proc_name = NULL;
  3915. remote_arg_t ra[3];
  3916. uint64_t phys = 0;
  3917. size_t size = 0;
  3918. int fds[3];
  3919. struct secure_vm *rhvm = &me->channel[fl->cid].rhvm;
  3920. struct {
  3921. int pgid;
  3922. unsigned int namelen;
  3923. unsigned int pageslen;
  3924. } inbuf;
  3925. unsigned long irq_flags = 0;
  3926. if (fl->dev_minor == MINOR_NUM_DEV) {
  3927. err = -ECONNREFUSED;
  3928. ADSPRPC_ERR(
  3929. "untrusted app trying to attach to audio PD\n");
  3930. return err;
  3931. }
  3932. VERIFY(err, init->memlen <= INIT_MEMLEN_MAX_STATIC);
  3933. if (err) {
  3934. ADSPRPC_ERR(
  3935. "init memory for static process %d is more than max allowed init len %d\n",
  3936. init->memlen, INIT_MEMLEN_MAX_STATIC);
  3937. err = -EFBIG;
  3938. goto bail;
  3939. }
  3940. if (!init->filelen)
  3941. goto bail;
  3942. proc_name = kzalloc(init->filelen + 1, GFP_KERNEL);
  3943. VERIFY(err, !IS_ERR_OR_NULL(proc_name));
  3944. if (err) {
  3945. err = -ENOMEM;
  3946. goto bail;
  3947. }
  3948. err = copy_from_user((void *)proc_name,
  3949. (void __user *)init->file, init->filelen);
  3950. if (err) {
  3951. err = -EFAULT;
  3952. goto bail;
  3953. }
  3954. fl->pd = FASTRPC_USER_PD;
  3955. inbuf.pgid = fl->tgid_frpc;
  3956. inbuf.namelen = init->filelen;
  3957. inbuf.pageslen = 0;
  3958. if (!strcmp(proc_name, "audiopd")) {
  3959. /*
  3960. * Remove any previous mappings in case process is trying
  3961. * to reconnect after a PD restart on remote subsystem.
  3962. */
  3963. err = fastrpc_mmap_remove_pdr(fl);
  3964. if (err)
  3965. goto bail;
  3966. } else {
  3967. ADSPRPC_ERR(
  3968. "Create static process is failed for proc_name %s",
  3969. proc_name);
  3970. goto bail;
  3971. }
  3972. if ((!me->staticpd_flags && !me->legacy_remote_heap)) {
  3973. inbuf.pageslen = 1;
  3974. if (!fastrpc_get_persistent_map(init->memlen, &mem)) {
  3975. mutex_lock(&fl->map_mutex);
  3976. err = fastrpc_mmap_create(fl, -1, NULL, 0, init->mem,
  3977. init->memlen, ADSP_MMAP_REMOTE_HEAP_ADDR, &mem);
  3978. mutex_unlock(&fl->map_mutex);
  3979. if (err || (!mem))
  3980. goto bail;
  3981. spin_lock_irqsave(&me->hlock, irq_flags);
  3982. mem->in_use = true;
  3983. spin_unlock_irqrestore(&me->hlock, irq_flags);
  3984. }
  3985. VERIFY(err, mem);
  3986. if (err)
  3987. goto bail;
  3988. phys = mem->phys;
  3989. size = mem->size;
  3990. /*
  3991. * If remote-heap VMIDs are defined in DTSI, then do
  3992. * hyp_assign from HLOS to those VMs (LPASS, ADSP).
  3993. */
  3994. if (rhvm->vmid && mem->refs == 1 && size) {
  3995. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  3996. struct qcom_scm_vmperm *dst_perms;
  3997. uint32_t i = 0;
  3998. VERIFY(err, NULL != (dst_perms = kcalloc(rhvm->vmcount,
  3999. sizeof(struct qcom_scm_vmperm), GFP_KERNEL)));
  4000. if (err)
  4001. goto bail;
  4002. for (i = 0; i < rhvm->vmcount; i++) {
  4003. dst_perms[i].vmid = rhvm->vmid[i];
  4004. dst_perms[i].perm = rhvm->vmperm[i];
  4005. }
  4006. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4007. &src_perms, dst_perms, rhvm->vmcount);
  4008. kfree(dst_perms);
  4009. if (err) {
  4010. ADSPRPC_ERR(
  4011. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  4012. err, phys, size);
  4013. err = -EADDRNOTAVAIL;
  4014. goto bail;
  4015. }
  4016. rh_hyp_done = 1;
  4017. }
  4018. me->staticpd_flags = 1;
  4019. mem->is_persistent = true;
  4020. }
  4021. /*
  4022. * Prepare remote arguments for static process create
  4023. * call to remote subsystem.
  4024. */
  4025. ra[0].buf.pv = (void *)&inbuf;
  4026. ra[0].buf.len = sizeof(inbuf);
  4027. fds[0] = -1;
  4028. ra[1].buf.pv = (void *)proc_name;
  4029. ra[1].buf.len = inbuf.namelen;
  4030. fds[1] = -1;
  4031. pages[0].addr = phys;
  4032. pages[0].size = size;
  4033. ra[2].buf.pv = (void *)pages;
  4034. ra[2].buf.len = sizeof(*pages);
  4035. fds[2] = -1;
  4036. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4037. ioctl.inv.sc = REMOTE_SCALARS_MAKE(8, 3, 0);
  4038. ioctl.inv.pra = ra;
  4039. ioctl.fds = NULL;
  4040. ioctl.attrs = NULL;
  4041. ioctl.crc = NULL;
  4042. ioctl.perf_kernel = NULL;
  4043. ioctl.perf_dsp = NULL;
  4044. ioctl.job = NULL;
  4045. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4046. if (err)
  4047. goto bail;
  4048. bail:
  4049. kfree(proc_name);
  4050. if (err) {
  4051. me->staticpd_flags = 0;
  4052. if (rh_hyp_done) {
  4053. int hyp_err = 0;
  4054. u64 src_perms = 0;
  4055. struct qcom_scm_vmperm dst_perms;
  4056. uint32_t i = 0;
  4057. for (i = 0; i < rhvm->vmcount; i++) {
  4058. src_perms |= BIT(rhvm->vmid[i]);
  4059. }
  4060. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4061. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4062. /* Assign memory back to HLOS in case of errors */
  4063. hyp_err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4064. &src_perms, &dst_perms, 1);
  4065. if (hyp_err)
  4066. ADSPRPC_WARN(
  4067. "rh hyp unassign failed with %d for phys 0x%llx of size %zu\n",
  4068. hyp_err, phys, size);
  4069. }
  4070. mutex_lock(&fl->map_mutex);
  4071. fastrpc_mmap_free(mem, 0);
  4072. mutex_unlock(&fl->map_mutex);
  4073. }
  4074. return err;
  4075. }
  4076. /*
  4077. * This function sets fastrpc service location name
  4078. * based on ioctl init flags.
  4079. */
  4080. static void fastrpc_set_servloc(struct fastrpc_file *fl,
  4081. struct fastrpc_ioctl_init *init)
  4082. {
  4083. char *proc_name = NULL;
  4084. int err = 0;
  4085. if (init->flags == FASTRPC_INIT_ATTACH_SENSORS) {
  4086. if (fl->cid == ADSP_DOMAIN_ID)
  4087. fl->servloc_name =
  4088. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME;
  4089. else if (fl->cid == SDSP_DOMAIN_ID)
  4090. fl->servloc_name =
  4091. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME;
  4092. } else if (init->flags == FASTRPC_INIT_CREATE_STATIC) {
  4093. if (!init->filelen)
  4094. goto bail;
  4095. proc_name = kzalloc(init->filelen + 1, GFP_KERNEL);
  4096. VERIFY(err, !IS_ERR_OR_NULL(proc_name));
  4097. if (err) {
  4098. err = -ENOMEM;
  4099. goto bail;
  4100. }
  4101. err = copy_from_user((void *)proc_name,
  4102. (void __user *)init->file, init->filelen);
  4103. if (err) {
  4104. err = -EFAULT;
  4105. goto bail;
  4106. }
  4107. if (!strcmp(proc_name, "audiopd"))
  4108. fl->servloc_name = AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME;
  4109. }
  4110. bail:
  4111. kfree(proc_name);
  4112. }
  4113. int fastrpc_init_process(struct fastrpc_file *fl,
  4114. struct fastrpc_ioctl_init_attrs *uproc)
  4115. {
  4116. int err = 0;
  4117. struct fastrpc_ioctl_init *init = &uproc->init;
  4118. int cid = fl->cid;
  4119. struct fastrpc_apps *me = &gfa;
  4120. struct fastrpc_channel_ctx *chan = NULL;
  4121. VERIFY(err, init->filelen < INIT_FILELEN_MAX
  4122. && init->memlen <= INIT_MEMLEN_MAX_DYNAMIC);
  4123. if (err) {
  4124. ADSPRPC_ERR(
  4125. "file size 0x%x or init memory 0x%x is more than max allowed file size 0x%x or init len 0x%x\n",
  4126. init->filelen, init->memlen,
  4127. INIT_FILELEN_MAX, INIT_MEMLEN_MAX_DYNAMIC);
  4128. err = -EFBIG;
  4129. goto bail;
  4130. }
  4131. VERIFY(err, VALID_FASTRPC_CID(cid));
  4132. if (err) {
  4133. err = -ECHRNG;
  4134. goto bail;
  4135. }
  4136. chan = &me->channel[cid];
  4137. if (chan->unsigned_support && fl->dev_minor == MINOR_NUM_DEV) {
  4138. /* Make sure third party applications */
  4139. /* can spawn only unsigned PD when */
  4140. /* channel configured as secure. */
  4141. if (chan->secure && !(fl->is_unsigned_pd)) {
  4142. err = -ECONNREFUSED;
  4143. goto bail;
  4144. }
  4145. }
  4146. if (fl->sharedcb == 1) {
  4147. // Only attach sensors pd use cases can share CB
  4148. VERIFY(err, init->flags == FASTRPC_INIT_ATTACH_SENSORS);
  4149. if (err) {
  4150. err = -EACCES;
  4151. goto bail;
  4152. }
  4153. }
  4154. fastrpc_set_servloc(fl, init);
  4155. err = fastrpc_set_tvm_remote_domain(fl, init);
  4156. if (err)
  4157. goto bail;
  4158. err = fastrpc_channel_open(fl, init->flags);
  4159. if (err)
  4160. goto bail;
  4161. fl->proc_flags = init->flags;
  4162. switch (init->flags) {
  4163. case FASTRPC_INIT_ATTACH:
  4164. case FASTRPC_INIT_ATTACH_SENSORS:
  4165. err = fastrpc_init_attach_process(fl, init);
  4166. break;
  4167. case FASTRPC_INIT_CREATE:
  4168. err = fastrpc_init_create_dynamic_process(fl, uproc);
  4169. break;
  4170. case FASTRPC_INIT_CREATE_STATIC:
  4171. err = fastrpc_init_create_static_process(fl, init);
  4172. break;
  4173. default:
  4174. err = -ENOTTY;
  4175. break;
  4176. }
  4177. if (err)
  4178. goto bail;
  4179. fl->dsp_proc_init = 1;
  4180. VERIFY(err, 0 == (err = fastrpc_device_create(fl)));
  4181. if (err)
  4182. goto bail;
  4183. bail:
  4184. return err;
  4185. }
  4186. static int fastrpc_send_cpuinfo_to_dsp(struct fastrpc_file *fl)
  4187. {
  4188. int err = 0;
  4189. uint64_t cpuinfo = 0;
  4190. struct fastrpc_apps *me = &gfa;
  4191. struct fastrpc_ioctl_invoke_async ioctl;
  4192. remote_arg_t ra[1];
  4193. int cid = -1;
  4194. if (!fl) {
  4195. err = -EBADF;
  4196. goto bail;
  4197. }
  4198. cid = fl->cid;
  4199. VERIFY(err, VALID_FASTRPC_CID(cid));
  4200. if (err) {
  4201. err = -ECHRNG;
  4202. ADSPRPC_ERR(
  4203. "invalid channel 0x%zx set for session\n",
  4204. cid);
  4205. goto bail;
  4206. }
  4207. cpuinfo = me->channel[cid].cpuinfo_todsp;
  4208. /* return success if already updated to remote processor */
  4209. if (me->channel[cid].cpuinfo_status)
  4210. return 0;
  4211. ra[0].buf.pv = (void *)&cpuinfo;
  4212. ra[0].buf.len = sizeof(cpuinfo);
  4213. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
  4214. ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
  4215. ioctl.inv.pra = ra;
  4216. ioctl.fds = NULL;
  4217. ioctl.attrs = NULL;
  4218. ioctl.crc = NULL;
  4219. ioctl.perf_kernel = NULL;
  4220. ioctl.perf_dsp = NULL;
  4221. ioctl.job = NULL;
  4222. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4223. if (!err)
  4224. me->channel[cid].cpuinfo_status = true;
  4225. bail:
  4226. return err;
  4227. }
  4228. int fastrpc_get_info_from_dsp(struct fastrpc_file *fl,
  4229. uint32_t *dsp_attr_buf,
  4230. uint32_t dsp_attr_buf_len,
  4231. uint32_t domain)
  4232. {
  4233. int err = 0;
  4234. struct fastrpc_ioctl_invoke_async ioctl;
  4235. remote_arg_t ra[2];
  4236. dsp_attr_buf[0] = 0; // Capability filled in userspace
  4237. // Fastrpc to modem not supported
  4238. if (domain == MDSP_DOMAIN_ID)
  4239. goto bail;
  4240. err = fastrpc_channel_open(fl, FASTRPC_INIT_NO_CREATE);
  4241. if (err)
  4242. goto bail;
  4243. ra[0].buf.pv = (void *)&dsp_attr_buf_len;
  4244. ra[0].buf.len = sizeof(dsp_attr_buf_len);
  4245. ra[1].buf.pv = (void *)(&dsp_attr_buf[1]);
  4246. ra[1].buf.len = dsp_attr_buf_len * sizeof(uint32_t);
  4247. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_DSP_UTILITIES;
  4248. ioctl.inv.sc = REMOTE_SCALARS_MAKE(0, 1, 1);
  4249. ioctl.inv.pra = ra;
  4250. ioctl.fds = NULL;
  4251. ioctl.attrs = NULL;
  4252. ioctl.crc = NULL;
  4253. ioctl.perf_kernel = NULL;
  4254. ioctl.perf_dsp = NULL;
  4255. ioctl.job = NULL;
  4256. err = fastrpc_internal_invoke(fl, FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl);
  4257. bail:
  4258. if (err)
  4259. ADSPRPC_ERR("could not obtain dsp information, err val %d\n",
  4260. err);
  4261. return err;
  4262. }
  4263. int fastrpc_get_info_from_kernel(
  4264. struct fastrpc_ioctl_capability *cap,
  4265. struct fastrpc_file *fl)
  4266. {
  4267. int err = 0;
  4268. uint32_t domain = cap->domain, attribute_ID = cap->attribute_ID;
  4269. uint32_t async_capability = 0;
  4270. struct fastrpc_dsp_capabilities *dsp_cap_ptr = NULL;
  4271. VERIFY(err, domain < NUM_CHANNELS);
  4272. if (err) {
  4273. err = -ECHRNG;
  4274. goto bail;
  4275. }
  4276. /*
  4277. * Check if number of attribute IDs obtained from userspace
  4278. * is less than the number of attribute IDs supported by
  4279. * kernel
  4280. */
  4281. if (attribute_ID >= FASTRPC_MAX_ATTRIBUTES) {
  4282. err = -EOVERFLOW;
  4283. goto bail;
  4284. }
  4285. dsp_cap_ptr = &gcinfo[domain].dsp_cap_kernel;
  4286. if (attribute_ID >= FASTRPC_MAX_DSP_ATTRIBUTES) {
  4287. // Driver capability, pass it to user
  4288. memcpy(&cap->capability,
  4289. &kernel_capabilities[attribute_ID -
  4290. FASTRPC_MAX_DSP_ATTRIBUTES],
  4291. sizeof(cap->capability));
  4292. } else if (!dsp_cap_ptr->is_cached) {
  4293. /*
  4294. * Information not on kernel, query device for information
  4295. * and cache on kernel
  4296. */
  4297. err = fastrpc_get_info_from_dsp(fl,
  4298. dsp_cap_ptr->dsp_attributes,
  4299. FASTRPC_MAX_DSP_ATTRIBUTES - 1,
  4300. domain);
  4301. if (err)
  4302. goto bail;
  4303. /* Async capability support depends on both kernel and DSP */
  4304. async_capability = IS_ASYNC_FASTRPC_AVAILABLE &&
  4305. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP];
  4306. dsp_cap_ptr->dsp_attributes[ASYNC_FASTRPC_CAP]
  4307. = async_capability;
  4308. memcpy(&cap->capability,
  4309. &dsp_cap_ptr->dsp_attributes[attribute_ID],
  4310. sizeof(cap->capability));
  4311. dsp_cap_ptr->is_cached = 1;
  4312. } else {
  4313. // Information on Kernel, pass it to user
  4314. memcpy(&cap->capability,
  4315. &dsp_cap_ptr->dsp_attributes[attribute_ID],
  4316. sizeof(cap->capability));
  4317. }
  4318. bail:
  4319. return err;
  4320. }
  4321. static int fastrpc_release_current_dsp_process(struct fastrpc_file *fl)
  4322. {
  4323. int err = 0;
  4324. struct fastrpc_ioctl_invoke_async ioctl;
  4325. remote_arg_t ra[1];
  4326. int tgid = 0;
  4327. int cid = -1;
  4328. unsigned long irq_flags = 0;
  4329. if (!fl) {
  4330. err = -EBADF;
  4331. goto bail;
  4332. }
  4333. cid = fl->cid;
  4334. VERIFY(err, VALID_FASTRPC_CID(cid));
  4335. if (err) {
  4336. err = -ECHRNG;
  4337. goto bail;
  4338. }
  4339. VERIFY(err, fl->sctx != NULL);
  4340. if (err) {
  4341. err = -EBADR;
  4342. goto bail;
  4343. }
  4344. err = verify_transport_device(cid, fl->tvm_remote_domain);
  4345. if (err)
  4346. goto bail;
  4347. VERIFY(err, fl->apps->channel[cid].subsystemstate != SUBSYSTEM_RESTARTING);
  4348. if (err) {
  4349. wait_for_completion(&fl->shutdown);
  4350. err = -ECONNRESET;
  4351. goto bail;
  4352. }
  4353. /* Send unique fastrpc process ID to dsp */
  4354. tgid = fl->tgid_frpc;
  4355. ra[0].buf.pv = (void *)&tgid;
  4356. ra[0].buf.len = sizeof(tgid);
  4357. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4358. ioctl.inv.sc = REMOTE_SCALARS_MAKE(1, 1, 0);
  4359. ioctl.inv.pra = ra;
  4360. ioctl.fds = NULL;
  4361. ioctl.attrs = NULL;
  4362. ioctl.crc = NULL;
  4363. ioctl.perf_kernel = NULL;
  4364. ioctl.perf_dsp = NULL;
  4365. ioctl.job = NULL;
  4366. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4367. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_INIT;
  4368. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4369. /*
  4370. * Pass 2 for "kernel" arg to send kernel msg to DSP
  4371. * with non-zero msg PID for the DSP to directly use
  4372. * that info to kill the remote process.
  4373. */
  4374. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4375. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_NONZERO_PID, &ioctl)));
  4376. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4377. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_COMPLETE;
  4378. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4379. if (err && fl->dsp_proc_init)
  4380. ADSPRPC_ERR(
  4381. "releasing DSP process failed with %d (0x%x) for %s\n",
  4382. err, err, current->comm);
  4383. bail:
  4384. if (err && fl && fl->apps) {
  4385. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  4386. fl->file_close = FASTRPC_PROCESS_DSP_EXIT_ERROR;
  4387. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  4388. }
  4389. return err;
  4390. }
  4391. static int fastrpc_mem_map_to_dsp(struct fastrpc_file *fl, int fd, int offset,
  4392. uint32_t flags, uintptr_t va, uint64_t phys,
  4393. size_t size, uintptr_t *raddr)
  4394. {
  4395. struct fastrpc_ioctl_invoke_async ioctl;
  4396. struct smq_phy_page page;
  4397. remote_arg_t ra[4];
  4398. int err = 0;
  4399. struct {
  4400. int pid;
  4401. int fd;
  4402. int offset;
  4403. uint32_t flags;
  4404. uint64_t vaddrin;
  4405. int num;
  4406. int data_len;
  4407. } inargs;
  4408. struct {
  4409. uint64_t vaddrout;
  4410. } routargs;
  4411. /* Send unique fastrpc process ID to dsp */
  4412. inargs.pid = fl->tgid_frpc;
  4413. inargs.fd = fd;
  4414. inargs.offset = offset;
  4415. inargs.vaddrin = (uintptr_t)va;
  4416. inargs.flags = flags;
  4417. inargs.num = sizeof(page);
  4418. inargs.data_len = 0;
  4419. ra[0].buf.pv = (void *)&inargs;
  4420. ra[0].buf.len = sizeof(inargs);
  4421. page.addr = phys;
  4422. page.size = size;
  4423. ra[1].buf.pv = (void *)&page;
  4424. ra[1].buf.len = sizeof(page);
  4425. ra[2].buf.pv = (void *)&page;
  4426. ra[2].buf.len = 0;
  4427. ra[3].buf.pv = (void *)&routargs;
  4428. ra[3].buf.len = sizeof(routargs);
  4429. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4430. ioctl.inv.sc = REMOTE_SCALARS_MAKE(10, 3, 1);
  4431. ioctl.inv.pra = ra;
  4432. ioctl.fds = NULL;
  4433. ioctl.attrs = NULL;
  4434. ioctl.crc = NULL;
  4435. ioctl.perf_kernel = NULL;
  4436. ioctl.perf_dsp = NULL;
  4437. ioctl.job = NULL;
  4438. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4439. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4440. if (err)
  4441. goto bail;
  4442. if (raddr)
  4443. *raddr = (uintptr_t)routargs.vaddrout;
  4444. bail:
  4445. return err;
  4446. }
  4447. static int fastrpc_mem_unmap_to_dsp(struct fastrpc_file *fl, int fd,
  4448. uint32_t flags, uintptr_t va,
  4449. uint64_t phys, size_t size)
  4450. {
  4451. struct fastrpc_ioctl_invoke_async ioctl;
  4452. remote_arg_t ra[1];
  4453. int err = 0;
  4454. struct {
  4455. int pid;
  4456. int fd;
  4457. uint64_t vaddrin;
  4458. uint64_t len;
  4459. } inargs;
  4460. /* Send unique fastrpc process ID to dsp */
  4461. inargs.pid = fl->tgid_frpc;
  4462. inargs.fd = fd;
  4463. inargs.vaddrin = (uint64_t)va;
  4464. inargs.len = (uint64_t)size;
  4465. ra[0].buf.pv = (void *)&inargs;
  4466. ra[0].buf.len = sizeof(inargs);
  4467. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4468. ioctl.inv.sc = REMOTE_SCALARS_MAKE(11, 1, 0);
  4469. ioctl.inv.pra = ra;
  4470. ioctl.fds = NULL;
  4471. ioctl.attrs = NULL;
  4472. ioctl.crc = NULL;
  4473. ioctl.perf_kernel = NULL;
  4474. ioctl.perf_dsp = NULL;
  4475. ioctl.job = NULL;
  4476. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4477. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4478. if (err)
  4479. goto bail;
  4480. bail:
  4481. return err;
  4482. }
  4483. static int fastrpc_unmap_on_dsp(struct fastrpc_file *fl,
  4484. uintptr_t raddr, uint64_t phys, size_t size, uint32_t flags)
  4485. {
  4486. struct fastrpc_ioctl_invoke_async ioctl;
  4487. remote_arg_t ra[1] = {};
  4488. int err = 0;
  4489. struct {
  4490. int pid;
  4491. uintptr_t vaddrout;
  4492. size_t size;
  4493. } inargs;
  4494. /* Send unique fastrpc process ID to dsp */
  4495. inargs.pid = fl->tgid_frpc;
  4496. inargs.size = size;
  4497. inargs.vaddrout = raddr;
  4498. ra[0].buf.pv = (void *)&inargs;
  4499. ra[0].buf.len = sizeof(inargs);
  4500. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4501. if (fl->apps->compat)
  4502. ioctl.inv.sc = REMOTE_SCALARS_MAKE(5, 1, 0);
  4503. else
  4504. ioctl.inv.sc = REMOTE_SCALARS_MAKE(3, 1, 0);
  4505. ioctl.inv.pra = ra;
  4506. ioctl.fds = NULL;
  4507. ioctl.attrs = NULL;
  4508. ioctl.crc = NULL;
  4509. ioctl.perf_kernel = NULL;
  4510. ioctl.perf_dsp = NULL;
  4511. ioctl.job = NULL;
  4512. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4513. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4514. if (err)
  4515. goto bail;
  4516. bail:
  4517. return err;
  4518. }
  4519. static int fastrpc_mmap_on_dsp(struct fastrpc_file *fl, uint32_t flags,
  4520. uintptr_t va, uint64_t phys,
  4521. size_t size, int refs, uintptr_t *raddr)
  4522. {
  4523. struct fastrpc_ioctl_invoke_async ioctl;
  4524. struct fastrpc_apps *me = &gfa;
  4525. struct smq_phy_page page;
  4526. int num = 1;
  4527. remote_arg_t ra[3];
  4528. int err = 0;
  4529. struct {
  4530. int pid;
  4531. uint32_t flags;
  4532. uintptr_t vaddrin;
  4533. int num;
  4534. } inargs;
  4535. struct {
  4536. uintptr_t vaddrout;
  4537. } routargs;
  4538. int cid = -1;
  4539. if (!fl) {
  4540. err = -EBADF;
  4541. goto bail;
  4542. }
  4543. cid = fl->cid;
  4544. /* Send unique fastrpc process ID to dsp */
  4545. inargs.pid = fl->tgid_frpc;
  4546. inargs.vaddrin = (uintptr_t)va;
  4547. inargs.flags = flags;
  4548. inargs.num = fl->apps->compat ? num * sizeof(page) : num;
  4549. ra[0].buf.pv = (void *)&inargs;
  4550. ra[0].buf.len = sizeof(inargs);
  4551. page.addr = phys;
  4552. page.size = size;
  4553. ra[1].buf.pv = (void *)&page;
  4554. ra[1].buf.len = num * sizeof(page);
  4555. ra[2].buf.pv = (void *)&routargs;
  4556. ra[2].buf.len = sizeof(routargs);
  4557. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4558. if (fl->apps->compat)
  4559. ioctl.inv.sc = REMOTE_SCALARS_MAKE(4, 2, 1);
  4560. else
  4561. ioctl.inv.sc = REMOTE_SCALARS_MAKE(2, 2, 1);
  4562. ioctl.inv.pra = ra;
  4563. ioctl.fds = NULL;
  4564. ioctl.attrs = NULL;
  4565. ioctl.crc = NULL;
  4566. ioctl.perf_kernel = NULL;
  4567. ioctl.perf_dsp = NULL;
  4568. ioctl.job = NULL;
  4569. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4570. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4571. *raddr = (uintptr_t)routargs.vaddrout;
  4572. if (err)
  4573. goto bail;
  4574. if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4575. VERIFY(err, VALID_FASTRPC_CID(cid));
  4576. if (err) {
  4577. err = -ECHRNG;
  4578. ADSPRPC_ERR(
  4579. "invalid channel 0x%zx set for session\n",
  4580. cid);
  4581. goto bail;
  4582. }
  4583. }
  4584. if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR
  4585. && me->channel[cid].rhvm.vmid && refs == 1) {
  4586. struct secure_vm *rhvm = &me->channel[cid].rhvm;
  4587. u64 src_perms = BIT(QCOM_SCM_VMID_HLOS);
  4588. struct qcom_scm_vmperm *dst_perms;
  4589. uint32_t i = 0;
  4590. VERIFY(err, NULL != (dst_perms = kcalloc(rhvm->vmcount,
  4591. sizeof(struct qcom_scm_vmperm), GFP_KERNEL)));
  4592. if (err)
  4593. goto bail;
  4594. for (i = 0; i < rhvm->vmcount; i++) {
  4595. dst_perms[i].vmid = rhvm->vmid[i];
  4596. dst_perms[i].perm = rhvm->vmperm[i];
  4597. }
  4598. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4599. &src_perms, dst_perms, rhvm->vmcount);
  4600. kfree(dst_perms);
  4601. if (err) {
  4602. int unmap_err = 0;
  4603. ADSPRPC_ERR(
  4604. "rh hyp assign failed with %d for phys 0x%llx, size %zu\n",
  4605. err, phys, size);
  4606. err = -EADDRNOTAVAIL;
  4607. unmap_err = fastrpc_unmap_on_dsp(fl,
  4608. *raddr, phys, size, flags);
  4609. if (unmap_err) {
  4610. ADSPRPC_ERR(
  4611. "failed to unmap %d for phys 0x%llx, size %zd\n",
  4612. unmap_err, phys, size);
  4613. }
  4614. goto bail;
  4615. }
  4616. }
  4617. bail:
  4618. return err;
  4619. }
  4620. static int fastrpc_munmap_on_dsp_rh(struct fastrpc_file *fl, uint64_t phys,
  4621. size_t size, uint32_t flags, int locked)
  4622. {
  4623. int err = 0;
  4624. int tgid = 0;
  4625. struct fastrpc_apps *me = &gfa;
  4626. int cid = -1;
  4627. struct fastrpc_ioctl_invoke_async ioctl;
  4628. remote_arg_t ra[2];
  4629. struct {
  4630. uint8_t skey;
  4631. } routargs;
  4632. if (!fl) {
  4633. err = -EBADF;
  4634. goto bail;
  4635. }
  4636. cid = fl->cid;
  4637. VERIFY(err, VALID_FASTRPC_CID(cid));
  4638. if (err) {
  4639. err = -ECHRNG;
  4640. ADSPRPC_ERR(
  4641. "invalid channel 0x%zx set for session\n",
  4642. cid);
  4643. goto bail;
  4644. }
  4645. /* Send unique fastrpc process ID to dsp */
  4646. tgid = fl->tgid_frpc;
  4647. ra[0].buf.pv = (void *)&tgid;
  4648. ra[0].buf.len = sizeof(tgid);
  4649. ra[1].buf.pv = (void *)&routargs;
  4650. ra[1].buf.len = sizeof(routargs);
  4651. ioctl.inv.handle = FASTRPC_STATIC_HANDLE_PROCESS_GROUP;
  4652. ioctl.inv.sc = REMOTE_SCALARS_MAKE(9, 1, 1);
  4653. ioctl.inv.pra = ra;
  4654. ioctl.fds = NULL;
  4655. ioctl.attrs = NULL;
  4656. ioctl.crc = NULL;
  4657. ioctl.perf_kernel = NULL;
  4658. ioctl.perf_dsp = NULL;
  4659. ioctl.job = NULL;
  4660. if (locked) {
  4661. mutex_unlock(&fl->map_mutex);
  4662. mutex_unlock(&me->channel[cid].smd_mutex);
  4663. }
  4664. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl,
  4665. FASTRPC_MODE_PARALLEL, KERNEL_MSG_WITH_ZERO_PID, &ioctl)));
  4666. if (locked) {
  4667. mutex_lock(&me->channel[cid].smd_mutex);
  4668. mutex_lock(&fl->map_mutex);
  4669. }
  4670. if (err)
  4671. goto bail;
  4672. bail:
  4673. return err;
  4674. }
  4675. static int fastrpc_munmap_rh(uint64_t phys, size_t size,
  4676. uint32_t flags)
  4677. {
  4678. int err = 0;
  4679. struct fastrpc_apps *me = &gfa;
  4680. struct secure_vm *rhvm = &me->channel[RH_CID].rhvm;
  4681. if ((rhvm->vmid)
  4682. && (me->channel[RH_CID].in_hib == 0)) {
  4683. u64 src_perms = 0;
  4684. struct qcom_scm_vmperm dst_perms = {0};
  4685. uint32_t i = 0;
  4686. for (i = 0; i < rhvm->vmcount; i++) {
  4687. src_perms |= BIT(rhvm->vmid[i]);
  4688. }
  4689. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4690. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4691. err = qcom_scm_assign_mem(phys,
  4692. (uint64_t)size, &src_perms, &dst_perms, 1);
  4693. if (err) {
  4694. ADSPRPC_ERR(
  4695. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  4696. err, phys, size);
  4697. err = -EADDRNOTAVAIL;
  4698. return err;
  4699. }
  4700. }
  4701. return err;
  4702. }
  4703. static int fastrpc_munmap_on_dsp(struct fastrpc_file *fl, uintptr_t raddr,
  4704. uint64_t phys, size_t size, uint32_t flags)
  4705. {
  4706. int err = 0;
  4707. VERIFY(err, 0 == (err = fastrpc_unmap_on_dsp(fl, raddr, phys,
  4708. size, flags)));
  4709. if (err)
  4710. goto bail;
  4711. if (flags == ADSP_MMAP_HEAP_ADDR) {
  4712. VERIFY(err, !(err = fastrpc_munmap_on_dsp_rh(fl, phys,
  4713. size, flags, 0)));
  4714. if (err)
  4715. goto bail;
  4716. } else if (flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4717. VERIFY(err, !(err = fastrpc_munmap_rh(phys,
  4718. size, flags)));
  4719. if (err)
  4720. goto bail;
  4721. }
  4722. bail:
  4723. return err;
  4724. }
  4725. static int fastrpc_mmap_remove_ssr(struct fastrpc_file *fl, int locked)
  4726. {
  4727. struct fastrpc_mmap *match = NULL, *map = NULL;
  4728. struct hlist_node *n = NULL;
  4729. int err = 0, ret = 0, lock = 0;
  4730. struct fastrpc_apps *me = &gfa;
  4731. struct qcom_dump_segment ramdump_segments_rh;
  4732. struct list_head head;
  4733. unsigned long irq_flags = 0;
  4734. INIT_LIST_HEAD(&head);
  4735. if (fl) {
  4736. VERIFY(err, fl->cid == RH_CID);
  4737. if (err) {
  4738. err = -EBADR;
  4739. goto bail;
  4740. }
  4741. }
  4742. spin_lock_irqsave(&me->hlock, irq_flags);
  4743. lock = 1;
  4744. hlist_for_each_entry_safe(map, n, &me->maps, hn) {
  4745. if (!lock) {
  4746. spin_lock_irqsave(&me->hlock, irq_flags);
  4747. lock = 1;
  4748. }
  4749. /* In hibernation suspend case fl is NULL, check !fl to cleanup */
  4750. if (!fl || (fl && map->servloc_name && fl->servloc_name
  4751. && !strcmp(map->servloc_name, fl->servloc_name))) {
  4752. match = map;
  4753. if (map->is_persistent && map->in_use) {
  4754. struct secure_vm *rhvm = &me->channel[RH_CID].rhvm;
  4755. uint64_t phys = map->phys;
  4756. size_t size = map->size;
  4757. if (lock) {
  4758. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4759. lock = 0;
  4760. }
  4761. //scm assign it back to HLOS
  4762. if (rhvm->vmid) {
  4763. u64 src_perms = 0;
  4764. struct qcom_scm_vmperm dst_perms = {0};
  4765. uint32_t i = 0;
  4766. for (i = 0; i < rhvm->vmcount; i++) {
  4767. src_perms |= BIT(rhvm->vmid[i]);
  4768. }
  4769. dst_perms.vmid = QCOM_SCM_VMID_HLOS;
  4770. dst_perms.perm = QCOM_SCM_PERM_RWX;
  4771. err = qcom_scm_assign_mem(phys, (uint64_t)size,
  4772. &src_perms, &dst_perms, 1);
  4773. }
  4774. if (err) {
  4775. ADSPRPC_ERR(
  4776. "rh hyp unassign failed with %d for phys 0x%llx, size %zu\n",
  4777. err, phys, size);
  4778. err = -EADDRNOTAVAIL;
  4779. goto bail;
  4780. }
  4781. if (!lock) {
  4782. spin_lock_irqsave(&me->hlock, irq_flags);
  4783. lock = 1;
  4784. }
  4785. map->in_use = false;
  4786. /*
  4787. * decrementing refcount for persistent mappings
  4788. * as incrementing it in fastrpc_get_persistent_map
  4789. */
  4790. map->refs--;
  4791. }
  4792. if (!match->is_persistent)
  4793. hlist_del_init(&map->hn);
  4794. }
  4795. if (lock) {
  4796. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4797. lock = 0;
  4798. }
  4799. if (match) {
  4800. if (!match->is_persistent) {
  4801. if (match->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  4802. err = fastrpc_munmap_rh(match->phys,
  4803. match->size, match->flags);
  4804. } else if (match->flags == ADSP_MMAP_HEAP_ADDR) {
  4805. if (fl)
  4806. err = fastrpc_munmap_on_dsp_rh(fl, match->phys,
  4807. match->size, match->flags, 0);
  4808. else {
  4809. pr_err("Cannot communicate with DSP, ADSP is down\n");
  4810. fastrpc_mmap_add(match);
  4811. }
  4812. }
  4813. }
  4814. memset(&ramdump_segments_rh, 0, sizeof(ramdump_segments_rh));
  4815. ramdump_segments_rh.da = match->phys;
  4816. ramdump_segments_rh.va = (void *)page_address((struct page *)match->va);
  4817. ramdump_segments_rh.size = match->size;
  4818. INIT_LIST_HEAD(&head);
  4819. list_add(&ramdump_segments_rh.node, &head);
  4820. if (me->dev && dump_enabled()) {
  4821. ret = qcom_elf_dump(&head, me->dev, ELF_CLASS);
  4822. if (ret < 0)
  4823. pr_err("adsprpc: %s: unable to dump heap (err %d)\n",
  4824. __func__, ret);
  4825. }
  4826. if (!match->is_persistent) {
  4827. if (!locked)
  4828. mutex_lock(&fl->map_mutex);
  4829. fastrpc_mmap_free(match, 0);
  4830. if (!locked)
  4831. mutex_unlock(&fl->map_mutex);
  4832. }
  4833. }
  4834. }
  4835. bail:
  4836. if (lock) {
  4837. spin_unlock_irqrestore(&me->hlock, irq_flags);
  4838. lock = 0;
  4839. }
  4840. if (err && match) {
  4841. if (!locked)
  4842. mutex_lock(&fl->map_mutex);
  4843. fastrpc_mmap_add(match);
  4844. if (!locked)
  4845. mutex_unlock(&fl->map_mutex);
  4846. }
  4847. return err;
  4848. }
  4849. static int fastrpc_mmap_remove_pdr(struct fastrpc_file *fl)
  4850. {
  4851. struct fastrpc_apps *me = &gfa;
  4852. int session = 0, err = 0, cid = -1;
  4853. if (!fl) {
  4854. err = -EBADF;
  4855. goto bail;
  4856. }
  4857. err = fastrpc_get_spd_session(fl->servloc_name,
  4858. &session, &cid);
  4859. if (err)
  4860. goto bail;
  4861. VERIFY(err, cid == fl->cid);
  4862. if (err) {
  4863. err = -EBADR;
  4864. goto bail;
  4865. }
  4866. if (atomic_read(&me->channel[cid].spd[session].ispdup) == 0) {
  4867. err = -ENOTCONN;
  4868. goto bail;
  4869. }
  4870. if (me->channel[cid].spd[session].pdrcount !=
  4871. me->channel[cid].spd[session].prevpdrcount) {
  4872. err = fastrpc_mmap_remove_ssr(fl, 0);
  4873. if (err)
  4874. ADSPRPC_WARN("failed to unmap remote heap (err %d)\n",
  4875. err);
  4876. me->channel[cid].spd[session].prevpdrcount =
  4877. me->channel[cid].spd[session].pdrcount;
  4878. }
  4879. bail:
  4880. return err;
  4881. }
  4882. static inline void get_fastrpc_ioctl_mmap_64(
  4883. struct fastrpc_ioctl_mmap_64 *mmap64,
  4884. struct fastrpc_ioctl_mmap *immap)
  4885. {
  4886. immap->fd = mmap64->fd;
  4887. immap->flags = mmap64->flags;
  4888. immap->vaddrin = (uintptr_t)mmap64->vaddrin;
  4889. immap->size = mmap64->size;
  4890. }
  4891. static inline void put_fastrpc_ioctl_mmap_64(
  4892. struct fastrpc_ioctl_mmap_64 *mmap64,
  4893. struct fastrpc_ioctl_mmap *immap)
  4894. {
  4895. mmap64->vaddrout = (uint64_t)immap->vaddrout;
  4896. }
  4897. static inline void get_fastrpc_ioctl_munmap_64(
  4898. struct fastrpc_ioctl_munmap_64 *munmap64,
  4899. struct fastrpc_ioctl_munmap *imunmap)
  4900. {
  4901. imunmap->vaddrout = (uintptr_t)munmap64->vaddrout;
  4902. imunmap->size = munmap64->size;
  4903. }
  4904. int fastrpc_internal_munmap(struct fastrpc_file *fl,
  4905. struct fastrpc_ioctl_munmap *ud)
  4906. {
  4907. int err = 0;
  4908. struct fastrpc_mmap *map = NULL;
  4909. struct fastrpc_buf *rbuf = NULL, *free = NULL;
  4910. struct hlist_node *n;
  4911. VERIFY(err, fl->dsp_proc_init == 1);
  4912. if (err) {
  4913. ADSPRPC_ERR(
  4914. "user application %s trying to unmap without initialization\n",
  4915. current->comm);
  4916. err = -EHOSTDOWN;
  4917. return err;
  4918. }
  4919. mutex_lock(&fl->internal_map_mutex);
  4920. spin_lock(&fl->hlock);
  4921. hlist_for_each_entry_safe(rbuf, n, &fl->remote_bufs, hn_rem) {
  4922. if (rbuf->raddr && ((rbuf->flags == ADSP_MMAP_ADD_PAGES) ||
  4923. (rbuf->flags == ADSP_MMAP_ADD_PAGES_LLC))) {
  4924. if ((rbuf->raddr == ud->vaddrout) &&
  4925. (rbuf->size == ud->size)) {
  4926. free = rbuf;
  4927. break;
  4928. }
  4929. }
  4930. }
  4931. spin_unlock(&fl->hlock);
  4932. if (free) {
  4933. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, free->raddr,
  4934. free->phys, free->size, free->flags)));
  4935. if (err)
  4936. goto bail;
  4937. fastrpc_buf_free(rbuf, 0);
  4938. mutex_unlock(&fl->internal_map_mutex);
  4939. return err;
  4940. }
  4941. mutex_lock(&fl->map_mutex);
  4942. VERIFY(err, !(err = fastrpc_mmap_remove(fl, -1, ud->vaddrout,
  4943. ud->size, &map)));
  4944. mutex_unlock(&fl->map_mutex);
  4945. if (err)
  4946. goto bail;
  4947. VERIFY(err, map != NULL);
  4948. if (err) {
  4949. err = -EINVAL;
  4950. goto bail;
  4951. }
  4952. if (!map->is_persistent) {
  4953. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, map->raddr,
  4954. map->phys, map->size, map->flags)));
  4955. }
  4956. if (err)
  4957. goto bail;
  4958. mutex_lock(&fl->map_mutex);
  4959. fastrpc_mmap_free(map, 0);
  4960. mutex_unlock(&fl->map_mutex);
  4961. bail:
  4962. if (err && map) {
  4963. mutex_lock(&fl->map_mutex);
  4964. fastrpc_mmap_add(map);
  4965. mutex_unlock(&fl->map_mutex);
  4966. }
  4967. mutex_unlock(&fl->internal_map_mutex);
  4968. return err;
  4969. }
  4970. /*
  4971. * fastrpc_internal_munmap_fd can only be used for buffers
  4972. * mapped with persist attributes. This can only be called
  4973. * once for any persist buffer
  4974. */
  4975. static int fastrpc_internal_munmap_fd(struct fastrpc_file *fl,
  4976. struct fastrpc_ioctl_munmap_fd *ud)
  4977. {
  4978. int err = 0;
  4979. struct fastrpc_mmap *map = NULL;
  4980. VERIFY(err, (fl && ud));
  4981. if (err) {
  4982. err = -EINVAL;
  4983. return err;
  4984. }
  4985. VERIFY(err, fl->dsp_proc_init == 1);
  4986. if (err) {
  4987. ADSPRPC_ERR(
  4988. "user application %s trying to unmap without initialization\n",
  4989. current->comm);
  4990. err = -EHOSTDOWN;
  4991. return err;
  4992. }
  4993. mutex_lock(&fl->internal_map_mutex);
  4994. mutex_lock(&fl->map_mutex);
  4995. err = fastrpc_mmap_find(fl, ud->fd, NULL, ud->va, ud->len, 0, 0, &map);
  4996. if (err) {
  4997. ADSPRPC_ERR(
  4998. "mapping not found to unmap fd 0x%x, va 0x%llx, len 0x%x, err %d\n",
  4999. ud->fd, (unsigned long long)ud->va,
  5000. (unsigned int)ud->len, err);
  5001. mutex_unlock(&fl->map_mutex);
  5002. goto bail;
  5003. }
  5004. if (map && (map->attr & FASTRPC_ATTR_KEEP_MAP)) {
  5005. map->attr = map->attr & (~FASTRPC_ATTR_KEEP_MAP);
  5006. fastrpc_mmap_free(map, 0);
  5007. }
  5008. mutex_unlock(&fl->map_mutex);
  5009. bail:
  5010. mutex_unlock(&fl->internal_map_mutex);
  5011. return err;
  5012. }
  5013. int fastrpc_internal_mem_map(struct fastrpc_file *fl,
  5014. struct fastrpc_ioctl_mem_map *ud)
  5015. {
  5016. int err = 0;
  5017. struct fastrpc_mmap *map = NULL;
  5018. VERIFY(err, fl->dsp_proc_init == 1);
  5019. if (err) {
  5020. pr_err("adsprpc: ERROR: %s: user application %s trying to map without initialization\n",
  5021. __func__, current->comm);
  5022. err = EBADR;
  5023. goto bail;
  5024. }
  5025. /* create SMMU mapping */
  5026. mutex_lock(&fl->map_mutex);
  5027. VERIFY(err, !(err = fastrpc_mmap_create(fl, ud->m.fd, NULL, ud->m.attrs,
  5028. ud->m.vaddrin, ud->m.length,
  5029. ud->m.flags, &map)));
  5030. mutex_unlock(&fl->map_mutex);
  5031. if (err)
  5032. goto bail;
  5033. if (map->raddr) {
  5034. err = -EEXIST;
  5035. goto bail;
  5036. }
  5037. /* create DSP mapping */
  5038. VERIFY(err, !(err = fastrpc_mem_map_to_dsp(fl, ud->m.fd, ud->m.offset,
  5039. ud->m.flags, map->va, map->phys, map->size, &map->raddr)));
  5040. if (err)
  5041. goto bail;
  5042. ud->m.vaddrout = map->raddr;
  5043. bail:
  5044. if (err) {
  5045. ADSPRPC_ERR("failed to map fd %d, len 0x%x, flags %d, map %pK, err %d\n",
  5046. ud->m.fd, ud->m.length, ud->m.flags, map, err);
  5047. if (map) {
  5048. mutex_lock(&fl->map_mutex);
  5049. fastrpc_mmap_free(map, 0);
  5050. mutex_unlock(&fl->map_mutex);
  5051. }
  5052. }
  5053. return err;
  5054. }
  5055. int fastrpc_internal_mem_unmap(struct fastrpc_file *fl,
  5056. struct fastrpc_ioctl_mem_unmap *ud)
  5057. {
  5058. int err = 0;
  5059. struct fastrpc_mmap *map = NULL;
  5060. size_t map_size = 0;
  5061. VERIFY(err, fl->dsp_proc_init == 1);
  5062. if (err) {
  5063. pr_err("adsprpc: ERROR: %s: user application %s trying to map without initialization\n",
  5064. __func__, current->comm);
  5065. err = EBADR;
  5066. goto bail;
  5067. }
  5068. mutex_lock(&fl->map_mutex);
  5069. VERIFY(err, !(err = fastrpc_mmap_remove(fl, ud->um.fd,
  5070. (uintptr_t)ud->um.vaddr, ud->um.length, &map)));
  5071. mutex_unlock(&fl->map_mutex);
  5072. if (err)
  5073. goto bail;
  5074. VERIFY(err, map->flags == FASTRPC_MAP_FD ||
  5075. map->flags == FASTRPC_MAP_FD_DELAYED ||
  5076. map->flags == FASTRPC_MAP_STATIC);
  5077. if (err) {
  5078. err = -EBADMSG;
  5079. goto bail;
  5080. }
  5081. map_size = map->size;
  5082. /* remove mapping on DSP */
  5083. VERIFY(err, !(err = fastrpc_mem_unmap_to_dsp(fl, map->fd, map->flags,
  5084. map->raddr, map->phys, map->size)));
  5085. if (err)
  5086. goto bail;
  5087. /* remove SMMU mapping */
  5088. mutex_lock(&fl->map_mutex);
  5089. fastrpc_mmap_free(map, 0);
  5090. mutex_unlock(&fl->map_mutex);
  5091. map = NULL;
  5092. bail:
  5093. if (err) {
  5094. ADSPRPC_ERR(
  5095. "failed to unmap fd %d addr 0x%llx length %zu map size %zu err 0x%x\n",
  5096. ud->um.fd, ud->um.vaddr, ud->um.length, map_size, err);
  5097. /* Add back to map list in case of error to unmap on DSP */
  5098. if (map) {
  5099. mutex_lock(&fl->map_mutex);
  5100. fastrpc_mmap_add(map);
  5101. mutex_unlock(&fl->map_mutex);
  5102. }
  5103. }
  5104. return err;
  5105. }
  5106. int fastrpc_internal_mmap(struct fastrpc_file *fl,
  5107. struct fastrpc_ioctl_mmap *ud)
  5108. {
  5109. struct fastrpc_mmap *map = NULL;
  5110. struct fastrpc_buf *rbuf = NULL;
  5111. unsigned long dma_attr = 0;
  5112. uintptr_t raddr = 0;
  5113. int err = 0;
  5114. VERIFY(err, fl->dsp_proc_init == 1);
  5115. if (err) {
  5116. ADSPRPC_ERR(
  5117. "user application %s trying to map without initialization\n",
  5118. current->comm);
  5119. err = -EHOSTDOWN;
  5120. return err;
  5121. }
  5122. mutex_lock(&fl->internal_map_mutex);
  5123. /* Pages for unsigned PD's user-heap should be allocated in userspace */
  5124. if (((ud->flags == ADSP_MMAP_ADD_PAGES) ||
  5125. (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)) && !fl->is_unsigned_pd) {
  5126. if (ud->vaddrin) {
  5127. err = -EINVAL;
  5128. ADSPRPC_ERR(
  5129. "adding user allocated pages is not supported\n");
  5130. goto bail;
  5131. }
  5132. dma_attr = DMA_ATTR_DELAYED_UNMAP | DMA_ATTR_NO_KERNEL_MAPPING;
  5133. if (ud->flags == ADSP_MMAP_ADD_PAGES_LLC)
  5134. dma_attr |= DMA_ATTR_SYS_CACHE_ONLY;
  5135. err = fastrpc_buf_alloc(fl, ud->size, dma_attr, ud->flags,
  5136. USERHEAP_BUF, &rbuf);
  5137. if (err)
  5138. goto bail;
  5139. err = fastrpc_mmap_on_dsp(fl, ud->flags, 0,
  5140. rbuf->phys, rbuf->size, 0, &raddr);
  5141. if (err)
  5142. goto bail;
  5143. rbuf->raddr = raddr;
  5144. } else {
  5145. uintptr_t va_to_dsp;
  5146. if (fl->is_unsigned_pd && ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR) {
  5147. err = -EINVAL;
  5148. ADSPRPC_ERR(
  5149. "Secure memory allocation is not supported in unsigned PD");
  5150. goto bail;
  5151. }
  5152. mutex_lock(&fl->map_mutex);
  5153. VERIFY(err, !(err = fastrpc_mmap_create(fl, ud->fd, NULL, 0,
  5154. (uintptr_t)ud->vaddrin, ud->size,
  5155. ud->flags, &map)));
  5156. mutex_unlock(&fl->map_mutex);
  5157. if (err)
  5158. goto bail;
  5159. if (ud->flags == ADSP_MMAP_HEAP_ADDR ||
  5160. ud->flags == ADSP_MMAP_REMOTE_HEAP_ADDR)
  5161. va_to_dsp = 0;
  5162. else
  5163. va_to_dsp = (uintptr_t)map->va;
  5164. VERIFY(err, 0 == (err = fastrpc_mmap_on_dsp(fl, ud->flags,
  5165. va_to_dsp, map->phys, map->size, map->refs, &raddr)));
  5166. if (err)
  5167. goto bail;
  5168. map->raddr = raddr;
  5169. }
  5170. ud->vaddrout = raddr;
  5171. bail:
  5172. if (err) {
  5173. if (map) {
  5174. mutex_lock(&fl->map_mutex);
  5175. fastrpc_mmap_free(map, 0);
  5176. mutex_unlock(&fl->map_mutex);
  5177. }
  5178. if (!IS_ERR_OR_NULL(rbuf))
  5179. fastrpc_buf_free(rbuf, 0);
  5180. }
  5181. mutex_unlock(&fl->internal_map_mutex);
  5182. return err;
  5183. }
  5184. static void fastrpc_context_list_dtor(struct fastrpc_file *fl);
  5185. static int fastrpc_session_alloc_locked(struct fastrpc_channel_ctx *chan,
  5186. int secure, int sharedcb, int pd_type, struct fastrpc_session_ctx **session)
  5187. {
  5188. struct fastrpc_apps *me = &gfa;
  5189. uint64_t idx = 0;
  5190. int err = 0;
  5191. /*
  5192. * PD type can be either unused(DEFAULT_UNUSED) (or) if PD type
  5193. * is used, choose the context bank with matching PD type.
  5194. */
  5195. if (chan->sesscount) {
  5196. for (idx = 0; idx < chan->sesscount; ++idx) {
  5197. if (!chan->session[idx].used &&
  5198. chan->session[idx].smmu.secure == secure &&
  5199. chan->session[idx].smmu.sharedcb == sharedcb &&
  5200. (pd_type == DEFAULT_UNUSED ||
  5201. chan->session[idx].smmu.pd_type == pd_type)) {
  5202. chan->session[idx].used = 1;
  5203. break;
  5204. }
  5205. }
  5206. if (idx >= chan->sesscount) {
  5207. err = -EUSERS;
  5208. goto bail;
  5209. }
  5210. chan->session[idx].smmu.faults = 0;
  5211. } else {
  5212. VERIFY(err, me->dev != NULL);
  5213. if (err) {
  5214. err = -ENODEV;
  5215. goto bail;
  5216. }
  5217. chan->session[0].dev = me->dev;
  5218. chan->session[0].smmu.dev = me->dev;
  5219. }
  5220. *session = &chan->session[idx];
  5221. bail:
  5222. return err;
  5223. }
  5224. static void handle_remote_signal(uint64_t msg, int cid)
  5225. {
  5226. struct fastrpc_apps *me = &gfa;
  5227. uint32_t pid = msg >> 32;
  5228. uint32_t signal_id = msg & 0xffffffff;
  5229. struct fastrpc_file *fl = NULL;
  5230. struct hlist_node *n = NULL;
  5231. unsigned long irq_flags = 0;
  5232. DSPSIGNAL_VERBOSE("Received queue signal %llx: PID %u, signal %u\n", msg, pid, signal_id);
  5233. if (signal_id >= DSPSIGNAL_NUM_SIGNALS) {
  5234. ADSPRPC_ERR("Received bad signal %u for PID %u\n", signal_id, pid);
  5235. return;
  5236. }
  5237. spin_lock_irqsave(&me->hlock, irq_flags);
  5238. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  5239. /* Response from DSP contains unique fastrpc process id, use unique fastrpc process ID to compare */
  5240. if ((fl->tgid_frpc == pid) && (fl->cid == cid)) {
  5241. unsigned long fflags = 0;
  5242. spin_lock_irqsave(&fl->dspsignals_lock, fflags);
  5243. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE]) {
  5244. struct fastrpc_dspsignal *group =
  5245. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  5246. struct fastrpc_dspsignal *sig =
  5247. &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  5248. if ((sig->state == DSPSIGNAL_STATE_PENDING) ||
  5249. (sig->state == DSPSIGNAL_STATE_SIGNALED)) {
  5250. DSPSIGNAL_VERBOSE("Signaling signal %u for PID %u\n",
  5251. signal_id, pid);
  5252. trace_fastrpc_dspsignal("complete", signal_id, sig->state, 0);
  5253. complete(&sig->comp);
  5254. sig->state = DSPSIGNAL_STATE_SIGNALED;
  5255. } else if (sig->state == DSPSIGNAL_STATE_UNUSED) {
  5256. ADSPRPC_ERR("Received unknown signal %u for PID %u\n",
  5257. signal_id, pid);
  5258. }
  5259. } else {
  5260. ADSPRPC_ERR("Received unknown signal %u for PID %u\n",
  5261. signal_id, pid);
  5262. }
  5263. spin_unlock_irqrestore(&fl->dspsignals_lock, fflags);
  5264. break;
  5265. }
  5266. }
  5267. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5268. }
  5269. int fastrpc_handle_rpc_response(void *data, int len, int cid)
  5270. {
  5271. struct smq_invoke_rsp *rsp = (struct smq_invoke_rsp *)data;
  5272. struct smq_notif_rspv3 *notif = (struct smq_notif_rspv3 *)data;
  5273. struct smq_invoke_rspv2 *rspv2 = NULL;
  5274. struct smq_invoke_ctx *ctx = NULL;
  5275. struct fastrpc_apps *me = &gfa;
  5276. uint32_t index, rsp_flags = 0, early_wake_time = 0, ver = 0;
  5277. int err = 0, ignore_rsp_err = 0;
  5278. struct fastrpc_channel_ctx *chan = NULL;
  5279. unsigned long irq_flags = 0;
  5280. int64_t ns = 0;
  5281. uint64_t xo_time_in_us = 0, dspsig_msg = 0;
  5282. xo_time_in_us = CONVERT_CNT_TO_US(__arch_counter_get_cntvct());
  5283. if (len == sizeof(uint64_t)) {
  5284. /*
  5285. * dspsignal message from the DSP
  5286. */
  5287. dspsig_msg = *((uint64_t *)data);
  5288. trace_fastrpc_transport_response(cid, dspsig_msg, 0, 0, 0);
  5289. handle_remote_signal(dspsig_msg, cid);
  5290. goto bail;
  5291. }
  5292. chan = &me->channel[cid];
  5293. VERIFY(err, (rsp && len >= sizeof(*rsp)));
  5294. if (err) {
  5295. err = -EINVAL;
  5296. goto bail;
  5297. }
  5298. if (notif->ctx == FASTRPC_NOTIF_CTX_RESERVED) {
  5299. VERIFY(err, (notif->type == STATUS_RESPONSE &&
  5300. len >= sizeof(*notif)));
  5301. if (err)
  5302. goto bail;
  5303. fastrpc_notif_find_process(cid, notif);
  5304. goto bail;
  5305. }
  5306. if (len >= sizeof(struct smq_invoke_rspv2))
  5307. rspv2 = (struct smq_invoke_rspv2 *)data;
  5308. if (rspv2) {
  5309. early_wake_time = rspv2->early_wake_time;
  5310. rsp_flags = rspv2->flags;
  5311. ver = rspv2->version;
  5312. }
  5313. trace_fastrpc_transport_response(cid, rsp->ctx,
  5314. rsp->retval, rsp_flags, early_wake_time);
  5315. ns = get_timestamp_in_ns();
  5316. fastrpc_update_rxmsg_buf(chan, rsp->ctx, rsp->retval,
  5317. rsp_flags, early_wake_time, ver, ns, xo_time_in_us);
  5318. index = (uint32_t)GET_TABLE_IDX_FROM_CTXID(rsp->ctx);
  5319. VERIFY(err, index < FASTRPC_CTX_MAX);
  5320. if (err)
  5321. goto bail;
  5322. spin_lock_irqsave(&chan->ctxlock, irq_flags);
  5323. ctx = chan->ctxtable[index];
  5324. VERIFY(err, !IS_ERR_OR_NULL(ctx) &&
  5325. (ctx->ctxid == GET_CTXID_FROM_RSP_CTX(rsp->ctx)) &&
  5326. ctx->magic == FASTRPC_CTX_MAGIC);
  5327. if (err) {
  5328. /*
  5329. * Received an anticipatory COMPLETE_SIGNAL from DSP for a
  5330. * context after CPU successfully polling on memory and
  5331. * completed processing of context. Ignore the message.
  5332. * Also ignore response for a call which was already
  5333. * completed by update of poll memory and the context was
  5334. * removed from the table and possibly reused for another call.
  5335. */
  5336. ignore_rsp_err = ((rsp_flags == COMPLETE_SIGNAL) || !ctx ||
  5337. (ctx && (ctx->ctxid != GET_CTXID_FROM_RSP_CTX(rsp->ctx)))) ? 1 : 0;
  5338. goto bail_unlock;
  5339. }
  5340. if (rspv2) {
  5341. VERIFY(err, rspv2->version == FASTRPC_RSP_VERSION2);
  5342. if (err)
  5343. goto bail_unlock;
  5344. }
  5345. VERIFY(err, VALID_FASTRPC_CID(ctx->fl->cid));
  5346. if (err) {
  5347. err = -ECHRNG;
  5348. goto bail_unlock;
  5349. }
  5350. context_notify_user(ctx, rsp->retval, rsp_flags, early_wake_time);
  5351. bail_unlock:
  5352. spin_unlock_irqrestore(&chan->ctxlock, irq_flags);
  5353. bail:
  5354. if (err) {
  5355. err = -ENOKEY;
  5356. if (!ignore_rsp_err)
  5357. ADSPRPC_ERR(
  5358. "invalid response data %pK, len %d from remote subsystem err %d\n",
  5359. data, len, err);
  5360. else {
  5361. err = 0;
  5362. me->duplicate_rsp_err_cnt++;
  5363. }
  5364. }
  5365. return err;
  5366. }
  5367. static int fastrpc_session_alloc_secure_memory(
  5368. struct fastrpc_channel_ctx *chan, int secure,
  5369. int sharedcb, int pd_type, struct fastrpc_session_ctx **session)
  5370. {
  5371. int err = 0;
  5372. struct fastrpc_apps *me = &gfa;
  5373. /*
  5374. * If PD type is configured for context banks,
  5375. * Use CPZ_USERPD, to allocate secure context bank type.
  5376. */
  5377. if (pd_type != DEFAULT_UNUSED && me->cb_pd_type)
  5378. pd_type = CPZ_USERPD;
  5379. mutex_lock(&chan->smd_mutex);
  5380. if (!*session)
  5381. err = fastrpc_session_alloc_locked(chan, secure, sharedcb, pd_type, session);
  5382. mutex_unlock(&chan->smd_mutex);
  5383. if (err == -EUSERS) {
  5384. ADSPRPC_WARN(
  5385. "max concurrent sessions limit (%d) already reached on %s err %d\n",
  5386. chan->sesscount, chan->subsys, err);
  5387. }
  5388. return err;
  5389. }
  5390. static void fastrpc_session_free(struct fastrpc_channel_ctx *chan,
  5391. struct fastrpc_session_ctx *session)
  5392. {
  5393. mutex_lock(&chan->smd_mutex);
  5394. session->used = 0;
  5395. mutex_unlock(&chan->smd_mutex);
  5396. }
  5397. static int fastrpc_file_free(struct fastrpc_file *fl)
  5398. {
  5399. struct hlist_node *n = NULL;
  5400. struct fastrpc_mmap *map = NULL, *lmap = NULL;
  5401. unsigned long flags;
  5402. int cid;
  5403. struct fastrpc_apps *me = &gfa;
  5404. bool is_driver_closed = false;
  5405. int err = 0;
  5406. unsigned long irq_flags = 0;
  5407. bool is_locked = false;
  5408. int i;
  5409. if (!fl)
  5410. return 0;
  5411. cid = fl->cid;
  5412. spin_lock_irqsave(&me->hlock, irq_flags);
  5413. if (fl->device) {
  5414. fl->device->dev_close = true;
  5415. if (fl->device->refs == 0) {
  5416. is_driver_closed = true;
  5417. hlist_del_init(&fl->device->hn);
  5418. }
  5419. }
  5420. fl->file_close = FASTRPC_PROCESS_EXIT_START;
  5421. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5422. (void)fastrpc_release_current_dsp_process(fl);
  5423. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5424. is_locked = true;
  5425. if (!fl->is_dma_invoke_pend)
  5426. goto skip_dmainvoke_wait;
  5427. is_locked = false;
  5428. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5429. wait_for_completion(&fl->dma_invoke);
  5430. skip_dmainvoke_wait:
  5431. if (!is_locked) {
  5432. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5433. is_locked = true;
  5434. }
  5435. if (!fl->is_ramdump_pend)
  5436. goto skip_dump_wait;
  5437. is_locked = false;
  5438. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5439. wait_for_completion(&fl->work);
  5440. skip_dump_wait:
  5441. if (!is_locked) {
  5442. spin_lock_irqsave(&fl->apps->hlock, irq_flags);
  5443. is_locked = true;
  5444. }
  5445. hlist_del_init(&fl->hn);
  5446. fl->is_ramdump_pend = false;
  5447. fl->is_dma_invoke_pend = false;
  5448. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  5449. /* Reset the tgid usage to false */
  5450. if (fl->tgid_frpc != -1)
  5451. frpc_tgid_usage_array[fl->tgid_frpc] = false;
  5452. is_locked = false;
  5453. spin_unlock_irqrestore(&fl->apps->hlock, irq_flags);
  5454. if (!fl->sctx) {
  5455. kfree(fl);
  5456. return 0;
  5457. }
  5458. //Dummy wake up to exit Async worker thread
  5459. spin_lock_irqsave(&fl->aqlock, flags);
  5460. atomic_add(1, &fl->async_queue_job_count);
  5461. wake_up_interruptible(&fl->async_wait_queue);
  5462. spin_unlock_irqrestore(&fl->aqlock, flags);
  5463. // Dummy wake up to exit notification worker thread
  5464. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  5465. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  5466. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  5467. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  5468. if (!IS_ERR_OR_NULL(fl->init_mem))
  5469. fastrpc_buf_free(fl->init_mem, 0);
  5470. fastrpc_context_list_dtor(fl);
  5471. fastrpc_cached_buf_list_free(fl);
  5472. if (!IS_ERR_OR_NULL(fl->hdr_bufs))
  5473. kfree(fl->hdr_bufs);
  5474. if (!IS_ERR_OR_NULL(fl->pers_hdr_buf))
  5475. fastrpc_buf_free(fl->pers_hdr_buf, 0);
  5476. mutex_lock(&fl->internal_map_mutex);
  5477. mutex_lock(&fl->map_mutex);
  5478. do {
  5479. lmap = NULL;
  5480. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5481. hlist_del_init(&map->hn);
  5482. lmap = map;
  5483. break;
  5484. }
  5485. fastrpc_mmap_free(lmap, 1);
  5486. } while (lmap);
  5487. mutex_unlock(&fl->map_mutex);
  5488. mutex_unlock(&fl->internal_map_mutex);
  5489. fastrpc_pm_relax(fl, gcinfo[fl->cid].secure);
  5490. if (fl->device && is_driver_closed)
  5491. device_unregister(&fl->device->dev);
  5492. VERIFY(err, VALID_FASTRPC_CID(cid));
  5493. if (!err && fl->sctx)
  5494. fastrpc_session_free(&fl->apps->channel[cid], fl->sctx);
  5495. if (!err && fl->secsctx)
  5496. fastrpc_session_free(&fl->apps->channel[cid], fl->secsctx);
  5497. for (i = 0; i < (DSPSIGNAL_NUM_SIGNALS / DSPSIGNAL_GROUP_SIZE); i++)
  5498. kfree(fl->signal_groups[i]);
  5499. mutex_destroy(&fl->signal_create_mutex);
  5500. fastrpc_remote_buf_list_free(fl);
  5501. mutex_destroy(&fl->map_mutex);
  5502. mutex_destroy(&fl->internal_map_mutex);
  5503. kfree(fl->dev_pm_qos_req);
  5504. kfree(fl->gidlist.gids);
  5505. kfree(fl);
  5506. return 0;
  5507. }
  5508. static int fastrpc_device_release(struct inode *inode, struct file *file)
  5509. {
  5510. struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
  5511. struct fastrpc_apps *me = &gfa;
  5512. unsigned int ii;
  5513. if (!fl)
  5514. return 0;
  5515. if (fl->qos_request && fl->dev_pm_qos_req) {
  5516. for (ii = 0; ii < me->lowest_capacity_core_count; ii++) {
  5517. if (!dev_pm_qos_request_active(&fl->dev_pm_qos_req[ii]))
  5518. continue;
  5519. dev_pm_qos_remove_request(&fl->dev_pm_qos_req[ii]);
  5520. }
  5521. }
  5522. debugfs_remove(fl->debugfs_file);
  5523. fastrpc_file_free(fl);
  5524. file->private_data = NULL;
  5525. return 0;
  5526. }
  5527. static ssize_t fastrpc_debugfs_read(struct file *filp, char __user *buffer,
  5528. size_t count, loff_t *position)
  5529. {
  5530. struct fastrpc_apps *me = &gfa;
  5531. struct fastrpc_file *fl = filp->private_data;
  5532. struct hlist_node *n;
  5533. struct fastrpc_buf *buf = NULL;
  5534. struct fastrpc_mmap *map = NULL;
  5535. struct fastrpc_mmap *gmaps = NULL;
  5536. struct smq_invoke_ctx *ictx = NULL;
  5537. struct fastrpc_channel_ctx *chan = NULL;
  5538. unsigned int len = 0;
  5539. int i, j, sess_used = 0, ret = 0;
  5540. char *fileinfo = NULL;
  5541. char single_line[] = "----------------";
  5542. char title[] = "=========================";
  5543. unsigned long irq_flags = 0;
  5544. fileinfo = kzalloc(DEBUGFS_SIZE, GFP_KERNEL);
  5545. if (!fileinfo) {
  5546. ret = -ENOMEM;
  5547. goto bail;
  5548. }
  5549. if (fl == NULL) {
  5550. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5551. "\n%s %s %s\n", title, " CHANNEL INFO ", title);
  5552. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5553. "%-7s|%-10s|%-15s|%-9s|%-13s\n",
  5554. "subsys", "sesscount", "subsystemstate",
  5555. "ssrcount", "session_used");
  5556. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5557. "-%s%s%s%s-\n", single_line, single_line,
  5558. single_line, single_line);
  5559. for (i = 0; i < NUM_CHANNELS; i++) {
  5560. sess_used = 0;
  5561. chan = &gcinfo[i];
  5562. len += scnprintf(fileinfo + len,
  5563. DEBUGFS_SIZE - len, "%-7s", chan->subsys);
  5564. len += scnprintf(fileinfo + len,
  5565. DEBUGFS_SIZE - len, "|%-10u",
  5566. chan->sesscount);
  5567. len += scnprintf(fileinfo + len,
  5568. DEBUGFS_SIZE - len, "|%-15d",
  5569. chan->subsystemstate);
  5570. len += scnprintf(fileinfo + len,
  5571. DEBUGFS_SIZE - len, "|%-9u",
  5572. chan->ssrcount);
  5573. for (j = 0; j < chan->sesscount; j++)
  5574. sess_used += chan->session[j].used;
  5575. len += scnprintf(fileinfo + len,
  5576. DEBUGFS_SIZE - len, "|%-13d\n", sess_used);
  5577. }
  5578. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5579. "\n%s%s%s\n", "=============",
  5580. " CMA HEAP ", "==============");
  5581. len += scnprintf(fileinfo + len,
  5582. DEBUGFS_SIZE - len, "%-20s|%-20s\n", "addr", "size");
  5583. len += scnprintf(fileinfo + len,
  5584. DEBUGFS_SIZE - len, "--%s%s---\n",
  5585. single_line, single_line);
  5586. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5587. "\n==========%s %s %s===========\n",
  5588. title, " GMAPS ", title);
  5589. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5590. "%-20s|%-20s|%-20s|%-20s\n",
  5591. "fd", "phys", "size", "va");
  5592. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5593. "%s%s%s%s%s\n", single_line, single_line,
  5594. single_line, single_line, single_line);
  5595. spin_lock_irqsave(&me->hlock, irq_flags);
  5596. hlist_for_each_entry_safe(gmaps, n, &me->maps, hn) {
  5597. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5598. "%-20d|0x%-18llX|0x%-18X|0x%-20lX\n\n",
  5599. gmaps->fd, gmaps->phys,
  5600. (uint32_t)gmaps->size,
  5601. gmaps->va);
  5602. }
  5603. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5604. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5605. "%-20s|%-20s|%-20s|%-20s\n",
  5606. "len", "refs", "raddr", "flags");
  5607. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5608. "%s%s%s%s%s\n", single_line, single_line,
  5609. single_line, single_line, single_line);
  5610. spin_lock_irqsave(&me->hlock, irq_flags);
  5611. hlist_for_each_entry_safe(gmaps, n, &me->maps, hn) {
  5612. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5613. "0x%-18X|%-20d|%-20lu|%-20u\n",
  5614. (uint32_t)gmaps->len, gmaps->refs,
  5615. gmaps->raddr, gmaps->flags);
  5616. }
  5617. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5618. } else {
  5619. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5620. "\n%s %13s %d\n", "cid", ":", fl->cid);
  5621. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5622. "%s %12s %d\n", "tgid", ":", fl->tgid);
  5623. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5624. "%s %14s %d\n", "tgid_frpc", ":", fl->tgid_frpc);
  5625. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5626. "%s %7s %d\n", "sessionid", ":", fl->sessionid);
  5627. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5628. "%s %8s %u\n", "ssrcount", ":", fl->ssrcount);
  5629. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5630. "%s %14s %d\n", "pd", ":", fl->pd);
  5631. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5632. "%s %9s %s\n", "servloc_name", ":", fl->servloc_name);
  5633. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5634. "%s %6s %d\n", "file_close", ":", fl->file_close);
  5635. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5636. "%s %9s %d\n", "profile", ":", fl->profile);
  5637. if (fl->sctx) {
  5638. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5639. "%s %3s %d\n", "smmu.coherent", ":",
  5640. fl->sctx->smmu.coherent);
  5641. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5642. "%s %4s %d\n", "smmu.enabled", ":",
  5643. fl->sctx->smmu.enabled);
  5644. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5645. "%s %9s %d\n", "smmu.cb", ":", fl->sctx->smmu.cb);
  5646. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5647. "%s %5s %d\n", "smmu.secure", ":",
  5648. fl->sctx->smmu.secure);
  5649. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5650. "%s %5s %d\n", "smmu.faults", ":",
  5651. fl->sctx->smmu.faults);
  5652. }
  5653. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5654. "\n=======%s %s %s======\n", title,
  5655. " LIST OF MAPS ", title);
  5656. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5657. "%-20s|%-20s|%-20s|%-20s\n", "va", "phys", "size", "flags");
  5658. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5659. "%s%s%s%s%s\n",
  5660. single_line, single_line, single_line,
  5661. single_line, single_line);
  5662. mutex_lock(&fl->map_mutex);
  5663. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5664. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5665. "0x%-20lX|0x%-20llX|0x%-20zu|0x%-17llX\n\n",
  5666. map->va, map->phys,
  5667. map->size, map->flags);
  5668. }
  5669. mutex_unlock(&fl->map_mutex);
  5670. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5671. "%-20s|%-20s|%-20s\n",
  5672. "len", "refs",
  5673. "raddr");
  5674. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5675. "%s%s%s%s%s\n",
  5676. single_line, single_line, single_line,
  5677. single_line, single_line);
  5678. mutex_lock(&fl->map_mutex);
  5679. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5680. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5681. "%-20zu|%-20d|0x%-20lX\n\n",
  5682. map->len, map->refs, map->raddr);
  5683. }
  5684. mutex_unlock(&fl->map_mutex);
  5685. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5686. "%-20s|%-20s\n", "secure", "attr");
  5687. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5688. "%s%s%s%s%s\n",
  5689. single_line, single_line, single_line,
  5690. single_line, single_line);
  5691. mutex_lock(&fl->map_mutex);
  5692. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  5693. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5694. "%-20d|0x%-20lX\n\n",
  5695. map->secure, map->attr);
  5696. }
  5697. mutex_unlock(&fl->map_mutex);
  5698. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5699. "\n======%s %s %s======\n", title,
  5700. " LIST OF BUFS ", title);
  5701. spin_lock(&fl->hlock);
  5702. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5703. "%-19s|%-19s|%-19s|%-19s\n",
  5704. "virt", "phys", "size", "flags");
  5705. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5706. "%s%s%s%s%s\n", single_line, single_line,
  5707. single_line, single_line, single_line);
  5708. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  5709. len += scnprintf(fileinfo + len,
  5710. DEBUGFS_SIZE - len,
  5711. "0x%-17p|0x%-17llX|%-19zu|0x%-17llX\n",
  5712. buf->virt, (uint64_t)buf->phys, buf->size, buf->flags);
  5713. }
  5714. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5715. "\n======%s %s %s======\n", title,
  5716. " LIST OF REMOTE BUFS ", title);
  5717. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5718. "%-19s|%-19s|%-19s|%-19s\n",
  5719. "virt", "phys", "size", "flags");
  5720. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5721. "%s%s%s%s%s\n", single_line, single_line,
  5722. single_line, single_line, single_line);
  5723. hlist_for_each_entry_safe(buf, n, &fl->remote_bufs, hn_rem) {
  5724. len += scnprintf(fileinfo + len,
  5725. DEBUGFS_SIZE - len,
  5726. "0x%-17p|0x%-17llX|%-19zu|0x%-17llX\n",
  5727. buf->virt, (uint64_t)buf->phys, buf->size, buf->flags);
  5728. }
  5729. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5730. "\n%s %s %s\n", title,
  5731. " LIST OF PENDING SMQCONTEXTS ", title);
  5732. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5733. "%-20s|%-10s|%-10s|%-10s|%-20s\n",
  5734. "sc", "pid", "tgid", "used", "ctxid");
  5735. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5736. "%s%s%s%s%s\n", single_line, single_line,
  5737. single_line, single_line, single_line);
  5738. hlist_for_each_entry_safe(ictx, n, &fl->clst.pending, hn) {
  5739. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5740. "0x%-18X|%-10d|%-10d|%-10zu|0x%-20llX\n\n",
  5741. ictx->sc, ictx->pid, ictx->tgid,
  5742. ictx->used, ictx->ctxid);
  5743. }
  5744. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5745. "\n%s %s %s\n", title,
  5746. " LIST OF INTERRUPTED SMQCONTEXTS ", title);
  5747. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5748. "%-20s|%-10s|%-10s|%-10s|%-20s\n",
  5749. "sc", "pid", "tgid", "used", "ctxid");
  5750. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5751. "%s%s%s%s%s\n", single_line, single_line,
  5752. single_line, single_line, single_line);
  5753. hlist_for_each_entry_safe(ictx, n, &fl->clst.interrupted, hn) {
  5754. len += scnprintf(fileinfo + len, DEBUGFS_SIZE - len,
  5755. "%-20u|%-20d|%-20d|%-20zu|0x%-20llX\n\n",
  5756. ictx->sc, ictx->pid, ictx->tgid,
  5757. ictx->used, ictx->ctxid);
  5758. }
  5759. spin_unlock(&fl->hlock);
  5760. }
  5761. if (len > DEBUGFS_SIZE)
  5762. len = DEBUGFS_SIZE;
  5763. ret = simple_read_from_buffer(buffer, count, position, fileinfo, len);
  5764. kfree(fileinfo);
  5765. bail:
  5766. return ret;
  5767. }
  5768. static const struct file_operations debugfs_fops = {
  5769. .open = simple_open,
  5770. .read = fastrpc_debugfs_read,
  5771. };
  5772. static int fastrpc_channel_open(struct fastrpc_file *fl, uint32_t flags)
  5773. {
  5774. struct fastrpc_apps *me = &gfa;
  5775. int cid = -1, err = 0;
  5776. VERIFY(err, fl && fl->sctx && fl->cid >= 0 && fl->cid < NUM_CHANNELS);
  5777. if (err) {
  5778. ADSPRPC_ERR("kernel session not initialized yet for %s\n",
  5779. current->comm);
  5780. err = -EBADR;
  5781. return err;
  5782. }
  5783. cid = fl->cid;
  5784. err = fastrpc_wait_for_transport_interrupt(cid, flags);
  5785. if (err)
  5786. goto bail;
  5787. err = verify_transport_device(cid, fl->tvm_remote_domain);
  5788. if (err)
  5789. goto bail;
  5790. mutex_lock(&me->channel[cid].smd_mutex);
  5791. if (me->channel[cid].ssrcount !=
  5792. me->channel[cid].prevssrcount) {
  5793. if (me->channel[cid].subsystemstate != SUBSYSTEM_UP) {
  5794. err = -ECONNREFUSED;
  5795. mutex_unlock(&me->channel[cid].smd_mutex);
  5796. goto bail;
  5797. }
  5798. }
  5799. fl->ssrcount = me->channel[cid].ssrcount;
  5800. if (cid == ADSP_DOMAIN_ID && me->channel[cid].ssrcount !=
  5801. me->channel[cid].prevssrcount) {
  5802. mutex_unlock(&me->channel[cid].smd_mutex);
  5803. mutex_lock(&fl->map_mutex);
  5804. err = fastrpc_mmap_remove_ssr(fl, 1);
  5805. mutex_unlock(&fl->map_mutex);
  5806. if (err)
  5807. ADSPRPC_WARN(
  5808. "failed to unmap remote heap for %s (err %d)\n",
  5809. me->channel[cid].subsys, err);
  5810. mutex_lock(&me->channel[cid].smd_mutex);
  5811. me->channel[cid].prevssrcount =
  5812. me->channel[cid].ssrcount;
  5813. }
  5814. me->channel[cid].in_hib = 0;
  5815. mutex_unlock(&me->channel[cid].smd_mutex);
  5816. bail:
  5817. return err;
  5818. }
  5819. static inline void fastrpc_register_wakeup_source(struct device *dev,
  5820. const char *client_name, struct wakeup_source **device_wake_source)
  5821. {
  5822. struct wakeup_source *wake_source = NULL;
  5823. wake_source = wakeup_source_register(dev, client_name);
  5824. if (IS_ERR_OR_NULL(wake_source)) {
  5825. ADSPRPC_ERR(
  5826. "wakeup_source_register failed for dev %s, client %s with err %ld\n",
  5827. dev_name(dev), client_name, PTR_ERR(wake_source));
  5828. return;
  5829. }
  5830. *device_wake_source = wake_source;
  5831. }
  5832. static int fastrpc_device_open(struct inode *inode, struct file *filp)
  5833. {
  5834. int err = 0;
  5835. struct fastrpc_file *fl = NULL;
  5836. struct fastrpc_apps *me = &gfa;
  5837. unsigned long irq_flags = 0;
  5838. /*
  5839. * Indicates the device node opened
  5840. * MINOR_NUM_DEV or MINOR_NUM_SECURE_DEV
  5841. */
  5842. int dev_minor = MINOR(inode->i_rdev);
  5843. VERIFY(err, ((dev_minor == MINOR_NUM_DEV) ||
  5844. (dev_minor == MINOR_NUM_SECURE_DEV)));
  5845. if (err) {
  5846. ADSPRPC_ERR("Invalid dev minor num %d\n",
  5847. dev_minor);
  5848. return err;
  5849. }
  5850. VERIFY(err, NULL != (fl = kzalloc(sizeof(*fl), GFP_KERNEL)));
  5851. if (err) {
  5852. err = -ENOMEM;
  5853. return err;
  5854. }
  5855. context_list_ctor(&fl->clst);
  5856. spin_lock_init(&fl->hlock);
  5857. spin_lock_init(&fl->aqlock);
  5858. spin_lock_init(&fl->proc_state_notif.nqlock);
  5859. INIT_HLIST_HEAD(&fl->maps);
  5860. INIT_HLIST_HEAD(&fl->cached_bufs);
  5861. fl->num_cached_buf = 0;
  5862. INIT_HLIST_HEAD(&fl->remote_bufs);
  5863. init_waitqueue_head(&fl->async_wait_queue);
  5864. init_waitqueue_head(&fl->proc_state_notif.notif_wait_queue);
  5865. INIT_HLIST_NODE(&fl->hn);
  5866. fl->sessionid = 0;
  5867. fl->tgid_open = current->tgid;
  5868. /* PD type is not known, when device is opened */
  5869. fl->pd_type = DEFAULT_UNUSED;
  5870. fl->apps = me;
  5871. fl->mode = FASTRPC_MODE_SERIAL;
  5872. fl->cid = -1;
  5873. fl->tgid_frpc = -1;
  5874. fl->tvm_remote_domain = -1;
  5875. fl->dev_minor = dev_minor;
  5876. fl->init_mem = NULL;
  5877. fl->qos_request = 0;
  5878. fl->dsp_proc_init = 0;
  5879. fl->is_ramdump_pend = false;
  5880. fl->dsp_process_state = PROCESS_CREATE_DEFAULT;
  5881. fl->is_unsigned_pd = false;
  5882. fl->is_compat = false;
  5883. fl->exit_notif = false;
  5884. fl->exit_async = false;
  5885. fl->multi_session_support = false;
  5886. fl->set_session_info = false;
  5887. init_completion(&fl->work);
  5888. init_completion(&fl->dma_invoke);
  5889. fl->file_close = FASTRPC_PROCESS_DEFAULT_STATE;
  5890. filp->private_data = fl;
  5891. fl->sharedbuf_info.buf_fd = -1;
  5892. mutex_init(&fl->internal_map_mutex);
  5893. mutex_init(&fl->map_mutex);
  5894. spin_lock_irqsave(&me->hlock, irq_flags);
  5895. hlist_add_head(&fl->hn, &me->drivers);
  5896. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5897. if (me->lowest_capacity_core_count)
  5898. fl->dev_pm_qos_req = kzalloc((me->lowest_capacity_core_count) *
  5899. sizeof(struct dev_pm_qos_request),
  5900. GFP_KERNEL);
  5901. spin_lock_init(&fl->dspsignals_lock);
  5902. mutex_init(&fl->signal_create_mutex);
  5903. init_completion(&fl->shutdown);
  5904. return 0;
  5905. }
  5906. static int fastrpc_get_process_gids(struct gid_list *gidlist)
  5907. {
  5908. struct group_info *group_info = get_current_groups();
  5909. int i = 0, err = 0, num_gids = group_info->ngroups + 1;
  5910. unsigned int *gids = NULL;
  5911. gids = kcalloc(num_gids, sizeof(unsigned int), GFP_KERNEL);
  5912. if (!gids) {
  5913. err = -ENOMEM;
  5914. goto bail;
  5915. }
  5916. /* Get the real GID */
  5917. gids[0] = __kgid_val(current_gid());
  5918. /* Get the supplemental GIDs */
  5919. for (i = 1; i < num_gids; i++)
  5920. gids[i] = __kgid_val(group_info->gid[i - 1]);
  5921. sort(gids, num_gids, sizeof(*gids), uint_cmp_func, NULL);
  5922. gidlist->gids = gids;
  5923. gidlist->gidcount = num_gids;
  5924. bail:
  5925. if (err)
  5926. kfree(gids);
  5927. return err;
  5928. }
  5929. // Generate a unique process ID to DSP process
  5930. static int get_unique_hlos_process_id(void)
  5931. {
  5932. int tgid_frpc = -1, tgid_index = 1;
  5933. struct fastrpc_apps *me = &gfa;
  5934. unsigned long irq_flags = 0;
  5935. spin_lock_irqsave(&me->hlock, irq_flags);
  5936. for (tgid_index = 1; tgid_index < MAX_FRPC_TGID; tgid_index++) {
  5937. if (!frpc_tgid_usage_array[tgid_index]) {
  5938. tgid_frpc = tgid_index;
  5939. /* Set the tgid usage to false */
  5940. frpc_tgid_usage_array[tgid_index] = true;
  5941. break;
  5942. }
  5943. }
  5944. spin_unlock_irqrestore(&me->hlock, irq_flags);
  5945. return tgid_frpc;
  5946. }
  5947. static int fastrpc_set_process_info(struct fastrpc_file *fl, uint32_t cid)
  5948. {
  5949. int err = 0, buf_size = 0;
  5950. char strpid[PID_SIZE];
  5951. char cur_comm[TASK_COMM_LEN];
  5952. memcpy(cur_comm, current->comm, TASK_COMM_LEN);
  5953. cur_comm[TASK_COMM_LEN-1] = '\0';
  5954. fl->tgid = current->tgid;
  5955. fl->tgid_frpc = get_unique_hlos_process_id();
  5956. VERIFY(err, fl->tgid_frpc != -1);
  5957. if (err) {
  5958. ADSPRPC_ERR("too many fastrpc clients, max %u allowed\n", MAX_FRPC_TGID);
  5959. err = -EUSERS;
  5960. goto bail;
  5961. }
  5962. ADSPRPC_INFO("HLOS pid %d, cid %d is mapped to unique sessions pid %d",
  5963. fl->tgid, cid, fl->tgid_frpc);
  5964. /*
  5965. * Third-party apps don't have permission to open the fastrpc device, so
  5966. * it is opened on their behalf by DSP HAL. This is detected by
  5967. * comparing current PID with the one stored during device open.
  5968. */
  5969. if (current->tgid != fl->tgid_open)
  5970. fl->untrusted_process = true;
  5971. snprintf(strpid, PID_SIZE, "%d", current->pid);
  5972. if (debugfs_root) {
  5973. VERIFY(err, VALID_FASTRPC_CID(cid));
  5974. if (err) {
  5975. err = -ECHRNG;
  5976. goto bail;
  5977. }
  5978. buf_size = strlen(cur_comm) + strlen("_") + strlen(strpid)
  5979. + strlen("_") + strlen(__TOSTR__(NUM_CHANNELS)) + 1;
  5980. spin_lock(&fl->hlock);
  5981. if (fl->debug_buf_alloced_attempted) {
  5982. spin_unlock(&fl->hlock);
  5983. return err;
  5984. }
  5985. fl->debug_buf_alloced_attempted = 1;
  5986. spin_unlock(&fl->hlock);
  5987. fl->debug_buf = kzalloc(buf_size, GFP_KERNEL);
  5988. if (!fl->debug_buf) {
  5989. err = -ENOMEM;
  5990. return err;
  5991. }
  5992. /* Use HLOS PID, unique fastrpc PID, CID in debugfs filename,
  5993. * for better ability to debug
  5994. */
  5995. snprintf(fl->debug_buf, buf_size, "%.10s%s%d%s%d%s%d",
  5996. cur_comm, "_", current->pid, "_", fl->tgid_frpc, "_", cid);
  5997. fl->debugfs_file = debugfs_create_file(fl->debug_buf, 0644,
  5998. debugfs_root, fl, &debugfs_fops);
  5999. if (IS_ERR_OR_NULL(fl->debugfs_file)) {
  6000. pr_warn("Error: %s: %s: failed to create debugfs file %s\n",
  6001. cur_comm, __func__, fl->debug_buf);
  6002. fl->debugfs_file = NULL;
  6003. }
  6004. kfree(fl->debug_buf);
  6005. fl->debug_buf = NULL;
  6006. }
  6007. bail:
  6008. return err;
  6009. }
  6010. int fastrpc_get_info(struct fastrpc_file *fl, uint32_t *info)
  6011. {
  6012. int err = 0;
  6013. uint32_t cid = *info;
  6014. struct fastrpc_apps *me = &gfa;
  6015. VERIFY(err, fl != NULL);
  6016. if (err) {
  6017. err = -EBADF;
  6018. goto bail;
  6019. }
  6020. spin_lock(&fl->hlock);
  6021. if (fl->set_session_info) {
  6022. spin_unlock(&fl->hlock);
  6023. ADSPRPC_ERR("Set session info invoked multiple times\n");
  6024. err = -EBADR;
  6025. goto bail;
  6026. }
  6027. // Set set_session_info to true
  6028. fl->set_session_info = true;
  6029. spin_unlock(&fl->hlock);
  6030. VERIFY(err, VALID_FASTRPC_CID(cid));
  6031. if (err) {
  6032. err = -ECHRNG;
  6033. goto bail;
  6034. }
  6035. fastrpc_get_process_gids(&fl->gidlist);
  6036. err = fastrpc_set_process_info(fl, cid);
  6037. if (err)
  6038. goto bail;
  6039. if (fl->cid == -1) {
  6040. struct fastrpc_channel_ctx *chan = NULL;
  6041. chan = &me->channel[cid];
  6042. /* Check to see if the device node is non-secure */
  6043. if (fl->dev_minor == MINOR_NUM_DEV) {
  6044. /*
  6045. * If an app is trying to offload to a secure remote
  6046. * channel by opening the non-secure device node, allow
  6047. * the access if the subsystem supports unsigned
  6048. * offload. Untrusted apps will be restricted from
  6049. * offloading to signed PD using DSP HAL.
  6050. */
  6051. if (chan->secure == SECURE_CHANNEL
  6052. && !chan->unsigned_support) {
  6053. ADSPRPC_ERR(
  6054. "cannot use domain %d with non-secure device\n",
  6055. cid);
  6056. err = -EACCES;
  6057. goto bail;
  6058. }
  6059. }
  6060. fl->cid = cid;
  6061. fl->ssrcount = fl->apps->channel[cid].ssrcount;
  6062. mutex_lock(&fl->apps->channel[cid].smd_mutex);
  6063. err = fastrpc_session_alloc_locked(&fl->apps->channel[cid],
  6064. 0, fl->sharedcb, fl->pd_type, &fl->sctx);
  6065. mutex_unlock(&fl->apps->channel[cid].smd_mutex);
  6066. if (err == -EUSERS) {
  6067. ADSPRPC_WARN(
  6068. "max concurrent sessions limit (%d) already reached on %s err %d\n",
  6069. chan->sesscount, chan->subsys, err);
  6070. }
  6071. if (err)
  6072. goto bail;
  6073. }
  6074. VERIFY(err, fl->sctx != NULL);
  6075. if (err) {
  6076. err = -EBADR;
  6077. goto bail;
  6078. }
  6079. *info = (fl->sctx->smmu.enabled ? 1 : 0);
  6080. bail:
  6081. return err;
  6082. }
  6083. static int fastrpc_manage_poll_mode(struct fastrpc_file *fl, uint32_t enable, uint32_t timeout)
  6084. {
  6085. int err = 0;
  6086. const unsigned int MAX_POLL_TIMEOUT_US = 10000;
  6087. if ((fl->cid != CDSP_DOMAIN_ID) || (fl->proc_flags != FASTRPC_INIT_CREATE)) {
  6088. err = -EPERM;
  6089. ADSPRPC_ERR("flags %d, cid %d, poll mode allowed only for dynamic CDSP process\n",
  6090. fl->proc_flags, fl->cid);
  6091. goto bail;
  6092. }
  6093. if (timeout > MAX_POLL_TIMEOUT_US) {
  6094. err = -EBADMSG;
  6095. ADSPRPC_ERR("poll timeout %u is greater than max allowed value %u\n",
  6096. timeout, MAX_POLL_TIMEOUT_US);
  6097. goto bail;
  6098. }
  6099. spin_lock(&fl->hlock);
  6100. if (enable) {
  6101. fl->poll_mode = true;
  6102. fl->poll_timeout = timeout;
  6103. } else {
  6104. fl->poll_mode = false;
  6105. fl->poll_timeout = 0;
  6106. }
  6107. spin_unlock(&fl->hlock);
  6108. ADSPRPC_INFO("updated poll mode to %d, timeout %u\n", enable, timeout);
  6109. bail:
  6110. return err;
  6111. }
  6112. int fastrpc_internal_control(struct fastrpc_file *fl,
  6113. struct fastrpc_ioctl_control *cp)
  6114. {
  6115. int err = 0;
  6116. unsigned int latency;
  6117. struct fastrpc_apps *me = &gfa;
  6118. unsigned int cpu;
  6119. unsigned long flags = 0;
  6120. VERIFY(err, !IS_ERR_OR_NULL(fl) && !IS_ERR_OR_NULL(fl->apps));
  6121. if (err) {
  6122. err = -EBADF;
  6123. goto bail;
  6124. }
  6125. VERIFY(err, !IS_ERR_OR_NULL(cp));
  6126. if (err) {
  6127. err = -EINVAL;
  6128. goto bail;
  6129. }
  6130. switch (cp->req) {
  6131. case FASTRPC_CONTROL_LATENCY:
  6132. latency = cp->lp.enable == FASTRPC_LATENCY_CTRL_ENB ?
  6133. fl->apps->latency : PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
  6134. VERIFY(err, latency != 0);
  6135. if (err) {
  6136. err = -EINVAL;
  6137. goto bail;
  6138. }
  6139. VERIFY(err, (me->lowest_capacity_core_count && fl->dev_pm_qos_req));
  6140. if (err) {
  6141. ADSPRPC_INFO("Skipping PM QoS latency voting, core count: %u\n",
  6142. me->lowest_capacity_core_count);
  6143. err = -EINVAL;
  6144. goto bail;
  6145. }
  6146. /*
  6147. * Add voting request for all possible cores corresponding to cluster
  6148. * id 0. If DT property 'qcom,single-core-latency-vote' is enabled
  6149. * then add voting request for only one core of cluster id 0.
  6150. */
  6151. for (cpu = 0; cpu < me->lowest_capacity_core_count; cpu++) {
  6152. if (!fl->qos_request) {
  6153. err = dev_pm_qos_add_request(
  6154. get_cpu_device(cpu),
  6155. &fl->dev_pm_qos_req[cpu],
  6156. DEV_PM_QOS_RESUME_LATENCY,
  6157. latency);
  6158. } else {
  6159. err = dev_pm_qos_update_request(
  6160. &fl->dev_pm_qos_req[cpu],
  6161. latency);
  6162. }
  6163. /* PM QoS request APIs return 0 or 1 on success */
  6164. if (err < 0) {
  6165. ADSPRPC_WARN("QoS with lat %u failed for CPU %d, err %d, req %d\n",
  6166. latency, cpu, err, fl->qos_request);
  6167. break;
  6168. }
  6169. }
  6170. if (err >= 0) {
  6171. fl->qos_request = 1;
  6172. err = 0;
  6173. }
  6174. /* Ensure CPU feature map updated to DSP for early WakeUp */
  6175. fastrpc_send_cpuinfo_to_dsp(fl);
  6176. break;
  6177. case FASTRPC_CONTROL_KALLOC:
  6178. cp->kalloc.kalloc_support = 1;
  6179. break;
  6180. case FASTRPC_CONTROL_WAKELOCK:
  6181. if (fl->dev_minor != MINOR_NUM_SECURE_DEV) {
  6182. ADSPRPC_ERR(
  6183. "PM voting not allowed for non-secure device node %d\n",
  6184. fl->dev_minor);
  6185. err = -EPERM;
  6186. goto bail;
  6187. }
  6188. fl->wake_enable = cp->wp.enable;
  6189. break;
  6190. case FASTRPC_CONTROL_PM:
  6191. if (!fl->wake_enable) {
  6192. /* Kernel PM voting not requested by this application */
  6193. err = -EACCES;
  6194. goto bail;
  6195. }
  6196. if (cp->pm.timeout > MAX_PM_TIMEOUT_MS)
  6197. fl->ws_timeout = MAX_PM_TIMEOUT_MS;
  6198. else
  6199. fl->ws_timeout = cp->pm.timeout;
  6200. VERIFY(err, VALID_FASTRPC_CID(fl->cid));
  6201. if (err) {
  6202. err = -ECHRNG;
  6203. goto bail;
  6204. }
  6205. fastrpc_pm_awake(fl, gcinfo[fl->cid].secure);
  6206. break;
  6207. case FASTRPC_CONTROL_DSPPROCESS_CLEAN:
  6208. (void)fastrpc_release_current_dsp_process(fl);
  6209. fastrpc_queue_pd_status(fl, fl->cid, FASTRPC_USER_PD_FORCE_KILL, fl->sessionid);
  6210. break;
  6211. case FASTRPC_CONTROL_RPC_POLL:
  6212. err = fastrpc_manage_poll_mode(fl, cp->lp.enable, cp->lp.latency);
  6213. if (err)
  6214. goto bail;
  6215. break;
  6216. case FASTRPC_CONTROL_SMMU:
  6217. fl->sharedcb = cp->smmu.sharedcb;
  6218. break;
  6219. case FASTRPC_CONTROL_ASYNC_WAKE:
  6220. fl->exit_async = true;
  6221. spin_lock_irqsave(&fl->aqlock, flags);
  6222. atomic_add(1, &fl->async_queue_job_count);
  6223. wake_up_interruptible(&fl->async_wait_queue);
  6224. spin_unlock_irqrestore(&fl->aqlock, flags);
  6225. break;
  6226. case FASTRPC_CONTROL_NOTIF_WAKE:
  6227. fl->exit_notif = true;
  6228. spin_lock_irqsave(&fl->proc_state_notif.nqlock, flags);
  6229. atomic_add(1, &fl->proc_state_notif.notif_queue_count);
  6230. wake_up_interruptible(&fl->proc_state_notif.notif_wait_queue);
  6231. spin_unlock_irqrestore(&fl->proc_state_notif.nqlock, flags);
  6232. break;
  6233. default:
  6234. err = -EBADRQC;
  6235. break;
  6236. }
  6237. bail:
  6238. return err;
  6239. }
  6240. /* Wait for PD to be up before audio or sensors daemons try connecting */
  6241. static int fastrpc_check_pd_status(struct fastrpc_file *fl, char *sloc_name)
  6242. {
  6243. int err = 0, session = -1, cid = -1;
  6244. struct fastrpc_apps *me = &gfa;
  6245. if (fl->servloc_name && sloc_name
  6246. && !strcmp(fl->servloc_name, sloc_name)) {
  6247. err = fastrpc_get_spd_session(sloc_name, &session, &cid);
  6248. if (err || cid != fl->cid)
  6249. goto bail;
  6250. #if IS_ENABLED(CONFIG_QCOM_PDR_HELPERS)
  6251. if (!strcmp(fl->servloc_name,
  6252. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME) || !strcmp(fl->servloc_name,
  6253. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME) ||
  6254. !strcmp(fl->servloc_name,
  6255. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME)) {
  6256. err = wait_event_interruptible(
  6257. me->channel[cid].spd[session].wait_for_pdup,
  6258. atomic_read(&me->channel[cid].spd[session].ispdup));
  6259. goto bail;
  6260. }
  6261. #else
  6262. (void)me;
  6263. #endif
  6264. }
  6265. bail:
  6266. return err;
  6267. }
  6268. int fastrpc_setmode(unsigned long ioctl_param,
  6269. struct fastrpc_file *fl)
  6270. {
  6271. int err = 0;
  6272. switch ((uint32_t)ioctl_param) {
  6273. case FASTRPC_MODE_PARALLEL:
  6274. case FASTRPC_MODE_SERIAL:
  6275. fl->mode = (uint32_t)ioctl_param;
  6276. break;
  6277. case FASTRPC_MODE_PROFILE:
  6278. fl->profile = (uint32_t)ioctl_param;
  6279. break;
  6280. case FASTRPC_MODE_SESSION:
  6281. if (!fl->multi_session_support)
  6282. fl->sessionid = 1;
  6283. break;
  6284. default:
  6285. err = -ENOTTY;
  6286. break;
  6287. }
  6288. return err;
  6289. }
  6290. int fastrpc_control(struct fastrpc_ioctl_control *cp,
  6291. void *param, struct fastrpc_file *fl)
  6292. {
  6293. int err = 0;
  6294. K_COPY_FROM_USER(err, 0, cp, param,
  6295. sizeof(*cp));
  6296. if (err) {
  6297. err = -EFAULT;
  6298. goto bail;
  6299. }
  6300. VERIFY(err, 0 == (err = fastrpc_internal_control(fl, cp)));
  6301. if (err)
  6302. goto bail;
  6303. if (cp->req == FASTRPC_CONTROL_KALLOC) {
  6304. K_COPY_TO_USER(err, 0, param, cp, sizeof(*cp));
  6305. if (err) {
  6306. err = -EFAULT;
  6307. goto bail;
  6308. }
  6309. }
  6310. bail:
  6311. return err;
  6312. }
  6313. static int fastrpc_get_dsp_info(
  6314. struct fastrpc_ioctl_capability *cap,
  6315. void *param, struct fastrpc_file *fl)
  6316. {
  6317. int err = 0;
  6318. K_COPY_FROM_USER(err, 0, cap, param,
  6319. sizeof(struct fastrpc_ioctl_capability));
  6320. VERIFY(err, cap->domain < NUM_CHANNELS);
  6321. if (err) {
  6322. err = -ECHRNG;
  6323. goto bail;
  6324. }
  6325. cap->capability = 0;
  6326. err = fastrpc_get_info_from_kernel(cap, fl);
  6327. if (err)
  6328. goto bail;
  6329. K_COPY_TO_USER(err, 0, &((struct fastrpc_ioctl_capability *)
  6330. param)->capability, &cap->capability, sizeof(cap->capability));
  6331. bail:
  6332. return err;
  6333. }
  6334. int fastrpc_dspsignal_signal(struct fastrpc_file *fl,
  6335. struct fastrpc_ioctl_dspsignal_signal *sig)
  6336. {
  6337. int err = 0, cid = -1;
  6338. struct fastrpc_channel_ctx *channel_ctx = NULL;
  6339. uint64_t msg = 0;
  6340. // We don't check if the signal has even been allocated since we don't
  6341. // track outgoing signals in the driver. The userspace library does a
  6342. // basic sanity check and any security validation needs to be done by
  6343. // the recipient.
  6344. DSPSIGNAL_VERBOSE("Send signal PID %u, unique fastrpc pid %u signal %u\n",
  6345. (unsigned int)fl->tgid, (unsigned int)fl->tgid_frpc,
  6346. (unsigned int)sig->signal_id);
  6347. VERIFY(err, sig->signal_id < DSPSIGNAL_NUM_SIGNALS);
  6348. if (err) {
  6349. ADSPRPC_ERR("Sending bad signal %u for PID %u",
  6350. sig->signal_id, (unsigned int)fl->tgid);
  6351. err = -EBADR;
  6352. goto bail;
  6353. }
  6354. cid = fl->cid;
  6355. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6356. if (err) {
  6357. err = -EBADR;
  6358. goto bail;
  6359. }
  6360. channel_ctx = &fl->apps->channel[cid];
  6361. mutex_lock(&channel_ctx->smd_mutex);
  6362. if (fl->ssrcount != channel_ctx->ssrcount) {
  6363. err = -ECONNRESET;
  6364. mutex_unlock(&channel_ctx->smd_mutex);
  6365. goto bail;
  6366. }
  6367. /* Use unique fastrpc pid, to signal DSP process */
  6368. msg = (((uint64_t)fl->tgid_frpc) << 32) | ((uint64_t)sig->signal_id);
  6369. err = fastrpc_transport_send(cid, (void *)&msg, sizeof(msg), fl->tvm_remote_domain);
  6370. mutex_unlock(&channel_ctx->smd_mutex);
  6371. trace_fastrpc_dspsignal("signal", sig->signal_id, 0, 0);
  6372. bail:
  6373. return err;
  6374. }
  6375. int fastrpc_dspsignal_wait(struct fastrpc_file *fl,
  6376. struct fastrpc_ioctl_dspsignal_wait *wait)
  6377. {
  6378. int err = 0, cid = -1;
  6379. unsigned long timeout = usecs_to_jiffies(wait->timeout_usec);
  6380. uint32_t signal_id = wait->signal_id;
  6381. struct fastrpc_dspsignal *s = NULL;
  6382. long ret = 0;
  6383. unsigned long irq_flags = 0;
  6384. DSPSIGNAL_VERBOSE("Wait for signal %u\n", signal_id);
  6385. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6386. if (err) {
  6387. ADSPRPC_ERR("Waiting on bad signal %u", signal_id);
  6388. err = -EINVAL;
  6389. goto bail;
  6390. }
  6391. cid = fl->cid;
  6392. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6393. if (err) {
  6394. err = -EBADR;
  6395. goto bail;
  6396. }
  6397. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6398. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6399. struct fastrpc_dspsignal *group =
  6400. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6401. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6402. }
  6403. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6404. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6405. ADSPRPC_ERR("Unknown signal id %u\n", signal_id);
  6406. err = -ENOENT;
  6407. goto bail;
  6408. }
  6409. if (s->state != DSPSIGNAL_STATE_PENDING) {
  6410. if ((s->state == DSPSIGNAL_STATE_CANCELED) || (s->state == DSPSIGNAL_STATE_UNUSED))
  6411. err = -EINTR;
  6412. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6413. DSPSIGNAL_VERBOSE("Signal %u in state %u, complete wait immediately",
  6414. signal_id, s->state);
  6415. goto bail;
  6416. }
  6417. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6418. trace_fastrpc_dspsignal("wait", signal_id, s->state, wait->timeout_usec);
  6419. if (timeout != 0xffffffff)
  6420. ret = wait_for_completion_interruptible_timeout(&s->comp, timeout);
  6421. else
  6422. ret = wait_for_completion_interruptible(&s->comp);
  6423. trace_fastrpc_dspsignal("wakeup", signal_id, s->state, wait->timeout_usec);
  6424. if (ret == 0) {
  6425. DSPSIGNAL_VERBOSE("Wait for signal %u timed out\n", signal_id);
  6426. err = -ETIMEDOUT;
  6427. goto bail;
  6428. } else if (ret < 0) {
  6429. ADSPRPC_ERR("Wait for signal %u failed %d\n", signal_id, (int)ret);
  6430. err = ret;
  6431. goto bail;
  6432. }
  6433. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6434. if (s->state == DSPSIGNAL_STATE_SIGNALED) {
  6435. s->state = DSPSIGNAL_STATE_PENDING;
  6436. DSPSIGNAL_VERBOSE("Signal %u completed\n", signal_id);
  6437. } else if ((s->state == DSPSIGNAL_STATE_CANCELED) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6438. DSPSIGNAL_VERBOSE("Signal %u cancelled or destroyed\n", signal_id);
  6439. err = -EINTR;
  6440. }
  6441. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6442. bail:
  6443. return err;
  6444. }
  6445. int fastrpc_dspsignal_create(struct fastrpc_file *fl,
  6446. struct fastrpc_ioctl_dspsignal_create *create)
  6447. {
  6448. int err = 0, cid = -1;
  6449. uint32_t signal_id = create->signal_id;
  6450. struct fastrpc_dspsignal *group, *sig;
  6451. unsigned long irq_flags = 0;
  6452. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6453. if (err) {
  6454. err = -EINVAL;
  6455. goto bail;
  6456. }
  6457. cid = fl->cid;
  6458. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6459. if (err) {
  6460. err = -EBADR;
  6461. goto bail;
  6462. }
  6463. // Use a separate mutex for creating signals. This avoids holding on
  6464. // to a spinlock if we need to allocate a whole group of signals. The
  6465. // mutex ensures nobody else will allocate the same group.
  6466. mutex_lock(&fl->signal_create_mutex);
  6467. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6468. group = fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6469. if (group == NULL) {
  6470. int i;
  6471. // Release the spinlock while we allocate a new group but take
  6472. // it back before taking the group into use. No other code
  6473. // allocates groups so the mutex is sufficient.
  6474. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6475. VERIFY(err, (group = kzalloc(DSPSIGNAL_GROUP_SIZE * sizeof(*group),
  6476. GFP_KERNEL)) != NULL);
  6477. if (err) {
  6478. ADSPRPC_ERR("Unable to allocate signal group\n");
  6479. err = -ENOMEM;
  6480. mutex_unlock(&fl->signal_create_mutex);
  6481. goto bail;
  6482. }
  6483. for (i = 0; i < DSPSIGNAL_GROUP_SIZE; i++) {
  6484. sig = &group[i];
  6485. init_completion(&sig->comp);
  6486. sig->state = DSPSIGNAL_STATE_UNUSED;
  6487. }
  6488. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6489. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] = group;
  6490. }
  6491. sig = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6492. if (sig->state != DSPSIGNAL_STATE_UNUSED) {
  6493. err = -EBUSY;
  6494. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6495. mutex_unlock(&fl->signal_create_mutex);
  6496. ADSPRPC_ERR("Attempting to create signal %u already in use (state %u)\n",
  6497. signal_id, sig->state);
  6498. goto bail;
  6499. }
  6500. sig->state = DSPSIGNAL_STATE_PENDING;
  6501. reinit_completion(&sig->comp);
  6502. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6503. mutex_unlock(&fl->signal_create_mutex);
  6504. DSPSIGNAL_VERBOSE("Signal %u created\n", signal_id);
  6505. bail:
  6506. return err;
  6507. }
  6508. int fastrpc_dspsignal_destroy(struct fastrpc_file *fl,
  6509. struct fastrpc_ioctl_dspsignal_destroy *destroy)
  6510. {
  6511. int err = 0, cid = -1;
  6512. uint32_t signal_id = destroy->signal_id;
  6513. struct fastrpc_dspsignal *s = NULL;
  6514. unsigned long irq_flags = 0;
  6515. DSPSIGNAL_VERBOSE("Destroy signal %u\n", signal_id);
  6516. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6517. if (err) {
  6518. err = -EINVAL;
  6519. goto bail;
  6520. }
  6521. cid = fl->cid;
  6522. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6523. if (err) {
  6524. err = -EBADR;
  6525. goto bail;
  6526. }
  6527. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6528. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6529. struct fastrpc_dspsignal *group =
  6530. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6531. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6532. }
  6533. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6534. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6535. ADSPRPC_ERR("Attempting to destroy unused signal %u\n", signal_id);
  6536. err = -ENOENT;
  6537. goto bail;
  6538. }
  6539. s->state = DSPSIGNAL_STATE_UNUSED;
  6540. complete_all(&s->comp);
  6541. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6542. DSPSIGNAL_VERBOSE("Signal %u destroyed\n", signal_id);
  6543. bail:
  6544. return err;
  6545. }
  6546. int fastrpc_dspsignal_cancel_wait(struct fastrpc_file *fl,
  6547. struct fastrpc_ioctl_dspsignal_cancel_wait *cancel)
  6548. {
  6549. int err = 0, cid = -1;
  6550. uint32_t signal_id = cancel->signal_id;
  6551. struct fastrpc_dspsignal *s = NULL;
  6552. unsigned long irq_flags = 0;
  6553. DSPSIGNAL_VERBOSE("Cancel wait for signal %u\n", signal_id);
  6554. VERIFY(err, signal_id < DSPSIGNAL_NUM_SIGNALS);
  6555. if (err) {
  6556. err = -EINVAL;
  6557. goto bail;
  6558. }
  6559. cid = fl->cid;
  6560. VERIFY(err, VALID_FASTRPC_CID(cid) && fl->sctx != NULL);
  6561. if (err) {
  6562. err = -EBADR;
  6563. goto bail;
  6564. }
  6565. spin_lock_irqsave(&fl->dspsignals_lock, irq_flags);
  6566. if (fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE] != NULL) {
  6567. struct fastrpc_dspsignal *group =
  6568. fl->signal_groups[signal_id / DSPSIGNAL_GROUP_SIZE];
  6569. s = &group[signal_id % DSPSIGNAL_GROUP_SIZE];
  6570. }
  6571. if ((s == NULL) || (s->state == DSPSIGNAL_STATE_UNUSED)) {
  6572. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6573. ADSPRPC_ERR("Attempting to cancel unused signal %u\n", signal_id);
  6574. err = -ENOENT;
  6575. goto bail;
  6576. }
  6577. if (s->state != DSPSIGNAL_STATE_CANCELED) {
  6578. s->state = DSPSIGNAL_STATE_CANCELED;
  6579. trace_fastrpc_dspsignal("cancel", signal_id, s->state, 0);
  6580. complete_all(&s->comp);
  6581. }
  6582. spin_unlock_irqrestore(&fl->dspsignals_lock, irq_flags);
  6583. DSPSIGNAL_VERBOSE("Signal %u cancelled\n", signal_id);
  6584. bail:
  6585. return err;
  6586. }
  6587. static inline int fastrpc_mmap_device_ioctl(struct fastrpc_file *fl,
  6588. unsigned int ioctl_num, union fastrpc_ioctl_param *p,
  6589. void *param)
  6590. {
  6591. union {
  6592. struct fastrpc_ioctl_mmap mmap;
  6593. struct fastrpc_ioctl_munmap munmap;
  6594. } i;
  6595. int err = 0;
  6596. switch (ioctl_num) {
  6597. case FASTRPC_IOCTL_MEM_MAP:
  6598. K_COPY_FROM_USER(err, 0, &p->mem_map, param,
  6599. sizeof(p->mem_map));
  6600. if (err) {
  6601. err = -EFAULT;
  6602. goto bail;
  6603. }
  6604. VERIFY(err, 0 == (err = fastrpc_internal_mem_map(fl,
  6605. &p->mem_map)));
  6606. if (err)
  6607. goto bail;
  6608. K_COPY_TO_USER(err, 0, param, &p->mem_map, sizeof(p->mem_map));
  6609. if (err) {
  6610. err = -EFAULT;
  6611. goto bail;
  6612. }
  6613. break;
  6614. case FASTRPC_IOCTL_MEM_UNMAP:
  6615. K_COPY_FROM_USER(err, 0, &p->mem_unmap, param,
  6616. sizeof(p->mem_unmap));
  6617. if (err) {
  6618. err = -EFAULT;
  6619. goto bail;
  6620. }
  6621. VERIFY(err, 0 == (err = fastrpc_internal_mem_unmap(fl,
  6622. &p->mem_unmap)));
  6623. if (err)
  6624. goto bail;
  6625. K_COPY_TO_USER(err, 0, param, &p->mem_unmap,
  6626. sizeof(p->mem_unmap));
  6627. if (err) {
  6628. err = -EFAULT;
  6629. goto bail;
  6630. }
  6631. break;
  6632. case FASTRPC_IOCTL_MMAP:
  6633. K_COPY_FROM_USER(err, 0, &p->mmap, param,
  6634. sizeof(p->mmap));
  6635. if (err) {
  6636. err = -EFAULT;
  6637. goto bail;
  6638. }
  6639. VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &p->mmap)));
  6640. if (err)
  6641. goto bail;
  6642. K_COPY_TO_USER(err, 0, param, &p->mmap, sizeof(p->mmap));
  6643. if (err) {
  6644. err = -EFAULT;
  6645. goto bail;
  6646. }
  6647. break;
  6648. case FASTRPC_IOCTL_MUNMAP:
  6649. K_COPY_FROM_USER(err, 0, &p->munmap, param,
  6650. sizeof(p->munmap));
  6651. if (err) {
  6652. err = -EFAULT;
  6653. goto bail;
  6654. }
  6655. VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl,
  6656. &p->munmap)));
  6657. if (err)
  6658. goto bail;
  6659. break;
  6660. case FASTRPC_IOCTL_MMAP_64:
  6661. K_COPY_FROM_USER(err, 0, &p->mmap64, param,
  6662. sizeof(p->mmap64));
  6663. if (err) {
  6664. err = -EFAULT;
  6665. goto bail;
  6666. }
  6667. get_fastrpc_ioctl_mmap_64(&p->mmap64, &i.mmap);
  6668. VERIFY(err, 0 == (err = fastrpc_internal_mmap(fl, &i.mmap)));
  6669. if (err)
  6670. goto bail;
  6671. put_fastrpc_ioctl_mmap_64(&p->mmap64, &i.mmap);
  6672. K_COPY_TO_USER(err, 0, param, &p->mmap64, sizeof(p->mmap64));
  6673. if (err) {
  6674. err = -EFAULT;
  6675. goto bail;
  6676. }
  6677. break;
  6678. case FASTRPC_IOCTL_MUNMAP_64:
  6679. K_COPY_FROM_USER(err, 0, &p->munmap64, param,
  6680. sizeof(p->munmap64));
  6681. if (err) {
  6682. err = -EFAULT;
  6683. goto bail;
  6684. }
  6685. get_fastrpc_ioctl_munmap_64(&p->munmap64, &i.munmap);
  6686. VERIFY(err, 0 == (err = fastrpc_internal_munmap(fl,
  6687. &i.munmap)));
  6688. if (err)
  6689. goto bail;
  6690. break;
  6691. case FASTRPC_IOCTL_MUNMAP_FD:
  6692. K_COPY_FROM_USER(err, 0, &p->munmap_fd, param,
  6693. sizeof(p->munmap_fd));
  6694. if (err) {
  6695. err = -EFAULT;
  6696. goto bail;
  6697. }
  6698. VERIFY(err, 0 == (err = fastrpc_internal_munmap_fd(fl,
  6699. &p->munmap_fd)));
  6700. if (err)
  6701. goto bail;
  6702. break;
  6703. default:
  6704. err = -ENOTTY;
  6705. pr_info("bad ioctl: %d\n", ioctl_num);
  6706. break;
  6707. }
  6708. bail:
  6709. return err;
  6710. }
  6711. static long fastrpc_device_ioctl(struct file *file, unsigned int ioctl_num,
  6712. unsigned long ioctl_param)
  6713. {
  6714. union fastrpc_ioctl_param p;
  6715. void *param = (char *)ioctl_param;
  6716. struct fastrpc_file *fl = (struct fastrpc_file *)file->private_data;
  6717. int size = 0, err = 0;
  6718. uint32_t info;
  6719. p.inv.fds = NULL;
  6720. p.inv.attrs = NULL;
  6721. p.inv.crc = NULL;
  6722. p.inv.perf_kernel = NULL;
  6723. p.inv.perf_dsp = NULL;
  6724. p.inv.job = NULL;
  6725. spin_lock(&fl->hlock);
  6726. if (fl->file_close >= FASTRPC_PROCESS_EXIT_START) {
  6727. err = -ESHUTDOWN;
  6728. pr_warn("adsprpc: fastrpc_device_release is happening, So not sending any new requests to DSP\n");
  6729. spin_unlock(&fl->hlock);
  6730. goto bail;
  6731. }
  6732. spin_unlock(&fl->hlock);
  6733. switch (ioctl_num) {
  6734. case FASTRPC_IOCTL_INVOKE:
  6735. size = sizeof(struct fastrpc_ioctl_invoke);
  6736. fallthrough;
  6737. case FASTRPC_IOCTL_INVOKE_FD:
  6738. if (!size)
  6739. size = sizeof(struct fastrpc_ioctl_invoke_fd);
  6740. fallthrough;
  6741. case FASTRPC_IOCTL_INVOKE_ATTRS:
  6742. if (!size)
  6743. size = sizeof(struct fastrpc_ioctl_invoke_attrs);
  6744. fallthrough;
  6745. case FASTRPC_IOCTL_INVOKE_CRC:
  6746. if (!size)
  6747. size = sizeof(struct fastrpc_ioctl_invoke_crc);
  6748. fallthrough;
  6749. case FASTRPC_IOCTL_INVOKE_PERF:
  6750. if (!size)
  6751. size = sizeof(struct fastrpc_ioctl_invoke_perf);
  6752. trace_fastrpc_msg("invoke: begin");
  6753. K_COPY_FROM_USER(err, 0, &p.inv, param, size);
  6754. if (err) {
  6755. err = -EFAULT;
  6756. goto bail;
  6757. }
  6758. VERIFY(err, 0 == (err = fastrpc_internal_invoke(fl, fl->mode,
  6759. USER_MSG, &p.inv)));
  6760. trace_fastrpc_msg("invoke: end");
  6761. if (err)
  6762. goto bail;
  6763. break;
  6764. case FASTRPC_IOCTL_INVOKE2:
  6765. K_COPY_FROM_USER(err, 0, &p.inv2, param,
  6766. sizeof(struct fastrpc_ioctl_invoke2));
  6767. if (err) {
  6768. err = -EFAULT;
  6769. goto bail;
  6770. }
  6771. VERIFY(err, 0 == (err = fastrpc_internal_invoke2(fl, &p.inv2)));
  6772. if (err)
  6773. goto bail;
  6774. break;
  6775. case FASTRPC_IOCTL_SETMODE:
  6776. err = fastrpc_setmode(ioctl_param, fl);
  6777. break;
  6778. case FASTRPC_IOCTL_CONTROL:
  6779. err = fastrpc_control(&p.cp, param, fl);
  6780. break;
  6781. case FASTRPC_IOCTL_GETINFO:
  6782. K_COPY_FROM_USER(err, 0, &info, param, sizeof(info));
  6783. if (err) {
  6784. err = -EFAULT;
  6785. goto bail;
  6786. }
  6787. VERIFY(err, 0 == (err = fastrpc_get_info(fl, &info)));
  6788. if (err)
  6789. goto bail;
  6790. K_COPY_TO_USER(err, 0, param, &info, sizeof(info));
  6791. if (err) {
  6792. err = -EFAULT;
  6793. goto bail;
  6794. }
  6795. break;
  6796. case FASTRPC_IOCTL_INIT:
  6797. p.init.attrs = 0;
  6798. p.init.siglen = 0;
  6799. size = sizeof(struct fastrpc_ioctl_init);
  6800. fallthrough;
  6801. case FASTRPC_IOCTL_INIT_ATTRS:
  6802. if (!size)
  6803. size = sizeof(struct fastrpc_ioctl_init_attrs);
  6804. K_COPY_FROM_USER(err, 0, &p.init, param, size);
  6805. if (err) {
  6806. err = -EFAULT;
  6807. goto bail;
  6808. }
  6809. VERIFY(err, 0 == (err = fastrpc_init_process(fl, &p.init)));
  6810. if (err)
  6811. goto bail;
  6812. break;
  6813. case FASTRPC_IOCTL_GET_DSP_INFO:
  6814. err = fastrpc_get_dsp_info(&p.cap, param, fl);
  6815. break;
  6816. case FASTRPC_IOCTL_MEM_MAP:
  6817. fallthrough;
  6818. case FASTRPC_IOCTL_MEM_UNMAP:
  6819. fallthrough;
  6820. case FASTRPC_IOCTL_MMAP:
  6821. fallthrough;
  6822. case FASTRPC_IOCTL_MUNMAP:
  6823. fallthrough;
  6824. case FASTRPC_IOCTL_MMAP_64:
  6825. fallthrough;
  6826. case FASTRPC_IOCTL_MUNMAP_64:
  6827. fallthrough;
  6828. case FASTRPC_IOCTL_MUNMAP_FD:
  6829. err = fastrpc_mmap_device_ioctl(fl, ioctl_num, &p, param);
  6830. break;
  6831. case FASTRPC_IOCTL_DSPSIGNAL_SIGNAL:
  6832. K_COPY_FROM_USER(err, 0, &p.sig, param,
  6833. sizeof(struct fastrpc_ioctl_dspsignal_signal));
  6834. if (err) {
  6835. err = -EFAULT;
  6836. goto bail;
  6837. }
  6838. VERIFY(err, 0 == (err = fastrpc_dspsignal_signal(fl, &p.sig)));
  6839. if (err)
  6840. goto bail;
  6841. break;
  6842. case FASTRPC_IOCTL_DSPSIGNAL_WAIT:
  6843. K_COPY_FROM_USER(err, 0, &p.wait, param,
  6844. sizeof(struct fastrpc_ioctl_dspsignal_wait));
  6845. if (err) {
  6846. err = -EFAULT;
  6847. goto bail;
  6848. }
  6849. VERIFY(err, 0 == (err = fastrpc_dspsignal_wait(fl, &p.wait)));
  6850. if (err)
  6851. goto bail;
  6852. break;
  6853. case FASTRPC_IOCTL_DSPSIGNAL_CREATE:
  6854. K_COPY_FROM_USER(err, 0, &p.cre, param,
  6855. sizeof(struct fastrpc_ioctl_dspsignal_create));
  6856. if (err) {
  6857. err = -EFAULT;
  6858. goto bail;
  6859. }
  6860. VERIFY(err, 0 == (err = fastrpc_dspsignal_create(fl, &p.cre)));
  6861. if (err)
  6862. goto bail;
  6863. break;
  6864. case FASTRPC_IOCTL_DSPSIGNAL_DESTROY:
  6865. K_COPY_FROM_USER(err, 0, &p.des, param,
  6866. sizeof(struct fastrpc_ioctl_dspsignal_destroy));
  6867. if (err) {
  6868. err = -EFAULT;
  6869. goto bail;
  6870. }
  6871. VERIFY(err, 0 == (err = fastrpc_dspsignal_destroy(fl, &p.des)));
  6872. if (err)
  6873. goto bail;
  6874. break;
  6875. case FASTRPC_IOCTL_DSPSIGNAL_CANCEL_WAIT:
  6876. K_COPY_FROM_USER(err, 0, &p.canc, param,
  6877. sizeof(struct fastrpc_ioctl_dspsignal_cancel_wait));
  6878. if (err) {
  6879. err = -EFAULT;
  6880. goto bail;
  6881. }
  6882. VERIFY(err, 0 == (err = fastrpc_dspsignal_cancel_wait(fl, &p.canc)));
  6883. if (err)
  6884. goto bail;
  6885. break;
  6886. default:
  6887. err = -ENOTTY;
  6888. pr_info("bad ioctl: %d\n", ioctl_num);
  6889. break;
  6890. }
  6891. bail:
  6892. return err;
  6893. }
  6894. /*
  6895. * fastrpc_smq_ctx_detail : Store smq_invoke_ctx structure parameter.
  6896. * Input :
  6897. * structure smq_invoke_ctx
  6898. * void* mini_dump_buff
  6899. */
  6900. static void fastrpc_smq_ctx_detail(struct smq_invoke_ctx *smq_ctx, int cid, void *mini_dump_buff)
  6901. {
  6902. int i = 0;
  6903. remote_arg64_t *rpra = NULL;
  6904. struct fastrpc_mmap *map = NULL;
  6905. if (!smq_ctx)
  6906. return;
  6907. if (smq_ctx->buf && smq_ctx->buf->virt)
  6908. rpra = smq_ctx->buf->virt;
  6909. for (i = 0; rpra &&
  6910. i < (REMOTE_SCALARS_INBUFS(smq_ctx->sc) + REMOTE_SCALARS_OUTBUFS(smq_ctx->sc));
  6911. ++i) {
  6912. map = smq_ctx->maps[i];
  6913. if (map) {
  6914. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  6915. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  6916. smq_invoke_ctx_params,
  6917. smq_ctx->pid, smq_ctx->tgid, smq_ctx->handle,
  6918. smq_ctx->sc, smq_ctx->fl, smq_ctx->fds,
  6919. smq_ctx->magic);
  6920. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  6921. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  6922. fastrpc_mmap_params,
  6923. map->fd, map->flags, map->buf,
  6924. map->phys, map->size, map->va,
  6925. map->raddr, map->len, map->refs,
  6926. map->secure);
  6927. } else {
  6928. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  6929. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  6930. smq_invoke_ctx_params, smq_ctx->pid, smq_ctx->tgid,
  6931. smq_ctx->handle, smq_ctx->sc, smq_ctx->fl, smq_ctx->fds,
  6932. smq_ctx->magic);
  6933. }
  6934. break;
  6935. }
  6936. }
  6937. /*
  6938. * fastrpc_print_fastrpcbuf : Print fastrpc_buf structure parameter.
  6939. * Input :
  6940. * structure fastrpc_buf
  6941. * void* buffer
  6942. */
  6943. static void fastrpc_print_fastrpcbuf(struct fastrpc_buf *buf, void *buffer)
  6944. {
  6945. if (!buf || !buffer)
  6946. return;
  6947. scnprintf(buffer + strlen(buffer),
  6948. MINI_DUMP_DBG_SIZE - strlen(buffer),
  6949. fastrpc_buf_params, buf->fl, buf->phys,
  6950. buf->virt, buf->size, buf->dma_attr, buf->raddr,
  6951. buf->flags, buf->type, buf->in_use);
  6952. }
  6953. /*
  6954. * fastrpc_print_debug_data : Print debug structure variable in CMA memory.
  6955. * Input cid: Channel id
  6956. */
  6957. static void fastrpc_print_debug_data(int cid)
  6958. {
  6959. unsigned int i = 0, count = 0, gmsg_log_iter = 3, err = 0, len = 0;
  6960. unsigned int tx_index = 0, rx_index = 0;
  6961. unsigned long flags = 0;
  6962. char *gmsg_log_tx = NULL;
  6963. char *gmsg_log_rx = NULL;
  6964. void *mini_dump_buff = NULL;
  6965. struct fastrpc_apps *me = &gfa;
  6966. struct smq_invoke_rspv2 *rsp = NULL;
  6967. struct fastrpc_file *fl = NULL;
  6968. struct fastrpc_channel_ctx *chan = NULL;
  6969. struct hlist_node *n = NULL;
  6970. struct smq_invoke_ctx *ictx = NULL;
  6971. struct fastrpc_tx_msg *tx_msg = NULL;
  6972. struct fastrpc_buf *buf = NULL;
  6973. struct fastrpc_mmap *map = NULL;
  6974. unsigned long irq_flags = 0;
  6975. VERIFY(err, NULL != (gmsg_log_tx = kzalloc(MD_GMSG_BUFFER, GFP_KERNEL)));
  6976. if (err) {
  6977. err = -ENOMEM;
  6978. return;
  6979. }
  6980. VERIFY(err, NULL != (gmsg_log_rx = kzalloc(MD_GMSG_BUFFER, GFP_KERNEL)));
  6981. if (err) {
  6982. err = -ENOMEM;
  6983. return;
  6984. }
  6985. chan = &me->channel[cid];
  6986. if ((!chan) || (!chan->buf))
  6987. return;
  6988. mini_dump_buff = chan->buf->virt;
  6989. if (!mini_dump_buff)
  6990. return;
  6991. if (chan) {
  6992. tx_index = chan->gmsg_log.tx_index;
  6993. rx_index = chan->gmsg_log.rx_index;
  6994. }
  6995. spin_lock_irqsave(&me->hlock, irq_flags);
  6996. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  6997. if (fl->cid == cid) {
  6998. scnprintf(mini_dump_buff +
  6999. strlen(mini_dump_buff),
  7000. MINI_DUMP_DBG_SIZE -
  7001. strlen(mini_dump_buff),
  7002. "\nfastrpc_file : %p\n", fl);
  7003. scnprintf(mini_dump_buff +
  7004. strlen(mini_dump_buff),
  7005. MINI_DUMP_DBG_SIZE -
  7006. strlen(mini_dump_buff),
  7007. fastrpc_file_params, fl->tgid,
  7008. fl->cid, fl->ssrcount, fl->pd,
  7009. fl->profile, fl->mode,
  7010. fl->tgid_open, fl->num_cached_buf,
  7011. fl->num_pers_hdrs, fl->sessionid,
  7012. fl->servloc_name, fl->file_close,
  7013. fl->dsp_proc_init, fl->apps,
  7014. fl->qos_request, fl->dev_minor,
  7015. fl->debug_buf,
  7016. fl->debug_buf_alloced_attempted,
  7017. fl->wake_enable,
  7018. fl->ws_timeout,
  7019. fl->untrusted_process);
  7020. scnprintf(mini_dump_buff +
  7021. strlen(mini_dump_buff),
  7022. MINI_DUMP_DBG_SIZE -
  7023. strlen(mini_dump_buff),
  7024. "\nSession Maps\n");
  7025. hlist_for_each_entry_safe(map, n, &fl->maps, hn) {
  7026. scnprintf(mini_dump_buff +
  7027. strlen(mini_dump_buff),
  7028. MINI_DUMP_DBG_SIZE -
  7029. strlen(mini_dump_buff),
  7030. fastrpc_mmap_params,
  7031. map->fd,
  7032. map->flags, map->buf,
  7033. map->phys, map->size,
  7034. map->va, map->raddr,
  7035. map->len, map->refs,
  7036. map->secure);
  7037. }
  7038. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7039. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7040. "\ncached_bufs\n");
  7041. hlist_for_each_entry_safe(buf, n, &fl->cached_bufs, hn) {
  7042. fastrpc_print_fastrpcbuf(buf, mini_dump_buff);
  7043. }
  7044. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7045. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7046. "\ninit_mem: %p\n", fl->init_mem);
  7047. fastrpc_print_fastrpcbuf(fl->init_mem, mini_dump_buff);
  7048. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7049. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7050. "\npers_hdr_buf: %p\n", fl->pers_hdr_buf);
  7051. fastrpc_print_fastrpcbuf(fl->pers_hdr_buf, mini_dump_buff);
  7052. snprintf(mini_dump_buff + strlen(mini_dump_buff),
  7053. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7054. "\nhdr_bufs: %p\n", fl->hdr_bufs);
  7055. fastrpc_print_fastrpcbuf(fl->hdr_bufs, mini_dump_buff);
  7056. if (fl->debugfs_file) {
  7057. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7058. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7059. "\nfl->debugfs_file.d_iname : %s\n",
  7060. fl->debugfs_file->d_iname);
  7061. }
  7062. if (fl->sctx) {
  7063. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7064. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7065. "\nfl->sctx->smmu.cb : %d\n",
  7066. fl->sctx->smmu.cb);
  7067. }
  7068. if (fl->secsctx) {
  7069. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7070. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7071. "\nfl->secsctx->smmu.cb : %d\n",
  7072. fl->secsctx->smmu.cb);
  7073. }
  7074. spin_lock(&fl->hlock);
  7075. scnprintf(mini_dump_buff +
  7076. strlen(mini_dump_buff),
  7077. MINI_DUMP_DBG_SIZE -
  7078. strlen(mini_dump_buff),
  7079. "\nPending Ctx:\n");
  7080. hlist_for_each_entry_safe(ictx, n, &fl->clst.pending, hn) {
  7081. fastrpc_smq_ctx_detail(ictx,
  7082. cid, mini_dump_buff);
  7083. }
  7084. scnprintf(mini_dump_buff +
  7085. strlen(mini_dump_buff),
  7086. MINI_DUMP_DBG_SIZE -
  7087. strlen(mini_dump_buff),
  7088. "\nInterrupted Ctx:\n");
  7089. hlist_for_each_entry_safe(ictx, n,
  7090. &fl->clst.interrupted,
  7091. hn) {
  7092. fastrpc_smq_ctx_detail(ictx,
  7093. cid, mini_dump_buff);
  7094. }
  7095. spin_unlock(&fl->hlock);
  7096. }
  7097. }
  7098. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7099. spin_lock_irqsave(&chan->gmsg_log.lock, flags);
  7100. if (rx_index) {
  7101. for (i = rx_index, count = 0, len = 0 ; i > 0 &&
  7102. count <= gmsg_log_iter; i--, count++) {
  7103. rsp = &chan->gmsg_log.rx_msgs[i].rsp;
  7104. len += scnprintf(gmsg_log_rx + len, MD_GMSG_BUFFER - len,
  7105. "ctx: 0x%x, retval: %d, flags: %d, early_wake_time: %d, version: %d\n",
  7106. rsp->ctx, rsp->retval, rsp->flags,
  7107. rsp->early_wake_time, rsp->version);
  7108. }
  7109. }
  7110. if (tx_index) {
  7111. for (i = tx_index, count = 0, len = 0;
  7112. i > 0 && count <= gmsg_log_iter;
  7113. i--, count++) {
  7114. tx_msg = &chan->gmsg_log.tx_msgs[i];
  7115. len += scnprintf(gmsg_log_tx + len, MD_GMSG_BUFFER - len,
  7116. "pid: %d, tid: %d, ctx: 0x%x, handle: 0x%x, sc: 0x%x, addr: 0x%x, size:%d\n",
  7117. tx_msg->msg.pid,
  7118. tx_msg->msg.tid,
  7119. tx_msg->msg.invoke.header.ctx,
  7120. tx_msg->msg.invoke.header.handle,
  7121. tx_msg->msg.invoke.header.sc,
  7122. tx_msg->msg.invoke.page.addr,
  7123. tx_msg->msg.invoke.page.size);
  7124. }
  7125. }
  7126. spin_unlock_irqrestore(&chan->gmsg_log.lock, flags);
  7127. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7128. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7129. "gmsg_log_tx:\n%s\n", gmsg_log_tx);
  7130. scnprintf(mini_dump_buff + strlen(mini_dump_buff),
  7131. MINI_DUMP_DBG_SIZE - strlen(mini_dump_buff),
  7132. "gmsg_log_rx:\n %s\n", gmsg_log_rx);
  7133. if (chan && chan->buf)
  7134. chan->buf->size = strlen(mini_dump_buff);
  7135. kfree(gmsg_log_tx);
  7136. kfree(gmsg_log_rx);
  7137. }
  7138. void fastrpc_restart_drivers(int cid)
  7139. {
  7140. struct fastrpc_apps *me = &gfa;
  7141. fastrpc_notify_drivers(me, cid);
  7142. me->channel[cid].ssrcount++;
  7143. }
  7144. static int fastrpc_restart_notifier_cb(struct notifier_block *nb,
  7145. unsigned long code,
  7146. void *data)
  7147. {
  7148. struct fastrpc_apps *me = &gfa;
  7149. struct fastrpc_channel_ctx *ctx;
  7150. struct fastrpc_file *fl;
  7151. struct hlist_node *n;
  7152. int cid = -1;
  7153. struct timespec64 startT = {0};
  7154. unsigned long irq_flags = 0;
  7155. ctx = container_of(nb, struct fastrpc_channel_ctx, nb);
  7156. cid = ctx - &me->channel[0];
  7157. switch (code) {
  7158. case QCOM_SSR_BEFORE_SHUTDOWN:
  7159. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7160. "QCOM_SSR_BEFORE_SHUTDOWN", "fastrpc_restart_notifier-enter");
  7161. pr_info("adsprpc: %s: %s subsystem is restarting\n",
  7162. __func__, gcinfo[cid].subsys);
  7163. mutex_lock(&me->channel[cid].smd_mutex);
  7164. ctx->ssrcount++;
  7165. ctx->subsystemstate = SUBSYSTEM_RESTARTING;
  7166. mutex_unlock(&me->channel[cid].smd_mutex);
  7167. if (cid == RH_CID)
  7168. me->staticpd_flags = 0;
  7169. break;
  7170. case QCOM_SSR_AFTER_SHUTDOWN:
  7171. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7172. "QCOM_SSR_AFTER_SHUTDOWN", "fastrpc_restart_notifier-enter");
  7173. spin_lock_irqsave(&me->hlock, irq_flags);
  7174. hlist_for_each_entry_safe(fl, n, &me->drivers, hn) {
  7175. if (fl->cid != cid)
  7176. continue;
  7177. complete(&fl->shutdown);
  7178. }
  7179. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7180. ctx->subsystemstate = SUBSYSTEM_DOWN;
  7181. pr_info("adsprpc: %s: received RAMDUMP notification for %s\n",
  7182. __func__, gcinfo[cid].subsys);
  7183. break;
  7184. case QCOM_SSR_BEFORE_POWERUP:
  7185. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7186. "QCOM_SSR_BEFORE_POWERUP", "fastrpc_restart_notifier-enter");
  7187. pr_info("adsprpc: %s: subsystem %s is about to start\n",
  7188. __func__, gcinfo[cid].subsys);
  7189. if (cid == CDSP_DOMAIN_ID && dump_enabled() &&
  7190. ctx->ssrcount)
  7191. fastrpc_update_ramdump_status(cid);
  7192. fastrpc_notify_drivers(me, cid);
  7193. /* Skip ram dump collection in first boot */
  7194. if (cid == CDSP_DOMAIN_ID && dump_enabled() &&
  7195. ctx->ssrcount) {
  7196. mutex_lock(&me->channel[cid].smd_mutex);
  7197. fastrpc_print_debug_data(cid);
  7198. mutex_unlock(&me->channel[cid].smd_mutex);
  7199. ktime_get_real_ts64(&startT);
  7200. fastrpc_ramdump_collection(cid);
  7201. pr_info("adsprpc: %s: fastrpc ramdump finished in %lu (us)\n",
  7202. __func__, getnstimediff(&startT));
  7203. }
  7204. break;
  7205. case QCOM_SSR_AFTER_POWERUP:
  7206. fastrpc_rproc_trace_events(gcinfo[cid].subsys,
  7207. "QCOM_SSR_AFTER_POWERUP", "fastrpc_restart_notifier-enter");
  7208. pr_info("adsprpc: %s: %s subsystem is up\n",
  7209. __func__, gcinfo[cid].subsys);
  7210. ctx->subsystemstate = SUBSYSTEM_UP;
  7211. break;
  7212. default:
  7213. break;
  7214. }
  7215. fastrpc_rproc_trace_events(dev_name(me->dev), "fastrpc_restart_notifier", "exit");
  7216. return NOTIFY_DONE;
  7217. }
  7218. static void fastrpc_pdr_cb(int state, char *service_path, void *priv)
  7219. {
  7220. struct fastrpc_apps *me = &gfa;
  7221. struct fastrpc_static_pd *spd;
  7222. int err = 0;
  7223. spd = priv;
  7224. VERIFY(err, spd);
  7225. if (err)
  7226. goto bail;
  7227. switch (state) {
  7228. case SERVREG_SERVICE_STATE_DOWN:
  7229. pr_info("adsprpc: %s: %s (%s) is down for PDR on %s\n",
  7230. __func__, spd->spdname,
  7231. spd->servloc_name,
  7232. gcinfo[spd->cid].subsys);
  7233. mutex_lock(&me->channel[spd->cid].smd_mutex);
  7234. spd->pdrcount++;
  7235. atomic_set(&spd->ispdup, 0);
  7236. mutex_unlock(&me->channel[spd->cid].smd_mutex);
  7237. if (!strcmp(spd->servloc_name,
  7238. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME))
  7239. me->staticpd_flags = 0;
  7240. fastrpc_notify_pdr_drivers(me, spd->servloc_name);
  7241. break;
  7242. case SERVREG_SERVICE_STATE_UP:
  7243. pr_info("adsprpc: %s: %s (%s) is up for PDR on %s\n",
  7244. __func__, spd->spdname,
  7245. spd->servloc_name,
  7246. gcinfo[spd->cid].subsys);
  7247. atomic_set(&spd->ispdup, 1);
  7248. wake_up_interruptible(&spd->wait_for_pdup);
  7249. break;
  7250. default:
  7251. break;
  7252. }
  7253. bail:
  7254. if (err) {
  7255. pr_err("adsprpc: %s: failed for path %s, state %d, spd %pK\n",
  7256. __func__, service_path, state, spd);
  7257. }
  7258. }
  7259. static const struct file_operations fops = {
  7260. .open = fastrpc_device_open,
  7261. .release = fastrpc_device_release,
  7262. .unlocked_ioctl = fastrpc_device_ioctl,
  7263. /* Only DSP service 64-bit app will interface with fastrpc TVM driver.
  7264. * There is not need to support 32-bit fastrpc driver on TVM.
  7265. */
  7266. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  7267. .compat_ioctl = NULL,
  7268. #else
  7269. .compat_ioctl = compat_fastrpc_device_ioctl,
  7270. #endif
  7271. };
  7272. static const struct of_device_id fastrpc_match_table[] = {
  7273. { .compatible = "qcom,msm-fastrpc-adsp", },
  7274. { .compatible = "qcom,msm-fastrpc-compute", },
  7275. { .compatible = "qcom,msm-fastrpc-compute-cb", },
  7276. { .compatible = "qcom,msm-adsprpc-mem-region", },
  7277. {}
  7278. };
  7279. static int fastrpc_cb_probe(struct device *dev)
  7280. {
  7281. struct fastrpc_channel_ctx *chan = NULL;
  7282. struct fastrpc_session_ctx *sess = NULL;
  7283. struct of_phandle_args iommuspec;
  7284. struct fastrpc_apps *me = &gfa;
  7285. struct fastrpc_buf *buf = NULL;
  7286. struct gen_pool *gen_pool = NULL;
  7287. struct iommu_domain *domain = NULL;
  7288. const char *name;
  7289. int err = 0, cid = -1, i = 0;
  7290. u32 sharedcb_count = 0, j = 0;
  7291. uint32_t dma_addr_pool[2] = {0, 0};
  7292. VERIFY(err, NULL != (name = of_get_property(dev->of_node,
  7293. "label", NULL)));
  7294. if (err) {
  7295. err = -EINVAL;
  7296. goto bail;
  7297. }
  7298. for (i = 0; i < NUM_CHANNELS; i++) {
  7299. if (!gcinfo[i].name)
  7300. continue;
  7301. if (!strcmp(name, gcinfo[i].name))
  7302. break;
  7303. }
  7304. VERIFY(err, i < NUM_CHANNELS);
  7305. if (err) {
  7306. err = -ECHRNG;
  7307. goto bail;
  7308. }
  7309. cid = i;
  7310. chan = &gcinfo[i];
  7311. VERIFY(err, chan->sesscount < NUM_SESSIONS);
  7312. if (err) {
  7313. err = -EINVAL;
  7314. goto bail;
  7315. }
  7316. err = of_parse_phandle_with_args(dev->of_node, "iommus",
  7317. "#iommu-cells", 0, &iommuspec);
  7318. if (err) {
  7319. pr_err("Error: adsprpc: %s: parsing iommu arguments failed for %s with err %d\n",
  7320. __func__, dev_name(dev), err);
  7321. goto bail;
  7322. }
  7323. sess = &chan->session[chan->sesscount];
  7324. sess->used = 0;
  7325. sess->smmu.coherent = of_property_read_bool(dev->of_node,
  7326. "dma-coherent");
  7327. sess->smmu.secure = of_property_read_bool(dev->of_node,
  7328. "qcom,secure-context-bank");
  7329. sess->smmu.cb = iommuspec.args[0] & 0xf;
  7330. sess->smmu.dev = dev;
  7331. sess->smmu.dev_name = dev_name(dev);
  7332. sess->smmu.enabled = 1;
  7333. if (!sess->smmu.dev->dma_parms)
  7334. sess->smmu.dev->dma_parms = devm_kzalloc(sess->smmu.dev,
  7335. sizeof(*sess->smmu.dev->dma_parms), GFP_KERNEL);
  7336. dma_set_max_seg_size(sess->smmu.dev, DMA_BIT_MASK(32));
  7337. dma_set_seg_boundary(sess->smmu.dev, (unsigned long)DMA_BIT_MASK(64));
  7338. of_property_read_u32_array(dev->of_node, "qcom,iommu-dma-addr-pool",
  7339. dma_addr_pool, 2);
  7340. me->max_size_limit = (dma_addr_pool[1] == 0 ? 0x78000000 :
  7341. dma_addr_pool[1]);
  7342. if (of_get_property(dev->of_node, "pd-type", NULL) != NULL) {
  7343. err = of_property_read_u32(dev->of_node, "pd-type",
  7344. &(sess->smmu.pd_type));
  7345. /* Set cb_pd_type, if the process type is set for context banks */
  7346. me->cb_pd_type = true;
  7347. if (err)
  7348. goto bail;
  7349. }
  7350. if (of_get_property(dev->of_node, "shared-cb", NULL) != NULL) {
  7351. sess->smmu.sharedcb = 1;
  7352. // Set share_securecb, if the secure context bank is shared
  7353. if (sess->smmu.secure)
  7354. me->share_securecb = 1;
  7355. err = of_property_read_u32(dev->of_node, "shared-cb",
  7356. &sharedcb_count);
  7357. if (err)
  7358. goto bail;
  7359. if (sharedcb_count > 0) {
  7360. struct fastrpc_session_ctx *dup_sess;
  7361. for (j = 1; j < sharedcb_count &&
  7362. chan->sesscount < NUM_SESSIONS; j++) {
  7363. chan->sesscount++;
  7364. dup_sess = &chan->session[chan->sesscount];
  7365. memcpy(dup_sess, sess,
  7366. sizeof(struct fastrpc_session_ctx));
  7367. }
  7368. }
  7369. }
  7370. if (of_get_property(dev->of_node, "qrtr-gen-pool", NULL) != NULL) {
  7371. u32 frpc_gen_addr_pool[2] = {0, 0};
  7372. struct sg_table sgt;
  7373. err = of_property_read_u32_array(dev->of_node, "frpc-gen-addr-pool",
  7374. frpc_gen_addr_pool, 2);
  7375. if (err) {
  7376. pr_err("Error: adsprpc: %s: parsing frpc-gen-addr-pool arguments failed for %s with err %d\n",
  7377. __func__, dev_name(dev), err);
  7378. goto bail;
  7379. }
  7380. sess->smmu.genpool_iova = frpc_gen_addr_pool[0];
  7381. sess->smmu.genpool_size = frpc_gen_addr_pool[1];
  7382. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  7383. if (err) {
  7384. err = -ENOMEM;
  7385. ADSPRPC_ERR(
  7386. "allocation failed for size 0x%zx\n", sizeof(*buf));
  7387. goto bail;
  7388. }
  7389. INIT_HLIST_NODE(&buf->hn);
  7390. buf->virt = NULL;
  7391. buf->phys = 0;
  7392. buf->size = frpc_gen_addr_pool[1];
  7393. buf->dma_attr = DMA_ATTR_DELAYED_UNMAP;
  7394. /* Allocate memory for adding to genpool */
  7395. buf->virt = dma_alloc_attrs(sess->smmu.dev, buf->size,
  7396. (dma_addr_t *)&buf->phys,
  7397. GFP_KERNEL, buf->dma_attr);
  7398. if (IS_ERR_OR_NULL(buf->virt)) {
  7399. ADSPRPC_ERR(
  7400. "dma_alloc_attrs failed for size 0x%zx, returned %pK\n",
  7401. buf->size, buf->virt);
  7402. err = -ENOBUFS;
  7403. goto dma_alloc_bail;
  7404. }
  7405. err = dma_get_sgtable_attrs(sess->smmu.dev, &sgt, buf->virt,
  7406. buf->phys, buf->size, buf->dma_attr);
  7407. if (err) {
  7408. ADSPRPC_ERR("dma_get_sgtable_attrs failed with err %d", err);
  7409. goto iommu_map_bail;
  7410. }
  7411. domain = iommu_get_domain_for_dev(sess->smmu.dev);
  7412. if (!domain) {
  7413. ADSPRPC_ERR("iommu_get_domain_for_dev failed ");
  7414. goto iommu_map_bail;
  7415. }
  7416. /* Map the allocated memory with fixed IOVA and is shared to remote subsystem */
  7417. err = iommu_map_sg(domain, frpc_gen_addr_pool[0], sgt.sgl,
  7418. sgt.nents, IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE);
  7419. if (err < 0) {
  7420. ADSPRPC_ERR("iommu_map_sg failed with err %d", err);
  7421. goto iommu_map_bail;
  7422. }
  7423. /* Create genpool using SMMU device */
  7424. gen_pool = devm_gen_pool_create(sess->smmu.dev, 0,
  7425. NUMA_NO_NODE, NULL);
  7426. if (IS_ERR(gen_pool)) {
  7427. err = PTR_ERR(gen_pool);
  7428. ADSPRPC_ERR("devm_gen_pool_create failed with err %d", err);
  7429. goto genpool_create_bail;
  7430. }
  7431. /* Add allocated memory to genpool */
  7432. err = gen_pool_add_virt(gen_pool, (unsigned long)buf->virt,
  7433. buf->phys, buf->size, NUMA_NO_NODE);
  7434. if (err) {
  7435. ADSPRPC_ERR("gen_pool_add_virt failed with err %d", err);
  7436. goto genpool_add_bail;
  7437. }
  7438. sess->smmu.frpc_genpool = gen_pool;
  7439. sess->smmu.frpc_genpool_buf = buf;
  7440. }
  7441. chan->sesscount++;
  7442. if (debugfs_root && !debugfs_global_file) {
  7443. debugfs_global_file = debugfs_create_file("global", 0644,
  7444. debugfs_root, NULL, &debugfs_fops);
  7445. if (IS_ERR_OR_NULL(debugfs_global_file)) {
  7446. pr_warn("Error: %s: %s: failed to create debugfs global file\n",
  7447. current->comm, __func__);
  7448. debugfs_global_file = NULL;
  7449. }
  7450. }
  7451. bail:
  7452. return err;
  7453. genpool_add_bail:
  7454. gen_pool_destroy(gen_pool);
  7455. genpool_create_bail:
  7456. iommu_unmap(domain, sess->smmu.genpool_iova,
  7457. sess->smmu.genpool_size);
  7458. iommu_map_bail:
  7459. dma_free_attrs(sess->smmu.dev, buf->size, buf->virt,
  7460. buf->phys, buf->dma_attr);
  7461. dma_alloc_bail:
  7462. kfree(buf);
  7463. return err;
  7464. }
  7465. static void init_secure_vmid_list(struct device *dev, char *prop_name,
  7466. struct secure_vm *destvm)
  7467. {
  7468. int err = 0;
  7469. u32 len = 0, i = 0;
  7470. u32 *rhvmlist = NULL;
  7471. u32 *rhvmpermlist = NULL;
  7472. if (!of_find_property(dev->of_node, prop_name, &len))
  7473. goto bail;
  7474. if (len == 0)
  7475. goto bail;
  7476. len /= sizeof(u32);
  7477. VERIFY(err, NULL != (rhvmlist = kcalloc(len, sizeof(u32), GFP_KERNEL)));
  7478. if (err)
  7479. goto bail;
  7480. VERIFY(err, NULL != (rhvmpermlist = kcalloc(len, sizeof(u32),
  7481. GFP_KERNEL)));
  7482. if (err)
  7483. goto bail;
  7484. for (i = 0; i < len; i++) {
  7485. err = of_property_read_u32_index(dev->of_node, prop_name, i,
  7486. &rhvmlist[i]);
  7487. if (err) {
  7488. pr_err("Error: adsprpc: %s: failed to read VMID\n",
  7489. __func__);
  7490. goto bail;
  7491. }
  7492. ADSPRPC_INFO("secure VMID = %d\n",
  7493. rhvmlist[i]);
  7494. rhvmpermlist[i] = QCOM_SCM_PERM_RWX;
  7495. }
  7496. destvm->vmid = rhvmlist;
  7497. destvm->vmperm = rhvmpermlist;
  7498. destvm->vmcount = len;
  7499. bail:
  7500. if (err) {
  7501. kfree(rhvmlist);
  7502. kfree(rhvmpermlist);
  7503. }
  7504. }
  7505. static void fastrpc_init_privileged_gids(struct device *dev, char *prop_name,
  7506. struct gid_list *gidlist)
  7507. {
  7508. int err = 0;
  7509. u32 len = 0, i = 0;
  7510. u32 *gids = NULL;
  7511. if (!of_find_property(dev->of_node, prop_name, &len))
  7512. goto bail;
  7513. if (len == 0)
  7514. goto bail;
  7515. len /= sizeof(u32);
  7516. gids = kcalloc(len, sizeof(u32), GFP_KERNEL);
  7517. if (!gids) {
  7518. err = ENOMEM;
  7519. goto bail;
  7520. }
  7521. for (i = 0; i < len; i++) {
  7522. err = of_property_read_u32_index(dev->of_node, prop_name,
  7523. i, &gids[i]);
  7524. if (err) {
  7525. pr_err("Error: adsprpc: %s: failed to read GID %u\n",
  7526. __func__, i);
  7527. goto bail;
  7528. }
  7529. pr_info("adsprpc: %s: privileged GID: %u\n", __func__, gids[i]);
  7530. }
  7531. sort(gids, len, sizeof(*gids), uint_cmp_func, NULL);
  7532. gidlist->gids = gids;
  7533. gidlist->gidcount = len;
  7534. bail:
  7535. if (err)
  7536. kfree(gids);
  7537. }
  7538. static void configure_secure_channels(uint32_t secure_domains)
  7539. {
  7540. struct fastrpc_apps *me = &gfa;
  7541. int ii = 0;
  7542. /*
  7543. * secure_domains contains the bitmask of the secure channels
  7544. * Bit 0 - ADSP
  7545. * Bit 1 - MDSP
  7546. * Bit 2 - SLPI
  7547. * Bit 3 - CDSP
  7548. */
  7549. for (ii = ADSP_DOMAIN_ID; ii <= CDSP_DOMAIN_ID; ++ii) {
  7550. int secure = (secure_domains >> ii) & 0x01;
  7551. me->channel[ii].secure = secure;
  7552. ADSPRPC_INFO("domain %d configured as secure %d\n", ii, secure);
  7553. }
  7554. }
  7555. /*
  7556. * This function is used to create the service locator required for
  7557. * registering for remote process restart (PDR) notifications if that
  7558. * PDR property has been enabled in the fastrpc node on the DTSI.
  7559. */
  7560. static int fastrpc_setup_service_locator(struct device *dev,
  7561. const char *propname,
  7562. char *client_name, char *service_name,
  7563. char *service_path)
  7564. {
  7565. int err = 0, session = -1, cid = -1;
  7566. struct fastrpc_apps *me = &gfa;
  7567. struct pdr_handle *handle = NULL;
  7568. struct pdr_service *service = NULL;
  7569. if (of_property_read_bool(dev->of_node, propname)) {
  7570. err = fastrpc_get_spd_session(client_name, &session, &cid);
  7571. if (err)
  7572. goto bail;
  7573. /* Register the service locator's callback function */
  7574. handle = pdr_handle_alloc(fastrpc_pdr_cb, &me->channel[cid].spd[session]);
  7575. if (IS_ERR_OR_NULL(handle)) {
  7576. err = PTR_ERR(handle);
  7577. goto bail;
  7578. }
  7579. me->channel[cid].spd[session].pdrhandle = handle;
  7580. service = pdr_add_lookup(handle, service_name, service_path);
  7581. if (IS_ERR_OR_NULL(service)) {
  7582. err = PTR_ERR(service);
  7583. goto bail;
  7584. }
  7585. pr_info("adsprpc: %s: pdr_add_lookup enabled for %s (%s, %s), DTSI (%s)\n",
  7586. __func__, service_name, client_name, service_path, propname);
  7587. }
  7588. bail:
  7589. if (err) {
  7590. pr_warn("adsprpc: %s: failed for %s (%s, %s), DTSI (%s) with err %d\n",
  7591. __func__, service_name, client_name, service_path, propname, err);
  7592. }
  7593. return err;
  7594. }
  7595. /*
  7596. * fastrpc_nsp_status_show() - Updates the buffer with remote nsp status
  7597. * by reading the fastrpc node.
  7598. * @dev : pointer to device node.
  7599. * @attr: pointer to device attribute.
  7600. * @buf : Output parameter to be updated with remote nsp status.
  7601. * Return : bytes written to buffer.
  7602. */
  7603. static ssize_t fastrpc_nsp_status_show(struct device *dev,
  7604. struct device_attribute *attr, char *buf)
  7605. {
  7606. struct fastrpc_apps *me = &gfa;
  7607. /*
  7608. * Default remote DSP status: 0
  7609. * driver possibly not probed yet or not the main device.
  7610. */
  7611. if (!dev || !dev->driver ||
  7612. !of_device_is_compatible(dev->of_node, "qcom,msm-fastrpc-compute")) {
  7613. ADSPRPC_ERR("Driver not probed yet or not the main device\n");
  7614. return 0;
  7615. }
  7616. return scnprintf(buf, PAGE_SIZE, "%d",
  7617. me->fastrpc_nsp_status);
  7618. }
  7619. /* Remote nsp status attribute declaration as read only */
  7620. static DEVICE_ATTR_RO(fastrpc_nsp_status);
  7621. /* Declaring attribute for remote dsp */
  7622. static struct attribute *msm_remote_dsp_attrs[] = {
  7623. &dev_attr_fastrpc_nsp_status.attr,
  7624. NULL
  7625. };
  7626. /* Defining remote dsp attributes in attributes group */
  7627. static struct attribute_group msm_remote_dsp_attr_group = {
  7628. .attrs = msm_remote_dsp_attrs,
  7629. };
  7630. static int fastrpc_probe(struct platform_device *pdev)
  7631. {
  7632. int err = 0;
  7633. struct fastrpc_apps *me = &gfa;
  7634. struct device *dev = &pdev->dev;
  7635. int ret = 0;
  7636. uint32_t secure_domains = 0;
  7637. if (of_device_is_compatible(dev->of_node,
  7638. "qcom,msm-fastrpc-compute")) {
  7639. err = sysfs_create_group(&pdev->dev.kobj, &msm_remote_dsp_attr_group);
  7640. if (err) {
  7641. ADSPRPC_ERR(
  7642. "Initialization of sysfs create group failed with %d\n",
  7643. err);
  7644. goto bail;
  7645. }
  7646. init_secure_vmid_list(dev, "qcom,adsp-remoteheap-vmid",
  7647. &gcinfo[0].rhvm);
  7648. fastrpc_init_privileged_gids(dev, "qcom,fastrpc-gids",
  7649. &me->gidlist);
  7650. /*
  7651. * Check if latency voting for only one core
  7652. * is enabled for the platform
  7653. */
  7654. me->single_core_latency_vote = of_property_read_bool(dev->of_node,
  7655. "qcom,single-core-latency-vote");
  7656. if (me->single_core_latency_vote)
  7657. me->lowest_capacity_core_count = 1;
  7658. of_property_read_u32(dev->of_node, "qcom,rpc-latency-us",
  7659. &me->latency);
  7660. of_property_read_u32(dev->of_node, "qcom,max-sessions",
  7661. &me->max_sess_per_proc);
  7662. if (of_get_property(dev->of_node,
  7663. "qcom,secure-domains", NULL) != NULL) {
  7664. VERIFY(err, !of_property_read_u32(dev->of_node,
  7665. "qcom,secure-domains",
  7666. &secure_domains));
  7667. if (!err)
  7668. configure_secure_channels(secure_domains);
  7669. else
  7670. pr_info("adsprpc: unable to read the domain configuration from dts\n");
  7671. }
  7672. }
  7673. if (of_device_is_compatible(dev->of_node,
  7674. "qcom,msm-fastrpc-compute-cb"))
  7675. return fastrpc_cb_probe(dev);
  7676. if (of_device_is_compatible(dev->of_node,
  7677. "qcom,msm-adsprpc-mem-region")) {
  7678. me->dev = dev;
  7679. ret = of_reserved_mem_device_init_by_idx(dev, dev->of_node, 0);
  7680. if (ret) {
  7681. pr_warn("adsprpc: Error: %s: initialization of memory region adsp_mem failed with %d\n",
  7682. __func__, ret);
  7683. }
  7684. goto bail;
  7685. }
  7686. me->legacy_remote_heap = of_property_read_bool(dev->of_node,
  7687. "qcom,fastrpc-legacy-remote-heap");
  7688. err = fastrpc_setup_service_locator(dev, AUDIO_PDR_ADSP_DTSI_PROPERTY_NAME,
  7689. AUDIO_PDR_SERVICE_LOCATION_CLIENT_NAME,
  7690. AUDIO_PDR_ADSP_SERVICE_NAME, ADSP_AUDIOPD_NAME);
  7691. if (err)
  7692. goto bail;
  7693. err = fastrpc_setup_service_locator(dev, SENSORS_PDR_ADSP_DTSI_PROPERTY_NAME,
  7694. SENSORS_PDR_ADSP_SERVICE_LOCATION_CLIENT_NAME,
  7695. SENSORS_PDR_ADSP_SERVICE_NAME, ADSP_SENSORPD_NAME);
  7696. if (err)
  7697. goto bail;
  7698. err = fastrpc_setup_service_locator(dev, SENSORS_PDR_SLPI_DTSI_PROPERTY_NAME,
  7699. SENSORS_PDR_SLPI_SERVICE_LOCATION_CLIENT_NAME,
  7700. SENSORS_PDR_SLPI_SERVICE_NAME, SLPI_SENSORPD_NAME);
  7701. if (err)
  7702. goto bail;
  7703. err = of_platform_populate(pdev->dev.of_node,
  7704. fastrpc_match_table,
  7705. NULL, &pdev->dev);
  7706. if (err)
  7707. goto bail;
  7708. bail:
  7709. return err;
  7710. }
  7711. /*
  7712. * Function to free fastrpc genpool buffer
  7713. */
  7714. static void fastrpc_genpool_free(struct fastrpc_session_ctx *sess)
  7715. {
  7716. struct fastrpc_buf *buf = NULL;
  7717. struct iommu_domain *domain = NULL;
  7718. if (!sess)
  7719. goto bail;
  7720. buf = sess->smmu.frpc_genpool_buf;
  7721. if (sess->smmu.frpc_genpool) {
  7722. gen_pool_destroy(sess->smmu.frpc_genpool);
  7723. sess->smmu.frpc_genpool = NULL;
  7724. }
  7725. if (buf && sess->smmu.dev) {
  7726. domain = iommu_get_domain_for_dev(sess->smmu.dev);
  7727. iommu_unmap(domain, sess->smmu.genpool_iova,
  7728. sess->smmu.genpool_size);
  7729. if (buf->phys)
  7730. dma_free_attrs(sess->smmu.dev, buf->size, buf->virt,
  7731. buf->phys, buf->dma_attr);
  7732. kfree(buf);
  7733. sess->smmu.frpc_genpool_buf = NULL;
  7734. }
  7735. bail:
  7736. return;
  7737. }
  7738. static void fastrpc_deinit(void)
  7739. {
  7740. struct fastrpc_channel_ctx *chan = gcinfo;
  7741. struct fastrpc_apps *me = &gfa;
  7742. int i, j;
  7743. for (i = 0; i < NUM_CHANNELS; i++, chan++) {
  7744. for (j = 0; j < NUM_SESSIONS; j++) {
  7745. struct fastrpc_session_ctx *sess = &chan->session[j];
  7746. fastrpc_genpool_free(sess);
  7747. if (sess->smmu.dev)
  7748. sess->smmu.dev = NULL;
  7749. }
  7750. kfree(chan->rhvm.vmid);
  7751. kfree(chan->rhvm.vmperm);
  7752. fastrpc_transport_session_deinit(i);
  7753. mutex_destroy(&chan->smd_mutex);
  7754. }
  7755. if (me->transport_initialized)
  7756. fastrpc_transport_deinit();
  7757. me->transport_initialized = 0;
  7758. mutex_destroy(&me->mut_uid);
  7759. }
  7760. #ifdef CONFIG_HIBERNATION
  7761. static bool hibernation;
  7762. static int fastrpc_hibernation_notifier(struct notifier_block *nb,
  7763. unsigned long event, void *dummy)
  7764. {
  7765. if (event == PM_HIBERNATION_PREPARE)
  7766. hibernation = true;
  7767. else if (event == PM_POST_HIBERNATION)
  7768. hibernation = false;
  7769. return NOTIFY_OK;
  7770. }
  7771. static struct notifier_block fastrpc_notif_block = {
  7772. .notifier_call = fastrpc_hibernation_notifier,
  7773. };
  7774. #endif
  7775. #ifdef CONFIG_PM_SLEEP
  7776. static int fastrpc_hibernation_suspend(struct device *dev)
  7777. {
  7778. int err = 0;
  7779. if (of_device_is_compatible(dev->of_node,
  7780. "qcom,msm-fastrpc-compute")) {
  7781. err = fastrpc_mmap_remove_ssr(NULL, 0);
  7782. if (err)
  7783. ADSPRPC_WARN("failed to unmap remote heap (err %d)\n",
  7784. err);
  7785. }
  7786. return err;
  7787. }
  7788. static int fastrpc_restore(struct device *dev)
  7789. {
  7790. struct fastrpc_apps *me = &gfa;
  7791. int cid;
  7792. pr_info("adsprpc: restore enter\n");
  7793. for (cid = 0; cid < NUM_CHANNELS; cid++)
  7794. me->channel[cid].in_hib = 1;
  7795. pr_info("adsprpc: restore exit\n");
  7796. return 0;
  7797. }
  7798. static const struct dev_pm_ops fastrpc_pm = {
  7799. .freeze = fastrpc_hibernation_suspend,
  7800. .restore = fastrpc_restore,
  7801. };
  7802. #endif
  7803. static struct platform_driver fastrpc_driver = {
  7804. .probe = fastrpc_probe,
  7805. .driver = {
  7806. .name = "fastrpc",
  7807. .of_match_table = fastrpc_match_table,
  7808. .suppress_bind_attrs = true,
  7809. #ifdef CONFIG_PM_SLEEP
  7810. .pm = &fastrpc_pm,
  7811. #endif
  7812. },
  7813. };
  7814. union fastrpc_dev_param {
  7815. struct fastrpc_dev_map_dma *map;
  7816. struct fastrpc_dev_unmap_dma *unmap;
  7817. struct fastrpc_dev_get_hlos_pid *hpid;
  7818. };
  7819. long fastrpc_dev_map_dma(struct fastrpc_device *dev, unsigned long invoke_param)
  7820. {
  7821. int err = 0;
  7822. union fastrpc_dev_param p;
  7823. struct fastrpc_file *fl = NULL;
  7824. struct fastrpc_mmap *map = NULL;
  7825. struct fastrpc_apps *me = &gfa;
  7826. uintptr_t raddr = 0;
  7827. unsigned long irq_flags = 0;
  7828. p.map = (struct fastrpc_dev_map_dma *)invoke_param;
  7829. spin_lock_irqsave(&me->hlock, irq_flags);
  7830. /* Verify if fastrpc device is closed*/
  7831. VERIFY(err, dev && !dev->dev_close);
  7832. if (err) {
  7833. err = -ESRCH;
  7834. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7835. return err;
  7836. }
  7837. fl = dev->fl;
  7838. /* Verify if fastrpc file is not NULL*/
  7839. if (!fl) {
  7840. err = -EBADF;
  7841. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7842. return err;
  7843. }
  7844. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7845. mutex_lock(&fl->internal_map_mutex);
  7846. spin_lock_irqsave(&me->hlock, irq_flags);
  7847. /* Verify if fastrpc file is being closed, holding device lock*/
  7848. if (fl->file_close) {
  7849. err = -ESRCH;
  7850. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7851. goto bail;
  7852. }
  7853. fl->is_dma_invoke_pend = true;
  7854. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7855. mutex_lock(&fl->map_mutex);
  7856. /* Map DMA buffer on SMMU device*/
  7857. err = fastrpc_mmap_create(fl, -1, p.map->buf,
  7858. p.map->attrs, 0, p.map->size,
  7859. ADSP_MMAP_DMA_BUFFER, &map);
  7860. mutex_unlock(&fl->map_mutex);
  7861. if (err)
  7862. goto bail;
  7863. /* Map DMA buffer on DSP*/
  7864. VERIFY(err, 0 == (err = fastrpc_mmap_on_dsp(fl,
  7865. map->flags, 0, map->phys, map->size, map->refs, &raddr)));
  7866. if (err)
  7867. goto bail;
  7868. map->raddr = raddr;
  7869. p.map->v_dsp_addr = raddr;
  7870. bail:
  7871. if (err && map) {
  7872. mutex_lock(&fl->map_mutex);
  7873. fastrpc_mmap_free(map, 0);
  7874. mutex_unlock(&fl->map_mutex);
  7875. }
  7876. if (fl) {
  7877. spin_lock_irqsave(&me->hlock, irq_flags);
  7878. if (fl->file_close && fl->is_dma_invoke_pend)
  7879. complete(&fl->dma_invoke);
  7880. fl->is_dma_invoke_pend = false;
  7881. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7882. }
  7883. mutex_unlock(&fl->internal_map_mutex);
  7884. return err;
  7885. }
  7886. long fastrpc_dev_unmap_dma(struct fastrpc_device *dev, unsigned long invoke_param)
  7887. {
  7888. int err = 0;
  7889. union fastrpc_dev_param p;
  7890. struct fastrpc_file *fl = NULL;
  7891. struct fastrpc_mmap *map = NULL;
  7892. struct fastrpc_apps *me = &gfa;
  7893. unsigned long irq_flags = 0;
  7894. p.unmap = (struct fastrpc_dev_unmap_dma *)invoke_param;
  7895. spin_lock_irqsave(&me->hlock, irq_flags);
  7896. /* Verify if fastrpc device is closed*/
  7897. VERIFY(err, dev && !dev->dev_close);
  7898. if (err) {
  7899. err = -ESRCH;
  7900. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7901. return err;
  7902. }
  7903. fl = dev->fl;
  7904. /* Verify if fastrpc file is not NULL*/
  7905. if (!fl) {
  7906. err = -EBADF;
  7907. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7908. return err;
  7909. }
  7910. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7911. mutex_lock(&fl->internal_map_mutex);
  7912. spin_lock_irqsave(&me->hlock, irq_flags);
  7913. /* Verify if fastrpc file is being closed, holding device lock*/
  7914. if (fl->file_close) {
  7915. err = -ESRCH;
  7916. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7917. goto bail;
  7918. }
  7919. fl->is_dma_invoke_pend = true;
  7920. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7921. mutex_lock(&fl->map_mutex);
  7922. if (!fastrpc_mmap_find(fl, -1, p.unmap->buf, 0, 0, ADSP_MMAP_DMA_BUFFER, 0, &map)) {
  7923. mutex_unlock(&fl->map_mutex);
  7924. if (err)
  7925. goto bail;
  7926. /* Un-map DMA buffer on DSP*/
  7927. VERIFY(err, !(err = fastrpc_munmap_on_dsp(fl, map->raddr,
  7928. map->phys, map->size, map->flags)));
  7929. if (err)
  7930. goto bail;
  7931. mutex_lock(&fl->map_mutex);
  7932. fastrpc_mmap_free(map, 0);
  7933. }
  7934. mutex_unlock(&fl->map_mutex);
  7935. bail:
  7936. if (fl) {
  7937. spin_lock_irqsave(&me->hlock, irq_flags);
  7938. if (fl->file_close && fl->is_dma_invoke_pend)
  7939. complete(&fl->dma_invoke);
  7940. fl->is_dma_invoke_pend = false;
  7941. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7942. }
  7943. mutex_unlock(&fl->internal_map_mutex);
  7944. return err;
  7945. }
  7946. long fastrpc_dev_get_hlos_pid(struct fastrpc_device *dev, unsigned long invoke_param)
  7947. {
  7948. int err = 0;
  7949. union fastrpc_dev_param p;
  7950. struct fastrpc_file *fl = NULL;
  7951. struct fastrpc_apps *me = &gfa;
  7952. unsigned long irq_flags = 0;
  7953. p.hpid = (struct fastrpc_dev_get_hlos_pid *)invoke_param;
  7954. spin_lock_irqsave(&me->hlock, irq_flags);
  7955. /* Verify if fastrpc device is closed*/
  7956. VERIFY(err, dev && !dev->dev_close);
  7957. if (err) {
  7958. err = -ESRCH;
  7959. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7960. return err;
  7961. }
  7962. fl = dev->fl;
  7963. /* Verify if fastrpc file is not NULL*/
  7964. if (!fl) {
  7965. err = -EBADF;
  7966. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7967. return err;
  7968. }
  7969. p.hpid->hlos_pid = fl->tgid;
  7970. spin_unlock_irqrestore(&me->hlock, irq_flags);
  7971. return err;
  7972. }
  7973. long fastrpc_driver_invoke(struct fastrpc_device *dev, unsigned int invoke_num,
  7974. unsigned long invoke_param)
  7975. {
  7976. int err = 0;
  7977. switch (invoke_num) {
  7978. case FASTRPC_DEV_MAP_DMA:
  7979. err = fastrpc_dev_map_dma(dev, invoke_param);
  7980. break;
  7981. case FASTRPC_DEV_UNMAP_DMA:
  7982. err = fastrpc_dev_unmap_dma(dev, invoke_param);
  7983. break;
  7984. case FASTRPC_DEV_GET_HLOS_PID:
  7985. err = fastrpc_dev_get_hlos_pid(dev, invoke_param);
  7986. break;
  7987. default:
  7988. err = -ENOTTY;
  7989. break;
  7990. }
  7991. return err;
  7992. }
  7993. EXPORT_SYMBOL(fastrpc_driver_invoke);
  7994. static struct device fastrpc_bus = {
  7995. .init_name = "fastrpc"
  7996. };
  7997. static int fastrpc_bus_match(struct device *dev, struct device_driver *driver)
  7998. {
  7999. struct fastrpc_apps *me = &gfa;
  8000. struct fastrpc_driver *frpc_driver = to_fastrpc_driver(driver);
  8001. struct fastrpc_device *frpc_device = to_fastrpc_device(dev);
  8002. unsigned long irq_flags = 0;
  8003. if (frpc_device->handle == frpc_driver->handle) {
  8004. spin_lock_irqsave(&me->hlock, irq_flags);
  8005. /* If device is being closed, fail the match */
  8006. if (frpc_device->dev_close) {
  8007. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8008. return 0;
  8009. }
  8010. frpc_device->refs++;
  8011. frpc_driver->device = dev;
  8012. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8013. return 1;
  8014. }
  8015. return 0;
  8016. }
  8017. static int fastrpc_bus_probe(struct device *dev)
  8018. {
  8019. struct fastrpc_device *frpc_dev = to_fastrpc_device(dev);
  8020. struct fastrpc_driver *frpc_drv = to_fastrpc_driver(dev->driver);
  8021. if (frpc_drv && frpc_drv->probe)
  8022. return frpc_drv->probe(frpc_dev);
  8023. return 0;
  8024. }
  8025. static void fastrpc_bus_remove(struct device *dev)
  8026. {
  8027. struct fastrpc_driver *frpc_drv = to_fastrpc_driver(dev->driver);
  8028. if (frpc_drv && frpc_drv->callback)
  8029. frpc_drv->callback(to_fastrpc_device(dev), FASTRPC_PROC_DOWN);
  8030. }
  8031. static struct bus_type fastrpc_bus_type = {
  8032. .name = "fastrpc",
  8033. .match = fastrpc_bus_match,
  8034. .probe = fastrpc_bus_probe,
  8035. .remove = fastrpc_bus_remove,
  8036. };
  8037. static void fastrpc_dev_release(struct device *dev)
  8038. {
  8039. kfree(to_fastrpc_device(dev));
  8040. }
  8041. static int fastrpc_device_create(struct fastrpc_file *fl)
  8042. {
  8043. int err = 0;
  8044. struct fastrpc_device *frpc_dev;
  8045. struct fastrpc_apps *me = &gfa;
  8046. unsigned long irq_flags = 0;
  8047. frpc_dev = kzalloc(sizeof(*frpc_dev), GFP_KERNEL);
  8048. if (!frpc_dev) {
  8049. err = -ENOMEM;
  8050. goto bail;
  8051. }
  8052. frpc_dev->dev.parent = &fastrpc_bus;
  8053. frpc_dev->dev.bus = &fastrpc_bus_type;
  8054. /* Use HLOS PID, unique fastrpc process ID and CID to create device file,
  8055. * Else names would conflict for multiple sessions
  8056. * And also for better ability to debug
  8057. */
  8058. dev_set_name(&frpc_dev->dev, "%s-%d-%d-%d",
  8059. dev_name(frpc_dev->dev.parent), fl->tgid, fl->tgid_frpc, fl->cid);
  8060. frpc_dev->dev.release = fastrpc_dev_release;
  8061. frpc_dev->fl = fl;
  8062. /* Use unique fastrpc tgid as handle */
  8063. frpc_dev->handle = fl->tgid_frpc;
  8064. err = device_register(&frpc_dev->dev);
  8065. if (err) {
  8066. put_device(&frpc_dev->dev);
  8067. ADSPRPC_ERR(
  8068. "fastrpc device register failed for process %d unique fastrpc tgid %d session %d with error %d\n",
  8069. fl->tgid, fl->tgid_frpc, fl->sessionid, err);
  8070. goto bail;
  8071. }
  8072. fl->device = frpc_dev;
  8073. spin_lock_irqsave(&me->hlock, irq_flags);
  8074. hlist_add_head(&frpc_dev->hn, &me->frpc_devices);
  8075. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8076. bail:
  8077. return err;
  8078. }
  8079. void fastrpc_driver_unregister(struct fastrpc_driver *frpc_driver)
  8080. {
  8081. struct fastrpc_apps *me = &gfa;
  8082. struct device *dev = NULL;
  8083. struct fastrpc_device *frpc_dev = NULL;
  8084. bool is_device_closed = false;
  8085. unsigned long irq_flags = 0;
  8086. spin_lock_irqsave(&me->hlock, irq_flags);
  8087. dev = frpc_driver->device;
  8088. if (dev) {
  8089. frpc_dev = to_fastrpc_device(dev);
  8090. if (frpc_dev->refs > 0)
  8091. frpc_dev->refs--;
  8092. else
  8093. ADSPRPC_ERR("Fastrpc device for driver %s is already freed\n",
  8094. frpc_driver->driver.name);
  8095. if (frpc_dev->dev_close) {
  8096. hlist_del_init(&frpc_dev->hn);
  8097. is_device_closed = true;
  8098. }
  8099. }
  8100. hlist_del_init(&frpc_driver->hn);
  8101. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8102. if (is_device_closed) {
  8103. ADSPRPC_INFO("un-registering fastrpc device with handle %d\n",
  8104. frpc_dev->handle);
  8105. device_unregister(dev);
  8106. }
  8107. driver_unregister(&frpc_driver->driver);
  8108. ADSPRPC_INFO("Un-registering fastrpc driver %s with handle %d\n",
  8109. frpc_driver->driver.name, frpc_driver->handle);
  8110. }
  8111. EXPORT_SYMBOL(fastrpc_driver_unregister);
  8112. int fastrpc_driver_register(struct fastrpc_driver *frpc_driver)
  8113. {
  8114. int err = 0;
  8115. struct fastrpc_apps *me = &gfa;
  8116. unsigned long irq_flags = 0;
  8117. frpc_driver->driver.bus = &fastrpc_bus_type;
  8118. frpc_driver->driver.owner = THIS_MODULE;
  8119. err = driver_register(&frpc_driver->driver);
  8120. if (err) {
  8121. ADSPRPC_ERR("fastrpc driver %s failed to register with error %d\n",
  8122. frpc_driver->driver.name, err);
  8123. goto bail;
  8124. }
  8125. ADSPRPC_INFO("fastrpc driver %s registered with handle %d\n",
  8126. frpc_driver->driver.name, frpc_driver->handle);
  8127. spin_lock_irqsave(&me->hlock, irq_flags);
  8128. hlist_add_head(&frpc_driver->hn, &me->frpc_drivers);
  8129. spin_unlock_irqrestore(&me->hlock, irq_flags);
  8130. bail:
  8131. return err;
  8132. }
  8133. EXPORT_SYMBOL(fastrpc_driver_register);
  8134. static int __init fastrpc_device_init(void)
  8135. {
  8136. struct fastrpc_apps *me = &gfa;
  8137. int err = 0, i;
  8138. uintptr_t attr = 0;
  8139. dma_addr_t region_phys = 0;
  8140. void *region_vaddr = NULL;
  8141. struct fastrpc_buf *buf = NULL;
  8142. debugfs_root = debugfs_create_dir("adsprpc", NULL);
  8143. if (IS_ERR_OR_NULL(debugfs_root)) {
  8144. pr_warn("Error: %s: %s: failed to create debugfs root dir\n",
  8145. current->comm, __func__);
  8146. debugfs_remove_recursive(debugfs_root);
  8147. debugfs_root = NULL;
  8148. }
  8149. memset(me, 0, sizeof(*me));
  8150. fastrpc_init(me);
  8151. fastrpc_get_nsp_status(me);
  8152. me->dev = NULL;
  8153. me->legacy_remote_heap = false;
  8154. err = bus_register(&fastrpc_bus_type);
  8155. if (err) {
  8156. ADSPRPC_ERR("fastrpc bus register failed with err %d\n",
  8157. err);
  8158. goto bus_register_bail;
  8159. }
  8160. err = device_register(&fastrpc_bus);
  8161. if (err) {
  8162. ADSPRPC_ERR("fastrpc bus device register failed with err %d\n",
  8163. err);
  8164. goto bus_device_register_bail;
  8165. }
  8166. me->fastrpc_bus_register = true;
  8167. fastrpc_lowest_capacity_corecount(me);
  8168. VERIFY(err, 0 == platform_driver_register(&fastrpc_driver));
  8169. if (err)
  8170. goto register_bail;
  8171. VERIFY(err, 0 == alloc_chrdev_region(&me->dev_no, 0, NUM_CHANNELS,
  8172. DEVICE_NAME));
  8173. if (err)
  8174. goto alloc_chrdev_bail;
  8175. cdev_init(&me->cdev, &fops);
  8176. me->cdev.owner = THIS_MODULE;
  8177. VERIFY(err, 0 == cdev_add(&me->cdev, MKDEV(MAJOR(me->dev_no), 0),
  8178. NUM_DEVICES));
  8179. if (err)
  8180. goto cdev_init_bail;
  8181. me->class = class_create(THIS_MODULE, "fastrpc");
  8182. VERIFY(err, !IS_ERR(me->class));
  8183. if (err)
  8184. goto class_create_bail;
  8185. #if IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  8186. me->compat = 1;
  8187. #else
  8188. me->compat = (fops.compat_ioctl == NULL) ? 0 : 1;
  8189. #endif
  8190. /*
  8191. * Create devices and register with sysfs
  8192. * Create first device with minor number 0
  8193. */
  8194. me->non_secure_dev = device_create(me->class, NULL,
  8195. MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV),
  8196. NULL, DEVICE_NAME);
  8197. VERIFY(err, !IS_ERR_OR_NULL(me->non_secure_dev));
  8198. if (err) {
  8199. err = -ENODEV;
  8200. goto device_create_bail;
  8201. }
  8202. /* Create secure device with minor number for secure device */
  8203. me->secure_dev = device_create(me->class, NULL,
  8204. MKDEV(MAJOR(me->dev_no), MINOR_NUM_SECURE_DEV),
  8205. NULL, DEVICE_NAME_SECURE);
  8206. VERIFY(err, !IS_ERR_OR_NULL(me->secure_dev));
  8207. if (err)
  8208. goto device_create_bail;
  8209. for (i = 0; i < NUM_CHANNELS; i++) {
  8210. me->jobid[i] = 1;
  8211. me->channel[i].dev = me->secure_dev;
  8212. me->channel[i].ssrcount = 0;
  8213. me->channel[i].in_hib = 0;
  8214. me->channel[i].prevssrcount = 0;
  8215. me->channel[i].subsystemstate = SUBSYSTEM_UP;
  8216. me->channel[i].rh_dump_dev = NULL;
  8217. me->channel[i].nb.notifier_call = fastrpc_restart_notifier_cb;
  8218. me->channel[i].handle = qcom_register_ssr_notifier(
  8219. gcinfo[i].subsys,
  8220. &me->channel[i].nb);
  8221. if (i == CDSP_DOMAIN_ID) {
  8222. me->channel[i].dev = me->non_secure_dev;
  8223. #if !IS_ENABLED(CONFIG_MSM_ADSPRPC_TRUSTED)
  8224. err = fastrpc_alloc_cma_memory(&region_phys,
  8225. &region_vaddr,
  8226. MINI_DUMP_DBG_SIZE,
  8227. (unsigned long)attr);
  8228. #endif
  8229. if (err)
  8230. ADSPRPC_WARN("%s: CMA alloc failed err 0x%x\n",
  8231. __func__, err);
  8232. VERIFY(err, NULL != (buf = kzalloc(sizeof(*buf), GFP_KERNEL)));
  8233. if (err) {
  8234. err = -ENOMEM;
  8235. ADSPRPC_ERR("%s: CMA alloc failed err 0x%x\n",
  8236. __func__, err);
  8237. goto device_create_bail;
  8238. }
  8239. INIT_HLIST_NODE(&buf->hn);
  8240. buf->virt = region_vaddr;
  8241. buf->phys = (uintptr_t)region_phys;
  8242. buf->size = MINI_DUMP_DBG_SIZE;
  8243. buf->dma_attr = attr;
  8244. buf->raddr = 0;
  8245. ktime_get_real_ts64(&buf->buf_start_time);
  8246. me->channel[i].buf = buf;
  8247. }
  8248. if (IS_ERR_OR_NULL(me->channel[i].handle))
  8249. pr_warn("adsprpc: %s: SSR notifier register failed for %s with err %d\n",
  8250. __func__, gcinfo[i].subsys,
  8251. PTR_ERR(me->channel[i].handle));
  8252. else
  8253. pr_info("adsprpc: %s: SSR notifier registered for %s\n",
  8254. __func__, gcinfo[i].subsys);
  8255. }
  8256. err = fastrpc_transport_init();
  8257. if (err)
  8258. goto device_create_bail;
  8259. me->transport_initialized = 1;
  8260. #ifdef CONFIG_HIBERNATION
  8261. err = register_pm_notifier(&fastrpc_notif_block);
  8262. if (err)
  8263. goto device_create_bail;
  8264. #endif
  8265. fastrpc_register_wakeup_source(me->non_secure_dev,
  8266. FASTRPC_NON_SECURE_WAKE_SOURCE_CLIENT_NAME,
  8267. &me->wake_source);
  8268. fastrpc_register_wakeup_source(me->secure_dev,
  8269. FASTRPC_SECURE_WAKE_SOURCE_CLIENT_NAME,
  8270. &me->wake_source_secure);
  8271. return 0;
  8272. device_create_bail:
  8273. for (i = 0; i < NUM_CHANNELS; i++) {
  8274. if (me->channel[i].handle)
  8275. qcom_unregister_ssr_notifier(me->channel[i].handle,
  8276. &me->channel[i].nb);
  8277. }
  8278. if (!IS_ERR_OR_NULL(me->non_secure_dev))
  8279. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8280. MINOR_NUM_DEV));
  8281. if (!IS_ERR_OR_NULL(me->secure_dev))
  8282. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8283. MINOR_NUM_SECURE_DEV));
  8284. class_destroy(me->class);
  8285. class_create_bail:
  8286. cdev_del(&me->cdev);
  8287. cdev_init_bail:
  8288. unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
  8289. alloc_chrdev_bail:
  8290. platform_driver_unregister(&fastrpc_driver);
  8291. register_bail:
  8292. device_unregister(&fastrpc_bus);
  8293. bus_device_register_bail:
  8294. bus_unregister(&fastrpc_bus_type);
  8295. bus_register_bail:
  8296. fastrpc_deinit();
  8297. return err;
  8298. }
  8299. static void __exit fastrpc_device_exit(void)
  8300. {
  8301. struct fastrpc_apps *me = &gfa;
  8302. int i;
  8303. fastrpc_file_list_dtor(me);
  8304. fastrpc_deinit();
  8305. wakeup_source_unregister(me->wake_source);
  8306. wakeup_source_unregister(me->wake_source_secure);
  8307. for (i = 0; i < NUM_CHANNELS; i++) {
  8308. if (i == CDSP_DOMAIN_ID)
  8309. kfree(me->channel[i].buf);
  8310. if (!gcinfo[i].name)
  8311. continue;
  8312. qcom_unregister_ssr_notifier(me->channel[i].handle,
  8313. &me->channel[i].nb);
  8314. }
  8315. /* Destroy the secure and non secure devices */
  8316. device_destroy(me->class, MKDEV(MAJOR(me->dev_no), MINOR_NUM_DEV));
  8317. device_destroy(me->class, MKDEV(MAJOR(me->dev_no),
  8318. MINOR_NUM_SECURE_DEV));
  8319. of_reserved_mem_device_release(me->dev);
  8320. class_destroy(me->class);
  8321. cdev_del(&me->cdev);
  8322. unregister_chrdev_region(me->dev_no, NUM_CHANNELS);
  8323. if (me->transport_initialized)
  8324. fastrpc_transport_deinit();
  8325. me->transport_initialized = 0;
  8326. if (me->fastrpc_bus_register) {
  8327. bus_unregister(&fastrpc_bus_type);
  8328. device_unregister(&fastrpc_bus);
  8329. }
  8330. kfree(me->gidlist.gids);
  8331. debugfs_remove_recursive(debugfs_root);
  8332. }
  8333. module_init(fastrpc_device_init);
  8334. module_exit(fastrpc_device_exit);
  8335. MODULE_LICENSE("GPL v2");