cvp_hfi.c 139 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587458845894590459145924593459445954596459745984599460046014602460346044605460646074608460946104611461246134614461546164617461846194620462146224623462446254626462746284629463046314632463346344635463646374638463946404641464246434644464546464647464846494650465146524653465446554656465746584659466046614662466346644665466646674668466946704671467246734674467546764677467846794680468146824683468446854686468746884689469046914692469346944695469646974698469947004701470247034704470547064707470847094710471147124713471447154716471747184719472047214722472347244725472647274728472947304731473247334734473547364737473847394740474147424743474447454746474747484749475047514752475347544755475647574758475947604761476247634764476547664767476847694770477147724773477447754776477747784779478047814782478347844785478647874788478947904791479247934794479547964797479847994800480148024803480448054806480748084809481048114812481348144815481648174818481948204821482248234824482548264827482848294830483148324833483448354836483748384839484048414842484348444845484648474848484948504851485248534854485548564857485848594860486148624863486448654866486748684869487048714872487348744875487648774878487948804881488248834884488548864887488848894890489148924893489448954896489748984899490049014902490349044905490649074908490949104911491249134914491549164917491849194920492149224923492449254926492749284929493049314932493349344935493649374938493949404941494249434944494549464947494849494950495149524953495449554956495749584959496049614962496349644965496649674968496949704971497249734974497549764977497849794980498149824983498449854986498749884989499049914992499349944995499649974998499950005001500250035004500550065007500850095010501150125013501450155016501750185019502050215022502350245025502650275028502950305031503250335034503550365037503850395040504150425043504450455046504750485049505050515052505350545055505650575058505950605061506250635064506550665067506850695070507150725073507450755076507750785079508050815082508350845085508650875088508950905091509250935094509550965097509850995100510151025103510451055106510751085109511051115112511351145115511651175118511951205121512251235124512551265127512851295130513151325133513451355136513751385139514051415142514351445145514651475148514951505151515251535154515551565157515851595160516151625163516451655166516751685169517051715172517351745175517651775178517951805181518251835184518551865187518851895190519151925193519451955196519751985199520052015202520352045205520652075208520952105211521252135214521552165217521852195220522152225223522452255226522752285229523052315232523352345235523652375238523952405241524252435244524552465247524852495250525152525253525452555256525752585259526052615262526352645265526652675268526952705271527252735274527552765277527852795280528152825283528452855286528752885289529052915292529352945295529652975298529953005301530253035304530553065307530853095310531153125313531453155316531753185319532053215322532353245325532653275328532953305331533253335334533553365337533853395340534153425343534453455346534753485349535053515352535353545355535653575358535953605361536253635364536553665367536853695370537153725373537453755376537753785379538053815382538353845385538653875388538953905391539253935394539553965397539853995400540154025403540454055406540754085409541054115412541354145415541654175418541954205421542254235424542554265427542854295430543154325433543454355436543754385439544054415442544354445445544654475448544954505451545254535454545554565457545854595460546154625463546454655466546754685469547054715472547354745475547654775478547954805481548254835484
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
  4. * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
  5. */
  6. #include <asm/memory.h>
  7. #include <linux/coresight-stm.h>
  8. #include <linux/delay.h>
  9. #include <linux/devfreq.h>
  10. #include <linux/hash.h>
  11. #include <linux/io.h>
  12. #include <linux/iommu.h>
  13. #include <linux/iopoll.h>
  14. #include <linux/of.h>
  15. #include <linux/pm_qos.h>
  16. #include <linux/regulator/consumer.h>
  17. #include <linux/slab.h>
  18. #include <linux/workqueue.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/soc/qcom/llcc-qcom.h>
  21. #include <linux/qcom_scm.h>
  22. #include <linux/soc/qcom/smem.h>
  23. #include <linux/dma-mapping.h>
  24. #include <linux/reset.h>
  25. #include <linux/pm_wakeup.h>
  26. #include "hfi_packetization.h"
  27. #include "msm_cvp_debug.h"
  28. #include "cvp_core_hfi.h"
  29. #include "cvp_hfi_helper.h"
  30. #include "cvp_hfi_io.h"
  31. #include "msm_cvp_dsp.h"
  32. #include "msm_cvp_clocks.h"
  33. #include "vm/cvp_vm.h"
  34. #include "cvp_dump.h"
  35. // ysi - added for debug
  36. #include <linux/clk/qcom.h>
  37. #include "msm_cvp_common.h"
  38. #define REG_ADDR_OFFSET_BITMASK 0x000FFFFF
  39. #define QDSS_IOVA_START 0x80001000
  40. #define MIN_PAYLOAD_SIZE 3
  41. struct cvp_tzbsp_memprot {
  42. u32 cp_start;
  43. u32 cp_size;
  44. u32 cp_nonpixel_start;
  45. u32 cp_nonpixel_size;
  46. };
  47. #define TZBSP_CVP_PAS_ID 26
  48. /* Poll interval in uS */
  49. #define POLL_INTERVAL_US 50
  50. enum tzbsp_subsys_state {
  51. TZ_SUBSYS_STATE_SUSPEND = 0,
  52. TZ_SUBSYS_STATE_RESUME = 1,
  53. TZ_SUBSYS_STATE_RESTORE_THRESHOLD = 2,
  54. };
  55. const struct msm_cvp_gov_data CVP_DEFAULT_BUS_VOTE = {
  56. .data = NULL,
  57. .data_count = 0,
  58. };
  59. const int cvp_max_packets = 32;
  60. static void iris_hfi_pm_handler(struct work_struct *work);
  61. static DECLARE_DELAYED_WORK(iris_hfi_pm_work, iris_hfi_pm_handler);
  62. static inline int __resume(struct iris_hfi_device *device);
  63. static inline int __suspend(struct iris_hfi_device *device);
  64. static int __disable_regulator(struct iris_hfi_device *device,
  65. const char *name);
  66. static int __enable_regulator(struct iris_hfi_device *device,
  67. const char *name);
  68. static void __flush_debug_queue(struct iris_hfi_device *device, u8 *packet);
  69. static int __initialize_packetization(struct iris_hfi_device *device);
  70. static struct cvp_hal_session *__get_session(struct iris_hfi_device *device,
  71. u32 session_id);
  72. static bool __is_session_valid(struct iris_hfi_device *device,
  73. struct cvp_hal_session *session, const char *func);
  74. static int __iface_cmdq_write(struct iris_hfi_device *device,
  75. void *pkt);
  76. static int __load_fw(struct iris_hfi_device *device);
  77. static int __power_on_init(struct iris_hfi_device *device);
  78. static void __unload_fw(struct iris_hfi_device *device);
  79. static int __tzbsp_set_cvp_state(enum tzbsp_subsys_state state);
  80. static int __enable_subcaches(struct iris_hfi_device *device);
  81. static int __set_subcaches(struct iris_hfi_device *device);
  82. static int __release_subcaches(struct iris_hfi_device *device);
  83. static int __disable_subcaches(struct iris_hfi_device *device);
  84. static int __power_collapse(struct iris_hfi_device *device, bool force);
  85. static int iris_hfi_noc_error_info(void *dev);
  86. static void interrupt_init_iris2(struct iris_hfi_device *device);
  87. static void setup_dsp_uc_memmap_vpu5(struct iris_hfi_device *device);
  88. static void clock_config_on_enable_vpu5(struct iris_hfi_device *device);
  89. static void power_off_iris2(struct iris_hfi_device *device);
  90. static int __set_ubwc_config(struct iris_hfi_device *device);
  91. static void __noc_error_info_iris2(struct iris_hfi_device *device);
  92. static int __enable_hw_power_collapse(struct iris_hfi_device *device);
  93. static int __disable_hw_power_collapse(struct iris_hfi_device *device);
  94. static int __power_off_controller(struct iris_hfi_device *device);
  95. static int __hwfence_regs_map(struct iris_hfi_device *device);
  96. static int __hwfence_regs_unmap(struct iris_hfi_device *device);
  97. static int __reset_control_assert_name(struct iris_hfi_device *device, const char *name);
  98. static int __reset_control_deassert_name(struct iris_hfi_device *device, const char *name);
  99. static int __reset_control_acquire(struct iris_hfi_device *device, const char *name);
  100. static int __reset_control_release(struct iris_hfi_device *device, const char *name);
  101. static struct iris_hfi_vpu_ops iris2_ops = {
  102. .interrupt_init = interrupt_init_iris2,
  103. .setup_dsp_uc_memmap = setup_dsp_uc_memmap_vpu5,
  104. .clock_config_on_enable = clock_config_on_enable_vpu5,
  105. .power_off = power_off_iris2,
  106. .noc_error_info = __noc_error_info_iris2,
  107. .reset_control_assert_name = __reset_control_assert_name,
  108. .reset_control_deassert_name = __reset_control_deassert_name,
  109. .reset_control_acquire_name = __reset_control_acquire,
  110. .reset_control_release_name = __reset_control_release,
  111. };
  112. /**
  113. * Utility function to enforce some of our assumptions. Spam calls to this
  114. * in hotspots in code to double check some of the assumptions that we hold.
  115. */
  116. static inline void __strict_check(struct iris_hfi_device *device)
  117. {
  118. msm_cvp_res_handle_fatal_hw_error(device->res,
  119. !mutex_is_locked(&device->lock));
  120. }
  121. static inline void __set_state(struct iris_hfi_device *device,
  122. enum iris_hfi_state state)
  123. {
  124. device->state = state;
  125. }
  126. static inline bool __core_in_valid_state(struct iris_hfi_device *device)
  127. {
  128. return device->state != IRIS_STATE_DEINIT;
  129. }
  130. static inline bool is_sys_cache_present(struct iris_hfi_device *device)
  131. {
  132. return device->res->sys_cache_present;
  133. }
  134. static int cvp_synx_recover(void)
  135. {
  136. #ifdef CVP_SYNX_ENABLED
  137. return synx_recover(SYNX_CLIENT_EVA_CTX0);
  138. #else
  139. return 0;
  140. #endif /* End of CVP_SYNX_ENABLED */
  141. }
  142. #define ROW_SIZE 32
  143. unsigned long long get_aon_time(void)
  144. {
  145. unsigned long long val;
  146. asm volatile("mrs %0, cntvct_el0" : "=r" (val));
  147. return val;
  148. }
  149. int get_hfi_version(void)
  150. {
  151. struct msm_cvp_core *core;
  152. struct iris_hfi_device *hfi;
  153. core = cvp_driver->cvp_core;
  154. hfi = (struct iris_hfi_device *)core->device->hfi_device_data;
  155. return hfi->version;
  156. }
  157. unsigned int get_msg_size(struct cvp_hfi_msg_session_hdr *hdr)
  158. {
  159. struct msm_cvp_core *core;
  160. struct iris_hfi_device *device;
  161. u32 minor_ver;
  162. core = cvp_driver->cvp_core;
  163. if (core)
  164. device = core->device->hfi_device_data;
  165. else
  166. return 0;
  167. if (!device) {
  168. dprintk(CVP_ERR, "%s: NULL device\n", __func__);
  169. return 0;
  170. }
  171. minor_ver = (device->version & HFI_VERSION_MINOR_MASK) >>
  172. HFI_VERSION_MINOR_SHIFT;
  173. if (minor_ver < 2)
  174. return sizeof(struct cvp_hfi_msg_session_hdr);
  175. if (hdr->packet_type == HFI_MSG_SESSION_CVP_FD)
  176. return sizeof(struct cvp_hfi_msg_session_hdr_ext);
  177. else
  178. return sizeof(struct cvp_hfi_msg_session_hdr);
  179. }
  180. unsigned int get_msg_session_id(void *msg)
  181. {
  182. struct cvp_hfi_msg_session_hdr *hdr =
  183. (struct cvp_hfi_msg_session_hdr *)msg;
  184. return hdr->session_id;
  185. }
  186. unsigned int get_msg_errorcode(void *msg)
  187. {
  188. struct cvp_hfi_msg_session_hdr *hdr =
  189. (struct cvp_hfi_msg_session_hdr *)msg;
  190. return hdr->error_type;
  191. }
  192. int get_msg_opconfigs(void *msg, unsigned int *session_id,
  193. unsigned int *error_type, unsigned int *config_id)
  194. {
  195. struct cvp_hfi_msg_session_op_cfg_packet *cfg =
  196. (struct cvp_hfi_msg_session_op_cfg_packet *)msg;
  197. *session_id = cfg->session_id;
  198. *error_type = cfg->error_type;
  199. *config_id = cfg->op_conf_id;
  200. return 0;
  201. }
  202. static void __dump_packet(u8 *packet, enum cvp_msg_prio log_level)
  203. {
  204. u32 c = 0, packet_size = *(u32 *)packet;
  205. /*
  206. * row must contain enough for 0xdeadbaad * 8 to be converted into
  207. * "de ad ba ab " * 8 + '\0'
  208. */
  209. char row[3 * ROW_SIZE];
  210. for (c = 0; c * ROW_SIZE < packet_size; ++c) {
  211. int bytes_to_read = ((c + 1) * ROW_SIZE > packet_size) ?
  212. packet_size % ROW_SIZE : ROW_SIZE;
  213. hex_dump_to_buffer(packet + c * ROW_SIZE, bytes_to_read,
  214. ROW_SIZE, 4, row, sizeof(row), false);
  215. dprintk(log_level, "%s\n", row);
  216. }
  217. }
  218. static int __dsp_suspend(struct iris_hfi_device *device, bool force)
  219. {
  220. int rc;
  221. if (msm_cvp_dsp_disable)
  222. return 0;
  223. dprintk(CVP_DSP, "%s: suspend dsp\n", __func__);
  224. rc = cvp_dsp_suspend(force);
  225. if (rc) {
  226. if (rc != -EBUSY)
  227. dprintk(CVP_ERR,
  228. "%s: dsp suspend failed with error %d\n",
  229. __func__, rc);
  230. return rc;
  231. }
  232. dprintk(CVP_DSP, "%s: dsp suspended\n", __func__);
  233. return 0;
  234. }
  235. static int __dsp_resume(struct iris_hfi_device *device)
  236. {
  237. int rc;
  238. if (msm_cvp_dsp_disable)
  239. return 0;
  240. dprintk(CVP_DSP, "%s: resume dsp\n", __func__);
  241. rc = cvp_dsp_resume();
  242. if (rc) {
  243. dprintk(CVP_ERR,
  244. "%s: dsp resume failed with error %d\n",
  245. __func__, rc);
  246. return rc;
  247. }
  248. dprintk(CVP_DSP, "%s: dsp resumed\n", __func__);
  249. return rc;
  250. }
  251. static int __dsp_shutdown(struct iris_hfi_device *device)
  252. {
  253. int rc;
  254. if (msm_cvp_dsp_disable)
  255. return 0;
  256. dprintk(CVP_DSP, "%s: shutdown dsp\n", __func__);
  257. rc = cvp_dsp_shutdown();
  258. if (rc) {
  259. dprintk(CVP_ERR,
  260. "%s: dsp shutdown failed with error %d\n",
  261. __func__, rc);
  262. WARN_ON(1);
  263. }
  264. dprintk(CVP_DSP, "%s: dsp shutdown successful\n", __func__);
  265. return rc;
  266. }
  267. static int __acquire_regulator(struct regulator_info *rinfo,
  268. struct iris_hfi_device *device)
  269. {
  270. int rc = 0;
  271. if (rinfo->has_hw_power_collapse) {
  272. rc = regulator_set_mode(rinfo->regulator,
  273. REGULATOR_MODE_NORMAL);
  274. if (rc) {
  275. /*
  276. * This is somewhat fatal, but nothing we can do
  277. * about it. We can't disable the regulator w/o
  278. * getting it back under s/w control
  279. */
  280. dprintk(CVP_WARN,
  281. "Failed to acquire regulator control: %s\n",
  282. rinfo->name);
  283. } else {
  284. dprintk(CVP_PWR,
  285. "Acquire regulator control from HW: %s\n",
  286. rinfo->name);
  287. }
  288. }
  289. if (!regulator_is_enabled(rinfo->regulator)) {
  290. dprintk(CVP_WARN, "Regulator is not enabled %s\n",
  291. rinfo->name);
  292. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  293. }
  294. return rc;
  295. }
  296. static int __hand_off_regulator(struct regulator_info *rinfo)
  297. {
  298. int rc = 0;
  299. if (rinfo->has_hw_power_collapse) {
  300. rc = regulator_set_mode(rinfo->regulator,
  301. REGULATOR_MODE_FAST);
  302. if (rc) {
  303. dprintk(CVP_WARN,
  304. "Failed to hand off regulator control: %s\n",
  305. rinfo->name);
  306. } else {
  307. dprintk(CVP_PWR,
  308. "Hand off regulator control to HW: %s\n",
  309. rinfo->name);
  310. }
  311. }
  312. return rc;
  313. }
  314. static int __hand_off_regulators(struct iris_hfi_device *device)
  315. {
  316. struct regulator_info *rinfo;
  317. int rc = 0, c = 0;
  318. iris_hfi_for_each_regulator(device, rinfo) {
  319. rc = __hand_off_regulator(rinfo);
  320. /*
  321. * If one regulator hand off failed, driver should take
  322. * the control for other regulators back.
  323. */
  324. if (rc)
  325. goto err_reg_handoff_failed;
  326. c++;
  327. }
  328. return rc;
  329. err_reg_handoff_failed:
  330. iris_hfi_for_each_regulator_reverse_continue(device, rinfo, c)
  331. __acquire_regulator(rinfo, device);
  332. return rc;
  333. }
  334. static int __take_back_regulators(struct iris_hfi_device *device)
  335. {
  336. struct regulator_info *rinfo;
  337. int rc = 0;
  338. iris_hfi_for_each_regulator(device, rinfo) {
  339. rc = __acquire_regulator(rinfo, device);
  340. /*
  341. * if one regulator hand off failed, driver should take
  342. * the control for other regulators back.
  343. */
  344. if (rc)
  345. return rc;
  346. }
  347. return rc;
  348. }
  349. static int __write_queue(struct cvp_iface_q_info *qinfo, u8 *packet,
  350. bool *rx_req_is_set)
  351. {
  352. struct cvp_hfi_queue_header *queue;
  353. struct cvp_hfi_cmd_session_hdr *cmd_pkt;
  354. u32 packet_size_in_words, new_write_idx;
  355. u32 empty_space, read_idx, write_idx;
  356. u32 *write_ptr;
  357. if (!qinfo || !packet) {
  358. dprintk(CVP_ERR, "Invalid Params\n");
  359. return -EINVAL;
  360. } else if (!qinfo->q_array.align_virtual_addr) {
  361. dprintk(CVP_WARN, "Queues have already been freed\n");
  362. return -EINVAL;
  363. }
  364. queue = (struct cvp_hfi_queue_header *) qinfo->q_hdr;
  365. if (!queue) {
  366. dprintk(CVP_ERR, "queue not present\n");
  367. return -ENOENT;
  368. }
  369. cmd_pkt = (struct cvp_hfi_cmd_session_hdr *)packet;
  370. if (cmd_pkt->size >= sizeof(struct cvp_hfi_cmd_session_hdr))
  371. dprintk(CVP_CMD, "%s: "
  372. "pkt_type %08x sess_id %08x trans_id %u ktid %llu\n",
  373. __func__, cmd_pkt->packet_type,
  374. cmd_pkt->session_id,
  375. cmd_pkt->client_data.transaction_id,
  376. cmd_pkt->client_data.kdata & (FENCE_BIT - 1));
  377. else
  378. dprintk(CVP_CMD, "%s: "
  379. "pkt_type %08x", __func__, cmd_pkt->packet_type);
  380. if (msm_cvp_debug & CVP_PKT) {
  381. dprintk(CVP_PKT, "%s: %pK\n", __func__, qinfo);
  382. __dump_packet(packet, CVP_PKT);
  383. }
  384. packet_size_in_words = (*(u32 *)packet) >> 2;
  385. if (!packet_size_in_words || packet_size_in_words >
  386. qinfo->q_array.mem_size>>2) {
  387. dprintk(CVP_ERR, "Invalid packet size\n");
  388. return -ENODATA;
  389. }
  390. spin_lock(&qinfo->hfi_lock);
  391. read_idx = queue->qhdr_read_idx;
  392. write_idx = queue->qhdr_write_idx;
  393. empty_space = (write_idx >= read_idx) ?
  394. ((qinfo->q_array.mem_size>>2) - (write_idx - read_idx)) :
  395. (read_idx - write_idx);
  396. if (empty_space <= packet_size_in_words) {
  397. queue->qhdr_tx_req = 1;
  398. spin_unlock(&qinfo->hfi_lock);
  399. dprintk(CVP_ERR, "Insufficient size (%d) to write (%d)\n",
  400. empty_space, packet_size_in_words);
  401. return -ENOTEMPTY;
  402. }
  403. queue->qhdr_tx_req = 0;
  404. new_write_idx = write_idx + packet_size_in_words;
  405. write_ptr = (u32 *)((qinfo->q_array.align_virtual_addr) +
  406. (write_idx << 2));
  407. if (write_ptr < (u32 *)qinfo->q_array.align_virtual_addr ||
  408. write_ptr > (u32 *)(qinfo->q_array.align_virtual_addr +
  409. qinfo->q_array.mem_size)) {
  410. spin_unlock(&qinfo->hfi_lock);
  411. dprintk(CVP_ERR, "Invalid write index\n");
  412. return -ENODATA;
  413. }
  414. if (new_write_idx < (qinfo->q_array.mem_size >> 2)) {
  415. memcpy(write_ptr, packet, packet_size_in_words << 2);
  416. } else {
  417. new_write_idx -= qinfo->q_array.mem_size >> 2;
  418. memcpy(write_ptr, packet, (packet_size_in_words -
  419. new_write_idx) << 2);
  420. memcpy((void *)qinfo->q_array.align_virtual_addr,
  421. packet + ((packet_size_in_words - new_write_idx) << 2),
  422. new_write_idx << 2);
  423. }
  424. /*
  425. * Memory barrier to make sure packet is written before updating the
  426. * write index
  427. */
  428. mb();
  429. queue->qhdr_write_idx = new_write_idx;
  430. if (rx_req_is_set)
  431. *rx_req_is_set = queue->qhdr_rx_req == 1;
  432. /*
  433. * Memory barrier to make sure write index is updated before an
  434. * interrupt is raised.
  435. */
  436. mb();
  437. spin_unlock(&qinfo->hfi_lock);
  438. return 0;
  439. }
  440. static int __read_queue(struct cvp_iface_q_info *qinfo, u8 *packet,
  441. u32 *pb_tx_req_is_set)
  442. {
  443. struct cvp_hfi_queue_header *queue;
  444. struct cvp_hfi_msg_session_hdr *msg_pkt;
  445. u32 packet_size_in_words, new_read_idx;
  446. u32 *read_ptr;
  447. u32 receive_request = 0;
  448. u32 read_idx, write_idx;
  449. int rc = 0;
  450. if (!qinfo || !packet || !pb_tx_req_is_set) {
  451. dprintk(CVP_ERR, "Invalid Params\n");
  452. return -EINVAL;
  453. } else if (!qinfo->q_array.align_virtual_addr) {
  454. dprintk(CVP_WARN, "Queues have already been freed\n");
  455. return -EINVAL;
  456. }
  457. /*
  458. * Memory barrier to make sure data is valid before
  459. *reading it
  460. */
  461. mb();
  462. queue = (struct cvp_hfi_queue_header *) qinfo->q_hdr;
  463. if (!queue) {
  464. dprintk(CVP_ERR, "Queue memory is not allocated\n");
  465. return -ENOMEM;
  466. }
  467. /*
  468. * Do not set receive request for debug queue, if set,
  469. * Iris generates interrupt for debug messages even
  470. * when there is no response message available.
  471. * In general debug queue will not become full as it
  472. * is being emptied out for every interrupt from Iris.
  473. * Iris will anyway generates interrupt if it is full.
  474. */
  475. spin_lock(&qinfo->hfi_lock);
  476. if (queue->qhdr_type & HFI_Q_ID_CTRL_TO_HOST_MSG_Q)
  477. receive_request = 1;
  478. read_idx = queue->qhdr_read_idx;
  479. write_idx = queue->qhdr_write_idx;
  480. if (read_idx == write_idx) {
  481. queue->qhdr_rx_req = receive_request;
  482. /*
  483. * mb() to ensure qhdr is updated in main memory
  484. * so that iris reads the updated header values
  485. */
  486. mb();
  487. *pb_tx_req_is_set = 0;
  488. if (write_idx != queue->qhdr_write_idx) {
  489. queue->qhdr_rx_req = 0;
  490. } else {
  491. spin_unlock(&qinfo->hfi_lock);
  492. dprintk(CVP_HFI,
  493. "%s queue is empty, rx_req = %u, tx_req = %u, read_idx = %u\n",
  494. receive_request ? "message" : "debug",
  495. queue->qhdr_rx_req, queue->qhdr_tx_req,
  496. queue->qhdr_read_idx);
  497. return -ENODATA;
  498. }
  499. }
  500. read_ptr = (u32 *)((qinfo->q_array.align_virtual_addr) +
  501. (read_idx << 2));
  502. if (read_ptr < (u32 *)qinfo->q_array.align_virtual_addr ||
  503. read_ptr > (u32 *)(qinfo->q_array.align_virtual_addr +
  504. qinfo->q_array.mem_size - sizeof(*read_ptr))) {
  505. spin_unlock(&qinfo->hfi_lock);
  506. dprintk(CVP_ERR, "Invalid read index\n");
  507. return -ENODATA;
  508. }
  509. packet_size_in_words = (*read_ptr) >> 2;
  510. if (!packet_size_in_words) {
  511. spin_unlock(&qinfo->hfi_lock);
  512. dprintk(CVP_ERR, "Zero packet size\n");
  513. return -ENODATA;
  514. }
  515. new_read_idx = read_idx + packet_size_in_words;
  516. if (((packet_size_in_words << 2) <= CVP_IFACEQ_VAR_HUGE_PKT_SIZE)
  517. && read_idx <= (qinfo->q_array.mem_size >> 2)) {
  518. if (new_read_idx < (qinfo->q_array.mem_size >> 2)) {
  519. memcpy(packet, read_ptr,
  520. packet_size_in_words << 2);
  521. } else {
  522. new_read_idx -= (qinfo->q_array.mem_size >> 2);
  523. memcpy(packet, read_ptr,
  524. (packet_size_in_words - new_read_idx) << 2);
  525. memcpy(packet + ((packet_size_in_words -
  526. new_read_idx) << 2),
  527. (u8 *)qinfo->q_array.align_virtual_addr,
  528. new_read_idx << 2);
  529. }
  530. } else {
  531. dprintk(CVP_WARN,
  532. "BAD packet received, read_idx: %#x, pkt_size: %d\n",
  533. read_idx, packet_size_in_words << 2);
  534. dprintk(CVP_WARN, "Dropping this packet\n");
  535. new_read_idx = write_idx;
  536. rc = -ENODATA;
  537. }
  538. if (new_read_idx != queue->qhdr_write_idx)
  539. queue->qhdr_rx_req = 0;
  540. else
  541. queue->qhdr_rx_req = receive_request;
  542. queue->qhdr_read_idx = new_read_idx;
  543. /*
  544. * mb() to ensure qhdr is updated in main memory
  545. * so that iris reads the updated header values
  546. */
  547. mb();
  548. *pb_tx_req_is_set = (queue->qhdr_tx_req == 1) ? 1 : 0;
  549. spin_unlock(&qinfo->hfi_lock);
  550. if (!(queue->qhdr_type & HFI_Q_ID_CTRL_TO_HOST_DEBUG_Q)) {
  551. msg_pkt = (struct cvp_hfi_msg_session_hdr *)packet;
  552. dprintk(CVP_CMD, "%s: "
  553. "pkt_type %08x sess_id %08x trans_id %u ktid %llu\n",
  554. __func__, msg_pkt->packet_type,
  555. msg_pkt->session_id,
  556. msg_pkt->client_data.transaction_id,
  557. msg_pkt->client_data.kdata & (FENCE_BIT - 1));
  558. }
  559. if ((msm_cvp_debug & CVP_PKT) &&
  560. !(queue->qhdr_type & HFI_Q_ID_CTRL_TO_HOST_DEBUG_Q)) {
  561. dprintk(CVP_PKT, "%s: %pK\n", __func__, qinfo);
  562. __dump_packet(packet, CVP_PKT);
  563. }
  564. return rc;
  565. }
  566. static int __smem_alloc(struct iris_hfi_device *dev, struct cvp_mem_addr *mem,
  567. u32 size, u32 align, u32 flags)
  568. {
  569. struct msm_cvp_smem *alloc = &mem->mem_data;
  570. int rc = 0;
  571. if (!dev || !mem || !size) {
  572. dprintk(CVP_ERR, "Invalid Params\n");
  573. return -EINVAL;
  574. }
  575. dprintk(CVP_INFO, "start to alloc size: %d, flags: %d\n", size, flags);
  576. alloc->flags = flags;
  577. rc = msm_cvp_smem_alloc(size, align, 1, (void *)dev->res, alloc);
  578. if (rc) {
  579. dprintk(CVP_ERR, "Alloc failed\n");
  580. rc = -ENOMEM;
  581. goto fail_smem_alloc;
  582. }
  583. dprintk(CVP_MEM, "%s: ptr = %pK, size = %d\n", __func__,
  584. alloc->kvaddr, size);
  585. mem->mem_size = alloc->size;
  586. mem->align_virtual_addr = alloc->kvaddr;
  587. mem->align_device_addr = alloc->device_addr;
  588. alloc->pkt_type = 0;
  589. alloc->buf_idx = 0;
  590. return rc;
  591. fail_smem_alloc:
  592. return rc;
  593. }
  594. static void __smem_free(struct iris_hfi_device *dev, struct msm_cvp_smem *mem)
  595. {
  596. if (!dev || !mem) {
  597. dprintk(CVP_ERR, "invalid param %pK %pK\n", dev, mem);
  598. return;
  599. }
  600. msm_cvp_smem_free(mem);
  601. }
  602. static void __write_register(struct iris_hfi_device *device,
  603. u32 reg, u32 value)
  604. {
  605. u32 hwiosymaddr = reg;
  606. u8 *base_addr;
  607. if (!device) {
  608. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  609. return;
  610. }
  611. __strict_check(device);
  612. if (!device->power_enabled) {
  613. dprintk(CVP_WARN,
  614. "HFI Write register failed : Power is OFF\n");
  615. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  616. return;
  617. }
  618. base_addr = device->cvp_hal_data->register_base;
  619. dprintk(CVP_REG, "Base addr: %pK, written to: %#x, Value: %#x...\n",
  620. base_addr, hwiosymaddr, value);
  621. base_addr += hwiosymaddr;
  622. writel_relaxed(value, base_addr);
  623. /*
  624. * Memory barrier to make sure value is written into the register.
  625. */
  626. wmb();
  627. }
  628. static int __read_gcc_register(struct iris_hfi_device *device, u32 reg)
  629. {
  630. int rc = 0;
  631. u8 *base_addr;
  632. if (!device) {
  633. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  634. return -EINVAL;
  635. }
  636. __strict_check(device);
  637. if (!device->power_enabled) {
  638. dprintk(CVP_WARN,
  639. "%s HFI Read register failed : Power is OFF\n",
  640. __func__);
  641. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  642. return -EINVAL;
  643. }
  644. base_addr = device->cvp_hal_data->gcc_reg_base;
  645. rc = readl_relaxed(base_addr + reg);
  646. /*
  647. * Memory barrier to make sure value is read correctly from the
  648. * register.
  649. */
  650. rmb();
  651. dprintk(CVP_REG,
  652. "GCC Base addr: %pK, read from: %#x, value: %#x...\n",
  653. base_addr, reg, rc);
  654. return rc;
  655. }
  656. static int __read_register(struct iris_hfi_device *device, u32 reg)
  657. {
  658. int rc = 0;
  659. u8 *base_addr;
  660. if (!device) {
  661. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  662. return -EINVAL;
  663. }
  664. __strict_check(device);
  665. if (!device->power_enabled) {
  666. dprintk(CVP_WARN,
  667. "HFI Read register failed : Power is OFF\n");
  668. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  669. return -EINVAL;
  670. }
  671. base_addr = device->cvp_hal_data->register_base;
  672. rc = readl_relaxed(base_addr + reg);
  673. /*
  674. * Memory barrier to make sure value is read correctly from the
  675. * register.
  676. */
  677. rmb();
  678. dprintk(CVP_REG, "Base addr: %pK, read from: %#x, value: %#x...\n",
  679. base_addr, reg, rc);
  680. return rc;
  681. }
  682. static int __set_registers(struct iris_hfi_device *device)
  683. {
  684. struct msm_cvp_core *core;
  685. struct msm_cvp_platform_data *pdata;
  686. struct reg_set *reg_set;
  687. int i;
  688. if (!device->res) {
  689. dprintk(CVP_ERR,
  690. "device resources null, cannot set registers\n");
  691. return -EINVAL ;
  692. }
  693. core = cvp_driver->cvp_core;
  694. pdata = core->platform_data;
  695. reg_set = &device->res->reg_set;
  696. for (i = 0; i < reg_set->count; i++) {
  697. __write_register(device, reg_set->reg_tbl[i].reg,
  698. reg_set->reg_tbl[i].value);
  699. dprintk(CVP_REG, "write_reg offset=%x, val=%x\n",
  700. reg_set->reg_tbl[i].reg,
  701. reg_set->reg_tbl[i].value);
  702. }
  703. i = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  704. if (i) {
  705. dprintk(CVP_WARN, "%s Fail acquire xo_reset\n", __func__);
  706. return -EINVAL;
  707. }
  708. __write_register(device, CVP_CPU_CS_AXI4_QOS,
  709. pdata->noc_qos->axi_qos);
  710. __write_register(device, CVP_NOC_RGE_PRIORITYLUT_LOW,
  711. pdata->noc_qos->prioritylut_low);
  712. __write_register(device, CVP_NOC_RGE_PRIORITYLUT_HIGH,
  713. pdata->noc_qos->prioritylut_high);
  714. __write_register(device, CVP_NOC_RGE_URGENCY_LOW,
  715. pdata->noc_qos->urgency_low);
  716. __write_register(device, CVP_NOC_RGE_DANGERLUT_LOW,
  717. pdata->noc_qos->dangerlut_low);
  718. __write_register(device, CVP_NOC_RGE_SAFELUT_LOW,
  719. pdata->noc_qos->safelut_low);
  720. __write_register(device, CVP_NOC_CDM_PRIORITYLUT_LOW,
  721. pdata->noc_qos->prioritylut_low);
  722. __write_register(device, CVP_NOC_CDM_PRIORITYLUT_HIGH,
  723. pdata->noc_qos->prioritylut_high);
  724. __write_register(device, CVP_NOC_CDM_URGENCY_LOW,
  725. pdata->noc_qos->urgency_low);
  726. __write_register(device, CVP_NOC_CDM_DANGERLUT_LOW,
  727. pdata->noc_qos->dangerlut_low);
  728. __write_register(device, CVP_NOC_CDM_SAFELUT_LOW,
  729. pdata->noc_qos->safelut_low);
  730. /* Below registers write moved from FW to SW to enable UBWC */
  731. __write_register(device, CVP_NOC_RGE_NIU_DECCTL_LOW,
  732. 0x1);
  733. __write_register(device, CVP_NOC_RGE_NIU_ENCCTL_LOW,
  734. 0x1);
  735. __write_register(device, CVP_NOC_GCE_VADL_TOF_NIU_DECCTL_LOW,
  736. 0x1);
  737. __write_register(device, CVP_NOC_GCE_VADL_TOF_NIU_ENCCTL_LOW,
  738. 0x1);
  739. __write_register(device, CVP_NOC_CORE_ERR_MAINCTL_LOW_OFFS,
  740. 0x3);
  741. __write_register(device, CVP_NOC_MAIN_SIDEBANDMANAGER_FAULTINEN0_LOW,
  742. 0x1);
  743. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  744. return 0;
  745. }
  746. /*
  747. * The existence of this function is a hack for 8996 (or certain Iris versions)
  748. * to overcome a hardware bug. Whenever the GDSCs momentarily power collapse
  749. * (after calling __hand_off_regulators()), the values of the threshold
  750. * registers (typically programmed by TZ) are incorrectly reset. As a result
  751. * reprogram these registers at certain agreed upon points.
  752. */
  753. static void __set_threshold_registers(struct iris_hfi_device *device)
  754. {
  755. u32 version = __read_register(device, CVP_WRAPPER_HW_VERSION);
  756. version &= ~GENMASK(15, 0);
  757. if (version != (0x3 << 28 | 0x43 << 16))
  758. return;
  759. if (__tzbsp_set_cvp_state(TZ_SUBSYS_STATE_RESTORE_THRESHOLD))
  760. dprintk(CVP_ERR, "Failed to restore threshold values\n");
  761. }
  762. static int __unvote_buses(struct iris_hfi_device *device)
  763. {
  764. int rc = 0;
  765. struct bus_info *bus = NULL;
  766. kfree(device->bus_vote.data);
  767. device->bus_vote.data = NULL;
  768. device->bus_vote.data_count = 0;
  769. iris_hfi_for_each_bus(device, bus) {
  770. rc = msm_cvp_set_bw(bus, 0);
  771. if (rc) {
  772. dprintk(CVP_ERR,
  773. "%s: Failed unvoting bus\n", __func__);
  774. goto err_unknown_device;
  775. }
  776. }
  777. err_unknown_device:
  778. return rc;
  779. }
  780. static int __vote_buses(struct iris_hfi_device *device,
  781. struct cvp_bus_vote_data *data, int num_data)
  782. {
  783. int rc = 0;
  784. struct bus_info *bus = NULL;
  785. struct cvp_bus_vote_data *new_data = NULL;
  786. if (!num_data) {
  787. dprintk(CVP_PWR, "No vote data available\n");
  788. goto no_data_count;
  789. } else if (!data) {
  790. dprintk(CVP_ERR, "Invalid voting data\n");
  791. return -EINVAL;
  792. }
  793. new_data = kmemdup(data, num_data * sizeof(*new_data), GFP_KERNEL);
  794. if (!new_data) {
  795. dprintk(CVP_ERR, "Can't alloc memory to cache bus votes\n");
  796. rc = -ENOMEM;
  797. goto err_no_mem;
  798. }
  799. no_data_count:
  800. kfree(device->bus_vote.data);
  801. device->bus_vote.data = new_data;
  802. device->bus_vote.data_count = num_data;
  803. iris_hfi_for_each_bus(device, bus) {
  804. if (bus) {
  805. rc = msm_cvp_set_bw(bus, bus->range[1]);
  806. if (rc)
  807. dprintk(CVP_ERR,
  808. "Failed voting bus %s to ab %u\n",
  809. bus->name, bus->range[1]*1000);
  810. }
  811. }
  812. err_no_mem:
  813. return rc;
  814. }
  815. static int iris_hfi_vote_buses(void *dev, struct cvp_bus_vote_data *d, int n)
  816. {
  817. int rc = 0;
  818. struct iris_hfi_device *device = dev;
  819. if (!device)
  820. return -EINVAL;
  821. mutex_lock(&device->lock);
  822. rc = __vote_buses(device, d, n);
  823. mutex_unlock(&device->lock);
  824. return rc;
  825. }
  826. static int __core_set_resource(struct iris_hfi_device *device,
  827. struct cvp_resource_hdr *resource_hdr, void *resource_value)
  828. {
  829. struct cvp_hfi_cmd_sys_set_resource_packet *pkt;
  830. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  831. int rc = 0;
  832. if (!device || !resource_hdr || !resource_value) {
  833. dprintk(CVP_ERR, "set_res: Invalid Params\n");
  834. return -EINVAL;
  835. }
  836. pkt = (struct cvp_hfi_cmd_sys_set_resource_packet *) packet;
  837. rc = call_hfi_pkt_op(device, sys_set_resource,
  838. pkt, resource_hdr, resource_value);
  839. if (rc) {
  840. dprintk(CVP_ERR, "set_res: failed to create packet\n");
  841. goto err_create_pkt;
  842. }
  843. rc = __iface_cmdq_write(device, pkt);
  844. if (rc)
  845. rc = -ENOTEMPTY;
  846. err_create_pkt:
  847. return rc;
  848. }
  849. static int __core_release_resource(struct iris_hfi_device *device,
  850. struct cvp_resource_hdr *resource_hdr)
  851. {
  852. struct cvp_hfi_cmd_sys_release_resource_packet *pkt;
  853. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  854. int rc = 0;
  855. if (!device || !resource_hdr) {
  856. dprintk(CVP_ERR, "release_res: Invalid Params\n");
  857. return -EINVAL;
  858. }
  859. pkt = (struct cvp_hfi_cmd_sys_release_resource_packet *) packet;
  860. rc = call_hfi_pkt_op(device, sys_release_resource,
  861. pkt, resource_hdr);
  862. if (rc) {
  863. dprintk(CVP_ERR, "release_res: failed to create packet\n");
  864. goto err_create_pkt;
  865. }
  866. rc = __iface_cmdq_write(device, pkt);
  867. if (rc)
  868. rc = -ENOTEMPTY;
  869. err_create_pkt:
  870. return rc;
  871. }
  872. static int __tzbsp_set_cvp_state(enum tzbsp_subsys_state state)
  873. {
  874. int rc = 0;
  875. rc = qcom_scm_set_remote_state(state, TZBSP_CVP_PAS_ID);
  876. dprintk(CVP_CORE, "Set state %d, resp %d\n", state, rc);
  877. if (rc) {
  878. dprintk(CVP_ERR, "Failed qcom_scm_set_remote_state %d\n", rc);
  879. return rc;
  880. }
  881. return 0;
  882. }
  883. /*
  884. * Based on fal10_veto, X2RPMh, core_pwr_on and PWAitMode value, infer
  885. * value of xtss_sw_reset. xtss_sw_reset is a TZ register bit. Driver
  886. * cannot access it directly.
  887. *
  888. * In __boot_firmware() function, the caller of this function. It checks
  889. * "core_pwr_on" == false, basically core powered off. So this function
  890. * doesn't check core_pwr_on. Assume core_pwr_on = false.
  891. *
  892. * fal10_veto = VPU_CPU_CS_X2RPMh[2] |
  893. * ( ~VPU_CPU_CS_X2RPMh[1] & core_pwr_on ) |
  894. * ( ~VPU_CPU_CS_X2RPMh[0] & ~( xtss_sw_reset | PWaitMode ) ) ;
  895. */
  896. static inline void check_tensilica_in_reset(struct iris_hfi_device *device)
  897. {
  898. u32 X2RPMh, fal10_veto, wait_mode;
  899. X2RPMh = __read_register(device, CVP_CPU_CS_X2RPMh);
  900. X2RPMh = X2RPMh & 0x7;
  901. /* wait_mode = 1: Tensilica is in WFI mode (PWaitMode = true) */
  902. wait_mode = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  903. wait_mode = wait_mode & 0x1;
  904. fal10_veto = __read_register(device, CVP_CPU_CS_X2RPMh_STATUS);
  905. fal10_veto = fal10_veto & 0x1;
  906. dprintk(CVP_WARN, "tensilica reset check %#x %#x %#x\n",
  907. X2RPMh, wait_mode, fal10_veto);
  908. }
  909. static const char boot_states[0x40][32] = {
  910. "NOT INIT",
  911. "RST_START",
  912. "INIT_MEMCTL",
  913. "INTENABLE_RST",
  914. "LITBASE_RST",
  915. "PREFETCH_EN",
  916. "MPU_INIT",
  917. "CTRL_INIT_READ",
  918. "MEMCTL_L1_FIX",
  919. "RESTORE_EXTRA_NW",
  920. "CORE_RESTORE",
  921. "COLD_BOOT",
  922. "DISABLE_CACHE",
  923. "BEFORE_MPU_C",
  924. "RET_MPU_C",
  925. "IN_MPU_C",
  926. "IN_MPU_DEFAULT",
  927. "IN_MPU_SYNX",
  928. "UCR_SIZE_FAIL",
  929. "UCR_ADDR_FAIL",
  930. "UCR1_SIZE_FAIL",
  931. "UCR1_ADDR_FAIL",
  932. "UCR_OVERLAPPED_UCR1",
  933. "UCR1_OVERLAPPED_UCR",
  934. "UCR_EQ_UCR1",
  935. "MPU_CHECK_DONE",
  936. "BEFORE_INT_LOCK",
  937. "AFTER_INT_LOCK",
  938. "BEFORE_INT_UNLOCK",
  939. "AFTER_INT_UNLOCK",
  940. "CALL_START",
  941. "MAIN_ENTRY",
  942. "VENUS_INIT_ENTRY",
  943. "VSYS_INIT_ENTRY",
  944. "BEFORE_XOS_CLK",
  945. "AFTER_XOS_CLK",
  946. "LOG_MUTEX_INIT",
  947. "CREATE_FRAMEWORK_ENTRY",
  948. "DTG_INIT",
  949. "IDLE_TASK_INIT",
  950. "VENUS_CORE_INIT",
  951. "HW_CORES_INIT",
  952. "RST_THREAD_INIT",
  953. "HOST_THREAD_INIT",
  954. "ALL_THREADS_INIT",
  955. "TASK_MEMPOOL",
  956. "SESSION_MUTEX",
  957. "SIGNALS_INIT",
  958. "RST_SIGNAL_INIT",
  959. "INTR_EN_HOST",
  960. "INTR_REG_HOST",
  961. "INTR_EN_DSP",
  962. "INTR_REG_DSP",
  963. "X2HSOFTINTEN",
  964. "H2XSOFTINTEN",
  965. "CPU2DSPINTEN",
  966. "DSP2CPUINT_SWRESET",
  967. "THREADS_START",
  968. "RST_THREAD_START",
  969. "HST_THREAD_START",
  970. "HST_THREAD_ENTRY"
  971. };
  972. static inline int __boot_firmware(struct iris_hfi_device *device)
  973. {
  974. int rc = 0, loop = 10;
  975. u32 ctrl_init_val = 0, ctrl_status = 0, count = 0, max_tries = 500;
  976. u32 reg_gdsc;
  977. /*
  978. * Hand off control of regulators to h/w _after_ enabling clocks.
  979. * Note that the GDSC will turn off when switching from normal
  980. * (s/w triggered) to fast (HW triggered) unless the h/w vote is
  981. * present. Since Iris isn't up yet, the GDSC will be off briefly.
  982. */
  983. if (__enable_hw_power_collapse(device))
  984. dprintk(CVP_ERR, "Failed to enabled inter-frame PC\n");
  985. if (!msm_cvp_fw_low_power_mode)
  986. goto skip_core_power_check;
  987. while (loop) {
  988. reg_gdsc = __read_register(device, CVP_CC_MVS1_GDSCR);
  989. if (reg_gdsc & 0x80000000) {
  990. usleep_range(100, 200);
  991. loop--;
  992. } else {
  993. break;
  994. }
  995. }
  996. if (!loop)
  997. dprintk(CVP_ERR, "fail to power off CORE during resume\n");
  998. skip_core_power_check:
  999. ctrl_init_val = BIT(0);
  1000. /* RUMI: CVP_CTRL_INIT in MPTest has bit 0 and 3 set */
  1001. __write_register(device, CVP_CTRL_INIT, ctrl_init_val);
  1002. while (!(ctrl_status & CVP_CTRL_INIT_STATUS__M) && count < max_tries) {
  1003. ctrl_status = __read_register(device, CVP_CTRL_STATUS);
  1004. if ((ctrl_status & CVP_CTRL_ERROR_STATUS__M) == 0x4) {
  1005. dprintk(CVP_ERR, "invalid setting for UC_REGION\n");
  1006. rc = -ENODATA;
  1007. break;
  1008. }
  1009. /* Reduce to 500, 1000 on silicon */
  1010. usleep_range(500, 1000);
  1011. count++;
  1012. }
  1013. if (!(ctrl_status & CVP_CTRL_INIT_STATUS__M)) {
  1014. ctrl_init_val = __read_register(device, CVP_CTRL_INIT);
  1015. dprintk(CVP_ERR,
  1016. "Failed to boot FW status: %x %x %s\n",
  1017. ctrl_status, ctrl_init_val,
  1018. boot_states[(ctrl_status >> 9) & 0x3f]);
  1019. check_tensilica_in_reset(device);
  1020. rc = -ENODEV;
  1021. }
  1022. /* Enable interrupt before sending commands to tensilica */
  1023. __write_register(device, CVP_CPU_CS_H2XSOFTINTEN, 0x1);
  1024. __write_register(device, CVP_CPU_CS_X2RPMh, 0x0);
  1025. return rc;
  1026. }
  1027. static int iris_hfi_resume(void *dev)
  1028. {
  1029. int rc = 0;
  1030. struct iris_hfi_device *device = (struct iris_hfi_device *) dev;
  1031. if (!device) {
  1032. dprintk(CVP_ERR, "%s invalid device\n", __func__);
  1033. return -EINVAL;
  1034. }
  1035. dprintk(CVP_CORE, "Resuming Iris\n");
  1036. mutex_lock(&device->lock);
  1037. rc = __resume(device);
  1038. mutex_unlock(&device->lock);
  1039. return rc;
  1040. }
  1041. static int iris_hfi_suspend(void *dev)
  1042. {
  1043. int rc = 0;
  1044. struct iris_hfi_device *device = (struct iris_hfi_device *) dev;
  1045. if (!device) {
  1046. dprintk(CVP_ERR, "%s invalid device\n", __func__);
  1047. return -EINVAL;
  1048. } else if (!device->res->sw_power_collapsible) {
  1049. return -ENOTSUPP;
  1050. }
  1051. dprintk(CVP_CORE, "Suspending Iris\n");
  1052. mutex_lock(&device->lock);
  1053. rc = __power_collapse(device, true);
  1054. if (rc) {
  1055. dprintk(CVP_WARN, "%s: Iris is busy\n", __func__);
  1056. rc = -EBUSY;
  1057. }
  1058. mutex_unlock(&device->lock);
  1059. /* Cancel pending delayed works if any */
  1060. if (!rc)
  1061. cancel_delayed_work(&iris_hfi_pm_work);
  1062. return rc;
  1063. }
  1064. static void cvp_dump_csr(struct iris_hfi_device *dev)
  1065. {
  1066. u32 reg;
  1067. if (!dev)
  1068. return;
  1069. if (!dev->power_enabled || dev->reg_dumped)
  1070. return;
  1071. reg = __read_register(dev, CVP_WRAPPER_CPU_STATUS);
  1072. dprintk(CVP_ERR, "CVP_WRAPPER_CPU_STATUS: %x\n", reg);
  1073. reg = __read_register(dev, CVP_CPU_CS_SCIACMDARG0);
  1074. dprintk(CVP_ERR, "CVP_CPU_CS_SCIACMDARG0: %x\n", reg);
  1075. reg = __read_register(dev, CVP_WRAPPER_INTR_STATUS);
  1076. dprintk(CVP_ERR, "CVP_WRAPPER_INTR_STATUS: %x\n", reg);
  1077. reg = __read_register(dev, CVP_CPU_CS_H2ASOFTINT);
  1078. dprintk(CVP_ERR, "CVP_CPU_CS_H2ASOFTINT: %x\n", reg);
  1079. reg = __read_register(dev, CVP_CPU_CS_A2HSOFTINT);
  1080. dprintk(CVP_ERR, "CVP_CPU_CS_A2HSOFTINT: %x\n", reg);
  1081. reg = __read_register(dev, CVP_CC_MVS1C_GDSCR);
  1082. dprintk(CVP_ERR, "CVP_CC_MVS1C_GDSCR: %x\n", reg);
  1083. reg = __read_register(dev, CVP_CC_MVS1C_CBCR);
  1084. dprintk(CVP_ERR, "CVP_CC_MVS1C_CBCR: %x\n", reg);
  1085. reg = __read_register(dev, CVP_WRAPPER_CPU_CLOCK_CONFIG);
  1086. dprintk(CVP_ERR, "CVP_WRAPPER_CPU_CLOCK_CONFIG: %x\n", reg);
  1087. reg = __read_register(dev, CVP_WRAPPER_CORE_CLOCK_CONFIG);
  1088. dprintk(CVP_ERR, "CVP_WRAPPER_CORE_CLOCK_CONFIG: %x\n", reg);
  1089. dev->reg_dumped = true;
  1090. }
  1091. static int iris_hfi_flush_debug_queue(void *dev)
  1092. {
  1093. int rc = 0;
  1094. struct iris_hfi_device *device = (struct iris_hfi_device *) dev;
  1095. if (!device) {
  1096. dprintk(CVP_ERR, "%s invalid device\n", __func__);
  1097. return -EINVAL;
  1098. }
  1099. mutex_lock(&device->lock);
  1100. if (!device->power_enabled) {
  1101. dprintk(CVP_WARN, "%s: iris power off\n", __func__);
  1102. rc = -EINVAL;
  1103. goto exit;
  1104. }
  1105. cvp_dump_csr(device);
  1106. __flush_debug_queue(device, NULL);
  1107. exit:
  1108. mutex_unlock(&device->lock);
  1109. return rc;
  1110. }
  1111. static int iris_hfi_scale_clocks(void *dev, u32 freq)
  1112. {
  1113. int rc = 0;
  1114. struct iris_hfi_device *device = dev;
  1115. if (!device) {
  1116. dprintk(CVP_ERR, "Invalid args: %pK\n", device);
  1117. return -EINVAL;
  1118. }
  1119. mutex_lock(&device->lock);
  1120. if (__resume(device)) {
  1121. dprintk(CVP_ERR, "Resume from power collapse failed\n");
  1122. rc = -ENODEV;
  1123. goto exit;
  1124. }
  1125. rc = msm_cvp_set_clocks_impl(device, freq);
  1126. exit:
  1127. mutex_unlock(&device->lock);
  1128. return rc;
  1129. }
  1130. /* Writes into cmdq without raising an interrupt */
  1131. static int __iface_cmdq_write_relaxed(struct iris_hfi_device *device,
  1132. void *pkt, bool *requires_interrupt)
  1133. {
  1134. struct cvp_iface_q_info *q_info;
  1135. struct cvp_hal_cmd_pkt_hdr *cmd_packet;
  1136. int result = -E2BIG;
  1137. if (!device || !pkt) {
  1138. dprintk(CVP_ERR, "Invalid Params\n");
  1139. return -EINVAL;
  1140. }
  1141. __strict_check(device);
  1142. if (!__core_in_valid_state(device)) {
  1143. dprintk(CVP_ERR, "%s - fw not in init state\n", __func__);
  1144. result = -EINVAL;
  1145. goto err_q_null;
  1146. }
  1147. cmd_packet = (struct cvp_hal_cmd_pkt_hdr *)pkt;
  1148. device->last_packet_type = cmd_packet->packet_type;
  1149. q_info = &device->iface_queues[CVP_IFACEQ_CMDQ_IDX];
  1150. if (!q_info) {
  1151. dprintk(CVP_ERR, "cannot write to shared Q's\n");
  1152. goto err_q_null;
  1153. }
  1154. if (!q_info->q_array.align_virtual_addr) {
  1155. dprintk(CVP_ERR, "cannot write to shared CMD Q's\n");
  1156. result = -ENODATA;
  1157. goto err_q_null;
  1158. }
  1159. if (__resume(device)) {
  1160. dprintk(CVP_ERR, "%s: Power on failed\n", __func__);
  1161. goto err_q_write;
  1162. }
  1163. if (!__write_queue(q_info, (u8 *)pkt, requires_interrupt)) {
  1164. if (device->res->sw_power_collapsible) {
  1165. cancel_delayed_work(&iris_hfi_pm_work);
  1166. if (!queue_delayed_work(device->iris_pm_workq,
  1167. &iris_hfi_pm_work,
  1168. msecs_to_jiffies(
  1169. device->res->msm_cvp_pwr_collapse_delay))) {
  1170. dprintk(CVP_PWR,
  1171. "PM work already scheduled\n");
  1172. }
  1173. }
  1174. result = 0;
  1175. } else {
  1176. dprintk(CVP_ERR, "__iface_cmdq_write: queue full\n");
  1177. }
  1178. err_q_write:
  1179. err_q_null:
  1180. return result;
  1181. }
  1182. static int __iface_cmdq_write(struct iris_hfi_device *device, void *pkt)
  1183. {
  1184. bool needs_interrupt = false;
  1185. int rc = __iface_cmdq_write_relaxed(device, pkt, &needs_interrupt);
  1186. if (!rc && needs_interrupt) {
  1187. /* Consumer of cmdq prefers that we raise an interrupt */
  1188. rc = 0;
  1189. __write_register(device, CVP_CPU_CS_H2ASOFTINT, 1);
  1190. }
  1191. return rc;
  1192. }
  1193. static int __iface_msgq_read(struct iris_hfi_device *device, void *pkt)
  1194. {
  1195. u32 tx_req_is_set = 0;
  1196. int rc = 0;
  1197. struct cvp_iface_q_info *q_info;
  1198. if (!pkt) {
  1199. dprintk(CVP_ERR, "Invalid Params\n");
  1200. return -EINVAL;
  1201. }
  1202. __strict_check(device);
  1203. if (!__core_in_valid_state(device)) {
  1204. dprintk(CVP_WARN, "%s - fw not in init state\n", __func__);
  1205. rc = -EINVAL;
  1206. goto read_error_null;
  1207. }
  1208. q_info = &device->iface_queues[CVP_IFACEQ_MSGQ_IDX];
  1209. if (q_info->q_array.align_virtual_addr == NULL) {
  1210. dprintk(CVP_ERR, "cannot read from shared MSG Q's\n");
  1211. rc = -ENODATA;
  1212. goto read_error_null;
  1213. }
  1214. if (!__read_queue(q_info, (u8 *)pkt, &tx_req_is_set)) {
  1215. if (tx_req_is_set)
  1216. __write_register(device, CVP_CPU_CS_H2ASOFTINT, 1);
  1217. rc = 0;
  1218. } else
  1219. rc = -ENODATA;
  1220. read_error_null:
  1221. return rc;
  1222. }
  1223. static int __iface_dbgq_read(struct iris_hfi_device *device, void *pkt)
  1224. {
  1225. u32 tx_req_is_set = 0;
  1226. int rc = 0;
  1227. struct cvp_iface_q_info *q_info;
  1228. if (!pkt) {
  1229. dprintk(CVP_ERR, "Invalid Params\n");
  1230. return -EINVAL;
  1231. }
  1232. __strict_check(device);
  1233. q_info = &device->iface_queues[CVP_IFACEQ_DBGQ_IDX];
  1234. if (q_info->q_array.align_virtual_addr == NULL) {
  1235. dprintk(CVP_ERR, "cannot read from shared DBG Q's\n");
  1236. rc = -ENODATA;
  1237. goto dbg_error_null;
  1238. }
  1239. if (!__read_queue(q_info, (u8 *)pkt, &tx_req_is_set)) {
  1240. if (tx_req_is_set)
  1241. __write_register(device, CVP_CPU_CS_H2ASOFTINT, 1);
  1242. rc = 0;
  1243. } else
  1244. rc = -ENODATA;
  1245. dbg_error_null:
  1246. return rc;
  1247. }
  1248. static void __set_queue_hdr_defaults(struct cvp_hfi_queue_header *q_hdr)
  1249. {
  1250. q_hdr->qhdr_status = 0x1;
  1251. q_hdr->qhdr_type = CVP_IFACEQ_DFLT_QHDR;
  1252. q_hdr->qhdr_q_size = CVP_IFACEQ_QUEUE_SIZE / 4;
  1253. q_hdr->qhdr_pkt_size = 0;
  1254. q_hdr->qhdr_rx_wm = 0x1;
  1255. q_hdr->qhdr_tx_wm = 0x1;
  1256. q_hdr->qhdr_rx_req = 0x1;
  1257. q_hdr->qhdr_tx_req = 0x0;
  1258. q_hdr->qhdr_rx_irq_status = 0x0;
  1259. q_hdr->qhdr_tx_irq_status = 0x0;
  1260. q_hdr->qhdr_read_idx = 0x0;
  1261. q_hdr->qhdr_write_idx = 0x0;
  1262. }
  1263. /*
  1264. *Unused, keep for reference
  1265. */
  1266. /*
  1267. static void __interface_dsp_queues_release(struct iris_hfi_device *device)
  1268. {
  1269. int i;
  1270. struct msm_cvp_smem *mem_data = &device->dsp_iface_q_table.mem_data;
  1271. struct context_bank_info *cb = mem_data->mapping_info.cb_info;
  1272. if (!device->dsp_iface_q_table.align_virtual_addr) {
  1273. dprintk(CVP_ERR, "%s: already released\n", __func__);
  1274. return;
  1275. }
  1276. dma_unmap_single_attrs(cb->dev, mem_data->device_addr,
  1277. mem_data->size, DMA_BIDIRECTIONAL, 0);
  1278. dma_free_coherent(device->res->mem_cdsp.dev, mem_data->size,
  1279. mem_data->kvaddr, mem_data->dma_handle);
  1280. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1281. device->dsp_iface_queues[i].q_hdr = NULL;
  1282. device->dsp_iface_queues[i].q_array.align_virtual_addr = NULL;
  1283. device->dsp_iface_queues[i].q_array.align_device_addr = 0;
  1284. }
  1285. device->dsp_iface_q_table.align_virtual_addr = NULL;
  1286. device->dsp_iface_q_table.align_device_addr = 0;
  1287. }
  1288. */
  1289. static int __interface_dsp_queues_init(struct iris_hfi_device *dev)
  1290. {
  1291. int rc = 0;
  1292. u32 i;
  1293. struct cvp_iface_q_info *iface_q;
  1294. int offset = 0;
  1295. phys_addr_t fw_bias = 0;
  1296. size_t q_size;
  1297. struct msm_cvp_smem *mem_data;
  1298. void *kvaddr;
  1299. dma_addr_t dma_handle;
  1300. dma_addr_t iova;
  1301. struct context_bank_info *cb;
  1302. q_size = ALIGN(QUEUE_SIZE, SZ_1M);
  1303. mem_data = &dev->dsp_iface_q_table.mem_data;
  1304. if (mem_data->kvaddr) {
  1305. memset((void *)mem_data->kvaddr, 0, q_size);
  1306. cvp_dsp_init_hfi_queue_hdr(dev);
  1307. return 0;
  1308. }
  1309. /* Allocate dsp queues from CDSP device memory */
  1310. kvaddr = dma_alloc_coherent(dev->res->mem_cdsp.dev, q_size,
  1311. &dma_handle, GFP_KERNEL);
  1312. if (IS_ERR_OR_NULL(kvaddr)) {
  1313. dprintk(CVP_ERR, "%s: failed dma allocation\n", __func__);
  1314. goto fail_dma_alloc;
  1315. }
  1316. cb = msm_cvp_smem_get_context_bank(dev->res, 0);
  1317. if (!cb) {
  1318. dprintk(CVP_ERR,
  1319. "%s: failed to get context bank\n", __func__);
  1320. goto fail_dma_map;
  1321. }
  1322. iova = dma_map_single_attrs(cb->dev, phys_to_virt(dma_handle),
  1323. q_size, DMA_BIDIRECTIONAL, 0);
  1324. if (dma_mapping_error(cb->dev, iova)) {
  1325. dprintk(CVP_ERR, "%s: failed dma mapping\n", __func__);
  1326. goto fail_dma_map;
  1327. }
  1328. dprintk(CVP_DSP,
  1329. "%s: kvaddr %pK dma_handle %#llx iova %#llx size %zd\n",
  1330. __func__, kvaddr, dma_handle, iova, q_size);
  1331. memset(mem_data, 0, sizeof(struct msm_cvp_smem));
  1332. mem_data->kvaddr = kvaddr;
  1333. mem_data->device_addr = iova;
  1334. mem_data->dma_handle = dma_handle;
  1335. mem_data->size = q_size;
  1336. mem_data->mapping_info.cb_info = cb;
  1337. if (!is_iommu_present(dev->res))
  1338. fw_bias = dev->cvp_hal_data->firmware_base;
  1339. dev->dsp_iface_q_table.align_virtual_addr = kvaddr;
  1340. dev->dsp_iface_q_table.align_device_addr = iova - fw_bias;
  1341. dev->dsp_iface_q_table.mem_size = CVP_IFACEQ_TABLE_SIZE;
  1342. offset = dev->dsp_iface_q_table.mem_size;
  1343. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1344. iface_q = &dev->dsp_iface_queues[i];
  1345. iface_q->q_array.align_device_addr = iova + offset - fw_bias;
  1346. iface_q->q_array.align_virtual_addr = kvaddr + offset;
  1347. iface_q->q_array.mem_size = CVP_IFACEQ_QUEUE_SIZE;
  1348. offset += iface_q->q_array.mem_size;
  1349. spin_lock_init(&iface_q->hfi_lock);
  1350. }
  1351. cvp_dsp_init_hfi_queue_hdr(dev);
  1352. return rc;
  1353. fail_dma_map:
  1354. dma_free_coherent(dev->res->mem_cdsp.dev, q_size, kvaddr, dma_handle);
  1355. fail_dma_alloc:
  1356. return -ENOMEM;
  1357. }
  1358. static void __interface_queues_release(struct iris_hfi_device *device)
  1359. {
  1360. #ifdef CONFIG_EVA_TVM
  1361. int i;
  1362. struct cvp_hfi_mem_map_table *qdss;
  1363. struct cvp_hfi_mem_map *mem_map;
  1364. int num_entries = device->res->qdss_addr_set.count;
  1365. unsigned long mem_map_table_base_addr;
  1366. struct context_bank_info *cb;
  1367. if (device->qdss.align_virtual_addr) {
  1368. qdss = (struct cvp_hfi_mem_map_table *)
  1369. device->qdss.align_virtual_addr;
  1370. qdss->mem_map_num_entries = num_entries;
  1371. mem_map_table_base_addr =
  1372. device->qdss.align_device_addr +
  1373. sizeof(struct cvp_hfi_mem_map_table);
  1374. qdss->mem_map_table_base_addr =
  1375. (u32)mem_map_table_base_addr;
  1376. if ((unsigned long)qdss->mem_map_table_base_addr !=
  1377. mem_map_table_base_addr) {
  1378. dprintk(CVP_ERR,
  1379. "Invalid mem_map_table_base_addr %#lx",
  1380. mem_map_table_base_addr);
  1381. }
  1382. mem_map = (struct cvp_hfi_mem_map *)(qdss + 1);
  1383. cb = msm_cvp_smem_get_context_bank(device->res, 0);
  1384. for (i = 0; cb && i < num_entries; i++) {
  1385. iommu_unmap(cb->domain,
  1386. mem_map[i].virtual_addr,
  1387. mem_map[i].size);
  1388. }
  1389. __smem_free(device, &device->qdss.mem_data);
  1390. }
  1391. __smem_free(device, &device->iface_q_table.mem_data);
  1392. __smem_free(device, &device->sfr.mem_data);
  1393. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1394. device->iface_queues[i].q_hdr = NULL;
  1395. device->iface_queues[i].q_array.align_virtual_addr = NULL;
  1396. device->iface_queues[i].q_array.align_device_addr = 0;
  1397. }
  1398. device->iface_q_table.align_virtual_addr = NULL;
  1399. device->iface_q_table.align_device_addr = 0;
  1400. device->qdss.align_virtual_addr = NULL;
  1401. device->qdss.align_device_addr = 0;
  1402. device->sfr.align_virtual_addr = NULL;
  1403. device->sfr.align_device_addr = 0;
  1404. device->mem_addr.align_virtual_addr = NULL;
  1405. device->mem_addr.align_device_addr = 0;
  1406. #endif
  1407. }
  1408. static int __get_qdss_iommu_virtual_addr(struct iris_hfi_device *dev,
  1409. struct cvp_hfi_mem_map *mem_map,
  1410. struct iommu_domain *domain)
  1411. {
  1412. int i;
  1413. int rc = 0;
  1414. dma_addr_t iova = QDSS_IOVA_START;
  1415. int num_entries = dev->res->qdss_addr_set.count;
  1416. struct addr_range *qdss_addr_tbl = dev->res->qdss_addr_set.addr_tbl;
  1417. if (!num_entries)
  1418. return -ENODATA;
  1419. for (i = 0; i < num_entries; i++) {
  1420. if (domain) {
  1421. rc = iommu_map(domain, iova,
  1422. qdss_addr_tbl[i].start,
  1423. qdss_addr_tbl[i].size,
  1424. IOMMU_READ | IOMMU_WRITE);
  1425. if (rc) {
  1426. dprintk(CVP_ERR,
  1427. "IOMMU QDSS mapping failed for addr %#x\n",
  1428. qdss_addr_tbl[i].start);
  1429. rc = -ENOMEM;
  1430. break;
  1431. }
  1432. } else {
  1433. iova = qdss_addr_tbl[i].start;
  1434. }
  1435. mem_map[i].virtual_addr = (u32)iova;
  1436. mem_map[i].physical_addr = qdss_addr_tbl[i].start;
  1437. mem_map[i].size = qdss_addr_tbl[i].size;
  1438. mem_map[i].attr = 0x0;
  1439. iova += mem_map[i].size;
  1440. }
  1441. if (i < num_entries) {
  1442. dprintk(CVP_ERR,
  1443. "QDSS mapping failed, Freeing other entries %d\n", i);
  1444. for (--i; domain && i >= 0; i--) {
  1445. iommu_unmap(domain,
  1446. mem_map[i].virtual_addr,
  1447. mem_map[i].size);
  1448. }
  1449. }
  1450. return rc;
  1451. }
  1452. static void __setup_ucregion_memory_map(struct iris_hfi_device *device)
  1453. {
  1454. __write_register(device, CVP_UC_REGION_ADDR,
  1455. (u32)device->iface_q_table.align_device_addr);
  1456. __write_register(device, CVP_UC_REGION_SIZE, SHARED_QSIZE);
  1457. __write_register(device, CVP_QTBL_ADDR,
  1458. (u32)device->iface_q_table.align_device_addr);
  1459. __write_register(device, CVP_QTBL_INFO, 0x01);
  1460. if (device->sfr.align_device_addr)
  1461. __write_register(device, CVP_SFR_ADDR,
  1462. (u32)device->sfr.align_device_addr);
  1463. if (device->qdss.align_device_addr)
  1464. __write_register(device, CVP_MMAP_ADDR,
  1465. (u32)device->qdss.align_device_addr);
  1466. call_iris_op(device, setup_dsp_uc_memmap, device);
  1467. }
  1468. static void __hfi_queue_init(struct iris_hfi_device *dev)
  1469. {
  1470. int i, offset = 0;
  1471. struct cvp_hfi_queue_table_header *q_tbl_hdr;
  1472. struct cvp_iface_q_info *iface_q;
  1473. struct cvp_hfi_queue_header *q_hdr;
  1474. if (!dev)
  1475. return;
  1476. offset += dev->iface_q_table.mem_size;
  1477. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1478. iface_q = &dev->iface_queues[i];
  1479. iface_q->q_array.align_device_addr =
  1480. dev->iface_q_table.align_device_addr + offset;
  1481. iface_q->q_array.align_virtual_addr =
  1482. dev->iface_q_table.align_virtual_addr + offset;
  1483. iface_q->q_array.mem_size = CVP_IFACEQ_QUEUE_SIZE;
  1484. offset += iface_q->q_array.mem_size;
  1485. iface_q->q_hdr = CVP_IFACEQ_GET_QHDR_START_ADDR(
  1486. dev->iface_q_table.align_virtual_addr, i);
  1487. __set_queue_hdr_defaults(iface_q->q_hdr);
  1488. spin_lock_init(&iface_q->hfi_lock);
  1489. }
  1490. q_tbl_hdr = (struct cvp_hfi_queue_table_header *)
  1491. dev->iface_q_table.align_virtual_addr;
  1492. q_tbl_hdr->qtbl_version = 0;
  1493. q_tbl_hdr->device_addr = (void *)dev;
  1494. strlcpy(q_tbl_hdr->name, "msm_cvp", sizeof(q_tbl_hdr->name));
  1495. q_tbl_hdr->qtbl_size = CVP_IFACEQ_TABLE_SIZE;
  1496. q_tbl_hdr->qtbl_qhdr0_offset =
  1497. sizeof(struct cvp_hfi_queue_table_header);
  1498. q_tbl_hdr->qtbl_qhdr_size = sizeof(struct cvp_hfi_queue_header);
  1499. q_tbl_hdr->qtbl_num_q = CVP_IFACEQ_NUMQ;
  1500. q_tbl_hdr->qtbl_num_active_q = CVP_IFACEQ_NUMQ;
  1501. iface_q = &dev->iface_queues[CVP_IFACEQ_CMDQ_IDX];
  1502. q_hdr = iface_q->q_hdr;
  1503. q_hdr->qhdr_start_addr = iface_q->q_array.align_device_addr;
  1504. q_hdr->qhdr_type |= HFI_Q_ID_HOST_TO_CTRL_CMD_Q;
  1505. iface_q = &dev->iface_queues[CVP_IFACEQ_MSGQ_IDX];
  1506. q_hdr = iface_q->q_hdr;
  1507. q_hdr->qhdr_start_addr = iface_q->q_array.align_device_addr;
  1508. q_hdr->qhdr_type |= HFI_Q_ID_CTRL_TO_HOST_MSG_Q;
  1509. iface_q = &dev->iface_queues[CVP_IFACEQ_DBGQ_IDX];
  1510. q_hdr = iface_q->q_hdr;
  1511. q_hdr->qhdr_start_addr = iface_q->q_array.align_device_addr;
  1512. q_hdr->qhdr_type |= HFI_Q_ID_CTRL_TO_HOST_DEBUG_Q;
  1513. /*
  1514. * Set receive request to zero on debug queue as there is no
  1515. * need of interrupt from cvp hardware for debug messages
  1516. */
  1517. q_hdr->qhdr_rx_req = 0;
  1518. }
  1519. static void __sfr_init(struct iris_hfi_device *dev)
  1520. {
  1521. struct cvp_hfi_sfr_struct *vsfr;
  1522. if (!dev)
  1523. return;
  1524. vsfr = (struct cvp_hfi_sfr_struct *) dev->sfr.align_virtual_addr;
  1525. if (vsfr)
  1526. vsfr->bufSize = ALIGNED_SFR_SIZE;
  1527. }
  1528. static int __interface_queues_init(struct iris_hfi_device *dev)
  1529. {
  1530. int rc = 0;
  1531. struct cvp_hfi_mem_map_table *qdss;
  1532. struct cvp_hfi_mem_map *mem_map;
  1533. struct cvp_mem_addr *mem_addr;
  1534. int num_entries = dev->res->qdss_addr_set.count;
  1535. phys_addr_t fw_bias = 0;
  1536. size_t q_size;
  1537. unsigned long mem_map_table_base_addr;
  1538. struct context_bank_info *cb;
  1539. q_size = SHARED_QSIZE - ALIGNED_SFR_SIZE - ALIGNED_QDSS_SIZE;
  1540. mem_addr = &dev->mem_addr;
  1541. if (!is_iommu_present(dev->res))
  1542. fw_bias = dev->cvp_hal_data->firmware_base;
  1543. if (dev->iface_q_table.align_virtual_addr) {
  1544. memset((void *)dev->iface_q_table.align_virtual_addr,
  1545. 0, q_size);
  1546. goto hfi_queue_init;
  1547. }
  1548. rc = __smem_alloc(dev, mem_addr, q_size, 1, SMEM_UNCACHED);
  1549. if (rc) {
  1550. dprintk(CVP_ERR, "iface_q_table_alloc_fail\n");
  1551. goto fail_alloc_queue;
  1552. }
  1553. dev->iface_q_table.align_virtual_addr = mem_addr->align_virtual_addr;
  1554. dev->iface_q_table.align_device_addr = mem_addr->align_device_addr -
  1555. fw_bias;
  1556. dev->iface_q_table.mem_size = CVP_IFACEQ_TABLE_SIZE;
  1557. dev->iface_q_table.mem_data = mem_addr->mem_data;
  1558. hfi_queue_init:
  1559. __hfi_queue_init(dev);
  1560. if (dev->sfr.align_virtual_addr) {
  1561. memset((void *)dev->sfr.align_virtual_addr,
  1562. 0, ALIGNED_SFR_SIZE);
  1563. goto sfr_init;
  1564. }
  1565. rc = __smem_alloc(dev, mem_addr, ALIGNED_SFR_SIZE, 1, SMEM_UNCACHED);
  1566. if (rc) {
  1567. dprintk(CVP_WARN, "sfr_alloc_fail: SFR not will work\n");
  1568. dev->sfr.align_device_addr = 0;
  1569. } else {
  1570. dev->sfr.align_device_addr = mem_addr->align_device_addr -
  1571. fw_bias;
  1572. dev->sfr.align_virtual_addr = mem_addr->align_virtual_addr;
  1573. dev->sfr.mem_size = ALIGNED_SFR_SIZE;
  1574. dev->sfr.mem_data = mem_addr->mem_data;
  1575. }
  1576. sfr_init:
  1577. __sfr_init(dev);
  1578. if (dev->qdss.align_virtual_addr)
  1579. goto dsp_hfi_queue_init;
  1580. if ((msm_cvp_fw_debug_mode & HFI_DEBUG_MODE_QDSS) && num_entries) {
  1581. rc = __smem_alloc(dev, mem_addr, ALIGNED_QDSS_SIZE, 1,
  1582. SMEM_UNCACHED);
  1583. if (rc) {
  1584. dprintk(CVP_WARN,
  1585. "qdss_alloc_fail: QDSS messages logging will not work\n");
  1586. dev->qdss.align_device_addr = 0;
  1587. } else {
  1588. dev->qdss.align_device_addr =
  1589. mem_addr->align_device_addr - fw_bias;
  1590. dev->qdss.align_virtual_addr =
  1591. mem_addr->align_virtual_addr;
  1592. dev->qdss.mem_size = ALIGNED_QDSS_SIZE;
  1593. dev->qdss.mem_data = mem_addr->mem_data;
  1594. }
  1595. }
  1596. if (dev->qdss.align_virtual_addr) {
  1597. qdss =
  1598. (struct cvp_hfi_mem_map_table *)dev->qdss.align_virtual_addr;
  1599. qdss->mem_map_num_entries = num_entries;
  1600. mem_map_table_base_addr = dev->qdss.align_device_addr +
  1601. sizeof(struct cvp_hfi_mem_map_table);
  1602. qdss->mem_map_table_base_addr = mem_map_table_base_addr;
  1603. mem_map = (struct cvp_hfi_mem_map *)(qdss + 1);
  1604. cb = msm_cvp_smem_get_context_bank(dev->res, 0);
  1605. if (!cb) {
  1606. dprintk(CVP_ERR,
  1607. "%s: failed to get context bank\n", __func__);
  1608. return -EINVAL;
  1609. }
  1610. rc = __get_qdss_iommu_virtual_addr(dev, mem_map, cb->domain);
  1611. if (rc) {
  1612. dprintk(CVP_ERR,
  1613. "IOMMU mapping failed, Freeing qdss memdata\n");
  1614. __smem_free(dev, &dev->qdss.mem_data);
  1615. dev->qdss.align_virtual_addr = NULL;
  1616. dev->qdss.align_device_addr = 0;
  1617. }
  1618. }
  1619. dsp_hfi_queue_init:
  1620. rc = __interface_dsp_queues_init(dev);
  1621. if (rc) {
  1622. dprintk(CVP_ERR, "dsp_queues_init failed\n");
  1623. goto fail_alloc_queue;
  1624. }
  1625. __setup_ucregion_memory_map(dev);
  1626. return 0;
  1627. fail_alloc_queue:
  1628. return -ENOMEM;
  1629. }
  1630. static int __sys_set_debug(struct iris_hfi_device *device, u32 debug)
  1631. {
  1632. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1633. int rc = 0;
  1634. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1635. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1636. rc = call_hfi_pkt_op(device, sys_debug_config, pkt, debug);
  1637. if (rc) {
  1638. dprintk(CVP_WARN,
  1639. "Debug mode setting to FW failed\n");
  1640. return -ENOTEMPTY;
  1641. }
  1642. if (__iface_cmdq_write(device, pkt))
  1643. return -ENOTEMPTY;
  1644. return 0;
  1645. }
  1646. static int __sys_set_idle_indicator(struct iris_hfi_device *device,
  1647. bool enable)
  1648. {
  1649. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1650. int rc = 0;
  1651. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1652. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1653. rc = call_hfi_pkt_op(device, sys_set_idle_indicator, pkt, enable);
  1654. if (__iface_cmdq_write(device, pkt))
  1655. return -ENOTEMPTY;
  1656. return 0;
  1657. }
  1658. static int __sys_set_coverage(struct iris_hfi_device *device, u32 mode)
  1659. {
  1660. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1661. int rc = 0;
  1662. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1663. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1664. rc = call_hfi_pkt_op(device, sys_coverage_config,
  1665. pkt, mode);
  1666. if (rc) {
  1667. dprintk(CVP_WARN,
  1668. "Coverage mode setting to FW failed\n");
  1669. return -ENOTEMPTY;
  1670. }
  1671. if (__iface_cmdq_write(device, pkt)) {
  1672. dprintk(CVP_WARN, "Failed to send coverage pkt to f/w\n");
  1673. return -ENOTEMPTY;
  1674. }
  1675. return 0;
  1676. }
  1677. static int __sys_set_power_control(struct iris_hfi_device *device,
  1678. bool enable)
  1679. {
  1680. struct regulator_info *rinfo;
  1681. bool supported = false;
  1682. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1683. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1684. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1685. iris_hfi_for_each_regulator(device, rinfo) {
  1686. if (rinfo->has_hw_power_collapse) {
  1687. supported = true;
  1688. break;
  1689. }
  1690. }
  1691. if (!supported)
  1692. return 0;
  1693. call_hfi_pkt_op(device, sys_power_control, pkt, enable);
  1694. if (__iface_cmdq_write(device, pkt))
  1695. return -ENOTEMPTY;
  1696. return 0;
  1697. }
  1698. static void cvp_pm_qos_update(struct iris_hfi_device *device, bool vote_on)
  1699. {
  1700. u32 latency, off_vote_cnt;
  1701. int i, err = 0;
  1702. spin_lock(&device->res->pm_qos.lock);
  1703. off_vote_cnt = device->res->pm_qos.off_vote_cnt;
  1704. spin_unlock(&device->res->pm_qos.lock);
  1705. if (vote_on && off_vote_cnt)
  1706. return;
  1707. latency = vote_on ? device->res->pm_qos.latency_us :
  1708. PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
  1709. if (device->res->pm_qos.latency_us && device->res->pm_qos.pm_qos_hdls)
  1710. for (i = 0; i < device->res->pm_qos.silver_count; i++) {
  1711. if (!cpu_possible(device->res->pm_qos.silver_cores[i]))
  1712. continue;
  1713. err = dev_pm_qos_update_request(
  1714. &device->res->pm_qos.pm_qos_hdls[i],
  1715. latency);
  1716. if (err < 0) {
  1717. if (vote_on) {
  1718. dprintk(CVP_WARN,
  1719. "pm qos on failed %d\n", err);
  1720. } else {
  1721. dprintk(CVP_WARN,
  1722. "pm qos off failed %d\n", err);
  1723. }
  1724. }
  1725. }
  1726. }
  1727. static int iris_pm_qos_update(void *device)
  1728. {
  1729. struct iris_hfi_device *dev;
  1730. if (!device) {
  1731. dprintk(CVP_ERR, "%s Invalid device\n", __func__);
  1732. return -ENODEV;
  1733. }
  1734. dev = device;
  1735. mutex_lock(&dev->lock);
  1736. cvp_pm_qos_update(dev, true);
  1737. mutex_unlock(&dev->lock);
  1738. return 0;
  1739. }
  1740. static int __hwfence_regs_map(struct iris_hfi_device *device)
  1741. {
  1742. int rc = 0;
  1743. struct context_bank_info *cb;
  1744. cb = msm_cvp_smem_get_context_bank(device->res, 0);
  1745. if (!cb) {
  1746. dprintk(CVP_ERR, "%s: fail to get cb\n", __func__);
  1747. return -EINVAL;
  1748. }
  1749. if (device->res->reg_mappings.ipclite_phyaddr != 0) {
  1750. rc = iommu_map(cb->domain,
  1751. device->res->reg_mappings.ipclite_iova,
  1752. device->res->reg_mappings.ipclite_phyaddr,
  1753. device->res->reg_mappings.ipclite_size,
  1754. IOMMU_READ | IOMMU_WRITE);
  1755. if (rc) {
  1756. dprintk(CVP_ERR, "map ipclite fail %d %#x %#x %#x\n",
  1757. rc, device->res->reg_mappings.ipclite_iova,
  1758. device->res->reg_mappings.ipclite_phyaddr,
  1759. device->res->reg_mappings.ipclite_size);
  1760. return rc;
  1761. }
  1762. }
  1763. if (device->res->reg_mappings.hwmutex_phyaddr != 0) {
  1764. rc = iommu_map(cb->domain,
  1765. device->res->reg_mappings.hwmutex_iova,
  1766. device->res->reg_mappings.hwmutex_phyaddr,
  1767. device->res->reg_mappings.hwmutex_size,
  1768. IOMMU_MMIO | IOMMU_READ | IOMMU_WRITE);
  1769. if (rc) {
  1770. dprintk(CVP_ERR, "map hwmutex fail %d %#x %#x %#x\n",
  1771. rc, device->res->reg_mappings.hwmutex_iova,
  1772. device->res->reg_mappings.hwmutex_phyaddr,
  1773. device->res->reg_mappings.hwmutex_size);
  1774. return rc;
  1775. }
  1776. }
  1777. if (device->res->reg_mappings.aon_phyaddr != 0) {
  1778. rc = iommu_map(cb->domain,
  1779. device->res->reg_mappings.aon_iova,
  1780. device->res->reg_mappings.aon_phyaddr,
  1781. device->res->reg_mappings.aon_size,
  1782. IOMMU_MMIO | IOMMU_READ | IOMMU_WRITE);
  1783. if (rc) {
  1784. dprintk(CVP_ERR, "map aon fail %d %#x %#x %#x\n",
  1785. rc, device->res->reg_mappings.aon_iova,
  1786. device->res->reg_mappings.aon_phyaddr,
  1787. device->res->reg_mappings.aon_size);
  1788. return rc;
  1789. }
  1790. }
  1791. if (device->res->reg_mappings.timer_phyaddr != 0) {
  1792. rc = iommu_map(cb->domain,
  1793. device->res->reg_mappings.timer_iova,
  1794. device->res->reg_mappings.timer_phyaddr,
  1795. device->res->reg_mappings.timer_size,
  1796. IOMMU_MMIO | IOMMU_READ | IOMMU_WRITE);
  1797. if (rc) {
  1798. dprintk(CVP_ERR, "map timer fail %d %#x %#x %#x\n",
  1799. rc, device->res->reg_mappings.timer_iova,
  1800. device->res->reg_mappings.timer_phyaddr,
  1801. device->res->reg_mappings.timer_size);
  1802. return rc;
  1803. }
  1804. }
  1805. return rc;
  1806. }
  1807. static int __hwfence_regs_unmap(struct iris_hfi_device *device)
  1808. {
  1809. int rc = 0;
  1810. struct context_bank_info *cb;
  1811. cb = msm_cvp_smem_get_context_bank(device->res, 0);
  1812. if (!cb) {
  1813. dprintk(CVP_ERR, "%s: fail to get cb\n", __func__);
  1814. return -EINVAL;
  1815. }
  1816. if (device->res->reg_mappings.ipclite_iova != 0) {
  1817. iommu_unmap(cb->domain,
  1818. device->res->reg_mappings.ipclite_iova,
  1819. device->res->reg_mappings.ipclite_size);
  1820. }
  1821. if (device->res->reg_mappings.hwmutex_iova != 0) {
  1822. iommu_unmap(cb->domain,
  1823. device->res->reg_mappings.hwmutex_iova,
  1824. device->res->reg_mappings.hwmutex_size);
  1825. }
  1826. if (device->res->reg_mappings.aon_iova != 0) {
  1827. iommu_unmap(cb->domain,
  1828. device->res->reg_mappings.aon_iova,
  1829. device->res->reg_mappings.aon_size);
  1830. }
  1831. if (device->res->reg_mappings.timer_iova != 0) {
  1832. iommu_unmap(cb->domain,
  1833. device->res->reg_mappings.timer_iova,
  1834. device->res->reg_mappings.timer_size);
  1835. }
  1836. return rc;
  1837. }
  1838. static int iris_hfi_core_init(void *device)
  1839. {
  1840. int rc = 0;
  1841. u32 ipcc_iova;
  1842. struct cvp_hfi_cmd_sys_init_packet pkt;
  1843. struct cvp_hfi_cmd_sys_get_property_packet version_pkt;
  1844. struct iris_hfi_device *dev;
  1845. if (!device) {
  1846. dprintk(CVP_ERR, "Invalid device\n");
  1847. return -ENODEV;
  1848. }
  1849. dev = device;
  1850. dprintk(CVP_CORE, "Core initializing\n");
  1851. pm_stay_awake(dev->res->pdev->dev.parent);
  1852. mutex_lock(&dev->lock);
  1853. dev->bus_vote.data =
  1854. kzalloc(sizeof(struct cvp_bus_vote_data), GFP_KERNEL);
  1855. if (!dev->bus_vote.data) {
  1856. dprintk(CVP_ERR, "Bus vote data memory is not allocated\n");
  1857. rc = -ENOMEM;
  1858. goto err_no_mem;
  1859. }
  1860. dev->bus_vote.data_count = 1;
  1861. dev->bus_vote.data->power_mode = CVP_POWER_TURBO;
  1862. __hwfence_regs_map(dev);
  1863. rc = __power_on_init(dev);
  1864. if (rc) {
  1865. dprintk(CVP_ERR, "Failed to power on init EVA\n");
  1866. goto err_load_fw;
  1867. }
  1868. rc = cvp_synx_recover();
  1869. if (rc) {
  1870. dprintk(CVP_ERR, "Failed to recover synx\n");
  1871. goto err_core_init;
  1872. }
  1873. /* mmrm registration */
  1874. if (msm_cvp_mmrm_enabled) {
  1875. rc = msm_cvp_mmrm_register(device);
  1876. if (rc) {
  1877. dprintk(CVP_ERR, "Failed to register mmrm client\n");
  1878. goto err_core_init;
  1879. }
  1880. }
  1881. __set_state(dev, IRIS_STATE_INIT);
  1882. dev->reg_dumped = false;
  1883. dprintk(CVP_CORE, "Dev_Virt: %pa, Reg_Virt: %pK\n",
  1884. &dev->cvp_hal_data->firmware_base,
  1885. dev->cvp_hal_data->register_base);
  1886. rc = __interface_queues_init(dev);
  1887. if (rc) {
  1888. dprintk(CVP_ERR, "failed to init queues\n");
  1889. rc = -ENOMEM;
  1890. goto err_core_init;
  1891. }
  1892. cvp_register_va_md_region();
  1893. // Add node for dev struct
  1894. add_va_node_to_list(CVP_QUEUE_DUMP, dev,
  1895. sizeof(struct iris_hfi_device),
  1896. "iris_hfi_device-dev", false);
  1897. add_queue_header_to_va_md_list((void*)dev);
  1898. add_hfi_queue_to_va_md_list((void*)dev);
  1899. rc = msm_cvp_map_ipcc_regs(&ipcc_iova);
  1900. if (!rc) {
  1901. dprintk(CVP_CORE, "IPCC iova 0x%x\n", ipcc_iova);
  1902. __write_register(dev, CVP_MMAP_ADDR, ipcc_iova);
  1903. }
  1904. rc = __load_fw(dev);
  1905. if (rc) {
  1906. dprintk(CVP_ERR, "Failed to load Iris FW\n");
  1907. goto err_core_init;
  1908. }
  1909. rc = __boot_firmware(dev);
  1910. if (rc) {
  1911. dprintk(CVP_ERR, "Failed to start core\n");
  1912. rc = -ENODEV;
  1913. goto err_core_init;
  1914. }
  1915. dev->version = __read_register(dev, CVP_VERSION_INFO);
  1916. rc = call_hfi_pkt_op(dev, sys_init, &pkt, 0);
  1917. if (rc) {
  1918. dprintk(CVP_ERR, "Failed to create sys init pkt\n");
  1919. goto err_core_init;
  1920. }
  1921. if (__iface_cmdq_write(dev, &pkt)) {
  1922. rc = -ENOTEMPTY;
  1923. goto err_core_init;
  1924. }
  1925. rc = call_hfi_pkt_op(dev, sys_image_version, &version_pkt);
  1926. if (rc || __iface_cmdq_write(dev, &version_pkt))
  1927. dprintk(CVP_WARN, "Failed to send image version pkt to f/w\n");
  1928. __sys_set_debug(device, msm_cvp_fw_debug);
  1929. __enable_subcaches(device);
  1930. __set_subcaches(device);
  1931. __set_ubwc_config(device);
  1932. __sys_set_idle_indicator(device, true);
  1933. if (dev->res->pm_qos.latency_us) {
  1934. int err = 0;
  1935. u32 i, cpu;
  1936. dev->res->pm_qos.pm_qos_hdls = kcalloc(
  1937. dev->res->pm_qos.silver_count,
  1938. sizeof(struct dev_pm_qos_request),
  1939. GFP_KERNEL);
  1940. if (!dev->res->pm_qos.pm_qos_hdls) {
  1941. dprintk(CVP_WARN, "Failed allocate pm_qos_hdls\n");
  1942. goto pm_qos_bail;
  1943. }
  1944. for (i = 0; i < dev->res->pm_qos.silver_count; i++) {
  1945. cpu = dev->res->pm_qos.silver_cores[i];
  1946. if (!cpu_possible(cpu))
  1947. continue;
  1948. err = dev_pm_qos_add_request(
  1949. get_cpu_device(cpu),
  1950. &dev->res->pm_qos.pm_qos_hdls[i],
  1951. DEV_PM_QOS_RESUME_LATENCY,
  1952. dev->res->pm_qos.latency_us);
  1953. if (err < 0)
  1954. dprintk(CVP_WARN,
  1955. "%s pm_qos_add_req %d failed\n",
  1956. __func__, i);
  1957. }
  1958. }
  1959. pm_qos_bail:
  1960. mutex_unlock(&dev->lock);
  1961. cvp_dsp_send_hfi_queue();
  1962. pm_relax(dev->res->pdev->dev.parent);
  1963. dprintk(CVP_CORE, "Core inited successfully\n");
  1964. return 0;
  1965. err_core_init:
  1966. __set_state(dev, IRIS_STATE_DEINIT);
  1967. __unload_fw(dev);
  1968. if (dev->mmrm_cvp)
  1969. {
  1970. msm_cvp_mmrm_deregister(dev);
  1971. }
  1972. err_load_fw:
  1973. __hwfence_regs_unmap(dev);
  1974. err_no_mem:
  1975. dprintk(CVP_ERR, "Core init failed\n");
  1976. mutex_unlock(&dev->lock);
  1977. pm_relax(dev->res->pdev->dev.parent);
  1978. return rc;
  1979. }
  1980. static int iris_hfi_core_release(void *dev)
  1981. {
  1982. int rc = 0, i;
  1983. struct iris_hfi_device *device = dev;
  1984. struct cvp_hal_session *session, *next;
  1985. struct dev_pm_qos_request *qos_hdl;
  1986. u32 ipcc_iova;
  1987. if (!device) {
  1988. dprintk(CVP_ERR, "invalid device\n");
  1989. return -ENODEV;
  1990. }
  1991. mutex_lock(&device->lock);
  1992. dprintk(CVP_WARN, "Core releasing\n");
  1993. if (device->res->pm_qos.latency_us &&
  1994. device->res->pm_qos.pm_qos_hdls) {
  1995. for (i = 0; i < device->res->pm_qos.silver_count; i++) {
  1996. if (!cpu_possible(device->res->pm_qos.silver_cores[i]))
  1997. continue;
  1998. qos_hdl = &device->res->pm_qos.pm_qos_hdls[i];
  1999. if ((qos_hdl != NULL) && dev_pm_qos_request_active(qos_hdl))
  2000. dev_pm_qos_remove_request(qos_hdl);
  2001. }
  2002. kfree(device->res->pm_qos.pm_qos_hdls);
  2003. device->res->pm_qos.pm_qos_hdls = NULL;
  2004. }
  2005. __resume(device);
  2006. __set_state(device, IRIS_STATE_DEINIT);
  2007. rc = __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_SUSPEND);
  2008. if (rc)
  2009. dprintk(CVP_WARN, "Failed to suspend cvp FW%d\n", rc);
  2010. __dsp_shutdown(device);
  2011. __disable_subcaches(device);
  2012. ipcc_iova = __read_register(device, CVP_MMAP_ADDR);
  2013. msm_cvp_unmap_ipcc_regs(ipcc_iova);
  2014. __unload_fw(device);
  2015. __hwfence_regs_unmap(device);
  2016. if (msm_cvp_mmrm_enabled) {
  2017. rc = msm_cvp_mmrm_deregister(device);
  2018. if (rc) {
  2019. dprintk(CVP_ERR,
  2020. "%s: Failed msm_cvp_mmrm_deregister:%d\n",
  2021. __func__, rc);
  2022. }
  2023. }
  2024. /* unlink all sessions from device */
  2025. list_for_each_entry_safe(session, next, &device->sess_head, list) {
  2026. list_del(&session->list);
  2027. session->device = NULL;
  2028. }
  2029. dprintk(CVP_CORE, "Core released successfully\n");
  2030. mutex_unlock(&device->lock);
  2031. return rc;
  2032. }
  2033. static void __core_clear_interrupt(struct iris_hfi_device *device)
  2034. {
  2035. u32 intr_status = 0, mask = 0;
  2036. if (!device) {
  2037. dprintk(CVP_ERR, "%s: NULL device\n", __func__);
  2038. return;
  2039. }
  2040. intr_status = __read_register(device, CVP_WRAPPER_INTR_STATUS);
  2041. mask = (CVP_WRAPPER_INTR_MASK_A2HCPU_BMSK | CVP_FATAL_INTR_BMSK);
  2042. if (intr_status & mask) {
  2043. device->intr_status |= intr_status;
  2044. device->reg_count++;
  2045. dprintk(CVP_CORE,
  2046. "INTERRUPT for device: %pK: times: %d status: %d\n",
  2047. device, device->reg_count, intr_status);
  2048. } else {
  2049. device->spur_count++;
  2050. }
  2051. __write_register(device, CVP_CPU_CS_A2HSOFTINTCLR, 1);
  2052. }
  2053. static int iris_hfi_core_trigger_ssr(void *device,
  2054. enum hal_ssr_trigger_type type)
  2055. {
  2056. struct cvp_hfi_cmd_sys_test_ssr_packet pkt;
  2057. int rc = 0;
  2058. struct iris_hfi_device *dev;
  2059. cvp_free_va_md_list();
  2060. if (!device) {
  2061. dprintk(CVP_ERR, "invalid device\n");
  2062. return -ENODEV;
  2063. }
  2064. dev = device;
  2065. if (mutex_trylock(&dev->lock)) {
  2066. rc = call_hfi_pkt_op(dev, ssr_cmd, type, &pkt);
  2067. if (rc) {
  2068. dprintk(CVP_ERR, "%s: failed to create packet\n",
  2069. __func__);
  2070. goto err_create_pkt;
  2071. }
  2072. if (__iface_cmdq_write(dev, &pkt))
  2073. rc = -ENOTEMPTY;
  2074. } else {
  2075. return -EAGAIN;
  2076. }
  2077. err_create_pkt:
  2078. mutex_unlock(&dev->lock);
  2079. return rc;
  2080. }
  2081. static void __set_default_sys_properties(struct iris_hfi_device *device)
  2082. {
  2083. if (__sys_set_debug(device, msm_cvp_fw_debug))
  2084. dprintk(CVP_WARN, "Setting fw_debug msg ON failed\n");
  2085. if (__sys_set_power_control(device, msm_cvp_fw_low_power_mode))
  2086. dprintk(CVP_WARN, "Setting h/w power collapse ON failed\n");
  2087. }
  2088. static void __session_clean(struct cvp_hal_session *session)
  2089. {
  2090. struct cvp_hal_session *temp, *next;
  2091. struct iris_hfi_device *device;
  2092. if (!session || !session->device) {
  2093. dprintk(CVP_WARN, "%s: invalid params\n", __func__);
  2094. return;
  2095. }
  2096. device = session->device;
  2097. dprintk(CVP_SESS, "deleted the session: %pK\n", session);
  2098. /*
  2099. * session might have been removed from the device list in
  2100. * core_release, so check and remove if it is in the list
  2101. */
  2102. list_for_each_entry_safe(temp, next, &device->sess_head, list) {
  2103. if (session == temp) {
  2104. list_del(&session->list);
  2105. break;
  2106. }
  2107. }
  2108. /* Poison the session handle with zeros */
  2109. *session = (struct cvp_hal_session){ {0} };
  2110. kfree(session);
  2111. }
  2112. static int iris_hfi_session_clean(void *session)
  2113. {
  2114. struct cvp_hal_session *sess_close;
  2115. struct iris_hfi_device *device;
  2116. if (!session) {
  2117. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2118. return -EINVAL;
  2119. }
  2120. sess_close = session;
  2121. device = sess_close->device;
  2122. if (!device) {
  2123. dprintk(CVP_ERR, "Invalid device handle %s\n", __func__);
  2124. return -EINVAL;
  2125. }
  2126. mutex_lock(&device->lock);
  2127. __session_clean(sess_close);
  2128. mutex_unlock(&device->lock);
  2129. return 0;
  2130. }
  2131. static int iris_debug_hook(void *device)
  2132. {
  2133. struct iris_hfi_device *dev = device;
  2134. u32 val;
  2135. if (!device) {
  2136. dprintk(CVP_ERR, "%s Invalid device\n", __func__);
  2137. return -ENODEV;
  2138. }
  2139. //__write_register(dev, CVP_WRAPPER_CORE_CLOCK_CONFIG, 0x11);
  2140. //__write_register(dev, CVP_WRAPPER_TZ_CPU_CLOCK_CONFIG, 0x1);
  2141. dprintk(CVP_ERR, "Halt Tensilica and core and axi\n");
  2142. return 0;
  2143. /******* FDU & MPU *****/
  2144. #define CVP0_CVP_SS_FDU_SECURE_ENABLE 0x90
  2145. #define CVP0_CVP_SS_MPU_SECURE_ENABLE 0x94
  2146. #define CVP0_CVP_SS_ARP_THREAD_0_SECURE_ENABLE 0xA0
  2147. #define CVP0_CVP_SS_ARP_THREAD_1_SECURE_ENABLE 0xA4
  2148. #define CVP0_CVP_SS_ARP_THREAD_2_SECURE_ENABLE 0xA8
  2149. #define CVP0_CVP_SS_ARP_THREAD_3_SECURE_ENABLE 0xAC
  2150. val = __read_register(dev, CVP0_CVP_SS_FDU_SECURE_ENABLE);
  2151. dprintk(CVP_ERR, "FDU_SECURE_ENABLE %#x\n", val);
  2152. val = __read_register(dev, CVP0_CVP_SS_MPU_SECURE_ENABLE);
  2153. dprintk(CVP_ERR, "MPU_SECURE_ENABLE %#x\n", val);
  2154. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_0_SECURE_ENABLE);
  2155. dprintk(CVP_ERR, "ARP_THREAD_0_SECURE_ENABLE %#x\n", val);
  2156. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_1_SECURE_ENABLE);
  2157. dprintk(CVP_ERR, "ARP_THREAD_1_SECURE_ENABLE %#x\n", val);
  2158. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_2_SECURE_ENABLE);
  2159. dprintk(CVP_ERR, "ARP_THREAD_2_SECURE_ENABLE %#x\n", val);
  2160. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_3_SECURE_ENABLE);
  2161. dprintk(CVP_ERR, "ARP_THREAD_3_SECURE_ENABLE %#x\n", val);
  2162. if (true)
  2163. return 0;
  2164. /***** GCE *******
  2165. * Bit 0 of below register is CDM secure enable for GCE
  2166. * CDM buffer will be in CB4 if set
  2167. */
  2168. #define CVP_GCE_GCE_SS_CP_CTL 0x51100
  2169. /* STATUS bit0 && CFG bit 4 of below register set,
  2170. * expect pixel buffers in CB3,
  2171. * otherwise in CB0
  2172. * CFG bit 9:8 b01 -> LMC input in CB3
  2173. * CFG bit 9:8 b10 -> LMC input in CB4
  2174. */
  2175. #define CVP_GCE0_CP_STATUS 0x51080
  2176. #define CVP_GCE0_BIU_RD_INPUT_IF_SECURITY_CFG 0x52020
  2177. val = __read_register(dev, CVP_GCE_GCE_SS_CP_CTL);
  2178. dprintk(CVP_ERR, "CVP_GCE_GCE_SS_CP_CTL %#x\n", val);
  2179. val = __read_register(dev, CVP_GCE0_CP_STATUS);
  2180. dprintk(CVP_ERR, "CVP_GCE0_CP_STATUS %#x\n", val);
  2181. val = __read_register(dev, CVP_GCE0_BIU_RD_INPUT_IF_SECURITY_CFG);
  2182. dprintk(CVP_ERR, "CVP_GCE0_BIU_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2183. /***** RGE *****
  2184. * Bit 0 of below regiser is CDM secure enable for RGE
  2185. * CDM buffer to be in CB4 i fset
  2186. */
  2187. #define CVP_RGE0_TOPRGE_CP_CTL 0x31010
  2188. /* CFG bit 4 && IN bit 0:
  2189. * if both are set, expect CB3 or CB4 depending on IN 6:4 field
  2190. * either is clear, expect CB0
  2191. */
  2192. #define CVP_RGE0_BUS_RD_INPUT_IF_SECURITY_CFG 0x32020
  2193. #define CVP_RGE0_TOPSPARE_IN 0x311F4
  2194. val = __read_register(dev, CVP_RGE0_TOPRGE_CP_CTL);
  2195. dprintk(CVP_ERR, "CVP_RGE0_TOPRGE_CP_CTL %#x\n", val);
  2196. val = __read_register(dev, CVP_RGE0_BUS_RD_INPUT_IF_SECURITY_CFG);
  2197. dprintk(CVP_ERR, "CVP_RGE0_BUS_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2198. val = __read_register(dev, CVP_RGE0_TOPSPARE_IN);
  2199. dprintk(CVP_ERR, "CVP_RGE0_TOPSPARE_IN %#x\n", val);
  2200. /****** VADL ******
  2201. * Bit 0 of below register is CDM secure enable for VADL
  2202. * CDM buffer will bei in CB4 if set
  2203. */
  2204. #define CVP_VADL0_VADL_SS_CP_CTL 0x21010
  2205. /* Below registers are used the same way as RGE */
  2206. #define CVP_VADL0_BUS_RD_INPUT_IF_SECURITY_CFG 0x22020
  2207. #define CVP_VADL0_SPARE_IN 0x211F4
  2208. val = __read_register(dev, CVP_VADL0_VADL_SS_CP_CTL);
  2209. dprintk(CVP_ERR, "CVP_VADL0_VADL_SS_CP_CTL %#x\n", val);
  2210. val = __read_register(dev, CVP_VADL0_BUS_RD_INPUT_IF_SECURITY_CFG);
  2211. dprintk(CVP_ERR, "CVP_VADL0_BUS_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2212. val = __read_register(dev, CVP_VADL0_SPARE_IN);
  2213. dprintk(CVP_ERR, "CVP_VADL0_SPARE_IN %#x\n", val);
  2214. /****** ITOF *****
  2215. * Below registers are used the same way as RGE
  2216. */
  2217. #define CVP_ITOF0_TOF_SS_CP_CTL 0x41010
  2218. #define CVP_ITOF0_BUS_RD_INPUT_IF_SECURITY_CFG 0x42020
  2219. #define CVP_ITOF0_TOF_SS_SPARE_IN 0x411F4
  2220. val = __read_register(dev, CVP_ITOF0_TOF_SS_CP_CTL);
  2221. dprintk(CVP_ERR, "CVP_ITOF0_TOF_SS_CP_CTL %#x\n", val);
  2222. val = __read_register(dev, CVP_ITOF0_BUS_RD_INPUT_IF_SECURITY_CFG);
  2223. dprintk(CVP_ERR, "CVP_ITOF0_BUS_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2224. val = __read_register(dev, CVP_ITOF0_TOF_SS_SPARE_IN);
  2225. dprintk(CVP_ERR, "CVP_ITOF0_TOF_SS_SPARE_IN %#x\n", val);
  2226. return 0;
  2227. }
  2228. static int iris_hfi_session_init(void *device, void *session_id,
  2229. void **new_session)
  2230. {
  2231. struct cvp_hfi_cmd_sys_session_init_packet pkt;
  2232. struct iris_hfi_device *dev;
  2233. struct cvp_hal_session *s;
  2234. if (!device || !new_session) {
  2235. dprintk(CVP_ERR, "%s - invalid input\n", __func__);
  2236. return -EINVAL;
  2237. }
  2238. dev = device;
  2239. mutex_lock(&dev->lock);
  2240. s = kzalloc(sizeof(*s), GFP_KERNEL);
  2241. if (!s) {
  2242. dprintk(CVP_ERR, "new session fail: Out of memory\n");
  2243. goto err_session_init_fail;
  2244. }
  2245. s->session_id = session_id;
  2246. s->device = dev;
  2247. dprintk(CVP_SESS,
  2248. "%s: inst %pK, session %pK\n", __func__, session_id, s);
  2249. list_add_tail(&s->list, &dev->sess_head);
  2250. __set_default_sys_properties(device);
  2251. if (call_hfi_pkt_op(dev, session_init, &pkt, s)) {
  2252. dprintk(CVP_ERR, "session_init: failed to create packet\n");
  2253. goto err_session_init_fail;
  2254. }
  2255. *new_session = s;
  2256. if (__iface_cmdq_write(dev, &pkt))
  2257. goto err_session_init_fail;
  2258. mutex_unlock(&dev->lock);
  2259. return 0;
  2260. err_session_init_fail:
  2261. if (s)
  2262. __session_clean(s);
  2263. *new_session = NULL;
  2264. mutex_unlock(&dev->lock);
  2265. return -EINVAL;
  2266. }
  2267. static int __send_session_cmd(struct cvp_hal_session *session, int pkt_type)
  2268. {
  2269. struct cvp_hal_session_cmd_pkt pkt;
  2270. int rc = 0;
  2271. struct iris_hfi_device *device = session->device;
  2272. if (!__is_session_valid(device, session, __func__))
  2273. return -ECONNRESET;
  2274. rc = call_hfi_pkt_op(device, session_cmd,
  2275. &pkt, pkt_type, session);
  2276. if (rc == -EPERM)
  2277. return 0;
  2278. if (rc) {
  2279. dprintk(CVP_ERR, "send session cmd: create pkt failed\n");
  2280. goto err_create_pkt;
  2281. }
  2282. if (__iface_cmdq_write(session->device, &pkt))
  2283. rc = -ENOTEMPTY;
  2284. err_create_pkt:
  2285. return rc;
  2286. }
  2287. static int iris_hfi_session_end(void *session)
  2288. {
  2289. struct cvp_hal_session *sess;
  2290. struct iris_hfi_device *device;
  2291. int rc = 0;
  2292. if (!session) {
  2293. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2294. return -EINVAL;
  2295. }
  2296. sess = session;
  2297. device = sess->device;
  2298. if (!device) {
  2299. dprintk(CVP_ERR, "Invalid session %s\n", __func__);
  2300. return -EINVAL;
  2301. }
  2302. mutex_lock(&device->lock);
  2303. if (msm_cvp_fw_coverage) {
  2304. if (__sys_set_coverage(sess->device, msm_cvp_fw_coverage))
  2305. dprintk(CVP_WARN, "Fw_coverage msg ON failed\n");
  2306. }
  2307. rc = __send_session_cmd(session, HFI_CMD_SYS_SESSION_END);
  2308. mutex_unlock(&device->lock);
  2309. return rc;
  2310. }
  2311. static int iris_hfi_session_abort(void *sess)
  2312. {
  2313. struct cvp_hal_session *session = sess;
  2314. struct iris_hfi_device *device;
  2315. int rc = 0;
  2316. if (!session || !session->device) {
  2317. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2318. return -EINVAL;
  2319. }
  2320. device = session->device;
  2321. mutex_lock(&device->lock);
  2322. rc = __send_session_cmd(session, HFI_CMD_SYS_SESSION_ABORT);
  2323. mutex_unlock(&device->lock);
  2324. return rc;
  2325. }
  2326. static int iris_hfi_session_set_buffers(void *sess, u32 iova, u32 size)
  2327. {
  2328. struct cvp_hfi_cmd_session_set_buffers_packet pkt;
  2329. int rc = 0;
  2330. struct cvp_hal_session *session = sess;
  2331. struct iris_hfi_device *device;
  2332. if (!session || !session->device || !iova || !size) {
  2333. dprintk(CVP_ERR, "Invalid Params\n");
  2334. return -EINVAL;
  2335. }
  2336. device = session->device;
  2337. mutex_lock(&device->lock);
  2338. if (!__is_session_valid(device, session, __func__)) {
  2339. rc = -ECONNRESET;
  2340. goto err_create_pkt;
  2341. }
  2342. rc = call_hfi_pkt_op(device, session_set_buffers,
  2343. &pkt, session, iova, size);
  2344. if (rc) {
  2345. dprintk(CVP_ERR, "set buffers: failed to create packet\n");
  2346. goto err_create_pkt;
  2347. }
  2348. if (__iface_cmdq_write(session->device, &pkt))
  2349. rc = -ENOTEMPTY;
  2350. err_create_pkt:
  2351. mutex_unlock(&device->lock);
  2352. return rc;
  2353. }
  2354. static int iris_hfi_session_release_buffers(void *sess)
  2355. {
  2356. struct cvp_session_release_buffers_packet pkt;
  2357. int rc = 0;
  2358. struct cvp_hal_session *session = sess;
  2359. struct iris_hfi_device *device;
  2360. if (!session || !session->device) {
  2361. dprintk(CVP_ERR, "Invalid Params\n");
  2362. return -EINVAL;
  2363. }
  2364. device = session->device;
  2365. mutex_lock(&device->lock);
  2366. if (!__is_session_valid(device, session, __func__)) {
  2367. rc = -ECONNRESET;
  2368. goto err_create_pkt;
  2369. }
  2370. rc = call_hfi_pkt_op(device, session_release_buffers, &pkt, session);
  2371. if (rc) {
  2372. dprintk(CVP_ERR, "release buffers: failed to create packet\n");
  2373. goto err_create_pkt;
  2374. }
  2375. if (__iface_cmdq_write(session->device, &pkt))
  2376. rc = -ENOTEMPTY;
  2377. err_create_pkt:
  2378. mutex_unlock(&device->lock);
  2379. return rc;
  2380. }
  2381. static int iris_hfi_session_send(void *sess,
  2382. struct eva_kmd_hfi_packet *in_pkt)
  2383. {
  2384. int rc = 0;
  2385. struct eva_kmd_hfi_packet pkt;
  2386. struct cvp_hal_session *session = sess;
  2387. struct iris_hfi_device *device;
  2388. if (!session || !session->device) {
  2389. dprintk(CVP_ERR, "invalid session");
  2390. return -ENODEV;
  2391. }
  2392. device = session->device;
  2393. mutex_lock(&device->lock);
  2394. if (!__is_session_valid(device, session, __func__)) {
  2395. rc = -ECONNRESET;
  2396. goto err_send_pkt;
  2397. }
  2398. rc = call_hfi_pkt_op(device, session_send,
  2399. &pkt, session, in_pkt);
  2400. if (rc) {
  2401. dprintk(CVP_ERR,
  2402. "failed to create pkt\n");
  2403. goto err_send_pkt;
  2404. }
  2405. if (__iface_cmdq_write(session->device, &pkt))
  2406. rc = -ENOTEMPTY;
  2407. err_send_pkt:
  2408. mutex_unlock(&device->lock);
  2409. return rc;
  2410. return rc;
  2411. }
  2412. static int iris_hfi_session_flush(void *sess)
  2413. {
  2414. struct cvp_hal_session *session = sess;
  2415. struct iris_hfi_device *device;
  2416. int rc = 0;
  2417. if (!session || !session->device) {
  2418. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2419. return -EINVAL;
  2420. }
  2421. device = session->device;
  2422. mutex_lock(&device->lock);
  2423. rc = __send_session_cmd(session, HFI_CMD_SESSION_CVP_FLUSH);
  2424. mutex_unlock(&device->lock);
  2425. return rc;
  2426. }
  2427. static int iris_hfi_session_start(void *sess)
  2428. {
  2429. struct cvp_hal_session *session = sess;
  2430. struct iris_hfi_device *device;
  2431. int rc = 0;
  2432. if (!session || !session->device) {
  2433. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2434. return -EINVAL;
  2435. }
  2436. device = session->device;
  2437. mutex_lock(&device->lock);
  2438. rc = __send_session_cmd(session, HFI_CMD_SESSION_EVA_START);
  2439. mutex_unlock(&device->lock);
  2440. return rc;
  2441. }
  2442. static int iris_hfi_session_stop(void *sess)
  2443. {
  2444. struct cvp_hal_session *session = sess;
  2445. struct iris_hfi_device *device;
  2446. int rc = 0;
  2447. if (!session || !session->device) {
  2448. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2449. return -EINVAL;
  2450. }
  2451. device = session->device;
  2452. mutex_lock(&device->lock);
  2453. rc = __send_session_cmd(session, HFI_CMD_SESSION_EVA_STOP);
  2454. mutex_unlock(&device->lock);
  2455. return rc;
  2456. }
  2457. static void __process_fatal_error(
  2458. struct iris_hfi_device *device)
  2459. {
  2460. struct msm_cvp_cb_cmd_done cmd_done = {0};
  2461. device->callback(HAL_SYS_ERROR, &cmd_done);
  2462. }
  2463. static int __prepare_pc(struct iris_hfi_device *device)
  2464. {
  2465. int rc = 0;
  2466. struct cvp_hfi_cmd_sys_pc_prep_packet pkt;
  2467. rc = call_hfi_pkt_op(device, sys_pc_prep, &pkt);
  2468. if (rc) {
  2469. dprintk(CVP_ERR, "Failed to create sys pc prep pkt\n");
  2470. goto err_pc_prep;
  2471. }
  2472. if (__iface_cmdq_write(device, &pkt))
  2473. rc = -ENOTEMPTY;
  2474. if (rc)
  2475. dprintk(CVP_ERR, "Failed to prepare iris for power off");
  2476. err_pc_prep:
  2477. return rc;
  2478. }
  2479. static void iris_hfi_pm_handler(struct work_struct *work)
  2480. {
  2481. int rc = 0;
  2482. struct msm_cvp_core *core;
  2483. struct iris_hfi_device *device;
  2484. core = cvp_driver->cvp_core;
  2485. if (core)
  2486. device = core->device->hfi_device_data;
  2487. else
  2488. return;
  2489. if (!device) {
  2490. dprintk(CVP_ERR, "%s: NULL device\n", __func__);
  2491. return;
  2492. }
  2493. dprintk(CVP_PWR,
  2494. "Entering %s\n", __func__);
  2495. /*
  2496. * It is ok to check this variable outside the lock since
  2497. * it is being updated in this context only
  2498. */
  2499. if (device->skip_pc_count >= CVP_MAX_PC_SKIP_COUNT) {
  2500. dprintk(CVP_WARN, "Failed to PC for %d times\n",
  2501. device->skip_pc_count);
  2502. device->skip_pc_count = 0;
  2503. __process_fatal_error(device);
  2504. return;
  2505. }
  2506. mutex_lock(&device->lock);
  2507. if (gfa_cv.state == DSP_SUSPEND)
  2508. rc = __power_collapse(device, true);
  2509. else
  2510. rc = __power_collapse(device, false);
  2511. mutex_unlock(&device->lock);
  2512. switch (rc) {
  2513. case 0:
  2514. device->skip_pc_count = 0;
  2515. /* Cancel pending delayed works if any */
  2516. cancel_delayed_work(&iris_hfi_pm_work);
  2517. dprintk(CVP_PWR, "%s: power collapse successful!\n",
  2518. __func__);
  2519. break;
  2520. case -EBUSY:
  2521. device->skip_pc_count = 0;
  2522. dprintk(CVP_PWR, "%s: retry PC as cvp is busy\n", __func__);
  2523. queue_delayed_work(device->iris_pm_workq,
  2524. &iris_hfi_pm_work, msecs_to_jiffies(
  2525. device->res->msm_cvp_pwr_collapse_delay));
  2526. break;
  2527. case -EAGAIN:
  2528. device->skip_pc_count++;
  2529. dprintk(CVP_WARN, "%s: retry power collapse (count %d)\n",
  2530. __func__, device->skip_pc_count);
  2531. queue_delayed_work(device->iris_pm_workq,
  2532. &iris_hfi_pm_work, msecs_to_jiffies(
  2533. device->res->msm_cvp_pwr_collapse_delay));
  2534. break;
  2535. default:
  2536. dprintk(CVP_ERR, "%s: power collapse failed\n", __func__);
  2537. break;
  2538. }
  2539. }
  2540. static int __power_collapse(struct iris_hfi_device *device, bool force)
  2541. {
  2542. int rc = 0;
  2543. u32 wfi_status = 0, idle_status = 0, pc_ready = 0;
  2544. int count = 0;
  2545. const int max_tries = 150;
  2546. if (!device) {
  2547. dprintk(CVP_ERR, "%s: invalid params\n", __func__);
  2548. return -EINVAL;
  2549. }
  2550. if (!device->power_enabled) {
  2551. dprintk(CVP_PWR, "%s: Power already disabled\n",
  2552. __func__);
  2553. goto exit;
  2554. }
  2555. rc = __core_in_valid_state(device);
  2556. if (!rc) {
  2557. dprintk(CVP_WARN,
  2558. "Core is in bad state, Skipping power collapse\n");
  2559. return -EINVAL;
  2560. }
  2561. rc = __dsp_suspend(device, force);
  2562. if (rc == -EBUSY)
  2563. goto exit;
  2564. else if (rc)
  2565. goto skip_power_off;
  2566. __flush_debug_queue(device, device->raw_packet);
  2567. pc_ready = __read_register(device, CVP_CTRL_STATUS) &
  2568. CVP_CTRL_STATUS_PC_READY;
  2569. if (!pc_ready) {
  2570. wfi_status = __read_register(device,
  2571. CVP_WRAPPER_CPU_STATUS);
  2572. idle_status = __read_register(device,
  2573. CVP_CTRL_STATUS);
  2574. if (!(wfi_status & BIT(0))) {
  2575. dprintk(CVP_WARN,
  2576. "Skipping PC as wfi_status (%#x) bit not set\n",
  2577. wfi_status);
  2578. goto skip_power_off;
  2579. }
  2580. if (!(idle_status & BIT(30))) {
  2581. dprintk(CVP_WARN,
  2582. "Skipping PC as idle_status (%#x) bit not set\n",
  2583. idle_status);
  2584. goto skip_power_off;
  2585. }
  2586. rc = __prepare_pc(device);
  2587. if (rc) {
  2588. dprintk(CVP_WARN, "Failed PC %d\n", rc);
  2589. goto skip_power_off;
  2590. }
  2591. while (count < max_tries) {
  2592. wfi_status = __read_register(device,
  2593. CVP_WRAPPER_CPU_STATUS);
  2594. pc_ready = __read_register(device,
  2595. CVP_CTRL_STATUS);
  2596. if ((wfi_status & BIT(0)) && (pc_ready &
  2597. CVP_CTRL_STATUS_PC_READY))
  2598. break;
  2599. usleep_range(150, 250);
  2600. count++;
  2601. }
  2602. if (count == max_tries) {
  2603. dprintk(CVP_ERR,
  2604. "Skip PC. Core is not ready (%#x, %#x)\n",
  2605. wfi_status, pc_ready);
  2606. goto skip_power_off;
  2607. }
  2608. } else {
  2609. wfi_status = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  2610. if (!(wfi_status & BIT(0))) {
  2611. dprintk(CVP_WARN,
  2612. "Skip PC as wfi_status (%#x) bit not set\n",
  2613. wfi_status);
  2614. goto skip_power_off;
  2615. }
  2616. }
  2617. rc = __suspend(device);
  2618. if (rc)
  2619. dprintk(CVP_ERR, "Failed __suspend\n");
  2620. exit:
  2621. return rc;
  2622. skip_power_off:
  2623. dprintk(CVP_PWR, "Skip PC(%#x, %#x, %#x)\n",
  2624. wfi_status, idle_status, pc_ready);
  2625. __flush_debug_queue(device, device->raw_packet);
  2626. return -EAGAIN;
  2627. }
  2628. static void __process_sys_error(struct iris_hfi_device *device)
  2629. {
  2630. struct cvp_hfi_sfr_struct *vsfr = NULL;
  2631. vsfr = (struct cvp_hfi_sfr_struct *)device->sfr.align_virtual_addr;
  2632. if (vsfr) {
  2633. void *p = memchr(vsfr->rg_data, '\0', vsfr->bufSize);
  2634. /*
  2635. * SFR isn't guaranteed to be NULL terminated
  2636. * since SYS_ERROR indicates that Iris is in the
  2637. * process of crashing.
  2638. */
  2639. if (p == NULL)
  2640. vsfr->rg_data[vsfr->bufSize - 1] = '\0';
  2641. dprintk(CVP_ERR, "SFR Message from FW: %s\n",
  2642. vsfr->rg_data);
  2643. }
  2644. }
  2645. static void __flush_debug_queue(struct iris_hfi_device *device, u8 *packet)
  2646. {
  2647. bool local_packet = false;
  2648. enum cvp_msg_prio log_level = CVP_FW;
  2649. if (!device) {
  2650. dprintk(CVP_ERR, "%s: Invalid params\n", __func__);
  2651. return;
  2652. }
  2653. if (!packet) {
  2654. packet = kzalloc(CVP_IFACEQ_VAR_HUGE_PKT_SIZE, GFP_KERNEL);
  2655. if (!packet) {
  2656. dprintk(CVP_ERR, "In %s() Fail to allocate mem\n",
  2657. __func__);
  2658. return;
  2659. }
  2660. local_packet = true;
  2661. /*
  2662. * Local packek is used when something FATAL occurred.
  2663. * It is good to print these logs by default.
  2664. */
  2665. log_level = CVP_ERR;
  2666. }
  2667. #define SKIP_INVALID_PKT(pkt_size, payload_size, pkt_hdr_size) ({ \
  2668. if (pkt_size < pkt_hdr_size || \
  2669. payload_size < MIN_PAYLOAD_SIZE || \
  2670. payload_size > \
  2671. (pkt_size - pkt_hdr_size + sizeof(u8))) { \
  2672. dprintk(CVP_ERR, \
  2673. "%s: invalid msg size - %d\n", \
  2674. __func__, pkt->msg_size); \
  2675. continue; \
  2676. } \
  2677. })
  2678. while (!__iface_dbgq_read(device, packet)) {
  2679. struct cvp_hfi_packet_header *pkt =
  2680. (struct cvp_hfi_packet_header *) packet;
  2681. if (pkt->size < sizeof(struct cvp_hfi_packet_header)) {
  2682. dprintk(CVP_ERR, "Invalid pkt size - %s\n",
  2683. __func__);
  2684. continue;
  2685. }
  2686. if (pkt->packet_type == HFI_MSG_SYS_DEBUG) {
  2687. struct cvp_hfi_msg_sys_debug_packet *pkt =
  2688. (struct cvp_hfi_msg_sys_debug_packet *) packet;
  2689. SKIP_INVALID_PKT(pkt->size,
  2690. pkt->msg_size, sizeof(*pkt));
  2691. /*
  2692. * All fw messages starts with new line character. This
  2693. * causes dprintk to print this message in two lines
  2694. * in the kernel log. Ignoring the first character
  2695. * from the message fixes this to print it in a single
  2696. * line.
  2697. */
  2698. pkt->rg_msg_data[pkt->msg_size-1] = '\0';
  2699. dprintk(log_level, "%s", &pkt->rg_msg_data[1]);
  2700. }
  2701. }
  2702. #undef SKIP_INVALID_PKT
  2703. if (local_packet)
  2704. kfree(packet);
  2705. }
  2706. static bool __is_session_valid(struct iris_hfi_device *device,
  2707. struct cvp_hal_session *session, const char *func)
  2708. {
  2709. struct cvp_hal_session *temp = NULL;
  2710. if (!device || !session)
  2711. goto invalid;
  2712. list_for_each_entry(temp, &device->sess_head, list)
  2713. if (session == temp)
  2714. return true;
  2715. invalid:
  2716. dprintk(CVP_WARN, "%s: device %pK, invalid session %pK\n",
  2717. func, device, session);
  2718. return false;
  2719. }
  2720. static struct cvp_hal_session *__get_session(struct iris_hfi_device *device,
  2721. u32 session_id)
  2722. {
  2723. struct cvp_hal_session *temp = NULL;
  2724. list_for_each_entry(temp, &device->sess_head, list) {
  2725. if (session_id == hash32_ptr(temp))
  2726. return temp;
  2727. }
  2728. return NULL;
  2729. }
  2730. #define _INVALID_MSG_ "Unrecognized MSG (%#x) session (%pK), discarding\n"
  2731. #define _INVALID_STATE_ "Ignore responses from %d to %d invalid state\n"
  2732. #define _DEVFREQ_FAIL_ "Failed to add devfreq device bus %s governor %s: %d\n"
  2733. static void process_system_msg(struct msm_cvp_cb_info *info,
  2734. struct iris_hfi_device *device,
  2735. void *raw_packet)
  2736. {
  2737. struct cvp_hal_sys_init_done sys_init_done = {0};
  2738. switch (info->response_type) {
  2739. case HAL_SYS_ERROR:
  2740. __process_sys_error(device);
  2741. break;
  2742. case HAL_SYS_RELEASE_RESOURCE_DONE:
  2743. dprintk(CVP_CORE, "Received SYS_RELEASE_RESOURCE\n");
  2744. break;
  2745. case HAL_SYS_INIT_DONE:
  2746. dprintk(CVP_CORE, "Received SYS_INIT_DONE\n");
  2747. sys_init_done.capabilities =
  2748. device->sys_init_capabilities;
  2749. cvp_hfi_process_sys_init_done_prop_read(
  2750. (struct cvp_hfi_msg_sys_init_done_packet *)
  2751. raw_packet, &sys_init_done);
  2752. info->response.cmd.data.sys_init_done = sys_init_done;
  2753. break;
  2754. default:
  2755. break;
  2756. }
  2757. }
  2758. static void **get_session_id(struct msm_cvp_cb_info *info)
  2759. {
  2760. void **session_id = NULL;
  2761. /* For session-related packets, validate session */
  2762. switch (info->response_type) {
  2763. case HAL_SESSION_INIT_DONE:
  2764. case HAL_SESSION_END_DONE:
  2765. case HAL_SESSION_ABORT_DONE:
  2766. case HAL_SESSION_START_DONE:
  2767. case HAL_SESSION_STOP_DONE:
  2768. case HAL_SESSION_FLUSH_DONE:
  2769. case HAL_SESSION_SET_BUFFER_DONE:
  2770. case HAL_SESSION_SUSPEND_DONE:
  2771. case HAL_SESSION_RESUME_DONE:
  2772. case HAL_SESSION_SET_PROP_DONE:
  2773. case HAL_SESSION_GET_PROP_DONE:
  2774. case HAL_SESSION_RELEASE_BUFFER_DONE:
  2775. case HAL_SESSION_REGISTER_BUFFER_DONE:
  2776. case HAL_SESSION_UNREGISTER_BUFFER_DONE:
  2777. case HAL_SESSION_PROPERTY_INFO:
  2778. case HAL_SESSION_EVENT_CHANGE:
  2779. case HAL_SESSION_DUMP_NOTIFY:
  2780. case HAL_SESSION_ERROR:
  2781. session_id = &info->response.cmd.session_id;
  2782. break;
  2783. case HAL_RESPONSE_UNUSED:
  2784. default:
  2785. session_id = NULL;
  2786. break;
  2787. }
  2788. return session_id;
  2789. }
  2790. static void print_msg_hdr(void *hdr)
  2791. {
  2792. struct cvp_hfi_msg_session_hdr *new_hdr =
  2793. (struct cvp_hfi_msg_session_hdr *)hdr;
  2794. dprintk(CVP_HFI, "HFI MSG received: %x %x %x %x %x %x %x %#llx\n",
  2795. new_hdr->size, new_hdr->packet_type,
  2796. new_hdr->session_id,
  2797. new_hdr->client_data.transaction_id,
  2798. new_hdr->client_data.data1,
  2799. new_hdr->client_data.data2,
  2800. new_hdr->error_type,
  2801. new_hdr->client_data.kdata);
  2802. }
  2803. static int __response_handler(struct iris_hfi_device *device)
  2804. {
  2805. struct msm_cvp_cb_info *packets;
  2806. int packet_count = 0;
  2807. u8 *raw_packet = NULL;
  2808. bool requeue_pm_work = true;
  2809. if (!device || device->state != IRIS_STATE_INIT)
  2810. return 0;
  2811. packets = device->response_pkt;
  2812. raw_packet = device->raw_packet;
  2813. if (!raw_packet || !packets) {
  2814. dprintk(CVP_ERR,
  2815. "%s: Invalid args : Res pkt = %pK, Raw pkt = %pK\n",
  2816. __func__, packets, raw_packet);
  2817. return 0;
  2818. }
  2819. if (device->intr_status & CVP_FATAL_INTR_BMSK) {
  2820. struct cvp_hfi_sfr_struct *vsfr = (struct cvp_hfi_sfr_struct *)
  2821. device->sfr.align_virtual_addr;
  2822. struct msm_cvp_cb_info info = {
  2823. .response_type = HAL_SYS_WATCHDOG_TIMEOUT,
  2824. .response.cmd = {
  2825. .device_id = 0,
  2826. }
  2827. };
  2828. if (vsfr)
  2829. dprintk(CVP_ERR, "SFR Message from FW: %s\n",
  2830. vsfr->rg_data);
  2831. if (device->intr_status & CVP_WRAPPER_INTR_MASK_CPU_NOC_BMSK)
  2832. dprintk(CVP_ERR, "Received Xtensa NOC error\n");
  2833. if (device->intr_status & CVP_WRAPPER_INTR_MASK_CORE_NOC_BMSK)
  2834. dprintk(CVP_ERR, "Received CVP core NOC error\n");
  2835. if (device->intr_status & CVP_WRAPPER_INTR_MASK_A2HWD_BMSK)
  2836. dprintk(CVP_ERR, "Received CVP watchdog timeout\n");
  2837. packets[packet_count++] = info;
  2838. goto exit;
  2839. }
  2840. /* Bleed the msg queue dry of packets */
  2841. while (!__iface_msgq_read(device, raw_packet)) {
  2842. void **session_id = NULL;
  2843. struct msm_cvp_cb_info *info = &packets[packet_count++];
  2844. struct cvp_hfi_msg_session_hdr *hdr =
  2845. (struct cvp_hfi_msg_session_hdr *)raw_packet;
  2846. int rc = 0;
  2847. print_msg_hdr(hdr);
  2848. rc = cvp_hfi_process_msg_packet(0, raw_packet, info);
  2849. if (rc) {
  2850. dprintk(CVP_WARN,
  2851. "Corrupt/unknown packet found, discarding\n");
  2852. --packet_count;
  2853. continue;
  2854. } else if (info->response_type == HAL_NO_RESP) {
  2855. --packet_count;
  2856. continue;
  2857. }
  2858. /* Process the packet types that we're interested in */
  2859. process_system_msg(info, device, raw_packet);
  2860. session_id = get_session_id(info);
  2861. /*
  2862. * hfi_process_msg_packet provides a session_id that's a hashed
  2863. * value of struct cvp_hal_session, we need to coerce the hashed
  2864. * value back to pointer that we can use. Ideally, hfi_process\
  2865. * _msg_packet should take care of this, but it doesn't have
  2866. * required information for it
  2867. */
  2868. if (session_id) {
  2869. struct cvp_hal_session *session = NULL;
  2870. if (upper_32_bits((uintptr_t)*session_id) != 0) {
  2871. dprintk(CVP_ERR,
  2872. "Upper 32-bits != 0 for sess_id=%pK\n",
  2873. *session_id);
  2874. }
  2875. session = __get_session(device,
  2876. (u32)(uintptr_t)*session_id);
  2877. if (!session) {
  2878. dprintk(CVP_ERR, _INVALID_MSG_,
  2879. info->response_type,
  2880. *session_id);
  2881. --packet_count;
  2882. continue;
  2883. }
  2884. *session_id = session->session_id;
  2885. }
  2886. if (packet_count >= cvp_max_packets) {
  2887. dprintk(CVP_WARN,
  2888. "Too many packets in message queue!\n");
  2889. break;
  2890. }
  2891. /* do not read packets after sys error packet */
  2892. if (info->response_type == HAL_SYS_ERROR)
  2893. break;
  2894. }
  2895. if (requeue_pm_work && device->res->sw_power_collapsible) {
  2896. cancel_delayed_work(&iris_hfi_pm_work);
  2897. if (!queue_delayed_work(device->iris_pm_workq,
  2898. &iris_hfi_pm_work,
  2899. msecs_to_jiffies(
  2900. device->res->msm_cvp_pwr_collapse_delay))) {
  2901. dprintk(CVP_ERR, "PM work already scheduled\n");
  2902. }
  2903. }
  2904. exit:
  2905. __flush_debug_queue(device, raw_packet);
  2906. return packet_count;
  2907. }
  2908. irqreturn_t iris_hfi_core_work_handler(int irq, void *data)
  2909. {
  2910. struct msm_cvp_core *core;
  2911. struct iris_hfi_device *device;
  2912. int num_responses = 0, i = 0;
  2913. u32 intr_status;
  2914. static bool warning_on = true;
  2915. core = cvp_driver->cvp_core;
  2916. if (core)
  2917. device = core->device->hfi_device_data;
  2918. else
  2919. return IRQ_HANDLED;
  2920. mutex_lock(&device->lock);
  2921. if (!__core_in_valid_state(device)) {
  2922. if (warning_on) {
  2923. dprintk(CVP_WARN, "%s Core not in init state\n",
  2924. __func__);
  2925. warning_on = false;
  2926. }
  2927. goto err_no_work;
  2928. }
  2929. warning_on = true;
  2930. if (!device->callback) {
  2931. dprintk(CVP_ERR, "No interrupt callback function: %pK\n",
  2932. device);
  2933. goto err_no_work;
  2934. }
  2935. if (__resume(device)) {
  2936. dprintk(CVP_ERR, "%s: Power enable failed\n", __func__);
  2937. goto err_no_work;
  2938. }
  2939. __core_clear_interrupt(device);
  2940. num_responses = __response_handler(device);
  2941. dprintk(CVP_HFI, "%s:: cvp_driver_debug num_responses = %d ",
  2942. __func__, num_responses);
  2943. err_no_work:
  2944. /* Keep the interrupt status before releasing device lock */
  2945. intr_status = device->intr_status;
  2946. mutex_unlock(&device->lock);
  2947. /*
  2948. * Issue the callbacks outside of the locked contex to preserve
  2949. * re-entrancy.
  2950. */
  2951. for (i = 0; !IS_ERR_OR_NULL(device->response_pkt) &&
  2952. i < num_responses; ++i) {
  2953. struct msm_cvp_cb_info *r = &device->response_pkt[i];
  2954. void *rsp = (void *)&r->response;
  2955. if (!__core_in_valid_state(device)) {
  2956. dprintk(CVP_ERR,
  2957. _INVALID_STATE_, (i + 1), num_responses);
  2958. break;
  2959. }
  2960. dprintk(CVP_HFI, "Processing response %d of %d, type %d\n",
  2961. (i + 1), num_responses, r->response_type);
  2962. /* callback = void cvp_handle_cmd_response() */
  2963. device->callback(r->response_type, rsp);
  2964. }
  2965. /* We need re-enable the irq which was disabled in ISR handler */
  2966. if (!(intr_status & CVP_WRAPPER_INTR_STATUS_A2HWD_BMSK))
  2967. enable_irq(device->cvp_hal_data->irq);
  2968. return IRQ_HANDLED;
  2969. }
  2970. irqreturn_t cvp_hfi_isr(int irq, void *dev)
  2971. {
  2972. disable_irq_nosync(irq);
  2973. return IRQ_WAKE_THREAD;
  2974. }
  2975. static void iris_hfi_wd_work_handler(struct work_struct *work)
  2976. {
  2977. struct msm_cvp_core *core;
  2978. struct iris_hfi_device *device;
  2979. struct msm_cvp_cb_cmd_done response = {0};
  2980. enum hal_command_response cmd = HAL_SYS_WATCHDOG_TIMEOUT;
  2981. core = cvp_driver->cvp_core;
  2982. if (core)
  2983. device = core->device->hfi_device_data;
  2984. else
  2985. return;
  2986. if (msm_cvp_hw_wd_recovery) {
  2987. dprintk(CVP_ERR, "Cleaning up as HW WD recovery is enable %d\n",
  2988. msm_cvp_hw_wd_recovery);
  2989. response.device_id = 0;
  2990. handle_sys_error(cmd, (void *) &response);
  2991. enable_irq(device->cvp_hal_data->irq_wd);
  2992. }
  2993. else {
  2994. dprintk(CVP_ERR, "Crashing the device as HW WD recovery is disable %d\n",
  2995. msm_cvp_hw_wd_recovery);
  2996. BUG_ON(1);
  2997. }
  2998. }
  2999. static DECLARE_WORK(iris_hfi_wd_work, iris_hfi_wd_work_handler);
  3000. irqreturn_t iris_hfi_isr_wd(int irq, void *dev)
  3001. {
  3002. struct iris_hfi_device *device = dev;
  3003. dprintk(CVP_ERR, "Got HW WDOG IRQ at %llu! \n", get_aon_time());
  3004. disable_irq_nosync(irq);
  3005. queue_work(device->cvp_workq, &iris_hfi_wd_work);
  3006. return IRQ_HANDLED;
  3007. }
  3008. static int __init_reset_clk(struct msm_cvp_platform_resources *res,
  3009. int reset_index)
  3010. {
  3011. int rc = 0;
  3012. struct reset_control *rst;
  3013. struct reset_info *rst_info;
  3014. struct reset_set *rst_set = &res->reset_set;
  3015. if (!rst_set->reset_tbl)
  3016. return 0;
  3017. rst_info = &rst_set->reset_tbl[reset_index];
  3018. rst = rst_info->rst;
  3019. dprintk(CVP_PWR, "reset_clk: name %s rst %pK required_stage=%d\n",
  3020. rst_set->reset_tbl[reset_index].name, rst, rst_info->required_stage);
  3021. if (rst)
  3022. goto skip_reset_init;
  3023. if (rst_info->required_stage == CVP_ON_USE) {
  3024. rst = reset_control_get_exclusive_released(&res->pdev->dev,
  3025. rst_set->reset_tbl[reset_index].name);
  3026. if (IS_ERR(rst)) {
  3027. rc = PTR_ERR(rst);
  3028. dprintk(CVP_ERR, "reset get exclusive fail %d\n", rc);
  3029. return rc;
  3030. }
  3031. dprintk(CVP_PWR, "reset_clk: name %s get exclusive rst %llx\n",
  3032. rst_set->reset_tbl[reset_index].name, rst);
  3033. } else if (rst_info->required_stage == CVP_ON_INIT) {
  3034. rst = devm_reset_control_get(&res->pdev->dev,
  3035. rst_set->reset_tbl[reset_index].name);
  3036. if (IS_ERR(rst)) {
  3037. rc = PTR_ERR(rst);
  3038. dprintk(CVP_ERR, "reset get fail %d\n", rc);
  3039. return rc;
  3040. }
  3041. dprintk(CVP_PWR, "reset_clk: name %s get rst %llx\n",
  3042. rst_set->reset_tbl[reset_index].name, rst);
  3043. } else {
  3044. dprintk(CVP_ERR, "Invalid reset stage\n");
  3045. return -EINVAL;
  3046. }
  3047. rst_set->reset_tbl[reset_index].rst = rst;
  3048. rst_info->state = RESET_INIT;
  3049. return 0;
  3050. skip_reset_init:
  3051. return rc;
  3052. }
  3053. static int __reset_control_assert_name(struct iris_hfi_device *device,
  3054. const char *name)
  3055. {
  3056. struct reset_info *rcinfo = NULL;
  3057. int rc = 0;
  3058. bool found = false;
  3059. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3060. if (strcmp(rcinfo->name, name))
  3061. continue;
  3062. found = true;
  3063. rc = reset_control_assert(rcinfo->rst);
  3064. if (rc)
  3065. dprintk(CVP_ERR,
  3066. "%s: failed to assert reset control (%s), rc = %d\n",
  3067. __func__, rcinfo->name, rc);
  3068. else
  3069. dprintk(CVP_PWR, "%s: assert reset control (%s)\n",
  3070. __func__, rcinfo->name);
  3071. break;
  3072. }
  3073. if (!found) {
  3074. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3075. __func__, name);
  3076. rc = -EINVAL;
  3077. }
  3078. return rc;
  3079. }
  3080. static int __reset_control_deassert_name(struct iris_hfi_device *device,
  3081. const char *name)
  3082. {
  3083. struct reset_info *rcinfo = NULL;
  3084. int rc = 0;
  3085. bool found = false;
  3086. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3087. if (strcmp(rcinfo->name, name))
  3088. continue;
  3089. found = true;
  3090. rc = reset_control_deassert(rcinfo->rst);
  3091. if (rc)
  3092. dprintk(CVP_ERR,
  3093. "%s: deassert reset control for (%s) failed, rc %d\n",
  3094. __func__, rcinfo->name, rc);
  3095. else
  3096. dprintk(CVP_PWR, "%s: deassert reset control (%s)\n",
  3097. __func__, rcinfo->name);
  3098. break;
  3099. }
  3100. if (!found) {
  3101. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3102. __func__, name);
  3103. rc = -EINVAL;
  3104. }
  3105. return rc;
  3106. }
  3107. static int __reset_control_acquire(struct iris_hfi_device *device,
  3108. const char *name)
  3109. {
  3110. struct reset_info *rcinfo = NULL;
  3111. int rc = 0;
  3112. bool found = false;
  3113. int max_retries = 10;
  3114. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3115. if (strcmp(rcinfo->name, name))
  3116. continue;
  3117. found = true;
  3118. if (rcinfo->state == RESET_ACQUIRED)
  3119. return rc;
  3120. acquire_again:
  3121. rc = reset_control_acquire(rcinfo->rst);
  3122. if (rc) {
  3123. if (rc == -EBUSY) {
  3124. usleep_range(500, 1000);
  3125. max_retries--;
  3126. if (max_retries) {
  3127. goto acquire_again;
  3128. } else {
  3129. dprintk(CVP_ERR,
  3130. "%s acquire %s -EBUSY\n",
  3131. __func__, rcinfo->name);
  3132. rc = -EINVAL;
  3133. }
  3134. } else {
  3135. dprintk(CVP_ERR,
  3136. "%s: acquire failed (%s) rc %d\n",
  3137. __func__, rcinfo->name, rc);
  3138. rc = -EINVAL;
  3139. }
  3140. } else {
  3141. dprintk(CVP_PWR, "%s: reset acquire succeed (%s)\n",
  3142. __func__, rcinfo->name);
  3143. rcinfo->state = RESET_ACQUIRED;
  3144. }
  3145. break;
  3146. }
  3147. if (!found) {
  3148. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3149. __func__, name);
  3150. rc = -EINVAL;
  3151. }
  3152. return rc;
  3153. }
  3154. static int __reset_control_release(struct iris_hfi_device *device,
  3155. const char *name)
  3156. {
  3157. struct reset_info *rcinfo = NULL;
  3158. int rc = 0;
  3159. bool found = false;
  3160. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3161. if (strcmp(rcinfo->name, name))
  3162. continue;
  3163. found = true;
  3164. if (rcinfo->state != RESET_ACQUIRED) {
  3165. dprintk(CVP_WARN, "Double releasing reset clk?\n");
  3166. return -EINVAL;
  3167. }
  3168. reset_control_release(rcinfo->rst);
  3169. dprintk(CVP_PWR, "%s: reset release succeed (%s)\n",
  3170. __func__, rcinfo->name);
  3171. rcinfo->state = RESET_RELEASED;
  3172. break;
  3173. }
  3174. if (!found) {
  3175. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3176. __func__, name);
  3177. rc = -EINVAL;
  3178. }
  3179. return rc;
  3180. }
  3181. static void __deinit_bus(struct iris_hfi_device *device)
  3182. {
  3183. struct bus_info *bus = NULL;
  3184. if (!device)
  3185. return;
  3186. kfree(device->bus_vote.data);
  3187. device->bus_vote = CVP_DEFAULT_BUS_VOTE;
  3188. iris_hfi_for_each_bus_reverse(device, bus) {
  3189. dev_set_drvdata(bus->dev, NULL);
  3190. icc_put(bus->client);
  3191. bus->client = NULL;
  3192. }
  3193. }
  3194. static int __init_bus(struct iris_hfi_device *device)
  3195. {
  3196. struct bus_info *bus = NULL;
  3197. int rc = 0;
  3198. if (!device)
  3199. return -EINVAL;
  3200. iris_hfi_for_each_bus(device, bus) {
  3201. /*
  3202. * This is stupid, but there's no other easy way to ahold
  3203. * of struct bus_info in iris_hfi_devfreq_*()
  3204. */
  3205. WARN(dev_get_drvdata(bus->dev), "%s's drvdata already set\n",
  3206. dev_name(bus->dev));
  3207. dev_set_drvdata(bus->dev, device);
  3208. bus->client = icc_get(&device->res->pdev->dev,
  3209. bus->master, bus->slave);
  3210. if (IS_ERR_OR_NULL(bus->client)) {
  3211. rc = PTR_ERR(bus->client) ?: -EBADHANDLE;
  3212. dprintk(CVP_ERR, "Failed to register bus %s: %d\n",
  3213. bus->name, rc);
  3214. bus->client = NULL;
  3215. goto err_add_dev;
  3216. }
  3217. }
  3218. return 0;
  3219. err_add_dev:
  3220. __deinit_bus(device);
  3221. return rc;
  3222. }
  3223. static void __deinit_regulators(struct iris_hfi_device *device)
  3224. {
  3225. struct regulator_info *rinfo = NULL;
  3226. iris_hfi_for_each_regulator_reverse(device, rinfo) {
  3227. if (rinfo->regulator) {
  3228. regulator_put(rinfo->regulator);
  3229. rinfo->regulator = NULL;
  3230. }
  3231. }
  3232. }
  3233. static int __init_regulators(struct iris_hfi_device *device)
  3234. {
  3235. int rc = 0;
  3236. struct regulator_info *rinfo = NULL;
  3237. iris_hfi_for_each_regulator(device, rinfo) {
  3238. rinfo->regulator = regulator_get(&device->res->pdev->dev,
  3239. rinfo->name);
  3240. if (IS_ERR_OR_NULL(rinfo->regulator)) {
  3241. rc = PTR_ERR(rinfo->regulator) ?: -EBADHANDLE;
  3242. dprintk(CVP_ERR, "Failed to get regulator: %s\n",
  3243. rinfo->name);
  3244. rinfo->regulator = NULL;
  3245. goto err_reg_get;
  3246. }
  3247. }
  3248. return 0;
  3249. err_reg_get:
  3250. __deinit_regulators(device);
  3251. return rc;
  3252. }
  3253. static void __deinit_subcaches(struct iris_hfi_device *device)
  3254. {
  3255. struct subcache_info *sinfo = NULL;
  3256. if (!device) {
  3257. dprintk(CVP_ERR, "deinit_subcaches: invalid device %pK\n",
  3258. device);
  3259. goto exit;
  3260. }
  3261. if (!is_sys_cache_present(device))
  3262. goto exit;
  3263. iris_hfi_for_each_subcache_reverse(device, sinfo) {
  3264. if (sinfo->subcache) {
  3265. dprintk(CVP_CORE, "deinit_subcaches: %s\n",
  3266. sinfo->name);
  3267. llcc_slice_putd(sinfo->subcache);
  3268. sinfo->subcache = NULL;
  3269. }
  3270. }
  3271. exit:
  3272. return;
  3273. }
  3274. static int __init_subcaches(struct iris_hfi_device *device)
  3275. {
  3276. int rc = 0;
  3277. struct subcache_info *sinfo = NULL;
  3278. if (!device) {
  3279. dprintk(CVP_ERR, "init_subcaches: invalid device %pK\n",
  3280. device);
  3281. return -EINVAL;
  3282. }
  3283. if (!is_sys_cache_present(device))
  3284. return 0;
  3285. iris_hfi_for_each_subcache(device, sinfo) {
  3286. if (!strcmp("cvp", sinfo->name)) {
  3287. sinfo->subcache = llcc_slice_getd(LLCC_CVP);
  3288. } else if (!strcmp("cvpfw", sinfo->name)) {
  3289. sinfo->subcache = llcc_slice_getd(LLCC_CVPFW);
  3290. } else {
  3291. dprintk(CVP_ERR, "Invalid subcache name %s\n",
  3292. sinfo->name);
  3293. }
  3294. if (IS_ERR_OR_NULL(sinfo->subcache)) {
  3295. rc = PTR_ERR(sinfo->subcache) ?
  3296. PTR_ERR(sinfo->subcache) : -EBADHANDLE;
  3297. dprintk(CVP_ERR,
  3298. "init_subcaches: invalid subcache: %s rc %d\n",
  3299. sinfo->name, rc);
  3300. sinfo->subcache = NULL;
  3301. goto err_subcache_get;
  3302. }
  3303. dprintk(CVP_CORE, "init_subcaches: %s\n",
  3304. sinfo->name);
  3305. }
  3306. return 0;
  3307. err_subcache_get:
  3308. __deinit_subcaches(device);
  3309. return rc;
  3310. }
  3311. static int __init_resources(struct iris_hfi_device *device,
  3312. struct msm_cvp_platform_resources *res)
  3313. {
  3314. int i, rc = 0;
  3315. rc = __init_regulators(device);
  3316. if (rc) {
  3317. dprintk(CVP_ERR, "Failed to get all regulators\n");
  3318. return -ENODEV;
  3319. }
  3320. rc = msm_cvp_init_clocks(device);
  3321. if (rc) {
  3322. dprintk(CVP_ERR, "Failed to init clocks\n");
  3323. rc = -ENODEV;
  3324. goto err_init_clocks;
  3325. }
  3326. for (i = 0; i < device->res->reset_set.count; i++) {
  3327. rc = __init_reset_clk(res, i);
  3328. if (rc) {
  3329. dprintk(CVP_ERR, "Failed to init reset clocks\n");
  3330. rc = -ENODEV;
  3331. goto err_init_reset_clk;
  3332. }
  3333. }
  3334. rc = __init_bus(device);
  3335. if (rc) {
  3336. dprintk(CVP_ERR, "Failed to init bus: %d\n", rc);
  3337. goto err_init_bus;
  3338. }
  3339. rc = __init_subcaches(device);
  3340. if (rc)
  3341. dprintk(CVP_WARN, "Failed to init subcaches: %d\n", rc);
  3342. device->sys_init_capabilities =
  3343. kzalloc(sizeof(struct msm_cvp_capability)
  3344. * CVP_MAX_SESSIONS, GFP_KERNEL);
  3345. return rc;
  3346. err_init_reset_clk:
  3347. err_init_bus:
  3348. msm_cvp_deinit_clocks(device);
  3349. err_init_clocks:
  3350. __deinit_regulators(device);
  3351. return rc;
  3352. }
  3353. static void __deinit_resources(struct iris_hfi_device *device)
  3354. {
  3355. __deinit_subcaches(device);
  3356. __deinit_bus(device);
  3357. msm_cvp_deinit_clocks(device);
  3358. __deinit_regulators(device);
  3359. kfree(device->sys_init_capabilities);
  3360. device->sys_init_capabilities = NULL;
  3361. }
  3362. static int __disable_regulator_impl(struct regulator_info *rinfo,
  3363. struct iris_hfi_device *device)
  3364. {
  3365. int rc = 0;
  3366. dprintk(CVP_PWR, "Disabling regulator %s\n", rinfo->name);
  3367. /*
  3368. * This call is needed. Driver needs to acquire the control back
  3369. * from HW in order to disable the regualtor. Else the behavior
  3370. * is unknown.
  3371. */
  3372. rc = __acquire_regulator(rinfo, device);
  3373. if (rc) {
  3374. /*
  3375. * This is somewhat fatal, but nothing we can do
  3376. * about it. We can't disable the regulator w/o
  3377. * getting it back under s/w control
  3378. */
  3379. dprintk(CVP_WARN,
  3380. "Failed to acquire control on %s\n",
  3381. rinfo->name);
  3382. goto disable_regulator_failed;
  3383. }
  3384. rc = regulator_disable(rinfo->regulator);
  3385. if (rc) {
  3386. dprintk(CVP_WARN,
  3387. "Failed to disable %s: %d\n",
  3388. rinfo->name, rc);
  3389. goto disable_regulator_failed;
  3390. }
  3391. return 0;
  3392. disable_regulator_failed:
  3393. /* Bring attention to this issue */
  3394. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  3395. return rc;
  3396. }
  3397. static int __disable_hw_power_collapse(struct iris_hfi_device *device)
  3398. {
  3399. int rc = 0;
  3400. if (!msm_cvp_fw_low_power_mode) {
  3401. dprintk(CVP_PWR, "Not enabling hardware power collapse\n");
  3402. return 0;
  3403. }
  3404. rc = __take_back_regulators(device);
  3405. if (rc)
  3406. dprintk(CVP_WARN,
  3407. "%s : Failed to disable HW power collapse %d\n",
  3408. __func__, rc);
  3409. return rc;
  3410. }
  3411. static int __enable_hw_power_collapse(struct iris_hfi_device *device)
  3412. {
  3413. int rc = 0;
  3414. if (!msm_cvp_fw_low_power_mode) {
  3415. dprintk(CVP_PWR, "Not enabling hardware power collapse\n");
  3416. return 0;
  3417. }
  3418. rc = __hand_off_regulators(device);
  3419. if (rc)
  3420. dprintk(CVP_WARN,
  3421. "%s : Failed to enable HW power collapse %d\n",
  3422. __func__, rc);
  3423. return rc;
  3424. }
  3425. static int __enable_regulator(struct iris_hfi_device *device,
  3426. const char *name)
  3427. {
  3428. int rc = 0;
  3429. struct regulator_info *rinfo;
  3430. iris_hfi_for_each_regulator(device, rinfo) {
  3431. if (strcmp(rinfo->name, name))
  3432. continue;
  3433. rc = regulator_enable(rinfo->regulator);
  3434. if (rc) {
  3435. dprintk(CVP_ERR, "Failed to enable %s: %d\n",
  3436. rinfo->name, rc);
  3437. return rc;
  3438. }
  3439. if (!regulator_is_enabled(rinfo->regulator)) {
  3440. dprintk(CVP_ERR,"%s: regulator %s not enabled\n",
  3441. __func__, rinfo->name);
  3442. regulator_disable(rinfo->regulator);
  3443. return -EINVAL;
  3444. }
  3445. dprintk(CVP_PWR, "Enabled regulator %s\n", rinfo->name);
  3446. return 0;
  3447. }
  3448. dprintk(CVP_ERR, "regulator %s not found\n", name);
  3449. return -EINVAL;
  3450. }
  3451. static int __disable_regulator(struct iris_hfi_device *device,
  3452. const char *name)
  3453. {
  3454. struct regulator_info *rinfo;
  3455. iris_hfi_for_each_regulator_reverse(device, rinfo) {
  3456. if (strcmp(rinfo->name, name))
  3457. continue;
  3458. __disable_regulator_impl(rinfo, device);
  3459. dprintk(CVP_PWR, "%s Disabled regulator %s\n", __func__, name);
  3460. return 0;
  3461. }
  3462. dprintk(CVP_ERR, "%s regulator %s not found\n", __func__, name);
  3463. return -EINVAL;
  3464. }
  3465. static int __enable_subcaches(struct iris_hfi_device *device)
  3466. {
  3467. int rc = 0;
  3468. u32 c = 0;
  3469. struct subcache_info *sinfo;
  3470. if (msm_cvp_syscache_disable || !is_sys_cache_present(device))
  3471. return 0;
  3472. /* Activate subcaches */
  3473. iris_hfi_for_each_subcache(device, sinfo) {
  3474. rc = llcc_slice_activate(sinfo->subcache);
  3475. if (rc) {
  3476. dprintk(CVP_WARN, "Failed to activate %s: %d\n",
  3477. sinfo->name, rc);
  3478. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  3479. goto err_activate_fail;
  3480. }
  3481. sinfo->isactive = true;
  3482. dprintk(CVP_CORE, "Activated subcache %s\n", sinfo->name);
  3483. c++;
  3484. }
  3485. dprintk(CVP_CORE, "Activated %d Subcaches to CVP\n", c);
  3486. return 0;
  3487. err_activate_fail:
  3488. __release_subcaches(device);
  3489. __disable_subcaches(device);
  3490. return 0;
  3491. }
  3492. static int __set_subcaches(struct iris_hfi_device *device)
  3493. {
  3494. int rc = 0;
  3495. u32 c = 0;
  3496. struct subcache_info *sinfo;
  3497. u32 resource[CVP_MAX_SUBCACHE_SIZE];
  3498. struct cvp_hfi_resource_syscache_info_type *sc_res_info;
  3499. struct cvp_hfi_resource_subcache_type *sc_res;
  3500. struct cvp_resource_hdr rhdr;
  3501. if (device->res->sys_cache_res_set || msm_cvp_syscache_disable) {
  3502. dprintk(CVP_CORE, "Subcaches already set or disabled\n");
  3503. return 0;
  3504. }
  3505. memset((void *)resource, 0x0, (sizeof(u32) * CVP_MAX_SUBCACHE_SIZE));
  3506. sc_res_info = (struct cvp_hfi_resource_syscache_info_type *)resource;
  3507. sc_res = &(sc_res_info->rg_subcache_entries[0]);
  3508. iris_hfi_for_each_subcache(device, sinfo) {
  3509. if (sinfo->isactive) {
  3510. sc_res[c].size = sinfo->subcache->slice_size;
  3511. sc_res[c].sc_id = sinfo->subcache->slice_id;
  3512. c++;
  3513. }
  3514. }
  3515. /* Set resource to CVP for activated subcaches */
  3516. if (c) {
  3517. dprintk(CVP_CORE, "Setting %d Subcaches\n", c);
  3518. rhdr.resource_handle = sc_res_info; /* cookie */
  3519. rhdr.resource_id = CVP_RESOURCE_SYSCACHE;
  3520. sc_res_info->num_entries = c;
  3521. rc = __core_set_resource(device, &rhdr, (void *)sc_res_info);
  3522. if (rc) {
  3523. dprintk(CVP_WARN, "Failed to set subcaches %d\n", rc);
  3524. goto err_fail_set_subacaches;
  3525. }
  3526. iris_hfi_for_each_subcache(device, sinfo) {
  3527. if (sinfo->isactive)
  3528. sinfo->isset = true;
  3529. }
  3530. dprintk(CVP_CORE, "Set Subcaches done to CVP\n");
  3531. device->res->sys_cache_res_set = true;
  3532. }
  3533. return 0;
  3534. err_fail_set_subacaches:
  3535. __disable_subcaches(device);
  3536. return 0;
  3537. }
  3538. static int __release_subcaches(struct iris_hfi_device *device)
  3539. {
  3540. struct subcache_info *sinfo;
  3541. int rc = 0;
  3542. u32 c = 0;
  3543. u32 resource[CVP_MAX_SUBCACHE_SIZE];
  3544. struct cvp_hfi_resource_syscache_info_type *sc_res_info;
  3545. struct cvp_hfi_resource_subcache_type *sc_res;
  3546. struct cvp_resource_hdr rhdr;
  3547. if (msm_cvp_syscache_disable || !is_sys_cache_present(device))
  3548. return 0;
  3549. memset((void *)resource, 0x0, (sizeof(u32) * CVP_MAX_SUBCACHE_SIZE));
  3550. sc_res_info = (struct cvp_hfi_resource_syscache_info_type *)resource;
  3551. sc_res = &(sc_res_info->rg_subcache_entries[0]);
  3552. /* Release resource command to Iris */
  3553. iris_hfi_for_each_subcache_reverse(device, sinfo) {
  3554. if (sinfo->isset) {
  3555. /* Update the entry */
  3556. sc_res[c].size = sinfo->subcache->slice_size;
  3557. sc_res[c].sc_id = sinfo->subcache->slice_id;
  3558. c++;
  3559. sinfo->isset = false;
  3560. }
  3561. }
  3562. if (c > 0) {
  3563. dprintk(CVP_CORE, "Releasing %d subcaches\n", c);
  3564. rhdr.resource_handle = sc_res_info; /* cookie */
  3565. rhdr.resource_id = CVP_RESOURCE_SYSCACHE;
  3566. rc = __core_release_resource(device, &rhdr);
  3567. if (rc)
  3568. dprintk(CVP_WARN,
  3569. "Failed to release %d subcaches\n", c);
  3570. }
  3571. device->res->sys_cache_res_set = false;
  3572. return 0;
  3573. }
  3574. static int __disable_subcaches(struct iris_hfi_device *device)
  3575. {
  3576. struct subcache_info *sinfo;
  3577. int rc = 0;
  3578. if (msm_cvp_syscache_disable || !is_sys_cache_present(device))
  3579. return 0;
  3580. /* De-activate subcaches */
  3581. iris_hfi_for_each_subcache_reverse(device, sinfo) {
  3582. if (sinfo->isactive) {
  3583. dprintk(CVP_CORE, "De-activate subcache %s\n",
  3584. sinfo->name);
  3585. rc = llcc_slice_deactivate(sinfo->subcache);
  3586. if (rc) {
  3587. dprintk(CVP_WARN,
  3588. "Failed to de-activate %s: %d\n",
  3589. sinfo->name, rc);
  3590. }
  3591. sinfo->isactive = false;
  3592. }
  3593. }
  3594. return 0;
  3595. }
  3596. static void interrupt_init_iris2(struct iris_hfi_device *device)
  3597. {
  3598. u32 mask_val = 0;
  3599. /* All interrupts should be disabled initially 0x1F6 : Reset value */
  3600. mask_val = __read_register(device, CVP_WRAPPER_INTR_MASK);
  3601. /* Write 0 to unmask CPU and WD interrupts */
  3602. mask_val &= ~(CVP_FATAL_INTR_BMSK | CVP_WRAPPER_INTR_MASK_A2HCPU_BMSK);
  3603. __write_register(device, CVP_WRAPPER_INTR_MASK, mask_val);
  3604. dprintk(CVP_REG, "Init irq: reg: %x, mask value %x\n",
  3605. CVP_WRAPPER_INTR_MASK, mask_val);
  3606. mask_val = 0;
  3607. mask_val = __read_register(device, CVP_SS_IRQ_MASK);
  3608. mask_val &= ~(CVP_SS_INTR_BMASK);
  3609. __write_register(device, CVP_SS_IRQ_MASK, mask_val);
  3610. dprintk(CVP_REG, "Init irq_wd: reg: %x, mask value %x\n",
  3611. CVP_SS_IRQ_MASK, mask_val);
  3612. }
  3613. static void setup_dsp_uc_memmap_vpu5(struct iris_hfi_device *device)
  3614. {
  3615. /* initialize DSP QTBL & UCREGION with CPU queues */
  3616. __write_register(device, HFI_DSP_QTBL_ADDR,
  3617. (u32)device->dsp_iface_q_table.align_device_addr);
  3618. __write_register(device, HFI_DSP_UC_REGION_ADDR,
  3619. (u32)device->dsp_iface_q_table.align_device_addr);
  3620. __write_register(device, HFI_DSP_UC_REGION_SIZE,
  3621. device->dsp_iface_q_table.mem_data.size);
  3622. }
  3623. static void clock_config_on_enable_vpu5(struct iris_hfi_device *device)
  3624. {
  3625. __write_register(device, CVP_WRAPPER_CPU_CLOCK_CONFIG, 0);
  3626. }
  3627. static int __set_ubwc_config(struct iris_hfi_device *device)
  3628. {
  3629. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  3630. int rc = 0;
  3631. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  3632. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  3633. if (!device->res->ubwc_config)
  3634. return 0;
  3635. rc = call_hfi_pkt_op(device, sys_ubwc_config, pkt,
  3636. device->res->ubwc_config);
  3637. if (rc) {
  3638. dprintk(CVP_WARN,
  3639. "ubwc config setting to FW failed\n");
  3640. rc = -ENOTEMPTY;
  3641. goto fail_to_set_ubwc_config;
  3642. }
  3643. if (__iface_cmdq_write(device, pkt)) {
  3644. rc = -ENOTEMPTY;
  3645. goto fail_to_set_ubwc_config;
  3646. }
  3647. fail_to_set_ubwc_config:
  3648. return rc;
  3649. }
  3650. static int __power_on_controller(struct iris_hfi_device *device)
  3651. {
  3652. int rc = 0;
  3653. rc = __enable_regulator(device, "cvp");
  3654. if (rc) {
  3655. dprintk(CVP_ERR, "Failed to enable ctrler: %d\n", rc);
  3656. return rc;
  3657. }
  3658. rc = msm_cvp_prepare_enable_clk(device, "sleep_clk");
  3659. if (rc) {
  3660. dprintk(CVP_ERR, "Failed to enable sleep clk: %d\n", rc);
  3661. goto fail_reset_clks;
  3662. }
  3663. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_axi_reset");
  3664. if (rc)
  3665. dprintk(CVP_ERR, "%s: assert cvp_axi_reset failed\n", __func__);
  3666. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_core_reset");
  3667. if (rc)
  3668. dprintk(CVP_ERR, "%s: assert cvp_core_reset failed\n", __func__);
  3669. /* wait for deassert */
  3670. usleep_range(1000, 1050);
  3671. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_axi_reset");
  3672. if (rc)
  3673. dprintk(CVP_ERR, "%s: de-assert cvp_axi_reset failed\n", __func__);
  3674. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_core_reset");
  3675. if (rc)
  3676. dprintk(CVP_ERR, "%s: de-assert cvp_core_reset failed\n", __func__);
  3677. rc = msm_cvp_prepare_enable_clk(device, "gcc_video_axi1");
  3678. if (rc) {
  3679. dprintk(CVP_ERR, "Failed to enable axi1 clk: %d\n", rc);
  3680. goto fail_reset_clks;
  3681. }
  3682. rc = msm_cvp_prepare_enable_clk(device, "cvp_clk");
  3683. if (rc) {
  3684. dprintk(CVP_ERR, "Failed to enable cvp_clk: %d\n", rc);
  3685. goto fail_enable_clk;
  3686. }
  3687. dprintk(CVP_PWR, "EVA controller powered on\n");
  3688. return 0;
  3689. fail_enable_clk:
  3690. msm_cvp_disable_unprepare_clk(device, "gcc_video_axi1");
  3691. fail_reset_clks:
  3692. __disable_regulator(device, "cvp");
  3693. return rc;
  3694. }
  3695. static int __power_on_core(struct iris_hfi_device *device)
  3696. {
  3697. int rc = 0;
  3698. rc = __enable_regulator(device, "cvp-core");
  3699. if (rc) {
  3700. dprintk(CVP_ERR, "Failed to enable core: %d\n", rc);
  3701. return rc;
  3702. }
  3703. rc = msm_cvp_prepare_enable_clk(device, "video_cc_mvs1_clk_src");
  3704. if (rc) {
  3705. dprintk(CVP_ERR, "Failed to enable video_cc_mvs1_clk_src:%d\n",
  3706. rc);
  3707. __disable_regulator(device, "cvp-core");
  3708. return rc;
  3709. }
  3710. rc = msm_cvp_prepare_enable_clk(device, "core_clk");
  3711. if (rc) {
  3712. dprintk(CVP_ERR, "Failed to enable core_clk: %d\n", rc);
  3713. __disable_regulator(device, "cvp-core");
  3714. return rc;
  3715. }
  3716. /*#ifdef CONFIG_EVA_PINEAPPLE
  3717. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_ARCG_CONTROL, 0);
  3718. __write_register(device, CVP_NOC_RCGCONTROLLER_HYSTERESIS_LOW, 0x2f);
  3719. __write_register(device, CVP_NOC_RCG_VNOC_NOC_CLK_FORCECLOCKON_LOW, 1);
  3720. __write_register(device, CVP_NOC_RCGCONTROLLER_MAINCTL_LOW, 1);
  3721. usleep_range(50, 100);
  3722. __write_register(device, CVP_NOC_RCG_VNOC_NOC_CLK_FORCECLOCKON_LOW, 0);
  3723. #endif*/
  3724. dprintk(CVP_PWR, "EVA core powered on\n");
  3725. return 0;
  3726. }
  3727. static int __iris_power_on(struct iris_hfi_device *device)
  3728. {
  3729. int rc = 0;
  3730. u32 reg_gdsc, reg_cbcr, spare_val;
  3731. if (device->power_enabled)
  3732. return 0;
  3733. /* Vote for all hardware resources */
  3734. rc = __vote_buses(device, device->bus_vote.data,
  3735. device->bus_vote.data_count);
  3736. if (rc) {
  3737. dprintk(CVP_ERR, "Failed to vote buses, err: %d\n", rc);
  3738. goto fail_vote_buses;
  3739. }
  3740. rc = __power_on_controller(device);
  3741. if (rc)
  3742. goto fail_enable_controller;
  3743. rc = __power_on_core(device);
  3744. if (rc)
  3745. goto fail_enable_core;
  3746. rc = msm_cvp_scale_clocks(device);
  3747. if (rc) {
  3748. dprintk(CVP_WARN,
  3749. "Failed to scale clocks, perf may regress\n");
  3750. rc = 0;
  3751. } else {
  3752. dprintk(CVP_PWR, "Done with scaling\n");
  3753. }
  3754. /*Do not access registers before this point!*/
  3755. device->power_enabled = true;
  3756. /*
  3757. * Re-program all of the registers that get reset as a result of
  3758. * regulator_disable() and _enable()
  3759. * calling below function requires CORE powered on
  3760. */
  3761. rc = __set_registers(device);
  3762. if (rc)
  3763. goto fail_enable_core;
  3764. dprintk(CVP_CORE, "Done with register set\n");
  3765. reg_gdsc = __read_register(device, CVP_CC_MVS1_GDSCR);
  3766. reg_cbcr = __read_register(device, CVP_CC_MVS1_CBCR);
  3767. if (!(reg_gdsc & 0x80000000) || (reg_cbcr & 0x80000000)) {
  3768. rc = -EINVAL;
  3769. dprintk(CVP_ERR, "CORE power on failed gdsc %x cbcr %x\n",
  3770. reg_gdsc, reg_cbcr);
  3771. goto fail_enable_core;
  3772. }
  3773. reg_gdsc = __read_register(device, CVP_CC_MVS1C_GDSCR);
  3774. reg_cbcr = __read_register(device, CVP_CC_MVS1C_CBCR);
  3775. if (!(reg_gdsc & 0x80000000) || (reg_cbcr & 0x80000000)) {
  3776. rc = -EINVAL;
  3777. dprintk(CVP_ERR, "CTRL power on failed gdsc %x cbcr %x\n",
  3778. reg_gdsc, reg_cbcr);
  3779. goto fail_enable_core;
  3780. }
  3781. spare_val = __read_register(device, CVP_AON_WRAPPER_SPARE);
  3782. if ((spare_val & 0x2) != 0) {
  3783. usleep_range(2000, 3000);
  3784. spare_val = __read_register(device, CVP_AON_WRAPPER_SPARE);
  3785. if ((spare_val & 0x2) != 0) {
  3786. dprintk(CVP_ERR, "WRAPPER_SPARE non-zero %#x\n", spare_val);
  3787. rc = -EINVAL;
  3788. goto fail_enable_core;
  3789. }
  3790. }
  3791. call_iris_op(device, interrupt_init, device);
  3792. dprintk(CVP_CORE, "Done with interrupt enabling\n");
  3793. device->intr_status = 0;
  3794. enable_irq(device->cvp_hal_data->irq);
  3795. __write_register(device,
  3796. CVP_WRAPPER_DEBUG_BRIDGE_LPI_CONTROL, 0x7);
  3797. pr_info_ratelimited(CVP_DBG_TAG "cvp (eva) powered on\n", "pwr");
  3798. return 0;
  3799. fail_enable_core:
  3800. __power_off_controller(device);
  3801. fail_enable_controller:
  3802. __unvote_buses(device);
  3803. fail_vote_buses:
  3804. device->power_enabled = false;
  3805. return rc;
  3806. }
  3807. static inline int __suspend(struct iris_hfi_device *device)
  3808. {
  3809. int rc = 0;
  3810. if (!device) {
  3811. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  3812. return -EINVAL;
  3813. } else if (!device->power_enabled) {
  3814. dprintk(CVP_PWR, "Power already disabled\n");
  3815. return 0;
  3816. }
  3817. dprintk(CVP_PWR, "Entering suspend\n");
  3818. rc = __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_SUSPEND);
  3819. if (rc) {
  3820. dprintk(CVP_WARN, "Failed to suspend cvp core %d\n", rc);
  3821. goto err_tzbsp_suspend;
  3822. }
  3823. __disable_subcaches(device);
  3824. call_iris_op(device, power_off, device);
  3825. if (device->res->pm_qos.latency_us && device->res->pm_qos.pm_qos_hdls)
  3826. cvp_pm_qos_update(device, false);
  3827. return rc;
  3828. err_tzbsp_suspend:
  3829. return rc;
  3830. }
  3831. static void __print_sidebandmanager_regs(struct iris_hfi_device *device)
  3832. {
  3833. u32 sbm_ln0_low, axi_cbcr;
  3834. u32 main_sbm_ln0_low = 0xdeadbeef, main_sbm_ln0_high = 0xdeadbeef;
  3835. u32 main_sbm_ln1_high = 0xdeadbeef, cpu_cs_x2rpmh;
  3836. int rc;
  3837. sbm_ln0_low =
  3838. __read_register(device, CVP_NOC_SBM_SENSELN0_LOW);
  3839. cpu_cs_x2rpmh = __read_register(device, CVP_CPU_CS_X2RPMh);
  3840. __write_register(device, CVP_CPU_CS_X2RPMh,
  3841. (cpu_cs_x2rpmh | CVP_CPU_CS_X2RPMh_SWOVERRIDE_BMSK));
  3842. usleep_range(500, 1000);
  3843. cpu_cs_x2rpmh = __read_register(device, CVP_CPU_CS_X2RPMh);
  3844. if (!(cpu_cs_x2rpmh & CVP_CPU_CS_X2RPMh_SWOVERRIDE_BMSK)) {
  3845. dprintk(CVP_WARN,
  3846. "failed set CVP_CPU_CS_X2RPMH mask %x\n",
  3847. cpu_cs_x2rpmh);
  3848. goto exit;
  3849. }
  3850. axi_cbcr = __read_gcc_register(device, CVP_GCC_VIDEO_AXI1_CBCR);
  3851. if (axi_cbcr & 0x80000000) {
  3852. dprintk(CVP_WARN, "failed to turn on AXI clock %x\n",
  3853. axi_cbcr);
  3854. goto exit;
  3855. }
  3856. rc = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  3857. if (rc) {
  3858. dprintk(CVP_WARN, "%s Fail acquire xo_reset\n", __func__);
  3859. goto exit;
  3860. }
  3861. main_sbm_ln0_low = __read_register(device,
  3862. CVP_NOC_MAIN_SIDEBANDMANAGER_SENSELN0_LOW);
  3863. main_sbm_ln0_high = __read_register(device,
  3864. CVP_NOC_MAIN_SIDEBANDMANAGER_SENSELN0_HIGH);
  3865. main_sbm_ln1_high = __read_register(device,
  3866. CVP_NOC_MAIN_SIDEBANDMANAGER_SENSELN1_HIGH);
  3867. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  3868. exit:
  3869. cpu_cs_x2rpmh = cpu_cs_x2rpmh & (~CVP_CPU_CS_X2RPMh_SWOVERRIDE_BMSK);
  3870. __write_register(device, CVP_CPU_CS_X2RPMh, cpu_cs_x2rpmh);
  3871. dprintk(CVP_WARN, "Sidebandmanager regs %x %x %x %x %x\n",
  3872. sbm_ln0_low, main_sbm_ln0_low,
  3873. main_sbm_ln0_high, main_sbm_ln1_high,
  3874. cpu_cs_x2rpmh);
  3875. }
  3876. static int __power_off_controller(struct iris_hfi_device *device)
  3877. {
  3878. u32 lpi_status, reg_status = 0, count = 0, max_count = 1000;
  3879. u32 sbm_ln0_low;
  3880. int rc;
  3881. u32 spare_val, spare_status;
  3882. /* HPG 6.2.2 Step 1 */
  3883. __write_register(device, CVP_CPU_CS_X2RPMh, 0x3);
  3884. /* HPG 6.2.2 Step 2, noc to low power */
  3885. /* New addition to put CPU/Tensilica to low power */
  3886. reg_status = 0;
  3887. count = 0;
  3888. __write_register(device, CVP_WRAPPER_CPU_NOC_LPI_CONTROL, 0x1);
  3889. while (!reg_status && count < max_count) {
  3890. lpi_status =
  3891. __read_register(device,
  3892. CVP_WRAPPER_CPU_NOC_LPI_STATUS);
  3893. reg_status = lpi_status & BIT(0);
  3894. /* Wait for CPU noc lpi status to be set */
  3895. usleep_range(50, 100);
  3896. count++;
  3897. }
  3898. sbm_ln0_low = __read_register(device, CVP_NOC_SBM_SENSELN0_LOW);
  3899. dprintk(CVP_PWR,
  3900. "CPU Noc: lpi_status %x noc_status %x (count %d) 0x%x\n",
  3901. lpi_status, reg_status, count, sbm_ln0_low);
  3902. if (count == max_count) {
  3903. u32 pc_ready, wfi_status;
  3904. wfi_status = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  3905. pc_ready = __read_register(device, CVP_CTRL_STATUS);
  3906. dprintk(CVP_WARN,
  3907. "CPU NOC not in qaccept status %x %x %x %x\n",
  3908. reg_status, lpi_status, wfi_status, pc_ready);
  3909. __print_sidebandmanager_regs(device);
  3910. }
  3911. /* HPG 6.2.2 Step 3, debug bridge to low power BYPASSED */
  3912. /* HPG 6.2.2 Step 4, debug bridge to lpi release */
  3913. __write_register(device,
  3914. CVP_WRAPPER_DEBUG_BRIDGE_LPI_CONTROL, 0x0);
  3915. lpi_status = 0x1;
  3916. count = 0;
  3917. while (lpi_status && count < max_count) {
  3918. lpi_status = __read_register(device,
  3919. CVP_WRAPPER_DEBUG_BRIDGE_LPI_STATUS);
  3920. usleep_range(50, 100);
  3921. count++;
  3922. }
  3923. dprintk(CVP_PWR,
  3924. "DBLP Release: lpi_status %d(count %d)\n",
  3925. lpi_status, count);
  3926. if (count == max_count) {
  3927. dprintk(CVP_WARN,
  3928. "DBLP Release: lpi_status %x\n", lpi_status);
  3929. }
  3930. /* PDXFIFO reset: addition for Kailua / Lanai */
  3931. __write_register(device, CVP_WRAPPER_AXI_CLOCK_CONFIG, 0x3);
  3932. __write_register(device, CVP_WRAPPER_QNS4PDXFIFO_RESET, 0x1);
  3933. __write_register(device, CVP_WRAPPER_QNS4PDXFIFO_RESET, 0x0);
  3934. __write_register(device, CVP_WRAPPER_AXI_CLOCK_CONFIG, 0x0);
  3935. /* HPG 6.2.2 Step 5 */
  3936. msm_cvp_disable_unprepare_clk(device, "cvp_clk");
  3937. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_axi_reset");
  3938. if (rc)
  3939. dprintk(CVP_ERR, "%s: assert cvp_axi_reset failed\n", __func__);
  3940. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_core_reset");
  3941. if (rc)
  3942. dprintk(CVP_ERR, "%s: assert cvp_core_reset failed\n", __func__);
  3943. /* wait for deassert */
  3944. usleep_range(1000, 1050);
  3945. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_axi_reset");
  3946. if (rc)
  3947. dprintk(CVP_ERR, "%s: de-assert cvp_axi_reset failed\n", __func__);
  3948. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_core_reset");
  3949. if (rc)
  3950. dprintk(CVP_ERR, "%s: de-assert cvp_core_reset failed\n", __func__);
  3951. /* disable EVA NoC clock */
  3952. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_CLK_CONTROL, 0x1);
  3953. /* enable EVA NoC reset */
  3954. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_SW_RESET, 0x1);
  3955. rc = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  3956. if (rc) {
  3957. dprintk(CVP_ERR, "FATAL ERROR, HPG step 17 to 20 will be bypassed\n");
  3958. goto skip_xo_reset;
  3959. }
  3960. spare_status = 0x1;
  3961. while (spare_status != 0x0) {
  3962. spare_val = __read_register(device, CVP_AON_WRAPPER_SPARE);
  3963. spare_status = spare_val & 0x2;
  3964. usleep_range(50, 100);
  3965. }
  3966. __write_register(device, CVP_AON_WRAPPER_SPARE, 0x1);
  3967. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_xo_reset");
  3968. if (rc)
  3969. dprintk(CVP_ERR, "%s: assert cvp_xo_reset failed\n", __func__);
  3970. /* de-assert EVA_NoC reset */
  3971. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_SW_RESET, 0x0);
  3972. /* de-assert EVA video_cc XO reset and enable video_cc XO clock after 80us */
  3973. usleep_range(80, 100);
  3974. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_xo_reset");
  3975. if (rc)
  3976. dprintk(CVP_ERR, "%s: de-assert cvp_xo_reset failed\n", __func__);
  3977. /* clear XO mask bit - this step was missing in previous sequence */
  3978. __write_register(device, CVP_AON_WRAPPER_SPARE, 0x0);
  3979. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  3980. skip_xo_reset:
  3981. /* enable EVA NoC clock */
  3982. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_CLK_CONTROL, 0x0);
  3983. /* De-assert EVA_CTL Force Sleep Retention */
  3984. usleep_range(400, 500);
  3985. /* HPG 6.2.2 Step 6 */
  3986. __disable_regulator(device, "cvp");
  3987. /* HPG 6.2.2 Step 7 */
  3988. rc = msm_cvp_disable_unprepare_clk(device, "gcc_video_axi1");
  3989. if (rc) {
  3990. dprintk(CVP_ERR, "Failed to enable axi1 clk: %d\n", rc);
  3991. }
  3992. rc = msm_cvp_disable_unprepare_clk(device, "sleep_clk");
  3993. if (rc) {
  3994. dprintk(CVP_ERR, "Failed to disable sleep clk: %d\n", rc);
  3995. }
  3996. return 0;
  3997. }
  3998. static int __power_off_core(struct iris_hfi_device *device)
  3999. {
  4000. u32 reg_status = 0, lpi_status, config, value = 0, count = 0;
  4001. u32 warn_flag = 0, max_count = 10;
  4002. value = __read_register(device, CVP_CC_MVS1_GDSCR);
  4003. if (!(value & 0x80000000)) {
  4004. /*
  4005. * Core has been powered off by f/w.
  4006. * Check NOC reset registers to ensure
  4007. * NO outstanding NoC transactions
  4008. */
  4009. value = __read_register(device, CVP_NOC_RESET_ACK);
  4010. if (value) {
  4011. dprintk(CVP_WARN,
  4012. "Core off with NOC RESET ACK non-zero %x\n",
  4013. value);
  4014. __print_sidebandmanager_regs(device);
  4015. }
  4016. __disable_regulator(device, "cvp-core");
  4017. msm_cvp_disable_unprepare_clk(device, "core_clk");
  4018. msm_cvp_disable_unprepare_clk(device, "video_cc_mvs1_clk_src");
  4019. return 0;
  4020. } else if (!(value & 0x2)) {
  4021. /*
  4022. * HW_CONTROL PC disabled, then core is powered on for
  4023. * CVP NoC access
  4024. */
  4025. __disable_regulator(device, "cvp-core");
  4026. msm_cvp_disable_unprepare_clk(device, "core_clk");
  4027. msm_cvp_disable_unprepare_clk(device, "video_cc_mvs1_clk_src");
  4028. return 0;
  4029. }
  4030. dprintk(CVP_PWR, "Driver controls Core power off now\n");
  4031. /*
  4032. * check to make sure core clock branch enabled else
  4033. * we cannot read core idle register
  4034. */
  4035. config = __read_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG);
  4036. if (config) {
  4037. dprintk(CVP_PWR,
  4038. "core clock config not enabled, enable it to access core\n");
  4039. __write_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG, 0);
  4040. }
  4041. /*
  4042. * add MNoC idle check before collapsing MVS1 per HPG update
  4043. * poll for NoC DMA idle -> HPG 6.2.1
  4044. *
  4045. */
  4046. do {
  4047. value = __read_register(device, CVP_SS_IDLE_STATUS);
  4048. if (value & 0x400000)
  4049. break;
  4050. else
  4051. usleep_range(1000, 2000);
  4052. count++;
  4053. } while (count < max_count);
  4054. if (count == max_count) {
  4055. dprintk(CVP_WARN, "Core fail to go idle %x\n", value);
  4056. warn_flag = 1;
  4057. }
  4058. count = 0;
  4059. max_count = 1000;
  4060. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_LPI_CONTROL, 0x1);
  4061. while (!reg_status && count < max_count) {
  4062. lpi_status =
  4063. __read_register(device,
  4064. CVP_AON_WRAPPER_CVP_NOC_LPI_STATUS);
  4065. reg_status = lpi_status & BIT(0);
  4066. /* Wait for Core noc lpi status to be set */
  4067. usleep_range(50, 100);
  4068. count++;
  4069. }
  4070. dprintk(CVP_PWR,
  4071. "Core Noc: lpi_status %x noc_status %x (count %d)\n",
  4072. lpi_status, reg_status, count);
  4073. if (count == max_count) {
  4074. u32 pc_ready, wfi_status;
  4075. wfi_status = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  4076. pc_ready = __read_register(device, CVP_CTRL_STATUS);
  4077. dprintk(CVP_WARN,
  4078. "Core NOC not in qaccept status %x %x %x %x\n",
  4079. reg_status, lpi_status, wfi_status, pc_ready);
  4080. __print_sidebandmanager_regs(device);
  4081. }
  4082. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_LPI_CONTROL, 0x0);
  4083. if (warn_flag)
  4084. __print_sidebandmanager_regs(device);
  4085. /* Reset both sides of 2 ahb2ahb_bridges (TZ and non-TZ) */
  4086. __write_register(device, CVP_AHB_BRIDGE_SYNC_RESET, 0x3);
  4087. __write_register(device, CVP_AHB_BRIDGE_SYNC_RESET, 0x2);
  4088. __write_register(device, CVP_AHB_BRIDGE_SYNC_RESET, 0x0);
  4089. __write_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG, config);
  4090. __disable_hw_power_collapse(device);
  4091. usleep_range(100, 200);
  4092. __disable_regulator(device, "cvp-core");
  4093. msm_cvp_disable_unprepare_clk(device, "core_clk");
  4094. msm_cvp_disable_unprepare_clk(device, "video_cc_mvs1_clk_src");
  4095. return 0;
  4096. }
  4097. static void power_off_iris2(struct iris_hfi_device *device)
  4098. {
  4099. if (!device->power_enabled || !device->res->sw_power_collapsible)
  4100. return;
  4101. if (!(device->intr_status & CVP_WRAPPER_INTR_STATUS_A2HWD_BMSK))
  4102. disable_irq_nosync(device->cvp_hal_data->irq);
  4103. device->intr_status = 0;
  4104. __power_off_core(device);
  4105. __power_off_controller(device);
  4106. if (__unvote_buses(device))
  4107. dprintk(CVP_WARN, "Failed to unvote for buses\n");
  4108. /*Do not access registers after this point!*/
  4109. device->power_enabled = false;
  4110. pr_info(CVP_DBG_TAG "cvp (eva) power collapsed\n", "pwr");
  4111. }
  4112. static inline int __resume(struct iris_hfi_device *device)
  4113. {
  4114. int rc = 0;
  4115. struct msm_cvp_core *core;
  4116. if (!device) {
  4117. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  4118. return -EINVAL;
  4119. } else if (device->power_enabled) {
  4120. goto exit;
  4121. } else if (!__core_in_valid_state(device)) {
  4122. dprintk(CVP_PWR, "iris_hfi_device in deinit state.");
  4123. return -EINVAL;
  4124. }
  4125. core = cvp_driver->cvp_core;
  4126. dprintk(CVP_PWR, "Resuming from power collapse\n");
  4127. rc = __iris_power_on(device);
  4128. if (rc) {
  4129. dprintk(CVP_ERR, "Failed to power on cvp\n");
  4130. goto err_iris_power_on;
  4131. }
  4132. __setup_ucregion_memory_map(device);
  4133. /* RUMI: set CVP_CTRL_INIT register to disable synx in FW */
  4134. /* Reboot the firmware */
  4135. rc = __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_RESUME);
  4136. if (rc) {
  4137. dprintk(CVP_ERR, "Failed to resume cvp core %d\n", rc);
  4138. goto err_set_cvp_state;
  4139. }
  4140. /* Wait for boot completion */
  4141. rc = __boot_firmware(device);
  4142. if (rc) {
  4143. dprintk(CVP_ERR, "Failed to reset cvp core\n");
  4144. goto err_reset_core;
  4145. }
  4146. /*
  4147. * Work around for H/W bug, need to reprogram these registers once
  4148. * firmware is out reset
  4149. */
  4150. __set_threshold_registers(device);
  4151. if (device->res->pm_qos.latency_us && device->res->pm_qos.pm_qos_hdls)
  4152. cvp_pm_qos_update(device, true);
  4153. __sys_set_debug(device, msm_cvp_fw_debug);
  4154. __enable_subcaches(device);
  4155. __set_subcaches(device);
  4156. __dsp_resume(device);
  4157. dprintk(CVP_PWR, "Resumed from power collapse\n");
  4158. exit:
  4159. /* Don't reset skip_pc_count for SYS_PC_PREP cmd */
  4160. if (device->last_packet_type != HFI_CMD_SYS_PC_PREP)
  4161. device->skip_pc_count = 0;
  4162. return rc;
  4163. err_reset_core:
  4164. __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_SUSPEND);
  4165. err_set_cvp_state:
  4166. call_iris_op(device, power_off, device);
  4167. err_iris_power_on:
  4168. dprintk(CVP_ERR, "Failed to resume from power collapse\n");
  4169. return rc;
  4170. }
  4171. static int __power_on_init(struct iris_hfi_device *device)
  4172. {
  4173. int rc = 0;
  4174. /* Initialize resources */
  4175. rc = __init_resources(device, device->res);
  4176. if (rc) {
  4177. dprintk(CVP_ERR, "Failed to init resources: %d\n", rc);
  4178. return rc;
  4179. }
  4180. rc = __initialize_packetization(device);
  4181. if (rc) {
  4182. dprintk(CVP_ERR, "Failed to initialize packetization\n");
  4183. goto fail_iris_init;
  4184. }
  4185. rc = __iris_power_on(device);
  4186. if (rc) {
  4187. dprintk(CVP_ERR, "Failed to power on iris in in load_fw\n");
  4188. goto fail_iris_init;
  4189. }
  4190. return rc;
  4191. fail_iris_init:
  4192. __deinit_resources(device);
  4193. return rc;
  4194. }
  4195. static int __load_fw(struct iris_hfi_device *device)
  4196. {
  4197. int rc = 0;
  4198. if ((!device->res->use_non_secure_pil && !device->res->firmware_base)
  4199. || device->res->use_non_secure_pil) {
  4200. rc = load_cvp_fw_impl(device);
  4201. if (rc)
  4202. goto fail_load_fw;
  4203. }
  4204. return rc;
  4205. fail_load_fw:
  4206. call_iris_op(device, power_off, device);
  4207. return rc;
  4208. }
  4209. static void __unload_fw(struct iris_hfi_device *device)
  4210. {
  4211. if (!device->resources.fw.cookie)
  4212. return;
  4213. cancel_delayed_work(&iris_hfi_pm_work);
  4214. if (device->state != IRIS_STATE_DEINIT)
  4215. flush_workqueue(device->iris_pm_workq);
  4216. unload_cvp_fw_impl(device);
  4217. __interface_queues_release(device);
  4218. call_iris_op(device, power_off, device);
  4219. __deinit_resources(device);
  4220. dprintk(CVP_WARN, "Firmware unloaded\n");
  4221. }
  4222. static int iris_hfi_get_fw_info(void *dev, struct cvp_hal_fw_info *fw_info)
  4223. {
  4224. int i = 0;
  4225. struct iris_hfi_device *device = dev;
  4226. if (!device || !fw_info) {
  4227. dprintk(CVP_ERR,
  4228. "%s Invalid parameter: device = %pK fw_info = %pK\n",
  4229. __func__, device, fw_info);
  4230. return -EINVAL;
  4231. }
  4232. mutex_lock(&device->lock);
  4233. while (cvp_driver->fw_version[i++] != 'V' && i < CVP_VERSION_LENGTH)
  4234. ;
  4235. if (i == CVP_VERSION_LENGTH - 1) {
  4236. dprintk(CVP_WARN, "Iris version string is not proper\n");
  4237. fw_info->version[0] = '\0';
  4238. goto fail_version_string;
  4239. }
  4240. memcpy(&fw_info->version[0], &cvp_driver->fw_version[0],
  4241. CVP_VERSION_LENGTH);
  4242. fw_info->version[CVP_VERSION_LENGTH - 1] = '\0';
  4243. fail_version_string:
  4244. dprintk(CVP_CORE, "F/W version retrieved : %s\n", fw_info->version);
  4245. fw_info->base_addr = device->cvp_hal_data->firmware_base;
  4246. fw_info->register_base = device->res->register_base;
  4247. fw_info->register_size = device->cvp_hal_data->register_size;
  4248. fw_info->irq = device->cvp_hal_data->irq;
  4249. mutex_unlock(&device->lock);
  4250. return 0;
  4251. }
  4252. static int iris_hfi_get_core_capabilities(void *dev)
  4253. {
  4254. dprintk(CVP_CORE, "%s not supported yet!\n", __func__);
  4255. return 0;
  4256. }
  4257. static const char * const mid_names[16] = {
  4258. "CVP_FW",
  4259. "ARP_DATA",
  4260. "CVP_MPU_PIXEL",
  4261. "CVP_MPU_NON_PIXEL",
  4262. "CVP_FDU_PIXEL",
  4263. "CVP_FDU_NON_PIXEL",
  4264. "CVP_GCE_PIXEL",
  4265. "CVP_GCE_NON_PIXEL",
  4266. "CVP_TOF_PIXEL",
  4267. "CVP_TOF_NON_PIXEL",
  4268. "CVP_VADL_PIXEL",
  4269. "CVP_VADL_NON_PIXEL",
  4270. "CVP_RGE_NON_PIXEL",
  4271. "CVP_CDM",
  4272. "Invalid",
  4273. "Invalid"
  4274. };
  4275. static void __print_reg_details(u32 val)
  4276. {
  4277. u32 mid, sid;
  4278. mid = (val >> 5) & 0xF;
  4279. sid = (val >> 2) & 0x7;
  4280. dprintk(CVP_ERR, "CVP_NOC_CORE_ERL_MAIN_ERRLOG3_LOW: %#x\n", val);
  4281. dprintk(CVP_ERR, "Sub-client:%s, SID: %d\n", mid_names[mid], sid);
  4282. }
  4283. static void __err_log(bool logging, u32 *data, const char *name, u32 val)
  4284. {
  4285. if (logging)
  4286. *data = val;
  4287. dprintk(CVP_ERR, "%s: %#x\n", name, val);
  4288. }
  4289. static void __noc_error_info_iris2(struct iris_hfi_device *device)
  4290. {
  4291. struct msm_cvp_core *core;
  4292. struct cvp_noc_log *noc_log;
  4293. u32 val = 0, regi, regii, regiii;
  4294. bool log_required = false;
  4295. int rc;
  4296. core = cvp_driver->cvp_core;
  4297. if (!core->ssr_count && core->resources.max_ssr_allowed > 1)
  4298. log_required = true;
  4299. noc_log = &core->log.noc_log;
  4300. if (noc_log->used) {
  4301. dprintk(CVP_WARN, "Data already in NoC log, skip logging\n");
  4302. return;
  4303. }
  4304. noc_log->used = 1;
  4305. __disable_hw_power_collapse(device);
  4306. val = __read_register(device, CVP_CC_MVS1_GDSCR);
  4307. regi = __read_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_CLK_CONTROL);
  4308. regii = __read_register(device, CVP_CC_MVS1_CBCR);
  4309. regiii = __read_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG);
  4310. dprintk(CVP_ERR, "noc reg check: %#x %#x %#x %#x\n",
  4311. val, regi, regii, regiii);
  4312. val = __read_register(device, CVP_NOC_ERR_SWID_LOW_OFFS);
  4313. __err_log(log_required, &noc_log->err_ctrl_swid_low,
  4314. "CVP_NOC_ERL_MAIN_SWID_LOW", val);
  4315. val = __read_register(device, CVP_NOC_ERR_SWID_HIGH_OFFS);
  4316. __err_log(log_required, &noc_log->err_ctrl_swid_high,
  4317. "CVP_NOC_ERL_MAIN_SWID_HIGH", val);
  4318. val = __read_register(device, CVP_NOC_ERR_MAINCTL_LOW_OFFS);
  4319. __err_log(log_required, &noc_log->err_ctrl_mainctl_low,
  4320. "CVP_NOC_ERL_MAIN_MAINCTL_LOW", val);
  4321. val = __read_register(device, CVP_NOC_ERR_ERRVLD_LOW_OFFS);
  4322. __err_log(log_required, &noc_log->err_ctrl_errvld_low,
  4323. "CVP_NOC_ERL_MAIN_ERRVLD_LOW", val);
  4324. val = __read_register(device, CVP_NOC_ERR_ERRCLR_LOW_OFFS);
  4325. __err_log(log_required, &noc_log->err_ctrl_errclr_low,
  4326. "CVP_NOC_ERL_MAIN_ERRCLR_LOW", val);
  4327. val = __read_register(device, CVP_NOC_ERR_ERRLOG0_LOW_OFFS);
  4328. __err_log(log_required, &noc_log->err_ctrl_errlog0_low,
  4329. "CVP_NOC_ERL_MAIN_ERRLOG0_LOW", val);
  4330. val = __read_register(device, CVP_NOC_ERR_ERRLOG0_HIGH_OFFS);
  4331. __err_log(log_required, &noc_log->err_ctrl_errlog0_high,
  4332. "CVP_NOC_ERL_MAIN_ERRLOG0_HIGH", val);
  4333. val = __read_register(device, CVP_NOC_ERR_ERRLOG1_LOW_OFFS);
  4334. __err_log(log_required, &noc_log->err_ctrl_errlog1_low,
  4335. "CVP_NOC_ERL_MAIN_ERRLOG1_LOW", val);
  4336. val = __read_register(device, CVP_NOC_ERR_ERRLOG1_HIGH_OFFS);
  4337. __err_log(log_required, &noc_log->err_ctrl_errlog1_high,
  4338. "CVP_NOC_ERL_MAIN_ERRLOG1_HIGH", val);
  4339. val = __read_register(device, CVP_NOC_ERR_ERRLOG2_LOW_OFFS);
  4340. __err_log(log_required, &noc_log->err_ctrl_errlog2_low,
  4341. "CVP_NOC_ERL_MAIN_ERRLOG2_LOW", val);
  4342. val = __read_register(device, CVP_NOC_ERR_ERRLOG2_HIGH_OFFS);
  4343. __err_log(log_required, &noc_log->err_ctrl_errlog2_high,
  4344. "CVP_NOC_ERL_MAIN_ERRLOG2_HIGH", val);
  4345. val = __read_register(device, CVP_NOC_ERR_ERRLOG3_LOW_OFFS);
  4346. __err_log(log_required, &noc_log->err_ctrl_errlog3_low,
  4347. "CVP_NOC_ERL_MAIN_ERRLOG3_LOW", val);
  4348. val = __read_register(device, CVP_NOC_ERR_ERRLOG3_HIGH_OFFS);
  4349. __err_log(log_required, &noc_log->err_ctrl_errlog3_high,
  4350. "CVP_NOC_ERL_MAIN_ERRLOG3_HIGH", val);
  4351. rc = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  4352. if (rc) {
  4353. dprintk(CVP_WARN, "%s Fail acquire xo_reset\n", __func__);
  4354. return;
  4355. }
  4356. val = __read_register(device, CVP_NOC_CORE_ERR_SWID_LOW_OFFS);
  4357. __err_log(log_required, &noc_log->err_core_swid_low,
  4358. "CVP_NOC__CORE_ERL_MAIN_SWID_LOW", val);
  4359. val = __read_register(device, CVP_NOC_CORE_ERR_SWID_HIGH_OFFS);
  4360. __err_log(log_required, &noc_log->err_core_swid_high,
  4361. "CVP_NOC_CORE_ERL_MAIN_SWID_HIGH", val);
  4362. val = __read_register(device, CVP_NOC_CORE_ERR_MAINCTL_LOW_OFFS);
  4363. __err_log(log_required, &noc_log->err_core_mainctl_low,
  4364. "CVP_NOC_CORE_ERL_MAIN_MAINCTL_LOW", val);
  4365. val = __read_register(device, CVP_NOC_CORE_ERR_ERRVLD_LOW_OFFS);
  4366. __err_log(log_required, &noc_log->err_core_errvld_low,
  4367. "CVP_NOC_CORE_ERL_MAIN_ERRVLD_LOW", val);
  4368. val = __read_register(device, CVP_NOC_CORE_ERR_ERRCLR_LOW_OFFS);
  4369. __err_log(log_required, &noc_log->err_core_errclr_low,
  4370. "CVP_NOC_CORE_ERL_MAIN_ERRCLR_LOW", val);
  4371. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG0_LOW_OFFS);
  4372. __err_log(log_required, &noc_log->err_core_errlog0_low,
  4373. "CVP_NOC_CORE_ERL_MAIN_ERRLOG0_LOW", val);
  4374. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG0_HIGH_OFFS);
  4375. __err_log(log_required, &noc_log->err_core_errlog0_high,
  4376. "CVP_NOC_CORE_ERL_MAIN_ERRLOG0_HIGH", val);
  4377. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG1_LOW_OFFS);
  4378. __err_log(log_required, &noc_log->err_core_errlog1_low,
  4379. "CVP_NOC_CORE_ERL_MAIN_ERRLOG1_LOW", val);
  4380. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG1_HIGH_OFFS);
  4381. __err_log(log_required, &noc_log->err_core_errlog1_high,
  4382. "CVP_NOC_CORE_ERL_MAIN_ERRLOG1_HIGH", val);
  4383. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG2_LOW_OFFS);
  4384. __err_log(log_required, &noc_log->err_core_errlog2_low,
  4385. "CVP_NOC_CORE_ERL_MAIN_ERRLOG2_LOW", val);
  4386. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG2_HIGH_OFFS);
  4387. __err_log(log_required, &noc_log->err_core_errlog2_high,
  4388. "CVP_NOC_CORE_ERL_MAIN_ERRLOG2_HIGH", val);
  4389. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG3_LOW_OFFS);
  4390. __err_log(log_required, &noc_log->err_core_errlog3_low,
  4391. "CORE ERRLOG3_LOW, below details", val);
  4392. __print_reg_details(val);
  4393. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG3_HIGH_OFFS);
  4394. __err_log(log_required, &noc_log->err_core_errlog3_high,
  4395. "CVP_NOC_CORE_ERL_MAIN_ERRLOG3_HIGH", val);
  4396. __write_register(device, CVP_NOC_CORE_ERR_ERRCLR_LOW_OFFS, 0x1);
  4397. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  4398. #define CVP_SS_CLK_HALT 0x8
  4399. #define CVP_SS_CLK_EN 0xC
  4400. #define CVP_SS_ARP_TEST_BUS_CONTROL 0x700
  4401. #define CVP_SS_ARP_TEST_BUS_REGISTER 0x704
  4402. #define CVP_DMA_TEST_BUS_CONTROL 0x66A0
  4403. #define CVP_DMA_TEST_BUS_REGISTER 0x66A4
  4404. #define CVP_VPU_WRAPPER_CORE_CONFIG 0xB0088
  4405. __write_register(device, CVP_SS_CLK_HALT, 0);
  4406. __write_register(device, CVP_SS_CLK_EN, 0x3f);
  4407. __write_register(device, CVP_VPU_WRAPPER_CORE_CONFIG, 0);
  4408. }
  4409. static int iris_hfi_noc_error_info(void *dev)
  4410. {
  4411. struct iris_hfi_device *device;
  4412. if (!dev) {
  4413. dprintk(CVP_ERR, "%s: null device\n", __func__);
  4414. return -EINVAL;
  4415. }
  4416. device = dev;
  4417. mutex_lock(&device->lock);
  4418. dprintk(CVP_ERR, "%s: non error information\n", __func__);
  4419. call_iris_op(device, noc_error_info, device);
  4420. mutex_unlock(&device->lock);
  4421. return 0;
  4422. }
  4423. static int __initialize_packetization(struct iris_hfi_device *device)
  4424. {
  4425. int rc = 0;
  4426. if (!device || !device->res) {
  4427. dprintk(CVP_ERR, "%s - invalid param\n", __func__);
  4428. return -EINVAL;
  4429. }
  4430. device->packetization_type = HFI_PACKETIZATION_4XX;
  4431. device->pkt_ops = cvp_hfi_get_pkt_ops_handle(
  4432. device->packetization_type);
  4433. if (!device->pkt_ops) {
  4434. rc = -EINVAL;
  4435. dprintk(CVP_ERR, "Failed to get pkt_ops handle\n");
  4436. }
  4437. return rc;
  4438. }
  4439. void __init_cvp_ops(struct iris_hfi_device *device)
  4440. {
  4441. device->vpu_ops = &iris2_ops;
  4442. }
  4443. static struct iris_hfi_device *__add_device(struct msm_cvp_platform_resources *res,
  4444. hfi_cmd_response_callback callback)
  4445. {
  4446. struct iris_hfi_device *hdevice = NULL;
  4447. int rc = 0;
  4448. if (!res || !callback) {
  4449. dprintk(CVP_ERR, "Invalid Parameters\n");
  4450. return NULL;
  4451. }
  4452. hdevice = kzalloc(sizeof(*hdevice), GFP_KERNEL);
  4453. if (!hdevice) {
  4454. dprintk(CVP_ERR, "failed to allocate new device\n");
  4455. goto exit;
  4456. }
  4457. hdevice->response_pkt = kmalloc_array(cvp_max_packets,
  4458. sizeof(*hdevice->response_pkt), GFP_KERNEL);
  4459. if (!hdevice->response_pkt) {
  4460. dprintk(CVP_ERR, "failed to allocate response_pkt\n");
  4461. goto err_cleanup;
  4462. }
  4463. hdevice->raw_packet =
  4464. kzalloc(CVP_IFACEQ_VAR_HUGE_PKT_SIZE, GFP_KERNEL);
  4465. if (!hdevice->raw_packet) {
  4466. dprintk(CVP_ERR, "failed to allocate raw packet\n");
  4467. goto err_cleanup;
  4468. }
  4469. rc = vm_manager.vm_ops->vm_init_reg_and_irq(hdevice, res);
  4470. if (rc)
  4471. goto err_cleanup;
  4472. hdevice->res = res;
  4473. hdevice->callback = callback;
  4474. __init_cvp_ops(hdevice);
  4475. hdevice->cvp_workq = create_singlethread_workqueue(
  4476. "msm_cvp_workerq_iris");
  4477. if (!hdevice->cvp_workq) {
  4478. dprintk(CVP_ERR, ": create cvp workq failed\n");
  4479. goto err_cleanup;
  4480. }
  4481. hdevice->iris_pm_workq = create_singlethread_workqueue(
  4482. "pm_workerq_iris");
  4483. if (!hdevice->iris_pm_workq) {
  4484. dprintk(CVP_ERR, ": create pm workq failed\n");
  4485. goto err_cleanup;
  4486. }
  4487. mutex_init(&hdevice->lock);
  4488. INIT_LIST_HEAD(&hdevice->sess_head);
  4489. return hdevice;
  4490. err_cleanup:
  4491. if (hdevice->iris_pm_workq)
  4492. destroy_workqueue(hdevice->iris_pm_workq);
  4493. if (hdevice->cvp_workq)
  4494. destroy_workqueue(hdevice->cvp_workq);
  4495. kfree(hdevice->response_pkt);
  4496. kfree(hdevice->raw_packet);
  4497. kfree(hdevice);
  4498. exit:
  4499. return NULL;
  4500. }
  4501. static struct iris_hfi_device *__get_device(struct msm_cvp_platform_resources *res,
  4502. hfi_cmd_response_callback callback)
  4503. {
  4504. if (!res || !callback) {
  4505. dprintk(CVP_ERR, "Invalid params: %pK %pK\n", res, callback);
  4506. return NULL;
  4507. }
  4508. return __add_device(res, callback);
  4509. }
  4510. void cvp_iris_hfi_delete_device(void *device)
  4511. {
  4512. struct msm_cvp_core *core;
  4513. struct iris_hfi_device *dev = NULL;
  4514. if (!device)
  4515. return;
  4516. core = cvp_driver->cvp_core;
  4517. if (core)
  4518. dev = core->device->hfi_device_data;
  4519. if (!dev)
  4520. return;
  4521. mutex_destroy(&dev->lock);
  4522. destroy_workqueue(dev->cvp_workq);
  4523. destroy_workqueue(dev->iris_pm_workq);
  4524. free_irq(dev->cvp_hal_data->irq, dev);
  4525. iounmap(dev->cvp_hal_data->register_base);
  4526. iounmap(dev->cvp_hal_data->gcc_reg_base);
  4527. kfree(dev->cvp_hal_data);
  4528. kfree(dev->response_pkt);
  4529. kfree(dev->raw_packet);
  4530. kfree(dev);
  4531. }
  4532. static int iris_hfi_validate_session(void *sess, const char *func)
  4533. {
  4534. struct cvp_hal_session *session = sess;
  4535. int rc = 0;
  4536. struct iris_hfi_device *device;
  4537. if (!session || !session->device) {
  4538. dprintk(CVP_ERR, " %s Invalid Params %pK\n", __func__, session);
  4539. return -EINVAL;
  4540. }
  4541. device = session->device;
  4542. mutex_lock(&device->lock);
  4543. if (!__is_session_valid(device, session, func))
  4544. rc = -ECONNRESET;
  4545. mutex_unlock(&device->lock);
  4546. return rc;
  4547. }
  4548. static void iris_init_hfi_callbacks(struct cvp_hfi_device *hdev)
  4549. {
  4550. hdev->core_init = iris_hfi_core_init;
  4551. hdev->core_release = iris_hfi_core_release;
  4552. hdev->core_trigger_ssr = iris_hfi_core_trigger_ssr;
  4553. hdev->session_init = iris_hfi_session_init;
  4554. hdev->session_end = iris_hfi_session_end;
  4555. hdev->session_start = iris_hfi_session_start;
  4556. hdev->session_stop = iris_hfi_session_stop;
  4557. hdev->session_abort = iris_hfi_session_abort;
  4558. hdev->session_clean = iris_hfi_session_clean;
  4559. hdev->session_set_buffers = iris_hfi_session_set_buffers;
  4560. hdev->session_release_buffers = iris_hfi_session_release_buffers;
  4561. hdev->session_send = iris_hfi_session_send;
  4562. hdev->session_flush = iris_hfi_session_flush;
  4563. hdev->scale_clocks = iris_hfi_scale_clocks;
  4564. hdev->vote_bus = iris_hfi_vote_buses;
  4565. hdev->get_fw_info = iris_hfi_get_fw_info;
  4566. hdev->get_core_capabilities = iris_hfi_get_core_capabilities;
  4567. hdev->suspend = iris_hfi_suspend;
  4568. hdev->resume = iris_hfi_resume;
  4569. hdev->flush_debug_queue = iris_hfi_flush_debug_queue;
  4570. hdev->noc_error_info = iris_hfi_noc_error_info;
  4571. hdev->validate_session = iris_hfi_validate_session;
  4572. hdev->pm_qos_update = iris_pm_qos_update;
  4573. hdev->debug_hook = iris_debug_hook;
  4574. }
  4575. int cvp_iris_hfi_initialize(struct cvp_hfi_device *hdev,
  4576. struct msm_cvp_platform_resources *res,
  4577. hfi_cmd_response_callback callback)
  4578. {
  4579. int rc = 0;
  4580. if (!hdev || !res || !callback) {
  4581. dprintk(CVP_ERR, "Invalid params: %pK %pK %pK\n",
  4582. hdev, res, callback);
  4583. rc = -EINVAL;
  4584. goto err_iris_hfi_init;
  4585. }
  4586. hdev->hfi_device_data = __get_device(res, callback);
  4587. if (IS_ERR_OR_NULL(hdev->hfi_device_data)) {
  4588. rc = PTR_ERR(hdev->hfi_device_data) ?: -EINVAL;
  4589. goto err_iris_hfi_init;
  4590. }
  4591. iris_init_hfi_callbacks(hdev);
  4592. err_iris_hfi_init:
  4593. return rc;
  4594. }