cvp_hfi.c 139 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2018-2021, The Linux Foundation. All rights reserved.
  4. */
  5. #include <asm/memory.h>
  6. #include <linux/coresight-stm.h>
  7. #include <linux/delay.h>
  8. #include <linux/devfreq.h>
  9. #include <linux/hash.h>
  10. #include <linux/io.h>
  11. #include <linux/iommu.h>
  12. #include <linux/iopoll.h>
  13. #include <linux/of.h>
  14. #include <linux/pm_qos.h>
  15. #include <linux/regulator/consumer.h>
  16. #include <linux/slab.h>
  17. #include <linux/workqueue.h>
  18. #include <linux/platform_device.h>
  19. #include <linux/soc/qcom/llcc-qcom.h>
  20. #include <linux/qcom_scm.h>
  21. #include <linux/soc/qcom/smem.h>
  22. #include <linux/dma-mapping.h>
  23. #include <linux/reset.h>
  24. #include <linux/pm_wakeup.h>
  25. #include "hfi_packetization.h"
  26. #include "msm_cvp_debug.h"
  27. #include "cvp_core_hfi.h"
  28. #include "cvp_hfi_helper.h"
  29. #include "cvp_hfi_io.h"
  30. #include "msm_cvp_dsp.h"
  31. #include "msm_cvp_clocks.h"
  32. #include "vm/cvp_vm.h"
  33. #include "cvp_dump.h"
  34. // ysi - added for debug
  35. #include <linux/clk/qcom.h>
  36. #include "msm_cvp_common.h"
  37. #define REG_ADDR_OFFSET_BITMASK 0x000FFFFF
  38. #define QDSS_IOVA_START 0x80001000
  39. #define MIN_PAYLOAD_SIZE 3
  40. struct cvp_tzbsp_memprot {
  41. u32 cp_start;
  42. u32 cp_size;
  43. u32 cp_nonpixel_start;
  44. u32 cp_nonpixel_size;
  45. };
  46. #define TZBSP_CVP_PAS_ID 26
  47. /* Poll interval in uS */
  48. #define POLL_INTERVAL_US 50
  49. enum tzbsp_subsys_state {
  50. TZ_SUBSYS_STATE_SUSPEND = 0,
  51. TZ_SUBSYS_STATE_RESUME = 1,
  52. TZ_SUBSYS_STATE_RESTORE_THRESHOLD = 2,
  53. };
  54. const struct msm_cvp_gov_data CVP_DEFAULT_BUS_VOTE = {
  55. .data = NULL,
  56. .data_count = 0,
  57. };
  58. const int cvp_max_packets = 32;
  59. static void iris_hfi_pm_handler(struct work_struct *work);
  60. static DECLARE_DELAYED_WORK(iris_hfi_pm_work, iris_hfi_pm_handler);
  61. static inline int __resume(struct iris_hfi_device *device);
  62. static inline int __suspend(struct iris_hfi_device *device);
  63. static int __disable_regulator(struct iris_hfi_device *device,
  64. const char *name);
  65. static int __enable_regulator(struct iris_hfi_device *device,
  66. const char *name);
  67. static void __flush_debug_queue(struct iris_hfi_device *device, u8 *packet);
  68. static int __initialize_packetization(struct iris_hfi_device *device);
  69. static struct cvp_hal_session *__get_session(struct iris_hfi_device *device,
  70. u32 session_id);
  71. static bool __is_session_valid(struct iris_hfi_device *device,
  72. struct cvp_hal_session *session, const char *func);
  73. static int __iface_cmdq_write(struct iris_hfi_device *device,
  74. void *pkt);
  75. static int __load_fw(struct iris_hfi_device *device);
  76. static int __power_on_init(struct iris_hfi_device *device);
  77. static void __unload_fw(struct iris_hfi_device *device);
  78. static int __tzbsp_set_cvp_state(enum tzbsp_subsys_state state);
  79. static int __enable_subcaches(struct iris_hfi_device *device);
  80. static int __set_subcaches(struct iris_hfi_device *device);
  81. static int __release_subcaches(struct iris_hfi_device *device);
  82. static int __disable_subcaches(struct iris_hfi_device *device);
  83. static int __power_collapse(struct iris_hfi_device *device, bool force);
  84. static int iris_hfi_noc_error_info(void *dev);
  85. static void interrupt_init_iris2(struct iris_hfi_device *device);
  86. static void setup_dsp_uc_memmap_vpu5(struct iris_hfi_device *device);
  87. static void clock_config_on_enable_vpu5(struct iris_hfi_device *device);
  88. static void power_off_iris2(struct iris_hfi_device *device);
  89. static int __set_ubwc_config(struct iris_hfi_device *device);
  90. static void __noc_error_info_iris2(struct iris_hfi_device *device);
  91. static int __enable_hw_power_collapse(struct iris_hfi_device *device);
  92. static int __disable_hw_power_collapse(struct iris_hfi_device *device);
  93. static int __power_off_controller(struct iris_hfi_device *device);
  94. static int __hwfence_regs_map(struct iris_hfi_device *device);
  95. static int __hwfence_regs_unmap(struct iris_hfi_device *device);
  96. static int __reset_control_assert_name(struct iris_hfi_device *device, const char *name);
  97. static int __reset_control_deassert_name(struct iris_hfi_device *device, const char *name);
  98. static int __reset_control_acquire(struct iris_hfi_device *device, const char *name);
  99. static int __reset_control_release(struct iris_hfi_device *device, const char *name);
  100. static struct iris_hfi_vpu_ops iris2_ops = {
  101. .interrupt_init = interrupt_init_iris2,
  102. .setup_dsp_uc_memmap = setup_dsp_uc_memmap_vpu5,
  103. .clock_config_on_enable = clock_config_on_enable_vpu5,
  104. .power_off = power_off_iris2,
  105. .noc_error_info = __noc_error_info_iris2,
  106. .reset_control_assert_name = __reset_control_assert_name,
  107. .reset_control_deassert_name = __reset_control_deassert_name,
  108. .reset_control_acquire_name = __reset_control_acquire,
  109. .reset_control_release_name = __reset_control_release,
  110. };
  111. /**
  112. * Utility function to enforce some of our assumptions. Spam calls to this
  113. * in hotspots in code to double check some of the assumptions that we hold.
  114. */
  115. static inline void __strict_check(struct iris_hfi_device *device)
  116. {
  117. msm_cvp_res_handle_fatal_hw_error(device->res,
  118. !mutex_is_locked(&device->lock));
  119. }
  120. static inline void __set_state(struct iris_hfi_device *device,
  121. enum iris_hfi_state state)
  122. {
  123. device->state = state;
  124. }
  125. static inline bool __core_in_valid_state(struct iris_hfi_device *device)
  126. {
  127. return device->state != IRIS_STATE_DEINIT;
  128. }
  129. static inline bool is_sys_cache_present(struct iris_hfi_device *device)
  130. {
  131. return device->res->sys_cache_present;
  132. }
  133. static int cvp_synx_recover(void)
  134. {
  135. #ifdef CVP_SYNX_ENABLED
  136. return synx_recover(SYNX_CLIENT_EVA_CTX0);
  137. #else
  138. return 0;
  139. #endif /* End of CVP_SYNX_ENABLED */
  140. }
  141. #define ROW_SIZE 32
  142. int get_hfi_version(void)
  143. {
  144. struct msm_cvp_core *core;
  145. struct iris_hfi_device *hfi;
  146. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  147. hfi = (struct iris_hfi_device *)core->device->hfi_device_data;
  148. return hfi->version;
  149. }
  150. unsigned int get_msg_size(struct cvp_hfi_msg_session_hdr *hdr)
  151. {
  152. struct msm_cvp_core *core;
  153. struct iris_hfi_device *device;
  154. u32 minor_ver;
  155. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  156. if (core)
  157. device = core->device->hfi_device_data;
  158. else
  159. return 0;
  160. if (!device) {
  161. dprintk(CVP_ERR, "%s: NULL device\n", __func__);
  162. return 0;
  163. }
  164. minor_ver = (device->version & HFI_VERSION_MINOR_MASK) >>
  165. HFI_VERSION_MINOR_SHIFT;
  166. if (minor_ver < 2)
  167. return sizeof(struct cvp_hfi_msg_session_hdr);
  168. if (hdr->packet_type == HFI_MSG_SESSION_CVP_FD)
  169. return sizeof(struct cvp_hfi_msg_session_hdr_ext);
  170. else
  171. return sizeof(struct cvp_hfi_msg_session_hdr);
  172. }
  173. unsigned int get_msg_session_id(void *msg)
  174. {
  175. struct cvp_hfi_msg_session_hdr *hdr =
  176. (struct cvp_hfi_msg_session_hdr *)msg;
  177. return hdr->session_id;
  178. }
  179. unsigned int get_msg_errorcode(void *msg)
  180. {
  181. struct cvp_hfi_msg_session_hdr *hdr =
  182. (struct cvp_hfi_msg_session_hdr *)msg;
  183. return hdr->error_type;
  184. }
  185. int get_msg_opconfigs(void *msg, unsigned int *session_id,
  186. unsigned int *error_type, unsigned int *config_id)
  187. {
  188. struct cvp_hfi_msg_session_op_cfg_packet *cfg =
  189. (struct cvp_hfi_msg_session_op_cfg_packet *)msg;
  190. *session_id = cfg->session_id;
  191. *error_type = cfg->error_type;
  192. *config_id = cfg->op_conf_id;
  193. return 0;
  194. }
  195. static void __dump_packet(u8 *packet, enum cvp_msg_prio log_level)
  196. {
  197. u32 c = 0, packet_size = *(u32 *)packet;
  198. /*
  199. * row must contain enough for 0xdeadbaad * 8 to be converted into
  200. * "de ad ba ab " * 8 + '\0'
  201. */
  202. char row[3 * ROW_SIZE];
  203. for (c = 0; c * ROW_SIZE < packet_size; ++c) {
  204. int bytes_to_read = ((c + 1) * ROW_SIZE > packet_size) ?
  205. packet_size % ROW_SIZE : ROW_SIZE;
  206. hex_dump_to_buffer(packet + c * ROW_SIZE, bytes_to_read,
  207. ROW_SIZE, 4, row, sizeof(row), false);
  208. dprintk(log_level, "%s\n", row);
  209. }
  210. }
  211. static int __dsp_suspend(struct iris_hfi_device *device, bool force)
  212. {
  213. int rc;
  214. if (msm_cvp_dsp_disable)
  215. return 0;
  216. dprintk(CVP_DSP, "%s: suspend dsp\n", __func__);
  217. rc = cvp_dsp_suspend(force);
  218. if (rc) {
  219. if (rc != -EBUSY)
  220. dprintk(CVP_ERR,
  221. "%s: dsp suspend failed with error %d\n",
  222. __func__, rc);
  223. return rc;
  224. }
  225. dprintk(CVP_DSP, "%s: dsp suspended\n", __func__);
  226. return 0;
  227. }
  228. static int __dsp_resume(struct iris_hfi_device *device)
  229. {
  230. int rc;
  231. if (msm_cvp_dsp_disable)
  232. return 0;
  233. dprintk(CVP_DSP, "%s: resume dsp\n", __func__);
  234. rc = cvp_dsp_resume();
  235. if (rc) {
  236. dprintk(CVP_ERR,
  237. "%s: dsp resume failed with error %d\n",
  238. __func__, rc);
  239. return rc;
  240. }
  241. dprintk(CVP_DSP, "%s: dsp resumed\n", __func__);
  242. return rc;
  243. }
  244. static int __dsp_shutdown(struct iris_hfi_device *device)
  245. {
  246. int rc;
  247. if (msm_cvp_dsp_disable)
  248. return 0;
  249. dprintk(CVP_DSP, "%s: shutdown dsp\n", __func__);
  250. rc = cvp_dsp_shutdown();
  251. if (rc) {
  252. dprintk(CVP_ERR,
  253. "%s: dsp shutdown failed with error %d\n",
  254. __func__, rc);
  255. WARN_ON(1);
  256. }
  257. dprintk(CVP_DSP, "%s: dsp shutdown successful\n", __func__);
  258. return rc;
  259. }
  260. static int __acquire_regulator(struct regulator_info *rinfo,
  261. struct iris_hfi_device *device)
  262. {
  263. int rc = 0;
  264. if (rinfo->has_hw_power_collapse) {
  265. rc = regulator_set_mode(rinfo->regulator,
  266. REGULATOR_MODE_NORMAL);
  267. if (rc) {
  268. /*
  269. * This is somewhat fatal, but nothing we can do
  270. * about it. We can't disable the regulator w/o
  271. * getting it back under s/w control
  272. */
  273. dprintk(CVP_WARN,
  274. "Failed to acquire regulator control: %s\n",
  275. rinfo->name);
  276. } else {
  277. dprintk(CVP_PWR,
  278. "Acquire regulator control from HW: %s\n",
  279. rinfo->name);
  280. }
  281. }
  282. if (!regulator_is_enabled(rinfo->regulator)) {
  283. dprintk(CVP_WARN, "Regulator is not enabled %s\n",
  284. rinfo->name);
  285. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  286. }
  287. return rc;
  288. }
  289. static int __hand_off_regulator(struct regulator_info *rinfo)
  290. {
  291. int rc = 0;
  292. if (rinfo->has_hw_power_collapse) {
  293. rc = regulator_set_mode(rinfo->regulator,
  294. REGULATOR_MODE_FAST);
  295. if (rc) {
  296. dprintk(CVP_WARN,
  297. "Failed to hand off regulator control: %s\n",
  298. rinfo->name);
  299. } else {
  300. dprintk(CVP_PWR,
  301. "Hand off regulator control to HW: %s\n",
  302. rinfo->name);
  303. }
  304. }
  305. return rc;
  306. }
  307. static int __hand_off_regulators(struct iris_hfi_device *device)
  308. {
  309. struct regulator_info *rinfo;
  310. int rc = 0, c = 0;
  311. iris_hfi_for_each_regulator(device, rinfo) {
  312. rc = __hand_off_regulator(rinfo);
  313. /*
  314. * If one regulator hand off failed, driver should take
  315. * the control for other regulators back.
  316. */
  317. if (rc)
  318. goto err_reg_handoff_failed;
  319. c++;
  320. }
  321. return rc;
  322. err_reg_handoff_failed:
  323. iris_hfi_for_each_regulator_reverse_continue(device, rinfo, c)
  324. __acquire_regulator(rinfo, device);
  325. return rc;
  326. }
  327. static int __take_back_regulators(struct iris_hfi_device *device)
  328. {
  329. struct regulator_info *rinfo;
  330. int rc = 0;
  331. iris_hfi_for_each_regulator(device, rinfo) {
  332. rc = __acquire_regulator(rinfo, device);
  333. /*
  334. * if one regulator hand off failed, driver should take
  335. * the control for other regulators back.
  336. */
  337. if (rc)
  338. return rc;
  339. }
  340. return rc;
  341. }
  342. static int __write_queue(struct cvp_iface_q_info *qinfo, u8 *packet,
  343. bool *rx_req_is_set)
  344. {
  345. struct cvp_hfi_queue_header *queue;
  346. struct cvp_hfi_cmd_session_hdr *cmd_pkt;
  347. u32 packet_size_in_words, new_write_idx;
  348. u32 empty_space, read_idx, write_idx;
  349. u32 *write_ptr;
  350. if (!qinfo || !packet) {
  351. dprintk(CVP_ERR, "Invalid Params\n");
  352. return -EINVAL;
  353. } else if (!qinfo->q_array.align_virtual_addr) {
  354. dprintk(CVP_WARN, "Queues have already been freed\n");
  355. return -EINVAL;
  356. }
  357. queue = (struct cvp_hfi_queue_header *) qinfo->q_hdr;
  358. if (!queue) {
  359. dprintk(CVP_ERR, "queue not present\n");
  360. return -ENOENT;
  361. }
  362. cmd_pkt = (struct cvp_hfi_cmd_session_hdr *)packet;
  363. if (cmd_pkt->size >= sizeof(struct cvp_hfi_cmd_session_hdr))
  364. dprintk(CVP_CMD, "%s: "
  365. "pkt_type %08x sess_id %08x trans_id %u ktid %llu\n",
  366. __func__, cmd_pkt->packet_type,
  367. cmd_pkt->session_id,
  368. cmd_pkt->client_data.transaction_id,
  369. cmd_pkt->client_data.kdata & (FENCE_BIT - 1));
  370. else
  371. dprintk(CVP_CMD, "%s: "
  372. "pkt_type %08x", __func__, cmd_pkt->packet_type);
  373. if (msm_cvp_debug & CVP_PKT) {
  374. dprintk(CVP_PKT, "%s: %pK\n", __func__, qinfo);
  375. __dump_packet(packet, CVP_PKT);
  376. }
  377. packet_size_in_words = (*(u32 *)packet) >> 2;
  378. if (!packet_size_in_words || packet_size_in_words >
  379. qinfo->q_array.mem_size>>2) {
  380. dprintk(CVP_ERR, "Invalid packet size\n");
  381. return -ENODATA;
  382. }
  383. spin_lock(&qinfo->hfi_lock);
  384. read_idx = queue->qhdr_read_idx;
  385. write_idx = queue->qhdr_write_idx;
  386. empty_space = (write_idx >= read_idx) ?
  387. ((qinfo->q_array.mem_size>>2) - (write_idx - read_idx)) :
  388. (read_idx - write_idx);
  389. if (empty_space <= packet_size_in_words) {
  390. queue->qhdr_tx_req = 1;
  391. spin_unlock(&qinfo->hfi_lock);
  392. dprintk(CVP_ERR, "Insufficient size (%d) to write (%d)\n",
  393. empty_space, packet_size_in_words);
  394. return -ENOTEMPTY;
  395. }
  396. queue->qhdr_tx_req = 0;
  397. new_write_idx = write_idx + packet_size_in_words;
  398. write_ptr = (u32 *)((qinfo->q_array.align_virtual_addr) +
  399. (write_idx << 2));
  400. if (write_ptr < (u32 *)qinfo->q_array.align_virtual_addr ||
  401. write_ptr > (u32 *)(qinfo->q_array.align_virtual_addr +
  402. qinfo->q_array.mem_size)) {
  403. spin_unlock(&qinfo->hfi_lock);
  404. dprintk(CVP_ERR, "Invalid write index\n");
  405. return -ENODATA;
  406. }
  407. if (new_write_idx < (qinfo->q_array.mem_size >> 2)) {
  408. memcpy(write_ptr, packet, packet_size_in_words << 2);
  409. } else {
  410. new_write_idx -= qinfo->q_array.mem_size >> 2;
  411. memcpy(write_ptr, packet, (packet_size_in_words -
  412. new_write_idx) << 2);
  413. memcpy((void *)qinfo->q_array.align_virtual_addr,
  414. packet + ((packet_size_in_words - new_write_idx) << 2),
  415. new_write_idx << 2);
  416. }
  417. /*
  418. * Memory barrier to make sure packet is written before updating the
  419. * write index
  420. */
  421. mb();
  422. queue->qhdr_write_idx = new_write_idx;
  423. if (rx_req_is_set)
  424. *rx_req_is_set = queue->qhdr_rx_req == 1;
  425. /*
  426. * Memory barrier to make sure write index is updated before an
  427. * interrupt is raised.
  428. */
  429. mb();
  430. spin_unlock(&qinfo->hfi_lock);
  431. return 0;
  432. }
  433. static int __read_queue(struct cvp_iface_q_info *qinfo, u8 *packet,
  434. u32 *pb_tx_req_is_set)
  435. {
  436. struct cvp_hfi_queue_header *queue;
  437. struct cvp_hfi_msg_session_hdr *msg_pkt;
  438. u32 packet_size_in_words, new_read_idx;
  439. u32 *read_ptr;
  440. u32 receive_request = 0;
  441. u32 read_idx, write_idx;
  442. int rc = 0;
  443. if (!qinfo || !packet || !pb_tx_req_is_set) {
  444. dprintk(CVP_ERR, "Invalid Params\n");
  445. return -EINVAL;
  446. } else if (!qinfo->q_array.align_virtual_addr) {
  447. dprintk(CVP_WARN, "Queues have already been freed\n");
  448. return -EINVAL;
  449. }
  450. /*
  451. * Memory barrier to make sure data is valid before
  452. *reading it
  453. */
  454. mb();
  455. queue = (struct cvp_hfi_queue_header *) qinfo->q_hdr;
  456. if (!queue) {
  457. dprintk(CVP_ERR, "Queue memory is not allocated\n");
  458. return -ENOMEM;
  459. }
  460. /*
  461. * Do not set receive request for debug queue, if set,
  462. * Iris generates interrupt for debug messages even
  463. * when there is no response message available.
  464. * In general debug queue will not become full as it
  465. * is being emptied out for every interrupt from Iris.
  466. * Iris will anyway generates interrupt if it is full.
  467. */
  468. spin_lock(&qinfo->hfi_lock);
  469. if (queue->qhdr_type & HFI_Q_ID_CTRL_TO_HOST_MSG_Q)
  470. receive_request = 1;
  471. read_idx = queue->qhdr_read_idx;
  472. write_idx = queue->qhdr_write_idx;
  473. if (read_idx == write_idx) {
  474. queue->qhdr_rx_req = receive_request;
  475. /*
  476. * mb() to ensure qhdr is updated in main memory
  477. * so that iris reads the updated header values
  478. */
  479. mb();
  480. *pb_tx_req_is_set = 0;
  481. if (write_idx != queue->qhdr_write_idx) {
  482. queue->qhdr_rx_req = 0;
  483. } else {
  484. spin_unlock(&qinfo->hfi_lock);
  485. dprintk(CVP_HFI,
  486. "%s queue is empty, rx_req = %u, tx_req = %u, read_idx = %u\n",
  487. receive_request ? "message" : "debug",
  488. queue->qhdr_rx_req, queue->qhdr_tx_req,
  489. queue->qhdr_read_idx);
  490. return -ENODATA;
  491. }
  492. }
  493. read_ptr = (u32 *)((qinfo->q_array.align_virtual_addr) +
  494. (read_idx << 2));
  495. if (read_ptr < (u32 *)qinfo->q_array.align_virtual_addr ||
  496. read_ptr > (u32 *)(qinfo->q_array.align_virtual_addr +
  497. qinfo->q_array.mem_size - sizeof(*read_ptr))) {
  498. spin_unlock(&qinfo->hfi_lock);
  499. dprintk(CVP_ERR, "Invalid read index\n");
  500. return -ENODATA;
  501. }
  502. packet_size_in_words = (*read_ptr) >> 2;
  503. if (!packet_size_in_words) {
  504. spin_unlock(&qinfo->hfi_lock);
  505. dprintk(CVP_ERR, "Zero packet size\n");
  506. return -ENODATA;
  507. }
  508. new_read_idx = read_idx + packet_size_in_words;
  509. if (((packet_size_in_words << 2) <= CVP_IFACEQ_VAR_HUGE_PKT_SIZE)
  510. && read_idx <= (qinfo->q_array.mem_size >> 2)) {
  511. if (new_read_idx < (qinfo->q_array.mem_size >> 2)) {
  512. memcpy(packet, read_ptr,
  513. packet_size_in_words << 2);
  514. } else {
  515. new_read_idx -= (qinfo->q_array.mem_size >> 2);
  516. memcpy(packet, read_ptr,
  517. (packet_size_in_words - new_read_idx) << 2);
  518. memcpy(packet + ((packet_size_in_words -
  519. new_read_idx) << 2),
  520. (u8 *)qinfo->q_array.align_virtual_addr,
  521. new_read_idx << 2);
  522. }
  523. } else {
  524. dprintk(CVP_WARN,
  525. "BAD packet received, read_idx: %#x, pkt_size: %d\n",
  526. read_idx, packet_size_in_words << 2);
  527. dprintk(CVP_WARN, "Dropping this packet\n");
  528. new_read_idx = write_idx;
  529. rc = -ENODATA;
  530. }
  531. if (new_read_idx != queue->qhdr_write_idx)
  532. queue->qhdr_rx_req = 0;
  533. else
  534. queue->qhdr_rx_req = receive_request;
  535. queue->qhdr_read_idx = new_read_idx;
  536. /*
  537. * mb() to ensure qhdr is updated in main memory
  538. * so that iris reads the updated header values
  539. */
  540. mb();
  541. *pb_tx_req_is_set = (queue->qhdr_tx_req == 1) ? 1 : 0;
  542. spin_unlock(&qinfo->hfi_lock);
  543. if (!(queue->qhdr_type & HFI_Q_ID_CTRL_TO_HOST_DEBUG_Q)) {
  544. msg_pkt = (struct cvp_hfi_msg_session_hdr *)packet;
  545. dprintk(CVP_CMD, "%s: "
  546. "pkt_type %08x sess_id %08x trans_id %u ktid %llu\n",
  547. __func__, msg_pkt->packet_type,
  548. msg_pkt->session_id,
  549. msg_pkt->client_data.transaction_id,
  550. msg_pkt->client_data.kdata & (FENCE_BIT - 1));
  551. }
  552. if ((msm_cvp_debug & CVP_PKT) &&
  553. !(queue->qhdr_type & HFI_Q_ID_CTRL_TO_HOST_DEBUG_Q)) {
  554. dprintk(CVP_PKT, "%s: %pK\n", __func__, qinfo);
  555. __dump_packet(packet, CVP_PKT);
  556. }
  557. return rc;
  558. }
  559. static int __smem_alloc(struct iris_hfi_device *dev, struct cvp_mem_addr *mem,
  560. u32 size, u32 align, u32 flags)
  561. {
  562. struct msm_cvp_smem *alloc = &mem->mem_data;
  563. int rc = 0;
  564. if (!dev || !mem || !size) {
  565. dprintk(CVP_ERR, "Invalid Params\n");
  566. return -EINVAL;
  567. }
  568. dprintk(CVP_INFO, "start to alloc size: %d, flags: %d\n", size, flags);
  569. alloc->flags = flags;
  570. rc = msm_cvp_smem_alloc(size, align, 1, (void *)dev->res, alloc);
  571. if (rc) {
  572. dprintk(CVP_ERR, "Alloc failed\n");
  573. rc = -ENOMEM;
  574. goto fail_smem_alloc;
  575. }
  576. dprintk(CVP_MEM, "%s: ptr = %pK, size = %d\n", __func__,
  577. alloc->kvaddr, size);
  578. mem->mem_size = alloc->size;
  579. mem->align_virtual_addr = alloc->kvaddr;
  580. mem->align_device_addr = alloc->device_addr;
  581. alloc->pkt_type = 0;
  582. alloc->buf_idx = 0;
  583. return rc;
  584. fail_smem_alloc:
  585. return rc;
  586. }
  587. static void __smem_free(struct iris_hfi_device *dev, struct msm_cvp_smem *mem)
  588. {
  589. if (!dev || !mem) {
  590. dprintk(CVP_ERR, "invalid param %pK %pK\n", dev, mem);
  591. return;
  592. }
  593. msm_cvp_smem_free(mem);
  594. }
  595. static void __write_register(struct iris_hfi_device *device,
  596. u32 reg, u32 value)
  597. {
  598. u32 hwiosymaddr = reg;
  599. u8 *base_addr;
  600. if (!device) {
  601. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  602. return;
  603. }
  604. __strict_check(device);
  605. if (!device->power_enabled) {
  606. dprintk(CVP_WARN,
  607. "HFI Write register failed : Power is OFF\n");
  608. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  609. return;
  610. }
  611. base_addr = device->cvp_hal_data->register_base;
  612. dprintk(CVP_REG, "Base addr: %pK, written to: %#x, Value: %#x...\n",
  613. base_addr, hwiosymaddr, value);
  614. base_addr += hwiosymaddr;
  615. writel_relaxed(value, base_addr);
  616. /*
  617. * Memory barrier to make sure value is written into the register.
  618. */
  619. wmb();
  620. }
  621. static int __read_gcc_register(struct iris_hfi_device *device, u32 reg)
  622. {
  623. int rc = 0;
  624. u8 *base_addr;
  625. if (!device) {
  626. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  627. return -EINVAL;
  628. }
  629. __strict_check(device);
  630. if (!device->power_enabled) {
  631. dprintk(CVP_WARN,
  632. "%s HFI Read register failed : Power is OFF\n",
  633. __func__);
  634. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  635. return -EINVAL;
  636. }
  637. base_addr = device->cvp_hal_data->gcc_reg_base;
  638. rc = readl_relaxed(base_addr + reg);
  639. /*
  640. * Memory barrier to make sure value is read correctly from the
  641. * register.
  642. */
  643. rmb();
  644. dprintk(CVP_REG,
  645. "GCC Base addr: %pK, read from: %#x, value: %#x...\n",
  646. base_addr, reg, rc);
  647. return rc;
  648. }
  649. static int __read_register(struct iris_hfi_device *device, u32 reg)
  650. {
  651. int rc = 0;
  652. u8 *base_addr;
  653. if (!device) {
  654. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  655. return -EINVAL;
  656. }
  657. __strict_check(device);
  658. if (!device->power_enabled) {
  659. dprintk(CVP_WARN,
  660. "HFI Read register failed : Power is OFF\n");
  661. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  662. return -EINVAL;
  663. }
  664. base_addr = device->cvp_hal_data->register_base;
  665. rc = readl_relaxed(base_addr + reg);
  666. /*
  667. * Memory barrier to make sure value is read correctly from the
  668. * register.
  669. */
  670. rmb();
  671. dprintk(CVP_REG, "Base addr: %pK, read from: %#x, value: %#x...\n",
  672. base_addr, reg, rc);
  673. return rc;
  674. }
  675. static int __set_registers(struct iris_hfi_device *device)
  676. {
  677. struct msm_cvp_core *core;
  678. struct msm_cvp_platform_data *pdata;
  679. struct reg_set *reg_set;
  680. int i;
  681. if (!device->res) {
  682. dprintk(CVP_ERR,
  683. "device resources null, cannot set registers\n");
  684. return -EINVAL ;
  685. }
  686. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  687. pdata = core->platform_data;
  688. reg_set = &device->res->reg_set;
  689. for (i = 0; i < reg_set->count; i++) {
  690. __write_register(device, reg_set->reg_tbl[i].reg,
  691. reg_set->reg_tbl[i].value);
  692. dprintk(CVP_REG, "write_reg offset=%x, val=%x\n",
  693. reg_set->reg_tbl[i].reg,
  694. reg_set->reg_tbl[i].value);
  695. }
  696. i = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  697. if (i) {
  698. dprintk(CVP_WARN, "%s Fail acquire xo_reset\n", __func__);
  699. return -EINVAL;
  700. }
  701. __write_register(device, CVP_CPU_CS_AXI4_QOS,
  702. pdata->noc_qos->axi_qos);
  703. __write_register(device, CVP_NOC_RGE_PRIORITYLUT_LOW,
  704. pdata->noc_qos->prioritylut_low);
  705. __write_register(device, CVP_NOC_RGE_PRIORITYLUT_HIGH,
  706. pdata->noc_qos->prioritylut_high);
  707. __write_register(device, CVP_NOC_RGE_URGENCY_LOW,
  708. pdata->noc_qos->urgency_low);
  709. __write_register(device, CVP_NOC_RGE_DANGERLUT_LOW,
  710. pdata->noc_qos->dangerlut_low);
  711. __write_register(device, CVP_NOC_RGE_SAFELUT_LOW,
  712. pdata->noc_qos->safelut_low);
  713. __write_register(device, CVP_NOC_CDM_PRIORITYLUT_LOW,
  714. pdata->noc_qos->prioritylut_low);
  715. __write_register(device, CVP_NOC_CDM_PRIORITYLUT_HIGH,
  716. pdata->noc_qos->prioritylut_high);
  717. __write_register(device, CVP_NOC_CDM_URGENCY_LOW,
  718. pdata->noc_qos->urgency_low);
  719. __write_register(device, CVP_NOC_CDM_DANGERLUT_LOW,
  720. pdata->noc_qos->dangerlut_low);
  721. __write_register(device, CVP_NOC_CDM_SAFELUT_LOW,
  722. pdata->noc_qos->safelut_low);
  723. /* Below registers write moved from FW to SW to enable UBWC */
  724. __write_register(device, CVP_NOC_RGE_NIU_DECCTL_LOW,
  725. 0x1);
  726. __write_register(device, CVP_NOC_RGE_NIU_ENCCTL_LOW,
  727. 0x1);
  728. __write_register(device, CVP_NOC_GCE_VADL_TOF_NIU_DECCTL_LOW,
  729. 0x1);
  730. __write_register(device, CVP_NOC_GCE_VADL_TOF_NIU_ENCCTL_LOW,
  731. 0x1);
  732. __write_register(device, CVP_NOC_CORE_ERR_MAINCTL_LOW_OFFS,
  733. 0x3);
  734. __write_register(device, CVP_NOC_MAIN_SIDEBANDMANAGER_FAULTINEN0_LOW,
  735. 0x1);
  736. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  737. return 0;
  738. }
  739. /*
  740. * The existence of this function is a hack for 8996 (or certain Iris versions)
  741. * to overcome a hardware bug. Whenever the GDSCs momentarily power collapse
  742. * (after calling __hand_off_regulators()), the values of the threshold
  743. * registers (typically programmed by TZ) are incorrectly reset. As a result
  744. * reprogram these registers at certain agreed upon points.
  745. */
  746. static void __set_threshold_registers(struct iris_hfi_device *device)
  747. {
  748. u32 version = __read_register(device, CVP_WRAPPER_HW_VERSION);
  749. version &= ~GENMASK(15, 0);
  750. if (version != (0x3 << 28 | 0x43 << 16))
  751. return;
  752. if (__tzbsp_set_cvp_state(TZ_SUBSYS_STATE_RESTORE_THRESHOLD))
  753. dprintk(CVP_ERR, "Failed to restore threshold values\n");
  754. }
  755. static int __unvote_buses(struct iris_hfi_device *device)
  756. {
  757. int rc = 0;
  758. struct bus_info *bus = NULL;
  759. kfree(device->bus_vote.data);
  760. device->bus_vote.data = NULL;
  761. device->bus_vote.data_count = 0;
  762. iris_hfi_for_each_bus(device, bus) {
  763. rc = msm_cvp_set_bw(bus, 0);
  764. if (rc) {
  765. dprintk(CVP_ERR,
  766. "%s: Failed unvoting bus\n", __func__);
  767. goto err_unknown_device;
  768. }
  769. }
  770. err_unknown_device:
  771. return rc;
  772. }
  773. static int __vote_buses(struct iris_hfi_device *device,
  774. struct cvp_bus_vote_data *data, int num_data)
  775. {
  776. int rc = 0;
  777. struct bus_info *bus = NULL;
  778. struct cvp_bus_vote_data *new_data = NULL;
  779. if (!num_data) {
  780. dprintk(CVP_PWR, "No vote data available\n");
  781. goto no_data_count;
  782. } else if (!data) {
  783. dprintk(CVP_ERR, "Invalid voting data\n");
  784. return -EINVAL;
  785. }
  786. new_data = kmemdup(data, num_data * sizeof(*new_data), GFP_KERNEL);
  787. if (!new_data) {
  788. dprintk(CVP_ERR, "Can't alloc memory to cache bus votes\n");
  789. rc = -ENOMEM;
  790. goto err_no_mem;
  791. }
  792. no_data_count:
  793. kfree(device->bus_vote.data);
  794. device->bus_vote.data = new_data;
  795. device->bus_vote.data_count = num_data;
  796. iris_hfi_for_each_bus(device, bus) {
  797. if (bus) {
  798. rc = msm_cvp_set_bw(bus, bus->range[1]);
  799. if (rc)
  800. dprintk(CVP_ERR,
  801. "Failed voting bus %s to ab %u\n",
  802. bus->name, bus->range[1]*1000);
  803. }
  804. }
  805. err_no_mem:
  806. return rc;
  807. }
  808. static int iris_hfi_vote_buses(void *dev, struct cvp_bus_vote_data *d, int n)
  809. {
  810. int rc = 0;
  811. struct iris_hfi_device *device = dev;
  812. if (!device)
  813. return -EINVAL;
  814. mutex_lock(&device->lock);
  815. rc = __vote_buses(device, d, n);
  816. mutex_unlock(&device->lock);
  817. return rc;
  818. }
  819. static int __core_set_resource(struct iris_hfi_device *device,
  820. struct cvp_resource_hdr *resource_hdr, void *resource_value)
  821. {
  822. struct cvp_hfi_cmd_sys_set_resource_packet *pkt;
  823. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  824. int rc = 0;
  825. if (!device || !resource_hdr || !resource_value) {
  826. dprintk(CVP_ERR, "set_res: Invalid Params\n");
  827. return -EINVAL;
  828. }
  829. pkt = (struct cvp_hfi_cmd_sys_set_resource_packet *) packet;
  830. rc = call_hfi_pkt_op(device, sys_set_resource,
  831. pkt, resource_hdr, resource_value);
  832. if (rc) {
  833. dprintk(CVP_ERR, "set_res: failed to create packet\n");
  834. goto err_create_pkt;
  835. }
  836. rc = __iface_cmdq_write(device, pkt);
  837. if (rc)
  838. rc = -ENOTEMPTY;
  839. err_create_pkt:
  840. return rc;
  841. }
  842. static int __core_release_resource(struct iris_hfi_device *device,
  843. struct cvp_resource_hdr *resource_hdr)
  844. {
  845. struct cvp_hfi_cmd_sys_release_resource_packet *pkt;
  846. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  847. int rc = 0;
  848. if (!device || !resource_hdr) {
  849. dprintk(CVP_ERR, "release_res: Invalid Params\n");
  850. return -EINVAL;
  851. }
  852. pkt = (struct cvp_hfi_cmd_sys_release_resource_packet *) packet;
  853. rc = call_hfi_pkt_op(device, sys_release_resource,
  854. pkt, resource_hdr);
  855. if (rc) {
  856. dprintk(CVP_ERR, "release_res: failed to create packet\n");
  857. goto err_create_pkt;
  858. }
  859. rc = __iface_cmdq_write(device, pkt);
  860. if (rc)
  861. rc = -ENOTEMPTY;
  862. err_create_pkt:
  863. return rc;
  864. }
  865. static int __tzbsp_set_cvp_state(enum tzbsp_subsys_state state)
  866. {
  867. int rc = 0;
  868. rc = qcom_scm_set_remote_state(state, TZBSP_CVP_PAS_ID);
  869. dprintk(CVP_CORE, "Set state %d, resp %d\n", state, rc);
  870. if (rc) {
  871. dprintk(CVP_ERR, "Failed qcom_scm_set_remote_state %d\n", rc);
  872. return rc;
  873. }
  874. return 0;
  875. }
  876. /*
  877. * Based on fal10_veto, X2RPMh, core_pwr_on and PWAitMode value, infer
  878. * value of xtss_sw_reset. xtss_sw_reset is a TZ register bit. Driver
  879. * cannot access it directly.
  880. *
  881. * In __boot_firmware() function, the caller of this function. It checks
  882. * "core_pwr_on" == false, basically core powered off. So this function
  883. * doesn't check core_pwr_on. Assume core_pwr_on = false.
  884. *
  885. * fal10_veto = VPU_CPU_CS_X2RPMh[2] |
  886. * ( ~VPU_CPU_CS_X2RPMh[1] & core_pwr_on ) |
  887. * ( ~VPU_CPU_CS_X2RPMh[0] & ~( xtss_sw_reset | PWaitMode ) ) ;
  888. */
  889. static inline void check_tensilica_in_reset(struct iris_hfi_device *device)
  890. {
  891. u32 X2RPMh, fal10_veto, wait_mode;
  892. X2RPMh = __read_register(device, CVP_CPU_CS_X2RPMh);
  893. X2RPMh = X2RPMh & 0x7;
  894. /* wait_mode = 1: Tensilica is in WFI mode (PWaitMode = true) */
  895. wait_mode = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  896. wait_mode = wait_mode & 0x1;
  897. fal10_veto = __read_register(device, CVP_CPU_CS_X2RPMh_STATUS);
  898. fal10_veto = fal10_veto & 0x1;
  899. dprintk(CVP_WARN, "tensilica reset check %#x %#x %#x\n",
  900. X2RPMh, wait_mode, fal10_veto);
  901. }
  902. static const char boot_states[0x40][32] = {
  903. "NOT INIT",
  904. "RST_START",
  905. "INIT_MEMCTL",
  906. "INTENABLE_RST",
  907. "LITBASE_RST",
  908. "PREFETCH_EN",
  909. "MPU_INIT",
  910. "CTRL_INIT_READ",
  911. "MEMCTL_L1_FIX",
  912. "RESTORE_EXTRA_NW",
  913. "CORE_RESTORE",
  914. "COLD_BOOT",
  915. "DISABLE_CACHE",
  916. "BEFORE_MPU_C",
  917. "RET_MPU_C",
  918. "IN_MPU_C",
  919. "IN_MPU_DEFAULT",
  920. "IN_MPU_SYNX",
  921. "UCR_SIZE_FAIL",
  922. "UCR_ADDR_FAIL",
  923. "UCR1_SIZE_FAIL",
  924. "UCR1_ADDR_FAIL",
  925. "UCR_OVERLAPPED_UCR1",
  926. "UCR1_OVERLAPPED_UCR",
  927. "UCR_EQ_UCR1",
  928. "MPU_CHECK_DONE",
  929. "BEFORE_INT_LOCK",
  930. "AFTER_INT_LOCK",
  931. "BEFORE_INT_UNLOCK",
  932. "AFTER_INT_UNLOCK",
  933. "CALL_START",
  934. "MAIN_ENTRY",
  935. "VENUS_INIT_ENTRY",
  936. "VSYS_INIT_ENTRY",
  937. "BEFORE_XOS_CLK",
  938. "AFTER_XOS_CLK",
  939. "LOG_MUTEX_INIT",
  940. "CREATE_FRAMEWORK_ENTRY",
  941. "DTG_INIT",
  942. "IDLE_TASK_INIT",
  943. "VENUS_CORE_INIT",
  944. "HW_CORES_INIT",
  945. "RST_THREAD_INIT",
  946. "HOST_THREAD_INIT",
  947. "ALL_THREADS_INIT",
  948. "TASK_MEMPOOL",
  949. "SESSION_MUTEX",
  950. "SIGNALS_INIT",
  951. "RST_SIGNAL_INIT",
  952. "INTR_EN_HOST",
  953. "INTR_REG_HOST",
  954. "INTR_EN_DSP",
  955. "INTR_REG_DSP",
  956. "X2HSOFTINTEN",
  957. "H2XSOFTINTEN",
  958. "CPU2DSPINTEN",
  959. "DSP2CPUINT_SWRESET",
  960. "THREADS_START",
  961. "RST_THREAD_START",
  962. "HST_THREAD_START",
  963. "HST_THREAD_ENTRY"
  964. };
  965. static inline int __boot_firmware(struct iris_hfi_device *device)
  966. {
  967. int rc = 0, loop = 10;
  968. u32 ctrl_init_val = 0, ctrl_status = 0, count = 0, max_tries = 500;
  969. u32 reg_gdsc;
  970. /*
  971. * Hand off control of regulators to h/w _after_ enabling clocks.
  972. * Note that the GDSC will turn off when switching from normal
  973. * (s/w triggered) to fast (HW triggered) unless the h/w vote is
  974. * present. Since Iris isn't up yet, the GDSC will be off briefly.
  975. */
  976. if (__enable_hw_power_collapse(device))
  977. dprintk(CVP_ERR, "Failed to enabled inter-frame PC\n");
  978. if (!msm_cvp_fw_low_power_mode)
  979. goto skip_core_power_check;
  980. while (loop) {
  981. reg_gdsc = __read_register(device, CVP_CC_MVS1_GDSCR);
  982. if (reg_gdsc & 0x80000000) {
  983. usleep_range(100, 200);
  984. loop--;
  985. } else {
  986. break;
  987. }
  988. }
  989. if (!loop)
  990. dprintk(CVP_ERR, "fail to power off CORE during resume\n");
  991. skip_core_power_check:
  992. ctrl_init_val = BIT(0);
  993. /* RUMI: CVP_CTRL_INIT in MPTest has bit 0 and 3 set */
  994. __write_register(device, CVP_CTRL_INIT, ctrl_init_val);
  995. while (!(ctrl_status & CVP_CTRL_INIT_STATUS__M) && count < max_tries) {
  996. ctrl_status = __read_register(device, CVP_CTRL_STATUS);
  997. if ((ctrl_status & CVP_CTRL_ERROR_STATUS__M) == 0x4) {
  998. dprintk(CVP_ERR, "invalid setting for UC_REGION\n");
  999. rc = -ENODATA;
  1000. break;
  1001. }
  1002. /* Reduce to 500, 1000 on silicon */
  1003. usleep_range(500, 1000);
  1004. count++;
  1005. }
  1006. if (!(ctrl_status & CVP_CTRL_INIT_STATUS__M)) {
  1007. ctrl_init_val = __read_register(device, CVP_CTRL_INIT);
  1008. dprintk(CVP_ERR,
  1009. "Failed to boot FW status: %x %x %s\n",
  1010. ctrl_status, ctrl_init_val,
  1011. boot_states[(ctrl_status >> 9) & 0x3f]);
  1012. check_tensilica_in_reset(device);
  1013. rc = -ENODEV;
  1014. }
  1015. /* Enable interrupt before sending commands to tensilica */
  1016. __write_register(device, CVP_CPU_CS_H2XSOFTINTEN, 0x1);
  1017. __write_register(device, CVP_CPU_CS_X2RPMh, 0x0);
  1018. return rc;
  1019. }
  1020. static int iris_hfi_resume(void *dev)
  1021. {
  1022. int rc = 0;
  1023. struct iris_hfi_device *device = (struct iris_hfi_device *) dev;
  1024. if (!device) {
  1025. dprintk(CVP_ERR, "%s invalid device\n", __func__);
  1026. return -EINVAL;
  1027. }
  1028. dprintk(CVP_CORE, "Resuming Iris\n");
  1029. mutex_lock(&device->lock);
  1030. rc = __resume(device);
  1031. mutex_unlock(&device->lock);
  1032. return rc;
  1033. }
  1034. static int iris_hfi_suspend(void *dev)
  1035. {
  1036. int rc = 0;
  1037. struct iris_hfi_device *device = (struct iris_hfi_device *) dev;
  1038. if (!device) {
  1039. dprintk(CVP_ERR, "%s invalid device\n", __func__);
  1040. return -EINVAL;
  1041. } else if (!device->res->sw_power_collapsible) {
  1042. return -ENOTSUPP;
  1043. }
  1044. dprintk(CVP_CORE, "Suspending Iris\n");
  1045. mutex_lock(&device->lock);
  1046. rc = __power_collapse(device, true);
  1047. if (rc) {
  1048. dprintk(CVP_WARN, "%s: Iris is busy\n", __func__);
  1049. rc = -EBUSY;
  1050. }
  1051. mutex_unlock(&device->lock);
  1052. /* Cancel pending delayed works if any */
  1053. if (!rc)
  1054. cancel_delayed_work(&iris_hfi_pm_work);
  1055. return rc;
  1056. }
  1057. static void cvp_dump_csr(struct iris_hfi_device *dev)
  1058. {
  1059. u32 reg;
  1060. if (!dev)
  1061. return;
  1062. if (!dev->power_enabled || dev->reg_dumped)
  1063. return;
  1064. reg = __read_register(dev, CVP_WRAPPER_CPU_STATUS);
  1065. dprintk(CVP_ERR, "CVP_WRAPPER_CPU_STATUS: %x\n", reg);
  1066. reg = __read_register(dev, CVP_CPU_CS_SCIACMDARG0);
  1067. dprintk(CVP_ERR, "CVP_CPU_CS_SCIACMDARG0: %x\n", reg);
  1068. reg = __read_register(dev, CVP_WRAPPER_INTR_STATUS);
  1069. dprintk(CVP_ERR, "CVP_WRAPPER_INTR_STATUS: %x\n", reg);
  1070. reg = __read_register(dev, CVP_CPU_CS_H2ASOFTINT);
  1071. dprintk(CVP_ERR, "CVP_CPU_CS_H2ASOFTINT: %x\n", reg);
  1072. reg = __read_register(dev, CVP_CPU_CS_A2HSOFTINT);
  1073. dprintk(CVP_ERR, "CVP_CPU_CS_A2HSOFTINT: %x\n", reg);
  1074. reg = __read_register(dev, CVP_CC_MVS1C_GDSCR);
  1075. dprintk(CVP_ERR, "CVP_CC_MVS1C_GDSCR: %x\n", reg);
  1076. reg = __read_register(dev, CVP_CC_MVS1C_CBCR);
  1077. dprintk(CVP_ERR, "CVP_CC_MVS1C_CBCR: %x\n", reg);
  1078. reg = __read_register(dev, CVP_WRAPPER_CPU_CLOCK_CONFIG);
  1079. dprintk(CVP_ERR, "CVP_WRAPPER_CPU_CLOCK_CONFIG: %x\n", reg);
  1080. reg = __read_register(dev, CVP_WRAPPER_CORE_CLOCK_CONFIG);
  1081. dprintk(CVP_ERR, "CVP_WRAPPER_CORE_CLOCK_CONFIG: %x\n", reg);
  1082. dev->reg_dumped = true;
  1083. }
  1084. static int iris_hfi_flush_debug_queue(void *dev)
  1085. {
  1086. int rc = 0;
  1087. struct iris_hfi_device *device = (struct iris_hfi_device *) dev;
  1088. if (!device) {
  1089. dprintk(CVP_ERR, "%s invalid device\n", __func__);
  1090. return -EINVAL;
  1091. }
  1092. mutex_lock(&device->lock);
  1093. if (!device->power_enabled) {
  1094. dprintk(CVP_WARN, "%s: iris power off\n", __func__);
  1095. rc = -EINVAL;
  1096. goto exit;
  1097. }
  1098. cvp_dump_csr(device);
  1099. __flush_debug_queue(device, NULL);
  1100. exit:
  1101. mutex_unlock(&device->lock);
  1102. return rc;
  1103. }
  1104. static int iris_hfi_scale_clocks(void *dev, u32 freq)
  1105. {
  1106. int rc = 0;
  1107. struct iris_hfi_device *device = dev;
  1108. if (!device) {
  1109. dprintk(CVP_ERR, "Invalid args: %pK\n", device);
  1110. return -EINVAL;
  1111. }
  1112. mutex_lock(&device->lock);
  1113. if (__resume(device)) {
  1114. dprintk(CVP_ERR, "Resume from power collapse failed\n");
  1115. rc = -ENODEV;
  1116. goto exit;
  1117. }
  1118. rc = msm_cvp_set_clocks_impl(device, freq);
  1119. exit:
  1120. mutex_unlock(&device->lock);
  1121. return rc;
  1122. }
  1123. /* Writes into cmdq without raising an interrupt */
  1124. static int __iface_cmdq_write_relaxed(struct iris_hfi_device *device,
  1125. void *pkt, bool *requires_interrupt)
  1126. {
  1127. struct cvp_iface_q_info *q_info;
  1128. struct cvp_hal_cmd_pkt_hdr *cmd_packet;
  1129. int result = -E2BIG;
  1130. if (!device || !pkt) {
  1131. dprintk(CVP_ERR, "Invalid Params\n");
  1132. return -EINVAL;
  1133. }
  1134. __strict_check(device);
  1135. if (!__core_in_valid_state(device)) {
  1136. dprintk(CVP_ERR, "%s - fw not in init state\n", __func__);
  1137. result = -EINVAL;
  1138. goto err_q_null;
  1139. }
  1140. cmd_packet = (struct cvp_hal_cmd_pkt_hdr *)pkt;
  1141. device->last_packet_type = cmd_packet->packet_type;
  1142. q_info = &device->iface_queues[CVP_IFACEQ_CMDQ_IDX];
  1143. if (!q_info) {
  1144. dprintk(CVP_ERR, "cannot write to shared Q's\n");
  1145. goto err_q_null;
  1146. }
  1147. if (!q_info->q_array.align_virtual_addr) {
  1148. dprintk(CVP_ERR, "cannot write to shared CMD Q's\n");
  1149. result = -ENODATA;
  1150. goto err_q_null;
  1151. }
  1152. if (__resume(device)) {
  1153. dprintk(CVP_ERR, "%s: Power on failed\n", __func__);
  1154. goto err_q_write;
  1155. }
  1156. if (!__write_queue(q_info, (u8 *)pkt, requires_interrupt)) {
  1157. if (device->res->sw_power_collapsible) {
  1158. cancel_delayed_work(&iris_hfi_pm_work);
  1159. if (!queue_delayed_work(device->iris_pm_workq,
  1160. &iris_hfi_pm_work,
  1161. msecs_to_jiffies(
  1162. device->res->msm_cvp_pwr_collapse_delay))) {
  1163. dprintk(CVP_PWR,
  1164. "PM work already scheduled\n");
  1165. }
  1166. }
  1167. result = 0;
  1168. } else {
  1169. dprintk(CVP_ERR, "__iface_cmdq_write: queue full\n");
  1170. }
  1171. err_q_write:
  1172. err_q_null:
  1173. return result;
  1174. }
  1175. static int __iface_cmdq_write(struct iris_hfi_device *device, void *pkt)
  1176. {
  1177. bool needs_interrupt = false;
  1178. int rc = __iface_cmdq_write_relaxed(device, pkt, &needs_interrupt);
  1179. if (!rc && needs_interrupt) {
  1180. /* Consumer of cmdq prefers that we raise an interrupt */
  1181. rc = 0;
  1182. __write_register(device, CVP_CPU_CS_H2ASOFTINT, 1);
  1183. }
  1184. return rc;
  1185. }
  1186. static int __iface_msgq_read(struct iris_hfi_device *device, void *pkt)
  1187. {
  1188. u32 tx_req_is_set = 0;
  1189. int rc = 0;
  1190. struct cvp_iface_q_info *q_info;
  1191. if (!pkt) {
  1192. dprintk(CVP_ERR, "Invalid Params\n");
  1193. return -EINVAL;
  1194. }
  1195. __strict_check(device);
  1196. if (!__core_in_valid_state(device)) {
  1197. dprintk(CVP_WARN, "%s - fw not in init state\n", __func__);
  1198. rc = -EINVAL;
  1199. goto read_error_null;
  1200. }
  1201. q_info = &device->iface_queues[CVP_IFACEQ_MSGQ_IDX];
  1202. if (q_info->q_array.align_virtual_addr == NULL) {
  1203. dprintk(CVP_ERR, "cannot read from shared MSG Q's\n");
  1204. rc = -ENODATA;
  1205. goto read_error_null;
  1206. }
  1207. if (!__read_queue(q_info, (u8 *)pkt, &tx_req_is_set)) {
  1208. if (tx_req_is_set)
  1209. __write_register(device, CVP_CPU_CS_H2ASOFTINT, 1);
  1210. rc = 0;
  1211. } else
  1212. rc = -ENODATA;
  1213. read_error_null:
  1214. return rc;
  1215. }
  1216. static int __iface_dbgq_read(struct iris_hfi_device *device, void *pkt)
  1217. {
  1218. u32 tx_req_is_set = 0;
  1219. int rc = 0;
  1220. struct cvp_iface_q_info *q_info;
  1221. if (!pkt) {
  1222. dprintk(CVP_ERR, "Invalid Params\n");
  1223. return -EINVAL;
  1224. }
  1225. __strict_check(device);
  1226. q_info = &device->iface_queues[CVP_IFACEQ_DBGQ_IDX];
  1227. if (q_info->q_array.align_virtual_addr == NULL) {
  1228. dprintk(CVP_ERR, "cannot read from shared DBG Q's\n");
  1229. rc = -ENODATA;
  1230. goto dbg_error_null;
  1231. }
  1232. if (!__read_queue(q_info, (u8 *)pkt, &tx_req_is_set)) {
  1233. if (tx_req_is_set)
  1234. __write_register(device, CVP_CPU_CS_H2ASOFTINT, 1);
  1235. rc = 0;
  1236. } else
  1237. rc = -ENODATA;
  1238. dbg_error_null:
  1239. return rc;
  1240. }
  1241. static void __set_queue_hdr_defaults(struct cvp_hfi_queue_header *q_hdr)
  1242. {
  1243. q_hdr->qhdr_status = 0x1;
  1244. q_hdr->qhdr_type = CVP_IFACEQ_DFLT_QHDR;
  1245. q_hdr->qhdr_q_size = CVP_IFACEQ_QUEUE_SIZE / 4;
  1246. q_hdr->qhdr_pkt_size = 0;
  1247. q_hdr->qhdr_rx_wm = 0x1;
  1248. q_hdr->qhdr_tx_wm = 0x1;
  1249. q_hdr->qhdr_rx_req = 0x1;
  1250. q_hdr->qhdr_tx_req = 0x0;
  1251. q_hdr->qhdr_rx_irq_status = 0x0;
  1252. q_hdr->qhdr_tx_irq_status = 0x0;
  1253. q_hdr->qhdr_read_idx = 0x0;
  1254. q_hdr->qhdr_write_idx = 0x0;
  1255. }
  1256. /*
  1257. *Unused, keep for reference
  1258. */
  1259. /*
  1260. static void __interface_dsp_queues_release(struct iris_hfi_device *device)
  1261. {
  1262. int i;
  1263. struct msm_cvp_smem *mem_data = &device->dsp_iface_q_table.mem_data;
  1264. struct context_bank_info *cb = mem_data->mapping_info.cb_info;
  1265. if (!device->dsp_iface_q_table.align_virtual_addr) {
  1266. dprintk(CVP_ERR, "%s: already released\n", __func__);
  1267. return;
  1268. }
  1269. dma_unmap_single_attrs(cb->dev, mem_data->device_addr,
  1270. mem_data->size, DMA_BIDIRECTIONAL, 0);
  1271. dma_free_coherent(device->res->mem_cdsp.dev, mem_data->size,
  1272. mem_data->kvaddr, mem_data->dma_handle);
  1273. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1274. device->dsp_iface_queues[i].q_hdr = NULL;
  1275. device->dsp_iface_queues[i].q_array.align_virtual_addr = NULL;
  1276. device->dsp_iface_queues[i].q_array.align_device_addr = 0;
  1277. }
  1278. device->dsp_iface_q_table.align_virtual_addr = NULL;
  1279. device->dsp_iface_q_table.align_device_addr = 0;
  1280. }
  1281. */
  1282. static int __interface_dsp_queues_init(struct iris_hfi_device *dev)
  1283. {
  1284. int rc = 0;
  1285. u32 i;
  1286. struct cvp_iface_q_info *iface_q;
  1287. int offset = 0;
  1288. phys_addr_t fw_bias = 0;
  1289. size_t q_size;
  1290. struct msm_cvp_smem *mem_data;
  1291. void *kvaddr;
  1292. dma_addr_t dma_handle;
  1293. dma_addr_t iova;
  1294. struct context_bank_info *cb;
  1295. q_size = ALIGN(QUEUE_SIZE, SZ_1M);
  1296. mem_data = &dev->dsp_iface_q_table.mem_data;
  1297. if (mem_data->kvaddr) {
  1298. memset((void *)mem_data->kvaddr, 0, q_size);
  1299. cvp_dsp_init_hfi_queue_hdr(dev);
  1300. return 0;
  1301. }
  1302. /* Allocate dsp queues from CDSP device memory */
  1303. kvaddr = dma_alloc_coherent(dev->res->mem_cdsp.dev, q_size,
  1304. &dma_handle, GFP_KERNEL);
  1305. if (IS_ERR_OR_NULL(kvaddr)) {
  1306. dprintk(CVP_ERR, "%s: failed dma allocation\n", __func__);
  1307. goto fail_dma_alloc;
  1308. }
  1309. cb = msm_cvp_smem_get_context_bank(dev->res, 0);
  1310. if (!cb) {
  1311. dprintk(CVP_ERR,
  1312. "%s: failed to get context bank\n", __func__);
  1313. goto fail_dma_map;
  1314. }
  1315. iova = dma_map_single_attrs(cb->dev, phys_to_virt(dma_handle),
  1316. q_size, DMA_BIDIRECTIONAL, 0);
  1317. if (dma_mapping_error(cb->dev, iova)) {
  1318. dprintk(CVP_ERR, "%s: failed dma mapping\n", __func__);
  1319. goto fail_dma_map;
  1320. }
  1321. dprintk(CVP_DSP,
  1322. "%s: kvaddr %pK dma_handle %#llx iova %#llx size %zd\n",
  1323. __func__, kvaddr, dma_handle, iova, q_size);
  1324. memset(mem_data, 0, sizeof(struct msm_cvp_smem));
  1325. mem_data->kvaddr = kvaddr;
  1326. mem_data->device_addr = iova;
  1327. mem_data->dma_handle = dma_handle;
  1328. mem_data->size = q_size;
  1329. mem_data->mapping_info.cb_info = cb;
  1330. if (!is_iommu_present(dev->res))
  1331. fw_bias = dev->cvp_hal_data->firmware_base;
  1332. dev->dsp_iface_q_table.align_virtual_addr = kvaddr;
  1333. dev->dsp_iface_q_table.align_device_addr = iova - fw_bias;
  1334. dev->dsp_iface_q_table.mem_size = CVP_IFACEQ_TABLE_SIZE;
  1335. offset = dev->dsp_iface_q_table.mem_size;
  1336. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1337. iface_q = &dev->dsp_iface_queues[i];
  1338. iface_q->q_array.align_device_addr = iova + offset - fw_bias;
  1339. iface_q->q_array.align_virtual_addr = kvaddr + offset;
  1340. iface_q->q_array.mem_size = CVP_IFACEQ_QUEUE_SIZE;
  1341. offset += iface_q->q_array.mem_size;
  1342. spin_lock_init(&iface_q->hfi_lock);
  1343. }
  1344. cvp_dsp_init_hfi_queue_hdr(dev);
  1345. return rc;
  1346. fail_dma_map:
  1347. dma_free_coherent(dev->res->mem_cdsp.dev, q_size, kvaddr, dma_handle);
  1348. fail_dma_alloc:
  1349. return -ENOMEM;
  1350. }
  1351. static void __interface_queues_release(struct iris_hfi_device *device)
  1352. {
  1353. #ifdef CONFIG_EVA_TVM
  1354. int i;
  1355. struct cvp_hfi_mem_map_table *qdss;
  1356. struct cvp_hfi_mem_map *mem_map;
  1357. int num_entries = device->res->qdss_addr_set.count;
  1358. unsigned long mem_map_table_base_addr;
  1359. struct context_bank_info *cb;
  1360. if (device->qdss.align_virtual_addr) {
  1361. qdss = (struct cvp_hfi_mem_map_table *)
  1362. device->qdss.align_virtual_addr;
  1363. qdss->mem_map_num_entries = num_entries;
  1364. mem_map_table_base_addr =
  1365. device->qdss.align_device_addr +
  1366. sizeof(struct cvp_hfi_mem_map_table);
  1367. qdss->mem_map_table_base_addr =
  1368. (u32)mem_map_table_base_addr;
  1369. if ((unsigned long)qdss->mem_map_table_base_addr !=
  1370. mem_map_table_base_addr) {
  1371. dprintk(CVP_ERR,
  1372. "Invalid mem_map_table_base_addr %#lx",
  1373. mem_map_table_base_addr);
  1374. }
  1375. mem_map = (struct cvp_hfi_mem_map *)(qdss + 1);
  1376. cb = msm_cvp_smem_get_context_bank(device->res, 0);
  1377. for (i = 0; cb && i < num_entries; i++) {
  1378. iommu_unmap(cb->domain,
  1379. mem_map[i].virtual_addr,
  1380. mem_map[i].size);
  1381. }
  1382. __smem_free(device, &device->qdss.mem_data);
  1383. }
  1384. __smem_free(device, &device->iface_q_table.mem_data);
  1385. __smem_free(device, &device->sfr.mem_data);
  1386. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1387. device->iface_queues[i].q_hdr = NULL;
  1388. device->iface_queues[i].q_array.align_virtual_addr = NULL;
  1389. device->iface_queues[i].q_array.align_device_addr = 0;
  1390. }
  1391. device->iface_q_table.align_virtual_addr = NULL;
  1392. device->iface_q_table.align_device_addr = 0;
  1393. device->qdss.align_virtual_addr = NULL;
  1394. device->qdss.align_device_addr = 0;
  1395. device->sfr.align_virtual_addr = NULL;
  1396. device->sfr.align_device_addr = 0;
  1397. device->mem_addr.align_virtual_addr = NULL;
  1398. device->mem_addr.align_device_addr = 0;
  1399. #endif
  1400. }
  1401. static int __get_qdss_iommu_virtual_addr(struct iris_hfi_device *dev,
  1402. struct cvp_hfi_mem_map *mem_map,
  1403. struct iommu_domain *domain)
  1404. {
  1405. int i;
  1406. int rc = 0;
  1407. dma_addr_t iova = QDSS_IOVA_START;
  1408. int num_entries = dev->res->qdss_addr_set.count;
  1409. struct addr_range *qdss_addr_tbl = dev->res->qdss_addr_set.addr_tbl;
  1410. if (!num_entries)
  1411. return -ENODATA;
  1412. for (i = 0; i < num_entries; i++) {
  1413. if (domain) {
  1414. rc = iommu_map(domain, iova,
  1415. qdss_addr_tbl[i].start,
  1416. qdss_addr_tbl[i].size,
  1417. IOMMU_READ | IOMMU_WRITE);
  1418. if (rc) {
  1419. dprintk(CVP_ERR,
  1420. "IOMMU QDSS mapping failed for addr %#x\n",
  1421. qdss_addr_tbl[i].start);
  1422. rc = -ENOMEM;
  1423. break;
  1424. }
  1425. } else {
  1426. iova = qdss_addr_tbl[i].start;
  1427. }
  1428. mem_map[i].virtual_addr = (u32)iova;
  1429. mem_map[i].physical_addr = qdss_addr_tbl[i].start;
  1430. mem_map[i].size = qdss_addr_tbl[i].size;
  1431. mem_map[i].attr = 0x0;
  1432. iova += mem_map[i].size;
  1433. }
  1434. if (i < num_entries) {
  1435. dprintk(CVP_ERR,
  1436. "QDSS mapping failed, Freeing other entries %d\n", i);
  1437. for (--i; domain && i >= 0; i--) {
  1438. iommu_unmap(domain,
  1439. mem_map[i].virtual_addr,
  1440. mem_map[i].size);
  1441. }
  1442. }
  1443. return rc;
  1444. }
  1445. static void __setup_ucregion_memory_map(struct iris_hfi_device *device)
  1446. {
  1447. __write_register(device, CVP_UC_REGION_ADDR,
  1448. (u32)device->iface_q_table.align_device_addr);
  1449. __write_register(device, CVP_UC_REGION_SIZE, SHARED_QSIZE);
  1450. __write_register(device, CVP_QTBL_ADDR,
  1451. (u32)device->iface_q_table.align_device_addr);
  1452. __write_register(device, CVP_QTBL_INFO, 0x01);
  1453. if (device->sfr.align_device_addr)
  1454. __write_register(device, CVP_SFR_ADDR,
  1455. (u32)device->sfr.align_device_addr);
  1456. if (device->qdss.align_device_addr)
  1457. __write_register(device, CVP_MMAP_ADDR,
  1458. (u32)device->qdss.align_device_addr);
  1459. call_iris_op(device, setup_dsp_uc_memmap, device);
  1460. }
  1461. static void __hfi_queue_init(struct iris_hfi_device *dev)
  1462. {
  1463. int i, offset = 0;
  1464. struct cvp_hfi_queue_table_header *q_tbl_hdr;
  1465. struct cvp_iface_q_info *iface_q;
  1466. struct cvp_hfi_queue_header *q_hdr;
  1467. if (!dev)
  1468. return;
  1469. offset += dev->iface_q_table.mem_size;
  1470. for (i = 0; i < CVP_IFACEQ_NUMQ; i++) {
  1471. iface_q = &dev->iface_queues[i];
  1472. iface_q->q_array.align_device_addr =
  1473. dev->iface_q_table.align_device_addr + offset;
  1474. iface_q->q_array.align_virtual_addr =
  1475. dev->iface_q_table.align_virtual_addr + offset;
  1476. iface_q->q_array.mem_size = CVP_IFACEQ_QUEUE_SIZE;
  1477. offset += iface_q->q_array.mem_size;
  1478. iface_q->q_hdr = CVP_IFACEQ_GET_QHDR_START_ADDR(
  1479. dev->iface_q_table.align_virtual_addr, i);
  1480. __set_queue_hdr_defaults(iface_q->q_hdr);
  1481. spin_lock_init(&iface_q->hfi_lock);
  1482. }
  1483. q_tbl_hdr = (struct cvp_hfi_queue_table_header *)
  1484. dev->iface_q_table.align_virtual_addr;
  1485. q_tbl_hdr->qtbl_version = 0;
  1486. q_tbl_hdr->device_addr = (void *)dev;
  1487. strlcpy(q_tbl_hdr->name, "msm_cvp", sizeof(q_tbl_hdr->name));
  1488. q_tbl_hdr->qtbl_size = CVP_IFACEQ_TABLE_SIZE;
  1489. q_tbl_hdr->qtbl_qhdr0_offset =
  1490. sizeof(struct cvp_hfi_queue_table_header);
  1491. q_tbl_hdr->qtbl_qhdr_size = sizeof(struct cvp_hfi_queue_header);
  1492. q_tbl_hdr->qtbl_num_q = CVP_IFACEQ_NUMQ;
  1493. q_tbl_hdr->qtbl_num_active_q = CVP_IFACEQ_NUMQ;
  1494. iface_q = &dev->iface_queues[CVP_IFACEQ_CMDQ_IDX];
  1495. q_hdr = iface_q->q_hdr;
  1496. q_hdr->qhdr_start_addr = iface_q->q_array.align_device_addr;
  1497. q_hdr->qhdr_type |= HFI_Q_ID_HOST_TO_CTRL_CMD_Q;
  1498. iface_q = &dev->iface_queues[CVP_IFACEQ_MSGQ_IDX];
  1499. q_hdr = iface_q->q_hdr;
  1500. q_hdr->qhdr_start_addr = iface_q->q_array.align_device_addr;
  1501. q_hdr->qhdr_type |= HFI_Q_ID_CTRL_TO_HOST_MSG_Q;
  1502. iface_q = &dev->iface_queues[CVP_IFACEQ_DBGQ_IDX];
  1503. q_hdr = iface_q->q_hdr;
  1504. q_hdr->qhdr_start_addr = iface_q->q_array.align_device_addr;
  1505. q_hdr->qhdr_type |= HFI_Q_ID_CTRL_TO_HOST_DEBUG_Q;
  1506. /*
  1507. * Set receive request to zero on debug queue as there is no
  1508. * need of interrupt from cvp hardware for debug messages
  1509. */
  1510. q_hdr->qhdr_rx_req = 0;
  1511. }
  1512. static void __sfr_init(struct iris_hfi_device *dev)
  1513. {
  1514. struct cvp_hfi_sfr_struct *vsfr;
  1515. if (!dev)
  1516. return;
  1517. vsfr = (struct cvp_hfi_sfr_struct *) dev->sfr.align_virtual_addr;
  1518. if (vsfr)
  1519. vsfr->bufSize = ALIGNED_SFR_SIZE;
  1520. }
  1521. static int __interface_queues_init(struct iris_hfi_device *dev)
  1522. {
  1523. int rc = 0;
  1524. struct cvp_hfi_mem_map_table *qdss;
  1525. struct cvp_hfi_mem_map *mem_map;
  1526. struct cvp_mem_addr *mem_addr;
  1527. int num_entries = dev->res->qdss_addr_set.count;
  1528. phys_addr_t fw_bias = 0;
  1529. size_t q_size;
  1530. unsigned long mem_map_table_base_addr;
  1531. struct context_bank_info *cb;
  1532. q_size = SHARED_QSIZE - ALIGNED_SFR_SIZE - ALIGNED_QDSS_SIZE;
  1533. mem_addr = &dev->mem_addr;
  1534. if (!is_iommu_present(dev->res))
  1535. fw_bias = dev->cvp_hal_data->firmware_base;
  1536. if (dev->iface_q_table.align_virtual_addr) {
  1537. memset((void *)dev->iface_q_table.align_virtual_addr,
  1538. 0, q_size);
  1539. goto hfi_queue_init;
  1540. }
  1541. rc = __smem_alloc(dev, mem_addr, q_size, 1, SMEM_UNCACHED);
  1542. if (rc) {
  1543. dprintk(CVP_ERR, "iface_q_table_alloc_fail\n");
  1544. goto fail_alloc_queue;
  1545. }
  1546. dev->iface_q_table.align_virtual_addr = mem_addr->align_virtual_addr;
  1547. dev->iface_q_table.align_device_addr = mem_addr->align_device_addr -
  1548. fw_bias;
  1549. dev->iface_q_table.mem_size = CVP_IFACEQ_TABLE_SIZE;
  1550. dev->iface_q_table.mem_data = mem_addr->mem_data;
  1551. hfi_queue_init:
  1552. __hfi_queue_init(dev);
  1553. if (dev->sfr.align_virtual_addr) {
  1554. memset((void *)dev->sfr.align_virtual_addr,
  1555. 0, ALIGNED_SFR_SIZE);
  1556. goto sfr_init;
  1557. }
  1558. rc = __smem_alloc(dev, mem_addr, ALIGNED_SFR_SIZE, 1, SMEM_UNCACHED);
  1559. if (rc) {
  1560. dprintk(CVP_WARN, "sfr_alloc_fail: SFR not will work\n");
  1561. dev->sfr.align_device_addr = 0;
  1562. } else {
  1563. dev->sfr.align_device_addr = mem_addr->align_device_addr -
  1564. fw_bias;
  1565. dev->sfr.align_virtual_addr = mem_addr->align_virtual_addr;
  1566. dev->sfr.mem_size = ALIGNED_SFR_SIZE;
  1567. dev->sfr.mem_data = mem_addr->mem_data;
  1568. }
  1569. sfr_init:
  1570. __sfr_init(dev);
  1571. if (dev->qdss.align_virtual_addr)
  1572. goto dsp_hfi_queue_init;
  1573. if ((msm_cvp_fw_debug_mode & HFI_DEBUG_MODE_QDSS) && num_entries) {
  1574. rc = __smem_alloc(dev, mem_addr, ALIGNED_QDSS_SIZE, 1,
  1575. SMEM_UNCACHED);
  1576. if (rc) {
  1577. dprintk(CVP_WARN,
  1578. "qdss_alloc_fail: QDSS messages logging will not work\n");
  1579. dev->qdss.align_device_addr = 0;
  1580. } else {
  1581. dev->qdss.align_device_addr =
  1582. mem_addr->align_device_addr - fw_bias;
  1583. dev->qdss.align_virtual_addr =
  1584. mem_addr->align_virtual_addr;
  1585. dev->qdss.mem_size = ALIGNED_QDSS_SIZE;
  1586. dev->qdss.mem_data = mem_addr->mem_data;
  1587. }
  1588. }
  1589. if (dev->qdss.align_virtual_addr) {
  1590. qdss =
  1591. (struct cvp_hfi_mem_map_table *)dev->qdss.align_virtual_addr;
  1592. qdss->mem_map_num_entries = num_entries;
  1593. mem_map_table_base_addr = dev->qdss.align_device_addr +
  1594. sizeof(struct cvp_hfi_mem_map_table);
  1595. qdss->mem_map_table_base_addr = mem_map_table_base_addr;
  1596. mem_map = (struct cvp_hfi_mem_map *)(qdss + 1);
  1597. cb = msm_cvp_smem_get_context_bank(dev->res, 0);
  1598. if (!cb) {
  1599. dprintk(CVP_ERR,
  1600. "%s: failed to get context bank\n", __func__);
  1601. return -EINVAL;
  1602. }
  1603. rc = __get_qdss_iommu_virtual_addr(dev, mem_map, cb->domain);
  1604. if (rc) {
  1605. dprintk(CVP_ERR,
  1606. "IOMMU mapping failed, Freeing qdss memdata\n");
  1607. __smem_free(dev, &dev->qdss.mem_data);
  1608. dev->qdss.align_virtual_addr = NULL;
  1609. dev->qdss.align_device_addr = 0;
  1610. }
  1611. }
  1612. dsp_hfi_queue_init:
  1613. rc = __interface_dsp_queues_init(dev);
  1614. if (rc) {
  1615. dprintk(CVP_ERR, "dsp_queues_init failed\n");
  1616. goto fail_alloc_queue;
  1617. }
  1618. __setup_ucregion_memory_map(dev);
  1619. return 0;
  1620. fail_alloc_queue:
  1621. return -ENOMEM;
  1622. }
  1623. static int __sys_set_debug(struct iris_hfi_device *device, u32 debug)
  1624. {
  1625. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1626. int rc = 0;
  1627. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1628. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1629. rc = call_hfi_pkt_op(device, sys_debug_config, pkt, debug);
  1630. if (rc) {
  1631. dprintk(CVP_WARN,
  1632. "Debug mode setting to FW failed\n");
  1633. return -ENOTEMPTY;
  1634. }
  1635. if (__iface_cmdq_write(device, pkt))
  1636. return -ENOTEMPTY;
  1637. return 0;
  1638. }
  1639. static int __sys_set_idle_indicator(struct iris_hfi_device *device,
  1640. bool enable)
  1641. {
  1642. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1643. int rc = 0;
  1644. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1645. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1646. rc = call_hfi_pkt_op(device, sys_set_idle_indicator, pkt, enable);
  1647. if (__iface_cmdq_write(device, pkt))
  1648. return -ENOTEMPTY;
  1649. return 0;
  1650. }
  1651. static int __sys_set_coverage(struct iris_hfi_device *device, u32 mode)
  1652. {
  1653. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1654. int rc = 0;
  1655. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1656. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1657. rc = call_hfi_pkt_op(device, sys_coverage_config,
  1658. pkt, mode);
  1659. if (rc) {
  1660. dprintk(CVP_WARN,
  1661. "Coverage mode setting to FW failed\n");
  1662. return -ENOTEMPTY;
  1663. }
  1664. if (__iface_cmdq_write(device, pkt)) {
  1665. dprintk(CVP_WARN, "Failed to send coverage pkt to f/w\n");
  1666. return -ENOTEMPTY;
  1667. }
  1668. return 0;
  1669. }
  1670. static int __sys_set_power_control(struct iris_hfi_device *device,
  1671. bool enable)
  1672. {
  1673. struct regulator_info *rinfo;
  1674. bool supported = false;
  1675. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  1676. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  1677. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  1678. iris_hfi_for_each_regulator(device, rinfo) {
  1679. if (rinfo->has_hw_power_collapse) {
  1680. supported = true;
  1681. break;
  1682. }
  1683. }
  1684. if (!supported)
  1685. return 0;
  1686. call_hfi_pkt_op(device, sys_power_control, pkt, enable);
  1687. if (__iface_cmdq_write(device, pkt))
  1688. return -ENOTEMPTY;
  1689. return 0;
  1690. }
  1691. static void cvp_pm_qos_update(struct iris_hfi_device *device, bool vote_on)
  1692. {
  1693. u32 latency, off_vote_cnt;
  1694. int i, err = 0;
  1695. spin_lock(&device->res->pm_qos.lock);
  1696. off_vote_cnt = device->res->pm_qos.off_vote_cnt;
  1697. spin_unlock(&device->res->pm_qos.lock);
  1698. if (vote_on && off_vote_cnt)
  1699. return;
  1700. latency = vote_on ? device->res->pm_qos.latency_us :
  1701. PM_QOS_RESUME_LATENCY_DEFAULT_VALUE;
  1702. if (device->res->pm_qos.latency_us && device->res->pm_qos.pm_qos_hdls)
  1703. for (i = 0; i < device->res->pm_qos.silver_count; i++) {
  1704. err = dev_pm_qos_update_request(
  1705. &device->res->pm_qos.pm_qos_hdls[i],
  1706. latency);
  1707. if (err < 0) {
  1708. if (vote_on) {
  1709. dprintk(CVP_WARN,
  1710. "pm qos on failed %d\n", err);
  1711. } else {
  1712. dprintk(CVP_WARN,
  1713. "pm qos off failed %d\n", err);
  1714. }
  1715. }
  1716. }
  1717. }
  1718. static int iris_pm_qos_update(void *device)
  1719. {
  1720. struct iris_hfi_device *dev;
  1721. if (!device) {
  1722. dprintk(CVP_ERR, "%s Invalid device\n", __func__);
  1723. return -ENODEV;
  1724. }
  1725. dev = device;
  1726. mutex_lock(&dev->lock);
  1727. cvp_pm_qos_update(dev, true);
  1728. mutex_unlock(&dev->lock);
  1729. return 0;
  1730. }
  1731. static int __hwfence_regs_map(struct iris_hfi_device *device)
  1732. {
  1733. int rc = 0;
  1734. struct context_bank_info *cb;
  1735. cb = msm_cvp_smem_get_context_bank(device->res, 0);
  1736. if (!cb) {
  1737. dprintk(CVP_ERR, "%s: fail to get cb\n", __func__);
  1738. return -EINVAL;
  1739. }
  1740. if (device->res->reg_mappings.ipclite_phyaddr != 0) {
  1741. rc = iommu_map(cb->domain,
  1742. device->res->reg_mappings.ipclite_iova,
  1743. device->res->reg_mappings.ipclite_phyaddr,
  1744. device->res->reg_mappings.ipclite_size,
  1745. IOMMU_READ | IOMMU_WRITE);
  1746. if (rc) {
  1747. dprintk(CVP_ERR, "map ipclite fail %d %#x %#x %#x\n",
  1748. rc, device->res->reg_mappings.ipclite_iova,
  1749. device->res->reg_mappings.ipclite_phyaddr,
  1750. device->res->reg_mappings.ipclite_size);
  1751. return rc;
  1752. }
  1753. }
  1754. if (device->res->reg_mappings.hwmutex_phyaddr != 0) {
  1755. rc = iommu_map(cb->domain,
  1756. device->res->reg_mappings.hwmutex_iova,
  1757. device->res->reg_mappings.hwmutex_phyaddr,
  1758. device->res->reg_mappings.hwmutex_size,
  1759. IOMMU_MMIO | IOMMU_READ | IOMMU_WRITE);
  1760. if (rc) {
  1761. dprintk(CVP_ERR, "map hwmutex fail %d %#x %#x %#x\n",
  1762. rc, device->res->reg_mappings.hwmutex_iova,
  1763. device->res->reg_mappings.hwmutex_phyaddr,
  1764. device->res->reg_mappings.hwmutex_size);
  1765. return rc;
  1766. }
  1767. }
  1768. if (device->res->reg_mappings.aon_phyaddr != 0) {
  1769. rc = iommu_map(cb->domain,
  1770. device->res->reg_mappings.aon_iova,
  1771. device->res->reg_mappings.aon_phyaddr,
  1772. device->res->reg_mappings.aon_size,
  1773. IOMMU_MMIO | IOMMU_READ | IOMMU_WRITE);
  1774. if (rc) {
  1775. dprintk(CVP_ERR, "map aon fail %d %#x %#x %#x\n",
  1776. rc, device->res->reg_mappings.aon_iova,
  1777. device->res->reg_mappings.aon_phyaddr,
  1778. device->res->reg_mappings.aon_size);
  1779. return rc;
  1780. }
  1781. }
  1782. if (device->res->reg_mappings.timer_phyaddr != 0) {
  1783. rc = iommu_map(cb->domain,
  1784. device->res->reg_mappings.timer_iova,
  1785. device->res->reg_mappings.timer_phyaddr,
  1786. device->res->reg_mappings.timer_size,
  1787. IOMMU_MMIO | IOMMU_READ | IOMMU_WRITE);
  1788. if (rc) {
  1789. dprintk(CVP_ERR, "map timer fail %d %#x %#x %#x\n",
  1790. rc, device->res->reg_mappings.timer_iova,
  1791. device->res->reg_mappings.timer_phyaddr,
  1792. device->res->reg_mappings.timer_size);
  1793. return rc;
  1794. }
  1795. }
  1796. return rc;
  1797. }
  1798. static int __hwfence_regs_unmap(struct iris_hfi_device *device)
  1799. {
  1800. int rc = 0;
  1801. struct context_bank_info *cb;
  1802. cb = msm_cvp_smem_get_context_bank(device->res, 0);
  1803. if (!cb) {
  1804. dprintk(CVP_ERR, "%s: fail to get cb\n", __func__);
  1805. return -EINVAL;
  1806. }
  1807. if (device->res->reg_mappings.ipclite_iova != 0) {
  1808. iommu_unmap(cb->domain,
  1809. device->res->reg_mappings.ipclite_iova,
  1810. device->res->reg_mappings.ipclite_size);
  1811. }
  1812. if (device->res->reg_mappings.hwmutex_iova != 0) {
  1813. iommu_unmap(cb->domain,
  1814. device->res->reg_mappings.hwmutex_iova,
  1815. device->res->reg_mappings.hwmutex_size);
  1816. }
  1817. if (device->res->reg_mappings.aon_iova != 0) {
  1818. iommu_unmap(cb->domain,
  1819. device->res->reg_mappings.aon_iova,
  1820. device->res->reg_mappings.aon_size);
  1821. }
  1822. if (device->res->reg_mappings.timer_iova != 0) {
  1823. iommu_unmap(cb->domain,
  1824. device->res->reg_mappings.timer_iova,
  1825. device->res->reg_mappings.timer_size);
  1826. }
  1827. return rc;
  1828. }
  1829. static int iris_hfi_core_init(void *device)
  1830. {
  1831. int rc = 0;
  1832. u32 ipcc_iova;
  1833. struct cvp_hfi_cmd_sys_init_packet pkt;
  1834. struct cvp_hfi_cmd_sys_get_property_packet version_pkt;
  1835. struct iris_hfi_device *dev;
  1836. if (!device) {
  1837. dprintk(CVP_ERR, "Invalid device\n");
  1838. return -ENODEV;
  1839. }
  1840. dev = device;
  1841. dprintk(CVP_CORE, "Core initializing\n");
  1842. pm_stay_awake(dev->res->pdev->dev.parent);
  1843. mutex_lock(&dev->lock);
  1844. dev->bus_vote.data =
  1845. kzalloc(sizeof(struct cvp_bus_vote_data), GFP_KERNEL);
  1846. if (!dev->bus_vote.data) {
  1847. dprintk(CVP_ERR, "Bus vote data memory is not allocated\n");
  1848. rc = -ENOMEM;
  1849. goto err_no_mem;
  1850. }
  1851. dev->bus_vote.data_count = 1;
  1852. dev->bus_vote.data->power_mode = CVP_POWER_TURBO;
  1853. __hwfence_regs_map(dev);
  1854. rc = __power_on_init(dev);
  1855. if (rc) {
  1856. dprintk(CVP_ERR, "Failed to power on init EVA\n");
  1857. goto err_load_fw;
  1858. }
  1859. rc = cvp_synx_recover();
  1860. if (rc) {
  1861. dprintk(CVP_ERR, "Failed to recover synx\n");
  1862. goto err_core_init;
  1863. }
  1864. /* mmrm registration */
  1865. if (msm_cvp_mmrm_enabled) {
  1866. rc = msm_cvp_mmrm_register(device);
  1867. if (rc) {
  1868. dprintk(CVP_ERR, "Failed to register mmrm client\n");
  1869. goto err_core_init;
  1870. }
  1871. }
  1872. __set_state(dev, IRIS_STATE_INIT);
  1873. dev->reg_dumped = false;
  1874. dprintk(CVP_CORE, "Dev_Virt: %pa, Reg_Virt: %pK\n",
  1875. &dev->cvp_hal_data->firmware_base,
  1876. dev->cvp_hal_data->register_base);
  1877. rc = __interface_queues_init(dev);
  1878. if (rc) {
  1879. dprintk(CVP_ERR, "failed to init queues\n");
  1880. rc = -ENOMEM;
  1881. goto err_core_init;
  1882. }
  1883. cvp_register_va_md_region();
  1884. // Add node for dev struct
  1885. add_va_node_to_list(CVP_QUEUE_DUMP, dev,
  1886. sizeof(struct iris_hfi_device),
  1887. "iris_hfi_device-dev", false);
  1888. add_queue_header_to_va_md_list((void*)dev);
  1889. add_hfi_queue_to_va_md_list((void*)dev);
  1890. rc = msm_cvp_map_ipcc_regs(&ipcc_iova);
  1891. if (!rc) {
  1892. dprintk(CVP_CORE, "IPCC iova 0x%x\n", ipcc_iova);
  1893. __write_register(dev, CVP_MMAP_ADDR, ipcc_iova);
  1894. }
  1895. rc = __load_fw(dev);
  1896. if (rc) {
  1897. dprintk(CVP_ERR, "Failed to load Iris FW\n");
  1898. goto err_core_init;
  1899. }
  1900. rc = __boot_firmware(dev);
  1901. if (rc) {
  1902. dprintk(CVP_ERR, "Failed to start core\n");
  1903. rc = -ENODEV;
  1904. goto err_core_init;
  1905. }
  1906. dev->version = __read_register(dev, CVP_VERSION_INFO);
  1907. rc = call_hfi_pkt_op(dev, sys_init, &pkt, 0);
  1908. if (rc) {
  1909. dprintk(CVP_ERR, "Failed to create sys init pkt\n");
  1910. goto err_core_init;
  1911. }
  1912. if (__iface_cmdq_write(dev, &pkt)) {
  1913. rc = -ENOTEMPTY;
  1914. goto err_core_init;
  1915. }
  1916. rc = call_hfi_pkt_op(dev, sys_image_version, &version_pkt);
  1917. if (rc || __iface_cmdq_write(dev, &version_pkt))
  1918. dprintk(CVP_WARN, "Failed to send image version pkt to f/w\n");
  1919. __sys_set_debug(device, msm_cvp_fw_debug);
  1920. __enable_subcaches(device);
  1921. __set_subcaches(device);
  1922. __set_ubwc_config(device);
  1923. __sys_set_idle_indicator(device, true);
  1924. if (dev->res->pm_qos.latency_us) {
  1925. int err = 0;
  1926. u32 i, cpu;
  1927. dev->res->pm_qos.pm_qos_hdls = kcalloc(
  1928. dev->res->pm_qos.silver_count,
  1929. sizeof(struct dev_pm_qos_request),
  1930. GFP_KERNEL);
  1931. if (!dev->res->pm_qos.pm_qos_hdls) {
  1932. dprintk(CVP_WARN, "Failed allocate pm_qos_hdls\n");
  1933. goto pm_qos_bail;
  1934. }
  1935. for (i = 0; i < dev->res->pm_qos.silver_count; i++) {
  1936. cpu = dev->res->pm_qos.silver_cores[i];
  1937. err = dev_pm_qos_add_request(
  1938. get_cpu_device(cpu),
  1939. &dev->res->pm_qos.pm_qos_hdls[i],
  1940. DEV_PM_QOS_RESUME_LATENCY,
  1941. dev->res->pm_qos.latency_us);
  1942. if (err < 0)
  1943. dprintk(CVP_WARN,
  1944. "%s pm_qos_add_req %d failed\n",
  1945. __func__, i);
  1946. }
  1947. }
  1948. pm_qos_bail:
  1949. mutex_unlock(&dev->lock);
  1950. cvp_dsp_send_hfi_queue();
  1951. pm_relax(dev->res->pdev->dev.parent);
  1952. dprintk(CVP_CORE, "Core inited successfully\n");
  1953. return 0;
  1954. err_core_init:
  1955. __set_state(dev, IRIS_STATE_DEINIT);
  1956. __unload_fw(dev);
  1957. if (dev->mmrm_cvp)
  1958. {
  1959. msm_cvp_mmrm_deregister(dev);
  1960. }
  1961. err_load_fw:
  1962. __hwfence_regs_unmap(dev);
  1963. err_no_mem:
  1964. dprintk(CVP_ERR, "Core init failed\n");
  1965. mutex_unlock(&dev->lock);
  1966. pm_relax(dev->res->pdev->dev.parent);
  1967. return rc;
  1968. }
  1969. static int iris_hfi_core_release(void *dev)
  1970. {
  1971. int rc = 0, i;
  1972. struct iris_hfi_device *device = dev;
  1973. struct cvp_hal_session *session, *next;
  1974. struct dev_pm_qos_request *qos_hdl;
  1975. if (!device) {
  1976. dprintk(CVP_ERR, "invalid device\n");
  1977. return -ENODEV;
  1978. }
  1979. mutex_lock(&device->lock);
  1980. dprintk(CVP_WARN, "Core releasing\n");
  1981. if (device->res->pm_qos.latency_us &&
  1982. device->res->pm_qos.pm_qos_hdls) {
  1983. for (i = 0; i < device->res->pm_qos.silver_count; i++) {
  1984. qos_hdl = &device->res->pm_qos.pm_qos_hdls[i];
  1985. if ((qos_hdl != NULL) && dev_pm_qos_request_active(qos_hdl))
  1986. dev_pm_qos_remove_request(qos_hdl);
  1987. }
  1988. kfree(device->res->pm_qos.pm_qos_hdls);
  1989. device->res->pm_qos.pm_qos_hdls = NULL;
  1990. }
  1991. __resume(device);
  1992. __set_state(device, IRIS_STATE_DEINIT);
  1993. rc = __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_SUSPEND);
  1994. if (rc)
  1995. dprintk(CVP_WARN, "Failed to suspend cvp FW%d\n", rc);
  1996. __dsp_shutdown(device);
  1997. __disable_subcaches(device);
  1998. __unload_fw(device);
  1999. __hwfence_regs_unmap(device);
  2000. if (msm_cvp_mmrm_enabled) {
  2001. rc = msm_cvp_mmrm_deregister(device);
  2002. if (rc) {
  2003. dprintk(CVP_ERR,
  2004. "%s: Failed msm_cvp_mmrm_deregister:%d\n",
  2005. __func__, rc);
  2006. }
  2007. }
  2008. /* unlink all sessions from device */
  2009. list_for_each_entry_safe(session, next, &device->sess_head, list) {
  2010. list_del(&session->list);
  2011. session->device = NULL;
  2012. }
  2013. dprintk(CVP_CORE, "Core released successfully\n");
  2014. mutex_unlock(&device->lock);
  2015. return rc;
  2016. }
  2017. static void __core_clear_interrupt(struct iris_hfi_device *device)
  2018. {
  2019. u32 intr_status = 0, mask = 0;
  2020. if (!device) {
  2021. dprintk(CVP_ERR, "%s: NULL device\n", __func__);
  2022. return;
  2023. }
  2024. intr_status = __read_register(device, CVP_WRAPPER_INTR_STATUS);
  2025. mask = (CVP_WRAPPER_INTR_MASK_A2HCPU_BMSK | CVP_FATAL_INTR_BMSK);
  2026. if (intr_status & mask) {
  2027. device->intr_status |= intr_status;
  2028. device->reg_count++;
  2029. dprintk(CVP_CORE,
  2030. "INTERRUPT for device: %pK: times: %d status: %d\n",
  2031. device, device->reg_count, intr_status);
  2032. } else {
  2033. device->spur_count++;
  2034. }
  2035. __write_register(device, CVP_CPU_CS_A2HSOFTINTCLR, 1);
  2036. }
  2037. static int iris_hfi_core_trigger_ssr(void *device,
  2038. enum hal_ssr_trigger_type type)
  2039. {
  2040. struct cvp_hfi_cmd_sys_test_ssr_packet pkt;
  2041. int rc = 0;
  2042. struct iris_hfi_device *dev;
  2043. cvp_free_va_md_list();
  2044. if (!device) {
  2045. dprintk(CVP_ERR, "invalid device\n");
  2046. return -ENODEV;
  2047. }
  2048. dev = device;
  2049. if (mutex_trylock(&dev->lock)) {
  2050. rc = call_hfi_pkt_op(dev, ssr_cmd, type, &pkt);
  2051. if (rc) {
  2052. dprintk(CVP_ERR, "%s: failed to create packet\n",
  2053. __func__);
  2054. goto err_create_pkt;
  2055. }
  2056. if (__iface_cmdq_write(dev, &pkt))
  2057. rc = -ENOTEMPTY;
  2058. } else {
  2059. return -EAGAIN;
  2060. }
  2061. err_create_pkt:
  2062. mutex_unlock(&dev->lock);
  2063. return rc;
  2064. }
  2065. static void __set_default_sys_properties(struct iris_hfi_device *device)
  2066. {
  2067. if (__sys_set_debug(device, msm_cvp_fw_debug))
  2068. dprintk(CVP_WARN, "Setting fw_debug msg ON failed\n");
  2069. if (__sys_set_power_control(device, msm_cvp_fw_low_power_mode))
  2070. dprintk(CVP_WARN, "Setting h/w power collapse ON failed\n");
  2071. }
  2072. static void __session_clean(struct cvp_hal_session *session)
  2073. {
  2074. struct cvp_hal_session *temp, *next;
  2075. struct iris_hfi_device *device;
  2076. if (!session || !session->device) {
  2077. dprintk(CVP_WARN, "%s: invalid params\n", __func__);
  2078. return;
  2079. }
  2080. device = session->device;
  2081. dprintk(CVP_SESS, "deleted the session: %pK\n", session);
  2082. /*
  2083. * session might have been removed from the device list in
  2084. * core_release, so check and remove if it is in the list
  2085. */
  2086. list_for_each_entry_safe(temp, next, &device->sess_head, list) {
  2087. if (session == temp) {
  2088. list_del(&session->list);
  2089. break;
  2090. }
  2091. }
  2092. /* Poison the session handle with zeros */
  2093. *session = (struct cvp_hal_session){ {0} };
  2094. kfree(session);
  2095. }
  2096. static int iris_hfi_session_clean(void *session)
  2097. {
  2098. struct cvp_hal_session *sess_close;
  2099. struct iris_hfi_device *device;
  2100. if (!session) {
  2101. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2102. return -EINVAL;
  2103. }
  2104. sess_close = session;
  2105. device = sess_close->device;
  2106. if (!device) {
  2107. dprintk(CVP_ERR, "Invalid device handle %s\n", __func__);
  2108. return -EINVAL;
  2109. }
  2110. mutex_lock(&device->lock);
  2111. __session_clean(sess_close);
  2112. mutex_unlock(&device->lock);
  2113. return 0;
  2114. }
  2115. static int iris_debug_hook(void *device)
  2116. {
  2117. struct iris_hfi_device *dev = device;
  2118. u32 val;
  2119. if (!device) {
  2120. dprintk(CVP_ERR, "%s Invalid device\n", __func__);
  2121. return -ENODEV;
  2122. }
  2123. /******* FDU & MPU *****/
  2124. #define CVP0_CVP_SS_FDU_SECURE_ENABLE 0x90
  2125. #define CVP0_CVP_SS_MPU_SECURE_ENABLE 0x94
  2126. #define CVP0_CVP_SS_ARP_THREAD_0_SECURE_ENABLE 0xA0
  2127. #define CVP0_CVP_SS_ARP_THREAD_1_SECURE_ENABLE 0xA4
  2128. #define CVP0_CVP_SS_ARP_THREAD_2_SECURE_ENABLE 0xA8
  2129. #define CVP0_CVP_SS_ARP_THREAD_3_SECURE_ENABLE 0xAC
  2130. val = __read_register(dev, CVP0_CVP_SS_FDU_SECURE_ENABLE);
  2131. dprintk(CVP_ERR, "FDU_SECURE_ENABLE %#x\n", val);
  2132. val = __read_register(dev, CVP0_CVP_SS_MPU_SECURE_ENABLE);
  2133. dprintk(CVP_ERR, "MPU_SECURE_ENABLE %#x\n", val);
  2134. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_0_SECURE_ENABLE);
  2135. dprintk(CVP_ERR, "ARP_THREAD_0_SECURE_ENABLE %#x\n", val);
  2136. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_1_SECURE_ENABLE);
  2137. dprintk(CVP_ERR, "ARP_THREAD_1_SECURE_ENABLE %#x\n", val);
  2138. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_2_SECURE_ENABLE);
  2139. dprintk(CVP_ERR, "ARP_THREAD_2_SECURE_ENABLE %#x\n", val);
  2140. val = __read_register(dev, CVP0_CVP_SS_ARP_THREAD_3_SECURE_ENABLE);
  2141. dprintk(CVP_ERR, "ARP_THREAD_3_SECURE_ENABLE %#x\n", val);
  2142. if (true)
  2143. return 0;
  2144. /***** GCE *******
  2145. * Bit 0 of below register is CDM secure enable for GCE
  2146. * CDM buffer will be in CB4 if set
  2147. */
  2148. #define CVP_GCE_GCE_SS_CP_CTL 0x51100
  2149. /* STATUS bit0 && CFG bit 4 of below register set,
  2150. * expect pixel buffers in CB3,
  2151. * otherwise in CB0
  2152. * CFG bit 9:8 b01 -> LMC input in CB3
  2153. * CFG bit 9:8 b10 -> LMC input in CB4
  2154. */
  2155. #define CVP_GCE0_CP_STATUS 0x51080
  2156. #define CVP_GCE0_BIU_RD_INPUT_IF_SECURITY_CFG 0x52020
  2157. val = __read_register(dev, CVP_GCE_GCE_SS_CP_CTL);
  2158. dprintk(CVP_ERR, "CVP_GCE_GCE_SS_CP_CTL %#x\n", val);
  2159. val = __read_register(dev, CVP_GCE0_CP_STATUS);
  2160. dprintk(CVP_ERR, "CVP_GCE0_CP_STATUS %#x\n", val);
  2161. val = __read_register(dev, CVP_GCE0_BIU_RD_INPUT_IF_SECURITY_CFG);
  2162. dprintk(CVP_ERR, "CVP_GCE0_BIU_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2163. /***** RGE *****
  2164. * Bit 0 of below regiser is CDM secure enable for RGE
  2165. * CDM buffer to be in CB4 i fset
  2166. */
  2167. #define CVP_RGE0_TOPRGE_CP_CTL 0x31010
  2168. /* CFG bit 4 && IN bit 0:
  2169. * if both are set, expect CB3 or CB4 depending on IN 6:4 field
  2170. * either is clear, expect CB0
  2171. */
  2172. #define CVP_RGE0_BUS_RD_INPUT_IF_SECURITY_CFG 0x32020
  2173. #define CVP_RGE0_TOPSPARE_IN 0x311F4
  2174. val = __read_register(dev, CVP_RGE0_TOPRGE_CP_CTL);
  2175. dprintk(CVP_ERR, "CVP_RGE0_TOPRGE_CP_CTL %#x\n", val);
  2176. val = __read_register(dev, CVP_RGE0_BUS_RD_INPUT_IF_SECURITY_CFG);
  2177. dprintk(CVP_ERR, "CVP_RGE0_BUS_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2178. val = __read_register(dev, CVP_RGE0_TOPSPARE_IN);
  2179. dprintk(CVP_ERR, "CVP_RGE0_TOPSPARE_IN %#x\n", val);
  2180. /****** VADL ******
  2181. * Bit 0 of below register is CDM secure enable for VADL
  2182. * CDM buffer will bei in CB4 if set
  2183. */
  2184. #define CVP_VADL0_VADL_SS_CP_CTL 0x21010
  2185. /* Below registers are used the same way as RGE */
  2186. #define CVP_VADL0_BUS_RD_INPUT_IF_SECURITY_CFG 0x22020
  2187. #define CVP_VADL0_SPARE_IN 0x211F4
  2188. val = __read_register(dev, CVP_VADL0_VADL_SS_CP_CTL);
  2189. dprintk(CVP_ERR, "CVP_VADL0_VADL_SS_CP_CTL %#x\n", val);
  2190. val = __read_register(dev, CVP_VADL0_BUS_RD_INPUT_IF_SECURITY_CFG);
  2191. dprintk(CVP_ERR, "CVP_VADL0_BUS_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2192. val = __read_register(dev, CVP_VADL0_SPARE_IN);
  2193. dprintk(CVP_ERR, "CVP_VADL0_SPARE_IN %#x\n", val);
  2194. /****** ITOF *****
  2195. * Below registers are used the same way as RGE
  2196. */
  2197. #define CVP_ITOF0_TOF_SS_CP_CTL 0x41010
  2198. #define CVP_ITOF0_BUS_RD_INPUT_IF_SECURITY_CFG 0x42020
  2199. #define CVP_ITOF0_TOF_SS_SPARE_IN 0x411F4
  2200. val = __read_register(dev, CVP_ITOF0_TOF_SS_CP_CTL);
  2201. dprintk(CVP_ERR, "CVP_ITOF0_TOF_SS_CP_CTL %#x\n", val);
  2202. val = __read_register(dev, CVP_ITOF0_BUS_RD_INPUT_IF_SECURITY_CFG);
  2203. dprintk(CVP_ERR, "CVP_ITOF0_BUS_RD_INPUT_IF_SECURITY_CFG %#x\n", val);
  2204. val = __read_register(dev, CVP_ITOF0_TOF_SS_SPARE_IN);
  2205. dprintk(CVP_ERR, "CVP_ITOF0_TOF_SS_SPARE_IN %#x\n", val);
  2206. return 0;
  2207. }
  2208. static int iris_hfi_session_init(void *device, void *session_id,
  2209. void **new_session)
  2210. {
  2211. struct cvp_hfi_cmd_sys_session_init_packet pkt;
  2212. struct iris_hfi_device *dev;
  2213. struct cvp_hal_session *s;
  2214. if (!device || !new_session) {
  2215. dprintk(CVP_ERR, "%s - invalid input\n", __func__);
  2216. return -EINVAL;
  2217. }
  2218. dev = device;
  2219. mutex_lock(&dev->lock);
  2220. s = kzalloc(sizeof(*s), GFP_KERNEL);
  2221. if (!s) {
  2222. dprintk(CVP_ERR, "new session fail: Out of memory\n");
  2223. goto err_session_init_fail;
  2224. }
  2225. s->session_id = session_id;
  2226. s->device = dev;
  2227. dprintk(CVP_SESS,
  2228. "%s: inst %pK, session %pK\n", __func__, session_id, s);
  2229. list_add_tail(&s->list, &dev->sess_head);
  2230. __set_default_sys_properties(device);
  2231. if (call_hfi_pkt_op(dev, session_init, &pkt, s)) {
  2232. dprintk(CVP_ERR, "session_init: failed to create packet\n");
  2233. goto err_session_init_fail;
  2234. }
  2235. *new_session = s;
  2236. if (__iface_cmdq_write(dev, &pkt))
  2237. goto err_session_init_fail;
  2238. mutex_unlock(&dev->lock);
  2239. return 0;
  2240. err_session_init_fail:
  2241. if (s)
  2242. __session_clean(s);
  2243. *new_session = NULL;
  2244. mutex_unlock(&dev->lock);
  2245. return -EINVAL;
  2246. }
  2247. static int __send_session_cmd(struct cvp_hal_session *session, int pkt_type)
  2248. {
  2249. struct cvp_hal_session_cmd_pkt pkt;
  2250. int rc = 0;
  2251. struct iris_hfi_device *device = session->device;
  2252. if (!__is_session_valid(device, session, __func__))
  2253. return -ECONNRESET;
  2254. rc = call_hfi_pkt_op(device, session_cmd,
  2255. &pkt, pkt_type, session);
  2256. if (rc == -EPERM)
  2257. return 0;
  2258. if (rc) {
  2259. dprintk(CVP_ERR, "send session cmd: create pkt failed\n");
  2260. goto err_create_pkt;
  2261. }
  2262. if (__iface_cmdq_write(session->device, &pkt))
  2263. rc = -ENOTEMPTY;
  2264. err_create_pkt:
  2265. return rc;
  2266. }
  2267. static int iris_hfi_session_end(void *session)
  2268. {
  2269. struct cvp_hal_session *sess;
  2270. struct iris_hfi_device *device;
  2271. int rc = 0;
  2272. if (!session) {
  2273. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2274. return -EINVAL;
  2275. }
  2276. sess = session;
  2277. device = sess->device;
  2278. if (!device) {
  2279. dprintk(CVP_ERR, "Invalid session %s\n", __func__);
  2280. return -EINVAL;
  2281. }
  2282. mutex_lock(&device->lock);
  2283. if (msm_cvp_fw_coverage) {
  2284. if (__sys_set_coverage(sess->device, msm_cvp_fw_coverage))
  2285. dprintk(CVP_WARN, "Fw_coverage msg ON failed\n");
  2286. }
  2287. rc = __send_session_cmd(session, HFI_CMD_SYS_SESSION_END);
  2288. mutex_unlock(&device->lock);
  2289. return rc;
  2290. }
  2291. static int iris_hfi_session_abort(void *sess)
  2292. {
  2293. struct cvp_hal_session *session = sess;
  2294. struct iris_hfi_device *device;
  2295. int rc = 0;
  2296. if (!session || !session->device) {
  2297. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2298. return -EINVAL;
  2299. }
  2300. device = session->device;
  2301. mutex_lock(&device->lock);
  2302. rc = __send_session_cmd(session, HFI_CMD_SYS_SESSION_ABORT);
  2303. mutex_unlock(&device->lock);
  2304. return rc;
  2305. }
  2306. static int iris_hfi_session_set_buffers(void *sess, u32 iova, u32 size)
  2307. {
  2308. struct cvp_hfi_cmd_session_set_buffers_packet pkt;
  2309. int rc = 0;
  2310. struct cvp_hal_session *session = sess;
  2311. struct iris_hfi_device *device;
  2312. if (!session || !session->device || !iova || !size) {
  2313. dprintk(CVP_ERR, "Invalid Params\n");
  2314. return -EINVAL;
  2315. }
  2316. device = session->device;
  2317. mutex_lock(&device->lock);
  2318. if (!__is_session_valid(device, session, __func__)) {
  2319. rc = -ECONNRESET;
  2320. goto err_create_pkt;
  2321. }
  2322. rc = call_hfi_pkt_op(device, session_set_buffers,
  2323. &pkt, session, iova, size);
  2324. if (rc) {
  2325. dprintk(CVP_ERR, "set buffers: failed to create packet\n");
  2326. goto err_create_pkt;
  2327. }
  2328. if (__iface_cmdq_write(session->device, &pkt))
  2329. rc = -ENOTEMPTY;
  2330. err_create_pkt:
  2331. mutex_unlock(&device->lock);
  2332. return rc;
  2333. }
  2334. static int iris_hfi_session_release_buffers(void *sess)
  2335. {
  2336. struct cvp_session_release_buffers_packet pkt;
  2337. int rc = 0;
  2338. struct cvp_hal_session *session = sess;
  2339. struct iris_hfi_device *device;
  2340. if (!session || !session->device) {
  2341. dprintk(CVP_ERR, "Invalid Params\n");
  2342. return -EINVAL;
  2343. }
  2344. device = session->device;
  2345. mutex_lock(&device->lock);
  2346. if (!__is_session_valid(device, session, __func__)) {
  2347. rc = -ECONNRESET;
  2348. goto err_create_pkt;
  2349. }
  2350. rc = call_hfi_pkt_op(device, session_release_buffers, &pkt, session);
  2351. if (rc) {
  2352. dprintk(CVP_ERR, "release buffers: failed to create packet\n");
  2353. goto err_create_pkt;
  2354. }
  2355. if (__iface_cmdq_write(session->device, &pkt))
  2356. rc = -ENOTEMPTY;
  2357. err_create_pkt:
  2358. mutex_unlock(&device->lock);
  2359. return rc;
  2360. }
  2361. static int iris_hfi_session_send(void *sess,
  2362. struct eva_kmd_hfi_packet *in_pkt)
  2363. {
  2364. int rc = 0;
  2365. struct eva_kmd_hfi_packet pkt;
  2366. struct cvp_hal_session *session = sess;
  2367. struct iris_hfi_device *device;
  2368. if (!session || !session->device) {
  2369. dprintk(CVP_ERR, "invalid session");
  2370. return -ENODEV;
  2371. }
  2372. device = session->device;
  2373. mutex_lock(&device->lock);
  2374. if (!__is_session_valid(device, session, __func__)) {
  2375. rc = -ECONNRESET;
  2376. goto err_send_pkt;
  2377. }
  2378. rc = call_hfi_pkt_op(device, session_send,
  2379. &pkt, session, in_pkt);
  2380. if (rc) {
  2381. dprintk(CVP_ERR,
  2382. "failed to create pkt\n");
  2383. goto err_send_pkt;
  2384. }
  2385. if (__iface_cmdq_write(session->device, &pkt))
  2386. rc = -ENOTEMPTY;
  2387. err_send_pkt:
  2388. mutex_unlock(&device->lock);
  2389. return rc;
  2390. return rc;
  2391. }
  2392. static int iris_hfi_session_flush(void *sess)
  2393. {
  2394. struct cvp_hal_session *session = sess;
  2395. struct iris_hfi_device *device;
  2396. int rc = 0;
  2397. if (!session || !session->device) {
  2398. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2399. return -EINVAL;
  2400. }
  2401. device = session->device;
  2402. mutex_lock(&device->lock);
  2403. rc = __send_session_cmd(session, HFI_CMD_SESSION_CVP_FLUSH);
  2404. mutex_unlock(&device->lock);
  2405. return rc;
  2406. }
  2407. static int iris_hfi_session_start(void *sess)
  2408. {
  2409. struct cvp_hal_session *session = sess;
  2410. struct iris_hfi_device *device;
  2411. int rc = 0;
  2412. if (!session || !session->device) {
  2413. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2414. return -EINVAL;
  2415. }
  2416. device = session->device;
  2417. mutex_lock(&device->lock);
  2418. rc = __send_session_cmd(session, HFI_CMD_SESSION_EVA_START);
  2419. mutex_unlock(&device->lock);
  2420. return rc;
  2421. }
  2422. static int iris_hfi_session_stop(void *sess)
  2423. {
  2424. struct cvp_hal_session *session = sess;
  2425. struct iris_hfi_device *device;
  2426. int rc = 0;
  2427. if (!session || !session->device) {
  2428. dprintk(CVP_ERR, "Invalid Params %s\n", __func__);
  2429. return -EINVAL;
  2430. }
  2431. device = session->device;
  2432. mutex_lock(&device->lock);
  2433. rc = __send_session_cmd(session, HFI_CMD_SESSION_EVA_STOP);
  2434. mutex_unlock(&device->lock);
  2435. return rc;
  2436. }
  2437. static void __process_fatal_error(
  2438. struct iris_hfi_device *device)
  2439. {
  2440. struct msm_cvp_cb_cmd_done cmd_done = {0};
  2441. cmd_done.device_id = device->device_id;
  2442. device->callback(HAL_SYS_ERROR, &cmd_done);
  2443. }
  2444. static int __prepare_pc(struct iris_hfi_device *device)
  2445. {
  2446. int rc = 0;
  2447. struct cvp_hfi_cmd_sys_pc_prep_packet pkt;
  2448. rc = call_hfi_pkt_op(device, sys_pc_prep, &pkt);
  2449. if (rc) {
  2450. dprintk(CVP_ERR, "Failed to create sys pc prep pkt\n");
  2451. goto err_pc_prep;
  2452. }
  2453. if (__iface_cmdq_write(device, &pkt))
  2454. rc = -ENOTEMPTY;
  2455. if (rc)
  2456. dprintk(CVP_ERR, "Failed to prepare iris for power off");
  2457. err_pc_prep:
  2458. return rc;
  2459. }
  2460. static void iris_hfi_pm_handler(struct work_struct *work)
  2461. {
  2462. int rc = 0;
  2463. struct msm_cvp_core *core;
  2464. struct iris_hfi_device *device;
  2465. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  2466. if (core)
  2467. device = core->device->hfi_device_data;
  2468. else
  2469. return;
  2470. if (!device) {
  2471. dprintk(CVP_ERR, "%s: NULL device\n", __func__);
  2472. return;
  2473. }
  2474. dprintk(CVP_PWR,
  2475. "Entering %s\n", __func__);
  2476. /*
  2477. * It is ok to check this variable outside the lock since
  2478. * it is being updated in this context only
  2479. */
  2480. if (device->skip_pc_count >= CVP_MAX_PC_SKIP_COUNT) {
  2481. dprintk(CVP_WARN, "Failed to PC for %d times\n",
  2482. device->skip_pc_count);
  2483. device->skip_pc_count = 0;
  2484. __process_fatal_error(device);
  2485. return;
  2486. }
  2487. mutex_lock(&device->lock);
  2488. if (gfa_cv.state == DSP_SUSPEND)
  2489. rc = __power_collapse(device, true);
  2490. else
  2491. rc = __power_collapse(device, false);
  2492. mutex_unlock(&device->lock);
  2493. switch (rc) {
  2494. case 0:
  2495. device->skip_pc_count = 0;
  2496. /* Cancel pending delayed works if any */
  2497. cancel_delayed_work(&iris_hfi_pm_work);
  2498. dprintk(CVP_PWR, "%s: power collapse successful!\n",
  2499. __func__);
  2500. break;
  2501. case -EBUSY:
  2502. device->skip_pc_count = 0;
  2503. dprintk(CVP_PWR, "%s: retry PC as cvp is busy\n", __func__);
  2504. queue_delayed_work(device->iris_pm_workq,
  2505. &iris_hfi_pm_work, msecs_to_jiffies(
  2506. device->res->msm_cvp_pwr_collapse_delay));
  2507. break;
  2508. case -EAGAIN:
  2509. device->skip_pc_count++;
  2510. dprintk(CVP_WARN, "%s: retry power collapse (count %d)\n",
  2511. __func__, device->skip_pc_count);
  2512. queue_delayed_work(device->iris_pm_workq,
  2513. &iris_hfi_pm_work, msecs_to_jiffies(
  2514. device->res->msm_cvp_pwr_collapse_delay));
  2515. break;
  2516. default:
  2517. dprintk(CVP_ERR, "%s: power collapse failed\n", __func__);
  2518. break;
  2519. }
  2520. }
  2521. static int __power_collapse(struct iris_hfi_device *device, bool force)
  2522. {
  2523. int rc = 0;
  2524. u32 wfi_status = 0, idle_status = 0, pc_ready = 0;
  2525. int count = 0;
  2526. const int max_tries = 150;
  2527. if (!device) {
  2528. dprintk(CVP_ERR, "%s: invalid params\n", __func__);
  2529. return -EINVAL;
  2530. }
  2531. if (!device->power_enabled) {
  2532. dprintk(CVP_PWR, "%s: Power already disabled\n",
  2533. __func__);
  2534. goto exit;
  2535. }
  2536. rc = __core_in_valid_state(device);
  2537. if (!rc) {
  2538. dprintk(CVP_WARN,
  2539. "Core is in bad state, Skipping power collapse\n");
  2540. return -EINVAL;
  2541. }
  2542. rc = __dsp_suspend(device, force);
  2543. if (rc == -EBUSY)
  2544. goto exit;
  2545. else if (rc)
  2546. goto skip_power_off;
  2547. __flush_debug_queue(device, device->raw_packet);
  2548. pc_ready = __read_register(device, CVP_CTRL_STATUS) &
  2549. CVP_CTRL_STATUS_PC_READY;
  2550. if (!pc_ready) {
  2551. wfi_status = __read_register(device,
  2552. CVP_WRAPPER_CPU_STATUS);
  2553. idle_status = __read_register(device,
  2554. CVP_CTRL_STATUS);
  2555. if (!(wfi_status & BIT(0))) {
  2556. dprintk(CVP_WARN,
  2557. "Skipping PC as wfi_status (%#x) bit not set\n",
  2558. wfi_status);
  2559. goto skip_power_off;
  2560. }
  2561. if (!(idle_status & BIT(30))) {
  2562. dprintk(CVP_WARN,
  2563. "Skipping PC as idle_status (%#x) bit not set\n",
  2564. idle_status);
  2565. goto skip_power_off;
  2566. }
  2567. rc = __prepare_pc(device);
  2568. if (rc) {
  2569. dprintk(CVP_WARN, "Failed PC %d\n", rc);
  2570. goto skip_power_off;
  2571. }
  2572. while (count < max_tries) {
  2573. wfi_status = __read_register(device,
  2574. CVP_WRAPPER_CPU_STATUS);
  2575. pc_ready = __read_register(device,
  2576. CVP_CTRL_STATUS);
  2577. if ((wfi_status & BIT(0)) && (pc_ready &
  2578. CVP_CTRL_STATUS_PC_READY))
  2579. break;
  2580. usleep_range(150, 250);
  2581. count++;
  2582. }
  2583. if (count == max_tries) {
  2584. dprintk(CVP_ERR,
  2585. "Skip PC. Core is not ready (%#x, %#x)\n",
  2586. wfi_status, pc_ready);
  2587. goto skip_power_off;
  2588. }
  2589. } else {
  2590. wfi_status = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  2591. if (!(wfi_status & BIT(0))) {
  2592. dprintk(CVP_WARN,
  2593. "Skip PC as wfi_status (%#x) bit not set\n",
  2594. wfi_status);
  2595. goto skip_power_off;
  2596. }
  2597. }
  2598. rc = __suspend(device);
  2599. if (rc)
  2600. dprintk(CVP_ERR, "Failed __suspend\n");
  2601. exit:
  2602. return rc;
  2603. skip_power_off:
  2604. dprintk(CVP_PWR, "Skip PC(%#x, %#x, %#x)\n",
  2605. wfi_status, idle_status, pc_ready);
  2606. __flush_debug_queue(device, device->raw_packet);
  2607. return -EAGAIN;
  2608. }
  2609. static void __process_sys_error(struct iris_hfi_device *device)
  2610. {
  2611. struct cvp_hfi_sfr_struct *vsfr = NULL;
  2612. vsfr = (struct cvp_hfi_sfr_struct *)device->sfr.align_virtual_addr;
  2613. if (vsfr) {
  2614. void *p = memchr(vsfr->rg_data, '\0', vsfr->bufSize);
  2615. /*
  2616. * SFR isn't guaranteed to be NULL terminated
  2617. * since SYS_ERROR indicates that Iris is in the
  2618. * process of crashing.
  2619. */
  2620. if (p == NULL)
  2621. vsfr->rg_data[vsfr->bufSize - 1] = '\0';
  2622. dprintk(CVP_ERR, "SFR Message from FW: %s\n",
  2623. vsfr->rg_data);
  2624. }
  2625. }
  2626. static void __flush_debug_queue(struct iris_hfi_device *device, u8 *packet)
  2627. {
  2628. bool local_packet = false;
  2629. enum cvp_msg_prio log_level = CVP_FW;
  2630. if (!device) {
  2631. dprintk(CVP_ERR, "%s: Invalid params\n", __func__);
  2632. return;
  2633. }
  2634. if (!packet) {
  2635. packet = kzalloc(CVP_IFACEQ_VAR_HUGE_PKT_SIZE, GFP_KERNEL);
  2636. if (!packet) {
  2637. dprintk(CVP_ERR, "In %s() Fail to allocate mem\n",
  2638. __func__);
  2639. return;
  2640. }
  2641. local_packet = true;
  2642. /*
  2643. * Local packek is used when something FATAL occurred.
  2644. * It is good to print these logs by default.
  2645. */
  2646. log_level = CVP_ERR;
  2647. }
  2648. #define SKIP_INVALID_PKT(pkt_size, payload_size, pkt_hdr_size) ({ \
  2649. if (pkt_size < pkt_hdr_size || \
  2650. payload_size < MIN_PAYLOAD_SIZE || \
  2651. payload_size > \
  2652. (pkt_size - pkt_hdr_size + sizeof(u8))) { \
  2653. dprintk(CVP_ERR, \
  2654. "%s: invalid msg size - %d\n", \
  2655. __func__, pkt->msg_size); \
  2656. continue; \
  2657. } \
  2658. })
  2659. while (!__iface_dbgq_read(device, packet)) {
  2660. struct cvp_hfi_packet_header *pkt =
  2661. (struct cvp_hfi_packet_header *) packet;
  2662. if (pkt->size < sizeof(struct cvp_hfi_packet_header)) {
  2663. dprintk(CVP_ERR, "Invalid pkt size - %s\n",
  2664. __func__);
  2665. continue;
  2666. }
  2667. if (pkt->packet_type == HFI_MSG_SYS_DEBUG) {
  2668. struct cvp_hfi_msg_sys_debug_packet *pkt =
  2669. (struct cvp_hfi_msg_sys_debug_packet *) packet;
  2670. SKIP_INVALID_PKT(pkt->size,
  2671. pkt->msg_size, sizeof(*pkt));
  2672. /*
  2673. * All fw messages starts with new line character. This
  2674. * causes dprintk to print this message in two lines
  2675. * in the kernel log. Ignoring the first character
  2676. * from the message fixes this to print it in a single
  2677. * line.
  2678. */
  2679. pkt->rg_msg_data[pkt->msg_size-1] = '\0';
  2680. dprintk(log_level, "%s", &pkt->rg_msg_data[1]);
  2681. }
  2682. }
  2683. #undef SKIP_INVALID_PKT
  2684. if (local_packet)
  2685. kfree(packet);
  2686. }
  2687. static bool __is_session_valid(struct iris_hfi_device *device,
  2688. struct cvp_hal_session *session, const char *func)
  2689. {
  2690. struct cvp_hal_session *temp = NULL;
  2691. if (!device || !session)
  2692. goto invalid;
  2693. list_for_each_entry(temp, &device->sess_head, list)
  2694. if (session == temp)
  2695. return true;
  2696. invalid:
  2697. dprintk(CVP_WARN, "%s: device %pK, invalid session %pK\n",
  2698. func, device, session);
  2699. return false;
  2700. }
  2701. static struct cvp_hal_session *__get_session(struct iris_hfi_device *device,
  2702. u32 session_id)
  2703. {
  2704. struct cvp_hal_session *temp = NULL;
  2705. list_for_each_entry(temp, &device->sess_head, list) {
  2706. if (session_id == hash32_ptr(temp))
  2707. return temp;
  2708. }
  2709. return NULL;
  2710. }
  2711. #define _INVALID_MSG_ "Unrecognized MSG (%#x) session (%pK), discarding\n"
  2712. #define _INVALID_STATE_ "Ignore responses from %d to %d invalid state\n"
  2713. #define _DEVFREQ_FAIL_ "Failed to add devfreq device bus %s governor %s: %d\n"
  2714. static void process_system_msg(struct msm_cvp_cb_info *info,
  2715. struct iris_hfi_device *device,
  2716. void *raw_packet)
  2717. {
  2718. struct cvp_hal_sys_init_done sys_init_done = {0};
  2719. switch (info->response_type) {
  2720. case HAL_SYS_ERROR:
  2721. __process_sys_error(device);
  2722. break;
  2723. case HAL_SYS_RELEASE_RESOURCE_DONE:
  2724. dprintk(CVP_CORE, "Received SYS_RELEASE_RESOURCE\n");
  2725. break;
  2726. case HAL_SYS_INIT_DONE:
  2727. dprintk(CVP_CORE, "Received SYS_INIT_DONE\n");
  2728. sys_init_done.capabilities =
  2729. device->sys_init_capabilities;
  2730. cvp_hfi_process_sys_init_done_prop_read(
  2731. (struct cvp_hfi_msg_sys_init_done_packet *)
  2732. raw_packet, &sys_init_done);
  2733. info->response.cmd.data.sys_init_done = sys_init_done;
  2734. break;
  2735. default:
  2736. break;
  2737. }
  2738. }
  2739. static void **get_session_id(struct msm_cvp_cb_info *info)
  2740. {
  2741. void **session_id = NULL;
  2742. /* For session-related packets, validate session */
  2743. switch (info->response_type) {
  2744. case HAL_SESSION_INIT_DONE:
  2745. case HAL_SESSION_END_DONE:
  2746. case HAL_SESSION_ABORT_DONE:
  2747. case HAL_SESSION_START_DONE:
  2748. case HAL_SESSION_STOP_DONE:
  2749. case HAL_SESSION_FLUSH_DONE:
  2750. case HAL_SESSION_SET_BUFFER_DONE:
  2751. case HAL_SESSION_SUSPEND_DONE:
  2752. case HAL_SESSION_RESUME_DONE:
  2753. case HAL_SESSION_SET_PROP_DONE:
  2754. case HAL_SESSION_GET_PROP_DONE:
  2755. case HAL_SESSION_RELEASE_BUFFER_DONE:
  2756. case HAL_SESSION_REGISTER_BUFFER_DONE:
  2757. case HAL_SESSION_UNREGISTER_BUFFER_DONE:
  2758. case HAL_SESSION_PROPERTY_INFO:
  2759. case HAL_SESSION_EVENT_CHANGE:
  2760. case HAL_SESSION_DUMP_NOTIFY:
  2761. session_id = &info->response.cmd.session_id;
  2762. break;
  2763. case HAL_SESSION_ERROR:
  2764. session_id = &info->response.data.session_id;
  2765. break;
  2766. case HAL_RESPONSE_UNUSED:
  2767. default:
  2768. session_id = NULL;
  2769. break;
  2770. }
  2771. return session_id;
  2772. }
  2773. static void print_msg_hdr(void *hdr)
  2774. {
  2775. struct cvp_hfi_msg_session_hdr *new_hdr =
  2776. (struct cvp_hfi_msg_session_hdr *)hdr;
  2777. dprintk(CVP_HFI, "HFI MSG received: %x %x %x %x %x %x %x %#llx\n",
  2778. new_hdr->size, new_hdr->packet_type,
  2779. new_hdr->session_id,
  2780. new_hdr->client_data.transaction_id,
  2781. new_hdr->client_data.data1,
  2782. new_hdr->client_data.data2,
  2783. new_hdr->error_type,
  2784. new_hdr->client_data.kdata);
  2785. }
  2786. static int __response_handler(struct iris_hfi_device *device)
  2787. {
  2788. struct msm_cvp_cb_info *packets;
  2789. int packet_count = 0;
  2790. u8 *raw_packet = NULL;
  2791. bool requeue_pm_work = true;
  2792. if (!device || device->state != IRIS_STATE_INIT)
  2793. return 0;
  2794. packets = device->response_pkt;
  2795. raw_packet = device->raw_packet;
  2796. if (!raw_packet || !packets) {
  2797. dprintk(CVP_ERR,
  2798. "%s: Invalid args : Res pkt = %pK, Raw pkt = %pK\n",
  2799. __func__, packets, raw_packet);
  2800. return 0;
  2801. }
  2802. if (device->intr_status & CVP_FATAL_INTR_BMSK) {
  2803. struct cvp_hfi_sfr_struct *vsfr = (struct cvp_hfi_sfr_struct *)
  2804. device->sfr.align_virtual_addr;
  2805. struct msm_cvp_cb_info info = {
  2806. .response_type = HAL_SYS_WATCHDOG_TIMEOUT,
  2807. .response.cmd = {
  2808. .device_id = device->device_id,
  2809. }
  2810. };
  2811. if (vsfr)
  2812. dprintk(CVP_ERR, "SFR Message from FW: %s\n",
  2813. vsfr->rg_data);
  2814. if (device->intr_status & CVP_WRAPPER_INTR_MASK_CPU_NOC_BMSK)
  2815. dprintk(CVP_ERR, "Received Xtensa NOC error\n");
  2816. if (device->intr_status & CVP_WRAPPER_INTR_MASK_CORE_NOC_BMSK)
  2817. dprintk(CVP_ERR, "Received CVP core NOC error\n");
  2818. if (device->intr_status & CVP_WRAPPER_INTR_MASK_A2HWD_BMSK)
  2819. dprintk(CVP_ERR, "Received CVP watchdog timeout\n");
  2820. packets[packet_count++] = info;
  2821. goto exit;
  2822. }
  2823. /* Bleed the msg queue dry of packets */
  2824. while (!__iface_msgq_read(device, raw_packet)) {
  2825. void **session_id = NULL;
  2826. struct msm_cvp_cb_info *info = &packets[packet_count++];
  2827. struct cvp_hfi_msg_session_hdr *hdr =
  2828. (struct cvp_hfi_msg_session_hdr *)raw_packet;
  2829. int rc = 0;
  2830. print_msg_hdr(hdr);
  2831. rc = cvp_hfi_process_msg_packet(device->device_id,
  2832. raw_packet, info);
  2833. if (rc) {
  2834. dprintk(CVP_WARN,
  2835. "Corrupt/unknown packet found, discarding\n");
  2836. --packet_count;
  2837. continue;
  2838. } else if (info->response_type == HAL_NO_RESP) {
  2839. --packet_count;
  2840. continue;
  2841. }
  2842. /* Process the packet types that we're interested in */
  2843. process_system_msg(info, device, raw_packet);
  2844. session_id = get_session_id(info);
  2845. /*
  2846. * hfi_process_msg_packet provides a session_id that's a hashed
  2847. * value of struct cvp_hal_session, we need to coerce the hashed
  2848. * value back to pointer that we can use. Ideally, hfi_process\
  2849. * _msg_packet should take care of this, but it doesn't have
  2850. * required information for it
  2851. */
  2852. if (session_id) {
  2853. struct cvp_hal_session *session = NULL;
  2854. if (upper_32_bits((uintptr_t)*session_id) != 0) {
  2855. dprintk(CVP_ERR,
  2856. "Upper 32-bits != 0 for sess_id=%pK\n",
  2857. *session_id);
  2858. }
  2859. session = __get_session(device,
  2860. (u32)(uintptr_t)*session_id);
  2861. if (!session) {
  2862. dprintk(CVP_ERR, _INVALID_MSG_,
  2863. info->response_type,
  2864. *session_id);
  2865. --packet_count;
  2866. continue;
  2867. }
  2868. *session_id = session->session_id;
  2869. }
  2870. if (packet_count >= cvp_max_packets) {
  2871. dprintk(CVP_WARN,
  2872. "Too many packets in message queue!\n");
  2873. break;
  2874. }
  2875. /* do not read packets after sys error packet */
  2876. if (info->response_type == HAL_SYS_ERROR)
  2877. break;
  2878. }
  2879. if (requeue_pm_work && device->res->sw_power_collapsible) {
  2880. cancel_delayed_work(&iris_hfi_pm_work);
  2881. if (!queue_delayed_work(device->iris_pm_workq,
  2882. &iris_hfi_pm_work,
  2883. msecs_to_jiffies(
  2884. device->res->msm_cvp_pwr_collapse_delay))) {
  2885. dprintk(CVP_ERR, "PM work already scheduled\n");
  2886. }
  2887. }
  2888. exit:
  2889. __flush_debug_queue(device, raw_packet);
  2890. return packet_count;
  2891. }
  2892. static void iris_hfi_core_work_handler(struct work_struct *work)
  2893. {
  2894. struct msm_cvp_core *core;
  2895. struct iris_hfi_device *device;
  2896. int num_responses = 0, i = 0;
  2897. u32 intr_status;
  2898. static bool warning_on = true;
  2899. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  2900. if (core)
  2901. device = core->device->hfi_device_data;
  2902. else
  2903. return;
  2904. mutex_lock(&device->lock);
  2905. if (!__core_in_valid_state(device)) {
  2906. if (warning_on) {
  2907. dprintk(CVP_WARN, "%s Core not in init state\n",
  2908. __func__);
  2909. warning_on = false;
  2910. }
  2911. goto err_no_work;
  2912. }
  2913. warning_on = true;
  2914. if (!device->callback) {
  2915. dprintk(CVP_ERR, "No interrupt callback function: %pK\n",
  2916. device);
  2917. goto err_no_work;
  2918. }
  2919. if (__resume(device)) {
  2920. dprintk(CVP_ERR, "%s: Power enable failed\n", __func__);
  2921. goto err_no_work;
  2922. }
  2923. __core_clear_interrupt(device);
  2924. num_responses = __response_handler(device);
  2925. dprintk(CVP_HFI, "%s:: cvp_driver_debug num_responses = %d ",
  2926. __func__, num_responses);
  2927. err_no_work:
  2928. /* Keep the interrupt status before releasing device lock */
  2929. intr_status = device->intr_status;
  2930. mutex_unlock(&device->lock);
  2931. /*
  2932. * Issue the callbacks outside of the locked contex to preserve
  2933. * re-entrancy.
  2934. */
  2935. for (i = 0; !IS_ERR_OR_NULL(device->response_pkt) &&
  2936. i < num_responses; ++i) {
  2937. struct msm_cvp_cb_info *r = &device->response_pkt[i];
  2938. void *rsp = (void *)&r->response;
  2939. if (!__core_in_valid_state(device)) {
  2940. dprintk(CVP_ERR,
  2941. _INVALID_STATE_, (i + 1), num_responses);
  2942. break;
  2943. }
  2944. dprintk(CVP_HFI, "Processing response %d of %d, type %d\n",
  2945. (i + 1), num_responses, r->response_type);
  2946. device->callback(r->response_type, rsp);
  2947. }
  2948. /* We need re-enable the irq which was disabled in ISR handler */
  2949. if (!(intr_status & CVP_WRAPPER_INTR_STATUS_A2HWD_BMSK))
  2950. enable_irq(device->cvp_hal_data->irq);
  2951. /*
  2952. * XXX: Don't add any code beyond here. Reacquiring locks after release
  2953. * it above doesn't guarantee the atomicity that we're aiming for.
  2954. */
  2955. }
  2956. static DECLARE_WORK(iris_hfi_work, iris_hfi_core_work_handler);
  2957. irqreturn_t cvp_hfi_isr(int irq, void *dev)
  2958. {
  2959. struct iris_hfi_device *device = dev;
  2960. disable_irq_nosync(irq);
  2961. queue_work(device->cvp_workq, &iris_hfi_work);
  2962. return IRQ_HANDLED;
  2963. }
  2964. static void iris_hfi_wd_work_handler(struct work_struct *work)
  2965. {
  2966. struct msm_cvp_core *core;
  2967. struct iris_hfi_device *device;
  2968. struct msm_cvp_cb_cmd_done response = {0};
  2969. enum hal_command_response cmd = HAL_SYS_WATCHDOG_TIMEOUT;
  2970. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  2971. if (core)
  2972. device = core->device->hfi_device_data;
  2973. else
  2974. return;
  2975. if (msm_cvp_hw_wd_recovery) {
  2976. dprintk(CVP_ERR, "Cleaning up as HW WD recovery is enable %d\n",
  2977. msm_cvp_hw_wd_recovery);
  2978. response.device_id = device->device_id;
  2979. handle_sys_error(cmd, (void *) &response);
  2980. enable_irq(device->cvp_hal_data->irq_wd);
  2981. }
  2982. else {
  2983. dprintk(CVP_ERR, "Crashing the device as HW WD recovery is disable %d\n",
  2984. msm_cvp_hw_wd_recovery);
  2985. BUG_ON(1);
  2986. }
  2987. }
  2988. static DECLARE_WORK(iris_hfi_wd_work, iris_hfi_wd_work_handler);
  2989. irqreturn_t iris_hfi_isr_wd(int irq, void *dev)
  2990. {
  2991. struct iris_hfi_device *device = dev;
  2992. dprintk(CVP_ERR, "Got HW WDOG IRQ! \n");
  2993. disable_irq_nosync(irq);
  2994. queue_work(device->cvp_workq, &iris_hfi_wd_work);
  2995. return IRQ_HANDLED;
  2996. }
  2997. static int __init_reset_clk(struct msm_cvp_platform_resources *res,
  2998. int reset_index)
  2999. {
  3000. int rc = 0;
  3001. struct reset_control *rst;
  3002. struct reset_info *rst_info;
  3003. struct reset_set *rst_set = &res->reset_set;
  3004. if (!rst_set->reset_tbl)
  3005. return 0;
  3006. rst_info = &rst_set->reset_tbl[reset_index];
  3007. rst = rst_info->rst;
  3008. dprintk(CVP_PWR, "reset_clk: name %s rst %pK required_stage=%d\n",
  3009. rst_set->reset_tbl[reset_index].name, rst, rst_info->required_stage);
  3010. if (rst)
  3011. goto skip_reset_init;
  3012. if (rst_info->required_stage == CVP_ON_USE) {
  3013. rst = reset_control_get_exclusive_released(&res->pdev->dev,
  3014. rst_set->reset_tbl[reset_index].name);
  3015. if (IS_ERR(rst)) {
  3016. rc = PTR_ERR(rst);
  3017. dprintk(CVP_ERR, "reset get exclusive fail %d\n", rc);
  3018. return rc;
  3019. }
  3020. dprintk(CVP_PWR, "reset_clk: name %s get exclusive rst %llx\n",
  3021. rst_set->reset_tbl[reset_index].name, rst);
  3022. } else if (rst_info->required_stage == CVP_ON_INIT) {
  3023. rst = devm_reset_control_get(&res->pdev->dev,
  3024. rst_set->reset_tbl[reset_index].name);
  3025. if (IS_ERR(rst)) {
  3026. rc = PTR_ERR(rst);
  3027. dprintk(CVP_ERR, "reset get fail %d\n", rc);
  3028. return rc;
  3029. }
  3030. dprintk(CVP_PWR, "reset_clk: name %s get rst %llx\n",
  3031. rst_set->reset_tbl[reset_index].name, rst);
  3032. } else {
  3033. dprintk(CVP_ERR, "Invalid reset stage\n");
  3034. return -EINVAL;
  3035. }
  3036. rst_set->reset_tbl[reset_index].rst = rst;
  3037. rst_info->state = RESET_INIT;
  3038. return 0;
  3039. skip_reset_init:
  3040. return rc;
  3041. }
  3042. static int __reset_control_assert_name(struct iris_hfi_device *device,
  3043. const char *name)
  3044. {
  3045. struct reset_info *rcinfo = NULL;
  3046. int rc = 0;
  3047. bool found = false;
  3048. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3049. if (strcmp(rcinfo->name, name))
  3050. continue;
  3051. found = true;
  3052. rc = reset_control_assert(rcinfo->rst);
  3053. if (rc)
  3054. dprintk(CVP_ERR,
  3055. "%s: failed to assert reset control (%s), rc = %d\n",
  3056. __func__, rcinfo->name, rc);
  3057. else
  3058. dprintk(CVP_PWR, "%s: assert reset control (%s)\n",
  3059. __func__, rcinfo->name);
  3060. break;
  3061. }
  3062. if (!found) {
  3063. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3064. __func__, name);
  3065. rc = -EINVAL;
  3066. }
  3067. return rc;
  3068. }
  3069. static int __reset_control_deassert_name(struct iris_hfi_device *device,
  3070. const char *name)
  3071. {
  3072. struct reset_info *rcinfo = NULL;
  3073. int rc = 0;
  3074. bool found = false;
  3075. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3076. if (strcmp(rcinfo->name, name))
  3077. continue;
  3078. found = true;
  3079. rc = reset_control_deassert(rcinfo->rst);
  3080. if (rc)
  3081. dprintk(CVP_ERR,
  3082. "%s: deassert reset control for (%s) failed, rc %d\n",
  3083. __func__, rcinfo->name, rc);
  3084. else
  3085. dprintk(CVP_PWR, "%s: deassert reset control (%s)\n",
  3086. __func__, rcinfo->name);
  3087. break;
  3088. }
  3089. if (!found) {
  3090. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3091. __func__, name);
  3092. rc = -EINVAL;
  3093. }
  3094. return rc;
  3095. }
  3096. static int __reset_control_acquire(struct iris_hfi_device *device,
  3097. const char *name)
  3098. {
  3099. struct reset_info *rcinfo = NULL;
  3100. int rc = 0;
  3101. bool found = false;
  3102. int max_retries = 10;
  3103. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3104. if (strcmp(rcinfo->name, name))
  3105. continue;
  3106. found = true;
  3107. if (rcinfo->state == RESET_ACQUIRED)
  3108. return rc;
  3109. acquire_again:
  3110. rc = reset_control_acquire(rcinfo->rst);
  3111. if (rc) {
  3112. if (rc == -EBUSY) {
  3113. usleep_range(500, 1000);
  3114. max_retries--;
  3115. if (max_retries) {
  3116. goto acquire_again;
  3117. } else {
  3118. dprintk(CVP_ERR,
  3119. "%s acquire %s -EBUSY\n",
  3120. __func__, rcinfo->name);
  3121. rc = -EINVAL;
  3122. }
  3123. } else {
  3124. dprintk(CVP_ERR,
  3125. "%s: acquire failed (%s) rc %d\n",
  3126. __func__, rcinfo->name, rc);
  3127. rc = -EINVAL;
  3128. }
  3129. } else {
  3130. dprintk(CVP_PWR, "%s: reset acquire succeed (%s)\n",
  3131. __func__, rcinfo->name);
  3132. rcinfo->state = RESET_ACQUIRED;
  3133. }
  3134. break;
  3135. }
  3136. if (!found) {
  3137. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3138. __func__, name);
  3139. rc = -EINVAL;
  3140. }
  3141. return rc;
  3142. }
  3143. static int __reset_control_release(struct iris_hfi_device *device,
  3144. const char *name)
  3145. {
  3146. struct reset_info *rcinfo = NULL;
  3147. int rc = 0;
  3148. bool found = false;
  3149. iris_hfi_for_each_reset_clock(device, rcinfo) {
  3150. if (strcmp(rcinfo->name, name))
  3151. continue;
  3152. found = true;
  3153. if (rcinfo->state != RESET_ACQUIRED) {
  3154. dprintk(CVP_WARN, "Double releasing reset clk?\n");
  3155. return -EINVAL;
  3156. }
  3157. reset_control_release(rcinfo->rst);
  3158. dprintk(CVP_PWR, "%s: reset release succeed (%s)\n",
  3159. __func__, rcinfo->name);
  3160. rcinfo->state = RESET_RELEASED;
  3161. break;
  3162. }
  3163. if (!found) {
  3164. dprintk(CVP_PWR, "%s: reset control (%s) not found\n",
  3165. __func__, name);
  3166. rc = -EINVAL;
  3167. }
  3168. return rc;
  3169. }
  3170. static void __deinit_bus(struct iris_hfi_device *device)
  3171. {
  3172. struct bus_info *bus = NULL;
  3173. if (!device)
  3174. return;
  3175. kfree(device->bus_vote.data);
  3176. device->bus_vote = CVP_DEFAULT_BUS_VOTE;
  3177. iris_hfi_for_each_bus_reverse(device, bus) {
  3178. dev_set_drvdata(bus->dev, NULL);
  3179. icc_put(bus->client);
  3180. bus->client = NULL;
  3181. }
  3182. }
  3183. static int __init_bus(struct iris_hfi_device *device)
  3184. {
  3185. struct bus_info *bus = NULL;
  3186. int rc = 0;
  3187. if (!device)
  3188. return -EINVAL;
  3189. iris_hfi_for_each_bus(device, bus) {
  3190. /*
  3191. * This is stupid, but there's no other easy way to ahold
  3192. * of struct bus_info in iris_hfi_devfreq_*()
  3193. */
  3194. WARN(dev_get_drvdata(bus->dev), "%s's drvdata already set\n",
  3195. dev_name(bus->dev));
  3196. dev_set_drvdata(bus->dev, device);
  3197. bus->client = icc_get(&device->res->pdev->dev,
  3198. bus->master, bus->slave);
  3199. if (IS_ERR_OR_NULL(bus->client)) {
  3200. rc = PTR_ERR(bus->client) ?: -EBADHANDLE;
  3201. dprintk(CVP_ERR, "Failed to register bus %s: %d\n",
  3202. bus->name, rc);
  3203. bus->client = NULL;
  3204. goto err_add_dev;
  3205. }
  3206. }
  3207. return 0;
  3208. err_add_dev:
  3209. __deinit_bus(device);
  3210. return rc;
  3211. }
  3212. static void __deinit_regulators(struct iris_hfi_device *device)
  3213. {
  3214. struct regulator_info *rinfo = NULL;
  3215. iris_hfi_for_each_regulator_reverse(device, rinfo) {
  3216. if (rinfo->regulator) {
  3217. regulator_put(rinfo->regulator);
  3218. rinfo->regulator = NULL;
  3219. }
  3220. }
  3221. }
  3222. static int __init_regulators(struct iris_hfi_device *device)
  3223. {
  3224. int rc = 0;
  3225. struct regulator_info *rinfo = NULL;
  3226. iris_hfi_for_each_regulator(device, rinfo) {
  3227. rinfo->regulator = regulator_get(&device->res->pdev->dev,
  3228. rinfo->name);
  3229. if (IS_ERR_OR_NULL(rinfo->regulator)) {
  3230. rc = PTR_ERR(rinfo->regulator) ?: -EBADHANDLE;
  3231. dprintk(CVP_ERR, "Failed to get regulator: %s\n",
  3232. rinfo->name);
  3233. rinfo->regulator = NULL;
  3234. goto err_reg_get;
  3235. }
  3236. }
  3237. return 0;
  3238. err_reg_get:
  3239. __deinit_regulators(device);
  3240. return rc;
  3241. }
  3242. static void __deinit_subcaches(struct iris_hfi_device *device)
  3243. {
  3244. struct subcache_info *sinfo = NULL;
  3245. if (!device) {
  3246. dprintk(CVP_ERR, "deinit_subcaches: invalid device %pK\n",
  3247. device);
  3248. goto exit;
  3249. }
  3250. if (!is_sys_cache_present(device))
  3251. goto exit;
  3252. iris_hfi_for_each_subcache_reverse(device, sinfo) {
  3253. if (sinfo->subcache) {
  3254. dprintk(CVP_CORE, "deinit_subcaches: %s\n",
  3255. sinfo->name);
  3256. llcc_slice_putd(sinfo->subcache);
  3257. sinfo->subcache = NULL;
  3258. }
  3259. }
  3260. exit:
  3261. return;
  3262. }
  3263. static int __init_subcaches(struct iris_hfi_device *device)
  3264. {
  3265. int rc = 0;
  3266. struct subcache_info *sinfo = NULL;
  3267. if (!device) {
  3268. dprintk(CVP_ERR, "init_subcaches: invalid device %pK\n",
  3269. device);
  3270. return -EINVAL;
  3271. }
  3272. if (!is_sys_cache_present(device))
  3273. return 0;
  3274. iris_hfi_for_each_subcache(device, sinfo) {
  3275. if (!strcmp("cvp", sinfo->name)) {
  3276. sinfo->subcache = llcc_slice_getd(LLCC_CVP);
  3277. } else if (!strcmp("cvpfw", sinfo->name)) {
  3278. sinfo->subcache = llcc_slice_getd(LLCC_CVPFW);
  3279. } else {
  3280. dprintk(CVP_ERR, "Invalid subcache name %s\n",
  3281. sinfo->name);
  3282. }
  3283. if (IS_ERR_OR_NULL(sinfo->subcache)) {
  3284. rc = PTR_ERR(sinfo->subcache) ?
  3285. PTR_ERR(sinfo->subcache) : -EBADHANDLE;
  3286. dprintk(CVP_ERR,
  3287. "init_subcaches: invalid subcache: %s rc %d\n",
  3288. sinfo->name, rc);
  3289. sinfo->subcache = NULL;
  3290. goto err_subcache_get;
  3291. }
  3292. dprintk(CVP_CORE, "init_subcaches: %s\n",
  3293. sinfo->name);
  3294. }
  3295. return 0;
  3296. err_subcache_get:
  3297. __deinit_subcaches(device);
  3298. return rc;
  3299. }
  3300. static int __init_resources(struct iris_hfi_device *device,
  3301. struct msm_cvp_platform_resources *res)
  3302. {
  3303. int i, rc = 0;
  3304. rc = __init_regulators(device);
  3305. if (rc) {
  3306. dprintk(CVP_ERR, "Failed to get all regulators\n");
  3307. return -ENODEV;
  3308. }
  3309. rc = msm_cvp_init_clocks(device);
  3310. if (rc) {
  3311. dprintk(CVP_ERR, "Failed to init clocks\n");
  3312. rc = -ENODEV;
  3313. goto err_init_clocks;
  3314. }
  3315. for (i = 0; i < device->res->reset_set.count; i++) {
  3316. rc = __init_reset_clk(res, i);
  3317. if (rc) {
  3318. dprintk(CVP_ERR, "Failed to init reset clocks\n");
  3319. rc = -ENODEV;
  3320. goto err_init_reset_clk;
  3321. }
  3322. }
  3323. rc = __init_bus(device);
  3324. if (rc) {
  3325. dprintk(CVP_ERR, "Failed to init bus: %d\n", rc);
  3326. goto err_init_bus;
  3327. }
  3328. rc = __init_subcaches(device);
  3329. if (rc)
  3330. dprintk(CVP_WARN, "Failed to init subcaches: %d\n", rc);
  3331. device->sys_init_capabilities =
  3332. kzalloc(sizeof(struct msm_cvp_capability)
  3333. * CVP_MAX_SESSIONS, GFP_KERNEL);
  3334. return rc;
  3335. err_init_reset_clk:
  3336. err_init_bus:
  3337. msm_cvp_deinit_clocks(device);
  3338. err_init_clocks:
  3339. __deinit_regulators(device);
  3340. return rc;
  3341. }
  3342. static void __deinit_resources(struct iris_hfi_device *device)
  3343. {
  3344. __deinit_subcaches(device);
  3345. __deinit_bus(device);
  3346. msm_cvp_deinit_clocks(device);
  3347. __deinit_regulators(device);
  3348. kfree(device->sys_init_capabilities);
  3349. device->sys_init_capabilities = NULL;
  3350. }
  3351. static int __disable_regulator_impl(struct regulator_info *rinfo,
  3352. struct iris_hfi_device *device)
  3353. {
  3354. int rc = 0;
  3355. dprintk(CVP_PWR, "Disabling regulator %s\n", rinfo->name);
  3356. /*
  3357. * This call is needed. Driver needs to acquire the control back
  3358. * from HW in order to disable the regualtor. Else the behavior
  3359. * is unknown.
  3360. */
  3361. rc = __acquire_regulator(rinfo, device);
  3362. if (rc) {
  3363. /*
  3364. * This is somewhat fatal, but nothing we can do
  3365. * about it. We can't disable the regulator w/o
  3366. * getting it back under s/w control
  3367. */
  3368. dprintk(CVP_WARN,
  3369. "Failed to acquire control on %s\n",
  3370. rinfo->name);
  3371. goto disable_regulator_failed;
  3372. }
  3373. rc = regulator_disable(rinfo->regulator);
  3374. if (rc) {
  3375. dprintk(CVP_WARN,
  3376. "Failed to disable %s: %d\n",
  3377. rinfo->name, rc);
  3378. goto disable_regulator_failed;
  3379. }
  3380. return 0;
  3381. disable_regulator_failed:
  3382. /* Bring attention to this issue */
  3383. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  3384. return rc;
  3385. }
  3386. static int __disable_hw_power_collapse(struct iris_hfi_device *device)
  3387. {
  3388. int rc = 0;
  3389. if (!msm_cvp_fw_low_power_mode) {
  3390. dprintk(CVP_PWR, "Not enabling hardware power collapse\n");
  3391. return 0;
  3392. }
  3393. rc = __take_back_regulators(device);
  3394. if (rc)
  3395. dprintk(CVP_WARN,
  3396. "%s : Failed to disable HW power collapse %d\n",
  3397. __func__, rc);
  3398. return rc;
  3399. }
  3400. static int __enable_hw_power_collapse(struct iris_hfi_device *device)
  3401. {
  3402. int rc = 0;
  3403. if (!msm_cvp_fw_low_power_mode) {
  3404. dprintk(CVP_PWR, "Not enabling hardware power collapse\n");
  3405. return 0;
  3406. }
  3407. rc = __hand_off_regulators(device);
  3408. if (rc)
  3409. dprintk(CVP_WARN,
  3410. "%s : Failed to enable HW power collapse %d\n",
  3411. __func__, rc);
  3412. return rc;
  3413. }
  3414. static int __enable_regulator(struct iris_hfi_device *device,
  3415. const char *name)
  3416. {
  3417. int rc = 0;
  3418. struct regulator_info *rinfo;
  3419. iris_hfi_for_each_regulator(device, rinfo) {
  3420. if (strcmp(rinfo->name, name))
  3421. continue;
  3422. rc = regulator_enable(rinfo->regulator);
  3423. if (rc) {
  3424. dprintk(CVP_ERR, "Failed to enable %s: %d\n",
  3425. rinfo->name, rc);
  3426. return rc;
  3427. }
  3428. if (!regulator_is_enabled(rinfo->regulator)) {
  3429. dprintk(CVP_ERR,"%s: regulator %s not enabled\n",
  3430. __func__, rinfo->name);
  3431. regulator_disable(rinfo->regulator);
  3432. return -EINVAL;
  3433. }
  3434. dprintk(CVP_PWR, "Enabled regulator %s\n", rinfo->name);
  3435. return 0;
  3436. }
  3437. dprintk(CVP_ERR, "regulator %s not found\n", name);
  3438. return -EINVAL;
  3439. }
  3440. static int __disable_regulator(struct iris_hfi_device *device,
  3441. const char *name)
  3442. {
  3443. struct regulator_info *rinfo;
  3444. iris_hfi_for_each_regulator_reverse(device, rinfo) {
  3445. if (strcmp(rinfo->name, name))
  3446. continue;
  3447. __disable_regulator_impl(rinfo, device);
  3448. dprintk(CVP_PWR, "%s Disabled regulator %s\n", __func__, name);
  3449. return 0;
  3450. }
  3451. dprintk(CVP_ERR, "%s regulator %s not found\n", __func__, name);
  3452. return -EINVAL;
  3453. }
  3454. static int __enable_subcaches(struct iris_hfi_device *device)
  3455. {
  3456. int rc = 0;
  3457. u32 c = 0;
  3458. struct subcache_info *sinfo;
  3459. if (msm_cvp_syscache_disable || !is_sys_cache_present(device))
  3460. return 0;
  3461. /* Activate subcaches */
  3462. iris_hfi_for_each_subcache(device, sinfo) {
  3463. rc = llcc_slice_activate(sinfo->subcache);
  3464. if (rc) {
  3465. dprintk(CVP_WARN, "Failed to activate %s: %d\n",
  3466. sinfo->name, rc);
  3467. msm_cvp_res_handle_fatal_hw_error(device->res, true);
  3468. goto err_activate_fail;
  3469. }
  3470. sinfo->isactive = true;
  3471. dprintk(CVP_CORE, "Activated subcache %s\n", sinfo->name);
  3472. c++;
  3473. }
  3474. dprintk(CVP_CORE, "Activated %d Subcaches to CVP\n", c);
  3475. return 0;
  3476. err_activate_fail:
  3477. __release_subcaches(device);
  3478. __disable_subcaches(device);
  3479. return 0;
  3480. }
  3481. static int __set_subcaches(struct iris_hfi_device *device)
  3482. {
  3483. int rc = 0;
  3484. u32 c = 0;
  3485. struct subcache_info *sinfo;
  3486. u32 resource[CVP_MAX_SUBCACHE_SIZE];
  3487. struct cvp_hfi_resource_syscache_info_type *sc_res_info;
  3488. struct cvp_hfi_resource_subcache_type *sc_res;
  3489. struct cvp_resource_hdr rhdr;
  3490. if (device->res->sys_cache_res_set || msm_cvp_syscache_disable) {
  3491. dprintk(CVP_CORE, "Subcaches already set or disabled\n");
  3492. return 0;
  3493. }
  3494. memset((void *)resource, 0x0, (sizeof(u32) * CVP_MAX_SUBCACHE_SIZE));
  3495. sc_res_info = (struct cvp_hfi_resource_syscache_info_type *)resource;
  3496. sc_res = &(sc_res_info->rg_subcache_entries[0]);
  3497. iris_hfi_for_each_subcache(device, sinfo) {
  3498. if (sinfo->isactive) {
  3499. sc_res[c].size = sinfo->subcache->slice_size;
  3500. sc_res[c].sc_id = sinfo->subcache->slice_id;
  3501. c++;
  3502. }
  3503. }
  3504. /* Set resource to CVP for activated subcaches */
  3505. if (c) {
  3506. dprintk(CVP_CORE, "Setting %d Subcaches\n", c);
  3507. rhdr.resource_handle = sc_res_info; /* cookie */
  3508. rhdr.resource_id = CVP_RESOURCE_SYSCACHE;
  3509. sc_res_info->num_entries = c;
  3510. rc = __core_set_resource(device, &rhdr, (void *)sc_res_info);
  3511. if (rc) {
  3512. dprintk(CVP_WARN, "Failed to set subcaches %d\n", rc);
  3513. goto err_fail_set_subacaches;
  3514. }
  3515. iris_hfi_for_each_subcache(device, sinfo) {
  3516. if (sinfo->isactive)
  3517. sinfo->isset = true;
  3518. }
  3519. dprintk(CVP_CORE, "Set Subcaches done to CVP\n");
  3520. device->res->sys_cache_res_set = true;
  3521. }
  3522. return 0;
  3523. err_fail_set_subacaches:
  3524. __disable_subcaches(device);
  3525. return 0;
  3526. }
  3527. static int __release_subcaches(struct iris_hfi_device *device)
  3528. {
  3529. struct subcache_info *sinfo;
  3530. int rc = 0;
  3531. u32 c = 0;
  3532. u32 resource[CVP_MAX_SUBCACHE_SIZE];
  3533. struct cvp_hfi_resource_syscache_info_type *sc_res_info;
  3534. struct cvp_hfi_resource_subcache_type *sc_res;
  3535. struct cvp_resource_hdr rhdr;
  3536. if (msm_cvp_syscache_disable || !is_sys_cache_present(device))
  3537. return 0;
  3538. memset((void *)resource, 0x0, (sizeof(u32) * CVP_MAX_SUBCACHE_SIZE));
  3539. sc_res_info = (struct cvp_hfi_resource_syscache_info_type *)resource;
  3540. sc_res = &(sc_res_info->rg_subcache_entries[0]);
  3541. /* Release resource command to Iris */
  3542. iris_hfi_for_each_subcache_reverse(device, sinfo) {
  3543. if (sinfo->isset) {
  3544. /* Update the entry */
  3545. sc_res[c].size = sinfo->subcache->slice_size;
  3546. sc_res[c].sc_id = sinfo->subcache->slice_id;
  3547. c++;
  3548. sinfo->isset = false;
  3549. }
  3550. }
  3551. if (c > 0) {
  3552. dprintk(CVP_CORE, "Releasing %d subcaches\n", c);
  3553. rhdr.resource_handle = sc_res_info; /* cookie */
  3554. rhdr.resource_id = CVP_RESOURCE_SYSCACHE;
  3555. rc = __core_release_resource(device, &rhdr);
  3556. if (rc)
  3557. dprintk(CVP_WARN,
  3558. "Failed to release %d subcaches\n", c);
  3559. }
  3560. device->res->sys_cache_res_set = false;
  3561. return 0;
  3562. }
  3563. static int __disable_subcaches(struct iris_hfi_device *device)
  3564. {
  3565. struct subcache_info *sinfo;
  3566. int rc = 0;
  3567. if (msm_cvp_syscache_disable || !is_sys_cache_present(device))
  3568. return 0;
  3569. /* De-activate subcaches */
  3570. iris_hfi_for_each_subcache_reverse(device, sinfo) {
  3571. if (sinfo->isactive) {
  3572. dprintk(CVP_CORE, "De-activate subcache %s\n",
  3573. sinfo->name);
  3574. rc = llcc_slice_deactivate(sinfo->subcache);
  3575. if (rc) {
  3576. dprintk(CVP_WARN,
  3577. "Failed to de-activate %s: %d\n",
  3578. sinfo->name, rc);
  3579. }
  3580. sinfo->isactive = false;
  3581. }
  3582. }
  3583. return 0;
  3584. }
  3585. static void interrupt_init_iris2(struct iris_hfi_device *device)
  3586. {
  3587. u32 mask_val = 0;
  3588. /* All interrupts should be disabled initially 0x1F6 : Reset value */
  3589. mask_val = __read_register(device, CVP_WRAPPER_INTR_MASK);
  3590. /* Write 0 to unmask CPU and WD interrupts */
  3591. mask_val &= ~(CVP_FATAL_INTR_BMSK | CVP_WRAPPER_INTR_MASK_A2HCPU_BMSK);
  3592. __write_register(device, CVP_WRAPPER_INTR_MASK, mask_val);
  3593. dprintk(CVP_REG, "Init irq: reg: %x, mask value %x\n",
  3594. CVP_WRAPPER_INTR_MASK, mask_val);
  3595. mask_val = 0;
  3596. mask_val = __read_register(device, CVP_SS_IRQ_MASK);
  3597. mask_val &= ~(CVP_SS_INTR_BMASK);
  3598. __write_register(device, CVP_SS_IRQ_MASK, mask_val);
  3599. dprintk(CVP_REG, "Init irq_wd: reg: %x, mask value %x\n",
  3600. CVP_SS_IRQ_MASK, mask_val);
  3601. }
  3602. static void setup_dsp_uc_memmap_vpu5(struct iris_hfi_device *device)
  3603. {
  3604. /* initialize DSP QTBL & UCREGION with CPU queues */
  3605. __write_register(device, HFI_DSP_QTBL_ADDR,
  3606. (u32)device->dsp_iface_q_table.align_device_addr);
  3607. __write_register(device, HFI_DSP_UC_REGION_ADDR,
  3608. (u32)device->dsp_iface_q_table.align_device_addr);
  3609. __write_register(device, HFI_DSP_UC_REGION_SIZE,
  3610. device->dsp_iface_q_table.mem_data.size);
  3611. }
  3612. static void clock_config_on_enable_vpu5(struct iris_hfi_device *device)
  3613. {
  3614. __write_register(device, CVP_WRAPPER_CPU_CLOCK_CONFIG, 0);
  3615. }
  3616. static int __set_ubwc_config(struct iris_hfi_device *device)
  3617. {
  3618. u8 packet[CVP_IFACEQ_VAR_SMALL_PKT_SIZE];
  3619. int rc = 0;
  3620. struct cvp_hfi_cmd_sys_set_property_packet *pkt =
  3621. (struct cvp_hfi_cmd_sys_set_property_packet *) &packet;
  3622. if (!device->res->ubwc_config)
  3623. return 0;
  3624. rc = call_hfi_pkt_op(device, sys_ubwc_config, pkt,
  3625. device->res->ubwc_config);
  3626. if (rc) {
  3627. dprintk(CVP_WARN,
  3628. "ubwc config setting to FW failed\n");
  3629. rc = -ENOTEMPTY;
  3630. goto fail_to_set_ubwc_config;
  3631. }
  3632. if (__iface_cmdq_write(device, pkt)) {
  3633. rc = -ENOTEMPTY;
  3634. goto fail_to_set_ubwc_config;
  3635. }
  3636. fail_to_set_ubwc_config:
  3637. return rc;
  3638. }
  3639. static int __power_on_controller(struct iris_hfi_device *device)
  3640. {
  3641. int rc = 0;
  3642. rc = __enable_regulator(device, "cvp");
  3643. if (rc) {
  3644. dprintk(CVP_ERR, "Failed to enable ctrler: %d\n", rc);
  3645. return rc;
  3646. }
  3647. rc = msm_cvp_prepare_enable_clk(device, "sleep_clk");
  3648. if (rc) {
  3649. dprintk(CVP_ERR, "Failed to enable sleep clk: %d\n", rc);
  3650. goto fail_reset_clks;
  3651. }
  3652. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_axi_reset");
  3653. if (rc)
  3654. dprintk(CVP_ERR, "%s: assert cvp_axi_reset failed\n", __func__);
  3655. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_core_reset");
  3656. if (rc)
  3657. dprintk(CVP_ERR, "%s: assert cvp_core_reset failed\n", __func__);
  3658. /* wait for deassert */
  3659. usleep_range(1000, 1050);
  3660. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_axi_reset");
  3661. if (rc)
  3662. dprintk(CVP_ERR, "%s: de-assert cvp_axi_reset failed\n", __func__);
  3663. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_core_reset");
  3664. if (rc)
  3665. dprintk(CVP_ERR, "%s: de-assert cvp_core_reset failed\n", __func__);
  3666. rc = msm_cvp_prepare_enable_clk(device, "gcc_video_axi1");
  3667. if (rc) {
  3668. dprintk(CVP_ERR, "Failed to enable axi1 clk: %d\n", rc);
  3669. goto fail_reset_clks;
  3670. }
  3671. rc = msm_cvp_prepare_enable_clk(device, "cvp_clk");
  3672. if (rc) {
  3673. dprintk(CVP_ERR, "Failed to enable cvp_clk: %d\n", rc);
  3674. goto fail_enable_clk;
  3675. }
  3676. dprintk(CVP_PWR, "EVA controller powered on\n");
  3677. return 0;
  3678. fail_enable_clk:
  3679. msm_cvp_disable_unprepare_clk(device, "gcc_video_axi1");
  3680. fail_reset_clks:
  3681. __disable_regulator(device, "cvp");
  3682. return rc;
  3683. }
  3684. static int __power_on_core(struct iris_hfi_device *device)
  3685. {
  3686. int rc = 0;
  3687. rc = __enable_regulator(device, "cvp-core");
  3688. if (rc) {
  3689. dprintk(CVP_ERR, "Failed to enable core: %d\n", rc);
  3690. return rc;
  3691. }
  3692. rc = msm_cvp_prepare_enable_clk(device, "video_cc_mvs1_clk_src");
  3693. if (rc) {
  3694. dprintk(CVP_ERR, "Failed to enable video_cc_mvs1_clk_src:%d\n",
  3695. rc);
  3696. __disable_regulator(device, "cvp-core");
  3697. return rc;
  3698. }
  3699. rc = msm_cvp_prepare_enable_clk(device, "core_clk");
  3700. if (rc) {
  3701. dprintk(CVP_ERR, "Failed to enable core_clk: %d\n", rc);
  3702. __disable_regulator(device, "cvp-core");
  3703. return rc;
  3704. }
  3705. /*#ifdef CONFIG_EVA_PINEAPPLE
  3706. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_ARCG_CONTROL, 0);
  3707. __write_register(device, CVP_NOC_RCGCONTROLLER_HYSTERESIS_LOW, 0x2f);
  3708. __write_register(device, CVP_NOC_RCG_VNOC_NOC_CLK_FORCECLOCKON_LOW, 1);
  3709. __write_register(device, CVP_NOC_RCGCONTROLLER_MAINCTL_LOW, 1);
  3710. usleep_range(50, 100);
  3711. __write_register(device, CVP_NOC_RCG_VNOC_NOC_CLK_FORCECLOCKON_LOW, 0);
  3712. #endif*/
  3713. dprintk(CVP_PWR, "EVA core powered on\n");
  3714. return 0;
  3715. }
  3716. static int __iris_power_on(struct iris_hfi_device *device)
  3717. {
  3718. int rc = 0;
  3719. if (device->power_enabled)
  3720. return 0;
  3721. /* Vote for all hardware resources */
  3722. rc = __vote_buses(device, device->bus_vote.data,
  3723. device->bus_vote.data_count);
  3724. if (rc) {
  3725. dprintk(CVP_ERR, "Failed to vote buses, err: %d\n", rc);
  3726. goto fail_vote_buses;
  3727. }
  3728. rc = __power_on_controller(device);
  3729. if (rc)
  3730. goto fail_enable_controller;
  3731. rc = __power_on_core(device);
  3732. if (rc)
  3733. goto fail_enable_core;
  3734. rc = msm_cvp_scale_clocks(device);
  3735. if (rc) {
  3736. dprintk(CVP_WARN,
  3737. "Failed to scale clocks, perf may regress\n");
  3738. rc = 0;
  3739. } else {
  3740. dprintk(CVP_PWR, "Done with scaling\n");
  3741. }
  3742. /*Do not access registers before this point!*/
  3743. device->power_enabled = true;
  3744. /*
  3745. * Re-program all of the registers that get reset as a result of
  3746. * regulator_disable() and _enable()
  3747. * calling below function requires CORE powered on
  3748. */
  3749. rc = __set_registers(device);
  3750. if (rc)
  3751. goto fail_enable_core;
  3752. dprintk(CVP_CORE, "Done with register set\n");
  3753. call_iris_op(device, interrupt_init, device);
  3754. dprintk(CVP_CORE, "Done with interrupt enabling\n");
  3755. device->intr_status = 0;
  3756. enable_irq(device->cvp_hal_data->irq);
  3757. __write_register(device,
  3758. CVP_WRAPPER_DEBUG_BRIDGE_LPI_CONTROL, 0x7);
  3759. pr_info_ratelimited(CVP_DBG_TAG "cvp (eva) powered on\n", "pwr");
  3760. return 0;
  3761. fail_enable_core:
  3762. __power_off_controller(device);
  3763. fail_enable_controller:
  3764. __unvote_buses(device);
  3765. fail_vote_buses:
  3766. device->power_enabled = false;
  3767. return rc;
  3768. }
  3769. static inline int __suspend(struct iris_hfi_device *device)
  3770. {
  3771. int rc = 0;
  3772. if (!device) {
  3773. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  3774. return -EINVAL;
  3775. } else if (!device->power_enabled) {
  3776. dprintk(CVP_PWR, "Power already disabled\n");
  3777. return 0;
  3778. }
  3779. dprintk(CVP_PWR, "Entering suspend\n");
  3780. rc = __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_SUSPEND);
  3781. if (rc) {
  3782. dprintk(CVP_WARN, "Failed to suspend cvp core %d\n", rc);
  3783. goto err_tzbsp_suspend;
  3784. }
  3785. __disable_subcaches(device);
  3786. call_iris_op(device, power_off, device);
  3787. if (device->res->pm_qos.latency_us && device->res->pm_qos.pm_qos_hdls)
  3788. cvp_pm_qos_update(device, false);
  3789. return rc;
  3790. err_tzbsp_suspend:
  3791. return rc;
  3792. }
  3793. static void __print_sidebandmanager_regs(struct iris_hfi_device *device)
  3794. {
  3795. u32 sbm_ln0_low, axi_cbcr;
  3796. u32 main_sbm_ln0_low = 0xdeadbeef, main_sbm_ln0_high = 0xdeadbeef;
  3797. u32 main_sbm_ln1_high = 0xdeadbeef, cpu_cs_x2rpmh;
  3798. int rc;
  3799. sbm_ln0_low =
  3800. __read_register(device, CVP_NOC_SBM_SENSELN0_LOW);
  3801. cpu_cs_x2rpmh = __read_register(device, CVP_CPU_CS_X2RPMh);
  3802. __write_register(device, CVP_CPU_CS_X2RPMh,
  3803. (cpu_cs_x2rpmh | CVP_CPU_CS_X2RPMh_SWOVERRIDE_BMSK));
  3804. usleep_range(500, 1000);
  3805. cpu_cs_x2rpmh = __read_register(device, CVP_CPU_CS_X2RPMh);
  3806. if (!(cpu_cs_x2rpmh & CVP_CPU_CS_X2RPMh_SWOVERRIDE_BMSK)) {
  3807. dprintk(CVP_WARN,
  3808. "failed set CVP_CPU_CS_X2RPMH mask %x\n",
  3809. cpu_cs_x2rpmh);
  3810. goto exit;
  3811. }
  3812. axi_cbcr = __read_gcc_register(device, CVP_GCC_VIDEO_AXI1_CBCR);
  3813. if (axi_cbcr & 0x80000000) {
  3814. dprintk(CVP_WARN, "failed to turn on AXI clock %x\n",
  3815. axi_cbcr);
  3816. goto exit;
  3817. }
  3818. rc = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  3819. if (rc) {
  3820. dprintk(CVP_WARN, "%s Fail acquire xo_reset\n", __func__);
  3821. goto exit;
  3822. }
  3823. main_sbm_ln0_low = __read_register(device,
  3824. CVP_NOC_MAIN_SIDEBANDMANAGER_SENSELN0_LOW);
  3825. main_sbm_ln0_high = __read_register(device,
  3826. CVP_NOC_MAIN_SIDEBANDMANAGER_SENSELN0_HIGH);
  3827. main_sbm_ln1_high = __read_register(device,
  3828. CVP_NOC_MAIN_SIDEBANDMANAGER_SENSELN1_HIGH);
  3829. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  3830. exit:
  3831. cpu_cs_x2rpmh = cpu_cs_x2rpmh & (~CVP_CPU_CS_X2RPMh_SWOVERRIDE_BMSK);
  3832. __write_register(device, CVP_CPU_CS_X2RPMh, cpu_cs_x2rpmh);
  3833. dprintk(CVP_WARN, "Sidebandmanager regs %x %x %x %x %x\n",
  3834. sbm_ln0_low, main_sbm_ln0_low,
  3835. main_sbm_ln0_high, main_sbm_ln1_high,
  3836. cpu_cs_x2rpmh);
  3837. }
  3838. static int __power_off_controller(struct iris_hfi_device *device)
  3839. {
  3840. u32 lpi_status, reg_status = 0, count = 0, max_count = 1000;
  3841. u32 sbm_ln0_low;
  3842. int rc;
  3843. u32 spare_val, spare_status;
  3844. /* HPG 6.2.2 Step 1 */
  3845. __write_register(device, CVP_CPU_CS_X2RPMh, 0x3);
  3846. /* HPG 6.2.2 Step 2, noc to low power */
  3847. /* New addition to put CPU/Tensilica to low power */
  3848. reg_status = 0;
  3849. count = 0;
  3850. __write_register(device, CVP_WRAPPER_CPU_NOC_LPI_CONTROL, 0x1);
  3851. while (!reg_status && count < max_count) {
  3852. lpi_status =
  3853. __read_register(device,
  3854. CVP_WRAPPER_CPU_NOC_LPI_STATUS);
  3855. reg_status = lpi_status & BIT(0);
  3856. /* Wait for CPU noc lpi status to be set */
  3857. usleep_range(50, 100);
  3858. count++;
  3859. }
  3860. sbm_ln0_low = __read_register(device, CVP_NOC_SBM_SENSELN0_LOW);
  3861. dprintk(CVP_PWR,
  3862. "CPU Noc: lpi_status %x noc_status %x (count %d) 0x%x\n",
  3863. lpi_status, reg_status, count, sbm_ln0_low);
  3864. if (count == max_count) {
  3865. u32 pc_ready, wfi_status;
  3866. wfi_status = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  3867. pc_ready = __read_register(device, CVP_CTRL_STATUS);
  3868. dprintk(CVP_WARN,
  3869. "CPU NOC not in qaccept status %x %x %x %x\n",
  3870. reg_status, lpi_status, wfi_status, pc_ready);
  3871. __print_sidebandmanager_regs(device);
  3872. }
  3873. /* HPG 6.2.2 Step 3, debug bridge to low power BYPASSED */
  3874. /* HPG 6.2.2 Step 4, debug bridge to lpi release */
  3875. __write_register(device,
  3876. CVP_WRAPPER_DEBUG_BRIDGE_LPI_CONTROL, 0x0);
  3877. lpi_status = 0x1;
  3878. count = 0;
  3879. while (lpi_status && count < max_count) {
  3880. lpi_status = __read_register(device,
  3881. CVP_WRAPPER_DEBUG_BRIDGE_LPI_STATUS);
  3882. usleep_range(50, 100);
  3883. count++;
  3884. }
  3885. dprintk(CVP_PWR,
  3886. "DBLP Release: lpi_status %d(count %d)\n",
  3887. lpi_status, count);
  3888. if (count == max_count) {
  3889. dprintk(CVP_WARN,
  3890. "DBLP Release: lpi_status %x\n", lpi_status);
  3891. }
  3892. /* PDXFIFO reset: addition for Kailua / Lanai */
  3893. __write_register(device, CVP_WRAPPER_AXI_CLOCK_CONFIG, 0x3);
  3894. __write_register(device, CVP_WRAPPER_QNS4PDXFIFO_RESET, 0x1);
  3895. __write_register(device, CVP_WRAPPER_QNS4PDXFIFO_RESET, 0x0);
  3896. __write_register(device, CVP_WRAPPER_AXI_CLOCK_CONFIG, 0x0);
  3897. /* HPG 6.2.2 Step 5 */
  3898. msm_cvp_disable_unprepare_clk(device, "cvp_clk");
  3899. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_axi_reset");
  3900. if (rc)
  3901. dprintk(CVP_ERR, "%s: assert cvp_axi_reset failed\n", __func__);
  3902. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_core_reset");
  3903. if (rc)
  3904. dprintk(CVP_ERR, "%s: assert cvp_core_reset failed\n", __func__);
  3905. /* wait for deassert */
  3906. usleep_range(1000, 1050);
  3907. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_axi_reset");
  3908. if (rc)
  3909. dprintk(CVP_ERR, "%s: de-assert cvp_axi_reset failed\n", __func__);
  3910. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_core_reset");
  3911. if (rc)
  3912. dprintk(CVP_ERR, "%s: de-assert cvp_core_reset failed\n", __func__);
  3913. /* disable EVA NoC clock */
  3914. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_CLK_CONTROL, 0x1);
  3915. /* enable EVA NoC reset */
  3916. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_SW_RESET, 0x1);
  3917. rc = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  3918. if (rc) {
  3919. dprintk(CVP_ERR, "FATAL ERROR, HPG step 17 to 20 will be bypassed\n");
  3920. goto skip_xo_reset;
  3921. }
  3922. spare_status = 0x1;
  3923. while (spare_status != 0x0) {
  3924. spare_val = __read_register(device, CVP_AON_WRAPPER_SPARE);
  3925. spare_status = spare_val & 0x2;
  3926. usleep_range(50, 100);
  3927. }
  3928. __write_register(device, CVP_AON_WRAPPER_SPARE, 0x1);
  3929. rc = call_iris_op(device, reset_control_assert_name, device, "cvp_xo_reset");
  3930. if (rc)
  3931. dprintk(CVP_ERR, "%s: assert cvp_xo_reset failed\n", __func__);
  3932. /* de-assert EVA_NoC reset */
  3933. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_SW_RESET, 0x0);
  3934. /* de-assert EVA video_cc XO reset and enable video_cc XO clock after 80us */
  3935. usleep_range(80, 100);
  3936. rc = call_iris_op(device, reset_control_deassert_name, device, "cvp_xo_reset");
  3937. if (rc)
  3938. dprintk(CVP_ERR, "%s: de-assert cvp_xo_reset failed\n", __func__);
  3939. /* clear XO mask bit - this step was missing in previous sequence */
  3940. __write_register(device, CVP_AON_WRAPPER_SPARE, 0x0);
  3941. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  3942. skip_xo_reset:
  3943. /* enable EVA NoC clock */
  3944. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_CLK_CONTROL, 0x0);
  3945. /* De-assert EVA_CTL Force Sleep Retention */
  3946. usleep_range(400, 500);
  3947. /* HPG 6.2.2 Step 6 */
  3948. __disable_regulator(device, "cvp");
  3949. /* HPG 6.2.2 Step 7 */
  3950. rc = msm_cvp_disable_unprepare_clk(device, "gcc_video_axi1");
  3951. if (rc) {
  3952. dprintk(CVP_ERR, "Failed to enable axi1 clk: %d\n", rc);
  3953. }
  3954. rc = msm_cvp_disable_unprepare_clk(device, "sleep_clk");
  3955. if (rc) {
  3956. dprintk(CVP_ERR, "Failed to disable sleep clk: %d\n", rc);
  3957. }
  3958. return 0;
  3959. }
  3960. static int __power_off_core(struct iris_hfi_device *device)
  3961. {
  3962. u32 reg_status = 0, lpi_status, config, value = 0, count = 0;
  3963. u32 warn_flag = 0, max_count = 10;
  3964. value = __read_register(device, CVP_CC_MVS1_GDSCR);
  3965. if (!(value & 0x80000000)) {
  3966. /*
  3967. * Core has been powered off by f/w.
  3968. * Check NOC reset registers to ensure
  3969. * NO outstanding NoC transactions
  3970. */
  3971. value = __read_register(device, CVP_NOC_RESET_ACK);
  3972. if (value) {
  3973. dprintk(CVP_WARN,
  3974. "Core off with NOC RESET ACK non-zero %x\n",
  3975. value);
  3976. __print_sidebandmanager_regs(device);
  3977. }
  3978. __disable_regulator(device, "cvp-core");
  3979. msm_cvp_disable_unprepare_clk(device, "core_clk");
  3980. msm_cvp_disable_unprepare_clk(device, "video_cc_mvs1_clk_src");
  3981. return 0;
  3982. } else if (!(value & 0x2)) {
  3983. /*
  3984. * HW_CONTROL PC disabled, then core is powered on for
  3985. * CVP NoC access
  3986. */
  3987. __disable_regulator(device, "cvp-core");
  3988. msm_cvp_disable_unprepare_clk(device, "core_clk");
  3989. msm_cvp_disable_unprepare_clk(device, "video_cc_mvs1_clk_src");
  3990. return 0;
  3991. }
  3992. dprintk(CVP_PWR, "Driver controls Core power off now\n");
  3993. /*
  3994. * check to make sure core clock branch enabled else
  3995. * we cannot read core idle register
  3996. */
  3997. config = __read_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG);
  3998. if (config) {
  3999. dprintk(CVP_PWR,
  4000. "core clock config not enabled, enable it to access core\n");
  4001. __write_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG, 0);
  4002. }
  4003. /*
  4004. * add MNoC idle check before collapsing MVS1 per HPG update
  4005. * poll for NoC DMA idle -> HPG 6.2.1
  4006. *
  4007. */
  4008. do {
  4009. value = __read_register(device, CVP_SS_IDLE_STATUS);
  4010. if (value & 0x400000)
  4011. break;
  4012. else
  4013. usleep_range(1000, 2000);
  4014. count++;
  4015. } while (count < max_count);
  4016. if (count == max_count) {
  4017. dprintk(CVP_WARN, "Core fail to go idle %x\n", value);
  4018. warn_flag = 1;
  4019. }
  4020. count = 0;
  4021. max_count = 1000;
  4022. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_LPI_CONTROL, 0x1);
  4023. while (!reg_status && count < max_count) {
  4024. lpi_status =
  4025. __read_register(device,
  4026. CVP_AON_WRAPPER_CVP_NOC_LPI_STATUS);
  4027. reg_status = lpi_status & BIT(0);
  4028. /* Wait for Core noc lpi status to be set */
  4029. usleep_range(50, 100);
  4030. count++;
  4031. }
  4032. dprintk(CVP_PWR,
  4033. "Core Noc: lpi_status %x noc_status %x (count %d)\n",
  4034. lpi_status, reg_status, count);
  4035. if (count == max_count) {
  4036. u32 pc_ready, wfi_status;
  4037. wfi_status = __read_register(device, CVP_WRAPPER_CPU_STATUS);
  4038. pc_ready = __read_register(device, CVP_CTRL_STATUS);
  4039. dprintk(CVP_WARN,
  4040. "Core NOC not in qaccept status %x %x %x %x\n",
  4041. reg_status, lpi_status, wfi_status, pc_ready);
  4042. __print_sidebandmanager_regs(device);
  4043. }
  4044. __write_register(device, CVP_AON_WRAPPER_CVP_NOC_LPI_CONTROL, 0x0);
  4045. if (warn_flag)
  4046. __print_sidebandmanager_regs(device);
  4047. /* Reset both sides of 2 ahb2ahb_bridges (TZ and non-TZ) */
  4048. __write_register(device, CVP_AHB_BRIDGE_SYNC_RESET, 0x3);
  4049. __write_register(device, CVP_AHB_BRIDGE_SYNC_RESET, 0x2);
  4050. __write_register(device, CVP_AHB_BRIDGE_SYNC_RESET, 0x0);
  4051. __write_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG, config);
  4052. __disable_hw_power_collapse(device);
  4053. usleep_range(100, 200);
  4054. __disable_regulator(device, "cvp-core");
  4055. msm_cvp_disable_unprepare_clk(device, "core_clk");
  4056. msm_cvp_disable_unprepare_clk(device, "video_cc_mvs1_clk_src");
  4057. return 0;
  4058. }
  4059. static void power_off_iris2(struct iris_hfi_device *device)
  4060. {
  4061. if (!device->power_enabled || !device->res->sw_power_collapsible)
  4062. return;
  4063. if (!(device->intr_status & CVP_WRAPPER_INTR_STATUS_A2HWD_BMSK))
  4064. disable_irq_nosync(device->cvp_hal_data->irq);
  4065. device->intr_status = 0;
  4066. __power_off_core(device);
  4067. __power_off_controller(device);
  4068. if (__unvote_buses(device))
  4069. dprintk(CVP_WARN, "Failed to unvote for buses\n");
  4070. /*Do not access registers after this point!*/
  4071. device->power_enabled = false;
  4072. pr_info(CVP_DBG_TAG "cvp (eva) power collapsed\n", "pwr");
  4073. }
  4074. static inline int __resume(struct iris_hfi_device *device)
  4075. {
  4076. int rc = 0;
  4077. u32 reg_gdsc, reg_cbcr;
  4078. struct msm_cvp_core *core;
  4079. if (!device) {
  4080. dprintk(CVP_ERR, "Invalid params: %pK\n", device);
  4081. return -EINVAL;
  4082. } else if (device->power_enabled) {
  4083. goto exit;
  4084. } else if (!__core_in_valid_state(device)) {
  4085. dprintk(CVP_PWR, "iris_hfi_device in deinit state.");
  4086. return -EINVAL;
  4087. }
  4088. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  4089. dprintk(CVP_PWR, "Resuming from power collapse\n");
  4090. rc = __iris_power_on(device);
  4091. if (rc) {
  4092. dprintk(CVP_ERR, "Failed to power on cvp\n");
  4093. goto err_iris_power_on;
  4094. }
  4095. reg_gdsc = __read_register(device, CVP_CC_MVS1C_GDSCR);
  4096. reg_cbcr = __read_register(device, CVP_CC_MVS1C_CBCR);
  4097. if (!(reg_gdsc & 0x80000000) || (reg_cbcr & 0x80000000))
  4098. dprintk(CVP_ERR, "CVP power on failed gdsc %x cbcr %x\n",
  4099. reg_gdsc, reg_cbcr);
  4100. __setup_ucregion_memory_map(device);
  4101. /* RUMI: set CVP_CTRL_INIT register to disable synx in FW */
  4102. /* Reboot the firmware */
  4103. rc = __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_RESUME);
  4104. if (rc) {
  4105. dprintk(CVP_ERR, "Failed to resume cvp core %d\n", rc);
  4106. goto err_set_cvp_state;
  4107. }
  4108. /* Wait for boot completion */
  4109. rc = __boot_firmware(device);
  4110. if (rc) {
  4111. dprintk(CVP_ERR, "Failed to reset cvp core\n");
  4112. msm_cvp_trigger_ssr(core, SSR_ERR_FATAL);
  4113. goto err_reset_core;
  4114. }
  4115. /*
  4116. * Work around for H/W bug, need to reprogram these registers once
  4117. * firmware is out reset
  4118. */
  4119. __set_threshold_registers(device);
  4120. if (device->res->pm_qos.latency_us && device->res->pm_qos.pm_qos_hdls)
  4121. cvp_pm_qos_update(device, true);
  4122. __sys_set_debug(device, msm_cvp_fw_debug);
  4123. __enable_subcaches(device);
  4124. __set_subcaches(device);
  4125. __dsp_resume(device);
  4126. dprintk(CVP_PWR, "Resumed from power collapse\n");
  4127. exit:
  4128. /* Don't reset skip_pc_count for SYS_PC_PREP cmd */
  4129. if (device->last_packet_type != HFI_CMD_SYS_PC_PREP)
  4130. device->skip_pc_count = 0;
  4131. return rc;
  4132. err_reset_core:
  4133. __tzbsp_set_cvp_state(TZ_SUBSYS_STATE_SUSPEND);
  4134. err_set_cvp_state:
  4135. call_iris_op(device, power_off, device);
  4136. err_iris_power_on:
  4137. dprintk(CVP_ERR, "Failed to resume from power collapse\n");
  4138. return rc;
  4139. }
  4140. static int __power_on_init(struct iris_hfi_device *device)
  4141. {
  4142. int rc = 0;
  4143. /* Initialize resources */
  4144. rc = __init_resources(device, device->res);
  4145. if (rc) {
  4146. dprintk(CVP_ERR, "Failed to init resources: %d\n", rc);
  4147. return rc;
  4148. }
  4149. rc = __initialize_packetization(device);
  4150. if (rc) {
  4151. dprintk(CVP_ERR, "Failed to initialize packetization\n");
  4152. goto fail_iris_init;
  4153. }
  4154. rc = __iris_power_on(device);
  4155. if (rc) {
  4156. dprintk(CVP_ERR, "Failed to power on iris in in load_fw\n");
  4157. goto fail_iris_init;
  4158. }
  4159. return rc;
  4160. fail_iris_init:
  4161. __deinit_resources(device);
  4162. return rc;
  4163. }
  4164. static int __load_fw(struct iris_hfi_device *device)
  4165. {
  4166. int rc = 0;
  4167. if ((!device->res->use_non_secure_pil && !device->res->firmware_base)
  4168. || device->res->use_non_secure_pil) {
  4169. rc = load_cvp_fw_impl(device);
  4170. if (rc)
  4171. goto fail_load_fw;
  4172. }
  4173. return rc;
  4174. fail_load_fw:
  4175. call_iris_op(device, power_off, device);
  4176. return rc;
  4177. }
  4178. static void __unload_fw(struct iris_hfi_device *device)
  4179. {
  4180. if (!device->resources.fw.cookie)
  4181. return;
  4182. cancel_delayed_work(&iris_hfi_pm_work);
  4183. if (device->state != IRIS_STATE_DEINIT)
  4184. flush_workqueue(device->iris_pm_workq);
  4185. unload_cvp_fw_impl(device);
  4186. __interface_queues_release(device);
  4187. call_iris_op(device, power_off, device);
  4188. __deinit_resources(device);
  4189. dprintk(CVP_WARN, "Firmware unloaded\n");
  4190. }
  4191. static int iris_hfi_get_fw_info(void *dev, struct cvp_hal_fw_info *fw_info)
  4192. {
  4193. int i = 0;
  4194. struct iris_hfi_device *device = dev;
  4195. if (!device || !fw_info) {
  4196. dprintk(CVP_ERR,
  4197. "%s Invalid parameter: device = %pK fw_info = %pK\n",
  4198. __func__, device, fw_info);
  4199. return -EINVAL;
  4200. }
  4201. mutex_lock(&device->lock);
  4202. while (cvp_driver->fw_version[i++] != 'V' && i < CVP_VERSION_LENGTH)
  4203. ;
  4204. if (i == CVP_VERSION_LENGTH - 1) {
  4205. dprintk(CVP_WARN, "Iris version string is not proper\n");
  4206. fw_info->version[0] = '\0';
  4207. goto fail_version_string;
  4208. }
  4209. memcpy(&fw_info->version[0], &cvp_driver->fw_version[0],
  4210. CVP_VERSION_LENGTH);
  4211. fw_info->version[CVP_VERSION_LENGTH - 1] = '\0';
  4212. fail_version_string:
  4213. dprintk(CVP_CORE, "F/W version retrieved : %s\n", fw_info->version);
  4214. fw_info->base_addr = device->cvp_hal_data->firmware_base;
  4215. fw_info->register_base = device->res->register_base;
  4216. fw_info->register_size = device->cvp_hal_data->register_size;
  4217. fw_info->irq = device->cvp_hal_data->irq;
  4218. mutex_unlock(&device->lock);
  4219. return 0;
  4220. }
  4221. static int iris_hfi_get_core_capabilities(void *dev)
  4222. {
  4223. dprintk(CVP_CORE, "%s not supported yet!\n", __func__);
  4224. return 0;
  4225. }
  4226. static const char * const mid_names[16] = {
  4227. "CVP_FW",
  4228. "ARP_DATA",
  4229. "CVP_MPU_PIXEL",
  4230. "CVP_MPU_NON_PIXEL",
  4231. "CVP_FDU_PIXEL",
  4232. "CVP_FDU_NON_PIXEL",
  4233. "CVP_GCE_PIXEL",
  4234. "CVP_GCE_NON_PIXEL",
  4235. "CVP_TOF_PIXEL",
  4236. "CVP_TOF_NON_PIXEL",
  4237. "CVP_VADL_PIXEL",
  4238. "CVP_VADL_NON_PIXEL",
  4239. "CVP_RGE_NON_PIXEL",
  4240. "CVP_CDM",
  4241. "Invalid",
  4242. "Invalid"
  4243. };
  4244. static void __print_reg_details(u32 val)
  4245. {
  4246. u32 mid, sid;
  4247. mid = (val >> 5) & 0xF;
  4248. sid = (val >> 2) & 0x7;
  4249. dprintk(CVP_ERR, "CVP_NOC_CORE_ERL_MAIN_ERRLOG3_LOW: %#x\n", val);
  4250. dprintk(CVP_ERR, "Sub-client:%s, SID: %d\n", mid_names[mid], sid);
  4251. }
  4252. static void __err_log(bool logging, u32 *data, const char *name, u32 val)
  4253. {
  4254. if (logging)
  4255. *data = val;
  4256. dprintk(CVP_ERR, "%s: %#x\n", name, val);
  4257. }
  4258. static void __noc_error_info_iris2(struct iris_hfi_device *device)
  4259. {
  4260. struct msm_cvp_core *core;
  4261. struct cvp_noc_log *noc_log;
  4262. u32 val = 0, regi, regii, regiii, i;
  4263. bool log_required = false;
  4264. int rc;
  4265. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  4266. if (!core->ssr_count && core->resources.max_ssr_allowed > 1)
  4267. log_required = true;
  4268. noc_log = &core->log.noc_log;
  4269. if (noc_log->used) {
  4270. dprintk(CVP_WARN, "Data already in NoC log, skip logging\n");
  4271. return;
  4272. }
  4273. noc_log->used = 1;
  4274. __disable_hw_power_collapse(device);
  4275. val = __read_register(device, CVP_CC_MVS1_GDSCR);
  4276. regi = __read_register(device, CVP_AON_WRAPPER_CVP_NOC_CORE_CLK_CONTROL);
  4277. regii = __read_register(device, CVP_CC_MVS1_CBCR);
  4278. regiii = __read_register(device, CVP_WRAPPER_CORE_CLOCK_CONFIG);
  4279. dprintk(CVP_ERR, "noc reg check: %#x %#x %#x %#x\n",
  4280. val, regi, regii, regiii);
  4281. val = __read_register(device, CVP_NOC_ERR_SWID_LOW_OFFS);
  4282. __err_log(log_required, &noc_log->err_ctrl_swid_low,
  4283. "CVP_NOC_ERL_MAIN_SWID_LOW", val);
  4284. val = __read_register(device, CVP_NOC_ERR_SWID_HIGH_OFFS);
  4285. __err_log(log_required, &noc_log->err_ctrl_swid_high,
  4286. "CVP_NOC_ERL_MAIN_SWID_HIGH", val);
  4287. val = __read_register(device, CVP_NOC_ERR_MAINCTL_LOW_OFFS);
  4288. __err_log(log_required, &noc_log->err_ctrl_mainctl_low,
  4289. "CVP_NOC_ERL_MAIN_MAINCTL_LOW", val);
  4290. val = __read_register(device, CVP_NOC_ERR_ERRVLD_LOW_OFFS);
  4291. __err_log(log_required, &noc_log->err_ctrl_errvld_low,
  4292. "CVP_NOC_ERL_MAIN_ERRVLD_LOW", val);
  4293. val = __read_register(device, CVP_NOC_ERR_ERRCLR_LOW_OFFS);
  4294. __err_log(log_required, &noc_log->err_ctrl_errclr_low,
  4295. "CVP_NOC_ERL_MAIN_ERRCLR_LOW", val);
  4296. val = __read_register(device, CVP_NOC_ERR_ERRLOG0_LOW_OFFS);
  4297. __err_log(log_required, &noc_log->err_ctrl_errlog0_low,
  4298. "CVP_NOC_ERL_MAIN_ERRLOG0_LOW", val);
  4299. val = __read_register(device, CVP_NOC_ERR_ERRLOG0_HIGH_OFFS);
  4300. __err_log(log_required, &noc_log->err_ctrl_errlog0_high,
  4301. "CVP_NOC_ERL_MAIN_ERRLOG0_HIGH", val);
  4302. val = __read_register(device, CVP_NOC_ERR_ERRLOG1_LOW_OFFS);
  4303. __err_log(log_required, &noc_log->err_ctrl_errlog1_low,
  4304. "CVP_NOC_ERL_MAIN_ERRLOG1_LOW", val);
  4305. val = __read_register(device, CVP_NOC_ERR_ERRLOG1_HIGH_OFFS);
  4306. __err_log(log_required, &noc_log->err_ctrl_errlog1_high,
  4307. "CVP_NOC_ERL_MAIN_ERRLOG1_HIGH", val);
  4308. val = __read_register(device, CVP_NOC_ERR_ERRLOG2_LOW_OFFS);
  4309. __err_log(log_required, &noc_log->err_ctrl_errlog2_low,
  4310. "CVP_NOC_ERL_MAIN_ERRLOG2_LOW", val);
  4311. val = __read_register(device, CVP_NOC_ERR_ERRLOG2_HIGH_OFFS);
  4312. __err_log(log_required, &noc_log->err_ctrl_errlog2_high,
  4313. "CVP_NOC_ERL_MAIN_ERRLOG2_HIGH", val);
  4314. val = __read_register(device, CVP_NOC_ERR_ERRLOG3_LOW_OFFS);
  4315. __err_log(log_required, &noc_log->err_ctrl_errlog3_low,
  4316. "CVP_NOC_ERL_MAIN_ERRLOG3_LOW", val);
  4317. val = __read_register(device, CVP_NOC_ERR_ERRLOG3_HIGH_OFFS);
  4318. __err_log(log_required, &noc_log->err_ctrl_errlog3_high,
  4319. "CVP_NOC_ERL_MAIN_ERRLOG3_HIGH", val);
  4320. rc = call_iris_op(device, reset_control_acquire_name, device, "cvp_xo_reset");
  4321. if (rc) {
  4322. dprintk(CVP_WARN, "%s Fail acquire xo_reset\n", __func__);
  4323. return;
  4324. }
  4325. val = __read_register(device, CVP_NOC_CORE_ERR_SWID_LOW_OFFS);
  4326. __err_log(log_required, &noc_log->err_core_swid_low,
  4327. "CVP_NOC__CORE_ERL_MAIN_SWID_LOW", val);
  4328. val = __read_register(device, CVP_NOC_CORE_ERR_SWID_HIGH_OFFS);
  4329. __err_log(log_required, &noc_log->err_core_swid_high,
  4330. "CVP_NOC_CORE_ERL_MAIN_SWID_HIGH", val);
  4331. val = __read_register(device, CVP_NOC_CORE_ERR_MAINCTL_LOW_OFFS);
  4332. __err_log(log_required, &noc_log->err_core_mainctl_low,
  4333. "CVP_NOC_CORE_ERL_MAIN_MAINCTL_LOW", val);
  4334. val = __read_register(device, CVP_NOC_CORE_ERR_ERRVLD_LOW_OFFS);
  4335. __err_log(log_required, &noc_log->err_core_errvld_low,
  4336. "CVP_NOC_CORE_ERL_MAIN_ERRVLD_LOW", val);
  4337. val = __read_register(device, CVP_NOC_CORE_ERR_ERRCLR_LOW_OFFS);
  4338. __err_log(log_required, &noc_log->err_core_errclr_low,
  4339. "CVP_NOC_CORE_ERL_MAIN_ERRCLR_LOW", val);
  4340. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG0_LOW_OFFS);
  4341. __err_log(log_required, &noc_log->err_core_errlog0_low,
  4342. "CVP_NOC_CORE_ERL_MAIN_ERRLOG0_LOW", val);
  4343. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG0_HIGH_OFFS);
  4344. __err_log(log_required, &noc_log->err_core_errlog0_high,
  4345. "CVP_NOC_CORE_ERL_MAIN_ERRLOG0_HIGH", val);
  4346. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG1_LOW_OFFS);
  4347. __err_log(log_required, &noc_log->err_core_errlog1_low,
  4348. "CVP_NOC_CORE_ERL_MAIN_ERRLOG1_LOW", val);
  4349. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG1_HIGH_OFFS);
  4350. __err_log(log_required, &noc_log->err_core_errlog1_high,
  4351. "CVP_NOC_CORE_ERL_MAIN_ERRLOG1_HIGH", val);
  4352. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG2_LOW_OFFS);
  4353. __err_log(log_required, &noc_log->err_core_errlog2_low,
  4354. "CVP_NOC_CORE_ERL_MAIN_ERRLOG2_LOW", val);
  4355. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG2_HIGH_OFFS);
  4356. __err_log(log_required, &noc_log->err_core_errlog2_high,
  4357. "CVP_NOC_CORE_ERL_MAIN_ERRLOG2_HIGH", val);
  4358. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG3_LOW_OFFS);
  4359. __err_log(log_required, &noc_log->err_core_errlog3_low,
  4360. "CORE ERRLOG3_LOW, below details", val);
  4361. __print_reg_details(val);
  4362. val = __read_register(device, CVP_NOC_CORE_ERR_ERRLOG3_HIGH_OFFS);
  4363. __err_log(log_required, &noc_log->err_core_errlog3_high,
  4364. "CVP_NOC_CORE_ERL_MAIN_ERRLOG3_HIGH", val);
  4365. __write_register(device, CVP_NOC_CORE_ERR_ERRCLR_LOW_OFFS, 0x1);
  4366. call_iris_op(device, reset_control_release_name, device, "cvp_xo_reset");
  4367. #define CVP_SS_CLK_HALT 0x8
  4368. #define CVP_SS_CLK_EN 0xC
  4369. #define CVP_SS_ARP_TEST_BUS_CONTROL 0x700
  4370. #define CVP_SS_ARP_TEST_BUS_REGISTER 0x704
  4371. #define CVP_DMA_TEST_BUS_CONTROL 0x66A0
  4372. #define CVP_DMA_TEST_BUS_REGISTER 0x66A4
  4373. #define CVP_VPU_WRAPPER_CORE_CONFIG 0xB0088
  4374. __write_register(device, CVP_SS_CLK_HALT, 0);
  4375. __write_register(device, CVP_SS_CLK_EN, 0x3f);
  4376. __write_register(device, CVP_VPU_WRAPPER_CORE_CONFIG, 0);
  4377. for (i = 0; i < 15; i++) {
  4378. regi = 0xC0000000 + i;
  4379. __write_register(device, CVP_SS_ARP_TEST_BUS_CONTROL, regi);
  4380. val = __read_register(device, CVP_SS_ARP_TEST_BUS_REGISTER);
  4381. noc_log->arp_test_bus[i] = val;
  4382. }
  4383. for (i = 0; i < 512; i++) {
  4384. regi = 0x40000000 + i;
  4385. __write_register(device, CVP_DMA_TEST_BUS_CONTROL, regi);
  4386. val = __read_register(device, CVP_DMA_TEST_BUS_REGISTER);
  4387. noc_log->dma_test_bus[i] = val;
  4388. }
  4389. }
  4390. static int iris_hfi_noc_error_info(void *dev)
  4391. {
  4392. struct iris_hfi_device *device;
  4393. if (!dev) {
  4394. dprintk(CVP_ERR, "%s: null device\n", __func__);
  4395. return -EINVAL;
  4396. }
  4397. device = dev;
  4398. mutex_lock(&device->lock);
  4399. dprintk(CVP_ERR, "%s: non error information\n", __func__);
  4400. call_iris_op(device, noc_error_info, device);
  4401. mutex_unlock(&device->lock);
  4402. return 0;
  4403. }
  4404. static int __initialize_packetization(struct iris_hfi_device *device)
  4405. {
  4406. int rc = 0;
  4407. if (!device || !device->res) {
  4408. dprintk(CVP_ERR, "%s - invalid param\n", __func__);
  4409. return -EINVAL;
  4410. }
  4411. device->packetization_type = HFI_PACKETIZATION_4XX;
  4412. device->pkt_ops = cvp_hfi_get_pkt_ops_handle(
  4413. device->packetization_type);
  4414. if (!device->pkt_ops) {
  4415. rc = -EINVAL;
  4416. dprintk(CVP_ERR, "Failed to get pkt_ops handle\n");
  4417. }
  4418. return rc;
  4419. }
  4420. void __init_cvp_ops(struct iris_hfi_device *device)
  4421. {
  4422. device->vpu_ops = &iris2_ops;
  4423. }
  4424. static struct iris_hfi_device *__add_device(u32 device_id,
  4425. struct msm_cvp_platform_resources *res,
  4426. hfi_cmd_response_callback callback)
  4427. {
  4428. struct iris_hfi_device *hdevice = NULL;
  4429. int rc = 0;
  4430. if (!res || !callback) {
  4431. dprintk(CVP_ERR, "Invalid Parameters\n");
  4432. return NULL;
  4433. }
  4434. dprintk(CVP_INFO, "%s: device_id: %d\n", __func__, device_id);
  4435. hdevice = kzalloc(sizeof(*hdevice), GFP_KERNEL);
  4436. if (!hdevice) {
  4437. dprintk(CVP_ERR, "failed to allocate new device\n");
  4438. goto exit;
  4439. }
  4440. hdevice->response_pkt = kmalloc_array(cvp_max_packets,
  4441. sizeof(*hdevice->response_pkt), GFP_KERNEL);
  4442. if (!hdevice->response_pkt) {
  4443. dprintk(CVP_ERR, "failed to allocate response_pkt\n");
  4444. goto err_cleanup;
  4445. }
  4446. hdevice->raw_packet =
  4447. kzalloc(CVP_IFACEQ_VAR_HUGE_PKT_SIZE, GFP_KERNEL);
  4448. if (!hdevice->raw_packet) {
  4449. dprintk(CVP_ERR, "failed to allocate raw packet\n");
  4450. goto err_cleanup;
  4451. }
  4452. rc = vm_manager.vm_ops->vm_init_reg_and_irq(hdevice, res);
  4453. if (rc)
  4454. goto err_cleanup;
  4455. hdevice->res = res;
  4456. hdevice->device_id = device_id;
  4457. hdevice->callback = callback;
  4458. __init_cvp_ops(hdevice);
  4459. hdevice->cvp_workq = create_singlethread_workqueue(
  4460. "msm_cvp_workerq_iris");
  4461. if (!hdevice->cvp_workq) {
  4462. dprintk(CVP_ERR, ": create cvp workq failed\n");
  4463. goto err_cleanup;
  4464. }
  4465. hdevice->iris_pm_workq = create_singlethread_workqueue(
  4466. "pm_workerq_iris");
  4467. if (!hdevice->iris_pm_workq) {
  4468. dprintk(CVP_ERR, ": create pm workq failed\n");
  4469. goto err_cleanup;
  4470. }
  4471. mutex_init(&hdevice->lock);
  4472. INIT_LIST_HEAD(&hdevice->sess_head);
  4473. return hdevice;
  4474. err_cleanup:
  4475. if (hdevice->iris_pm_workq)
  4476. destroy_workqueue(hdevice->iris_pm_workq);
  4477. if (hdevice->cvp_workq)
  4478. destroy_workqueue(hdevice->cvp_workq);
  4479. kfree(hdevice->response_pkt);
  4480. kfree(hdevice->raw_packet);
  4481. kfree(hdevice);
  4482. exit:
  4483. return NULL;
  4484. }
  4485. static struct iris_hfi_device *__get_device(u32 device_id,
  4486. struct msm_cvp_platform_resources *res,
  4487. hfi_cmd_response_callback callback)
  4488. {
  4489. if (!res || !callback) {
  4490. dprintk(CVP_ERR, "Invalid params: %pK %pK\n", res, callback);
  4491. return NULL;
  4492. }
  4493. return __add_device(device_id, res, callback);
  4494. }
  4495. void cvp_iris_hfi_delete_device(void *device)
  4496. {
  4497. struct msm_cvp_core *core;
  4498. struct iris_hfi_device *dev = NULL;
  4499. if (!device)
  4500. return;
  4501. core = list_first_entry(&cvp_driver->cores, struct msm_cvp_core, list);
  4502. if (core)
  4503. dev = core->device->hfi_device_data;
  4504. if (!dev)
  4505. return;
  4506. mutex_destroy(&dev->lock);
  4507. destroy_workqueue(dev->cvp_workq);
  4508. destroy_workqueue(dev->iris_pm_workq);
  4509. free_irq(dev->cvp_hal_data->irq, dev);
  4510. iounmap(dev->cvp_hal_data->register_base);
  4511. iounmap(dev->cvp_hal_data->gcc_reg_base);
  4512. kfree(dev->cvp_hal_data);
  4513. kfree(dev->response_pkt);
  4514. kfree(dev->raw_packet);
  4515. kfree(dev);
  4516. }
  4517. static int iris_hfi_validate_session(void *sess, const char *func)
  4518. {
  4519. struct cvp_hal_session *session = sess;
  4520. int rc = 0;
  4521. struct iris_hfi_device *device;
  4522. if (!session || !session->device) {
  4523. dprintk(CVP_ERR, " %s Invalid Params %pK\n", __func__, session);
  4524. return -EINVAL;
  4525. }
  4526. device = session->device;
  4527. mutex_lock(&device->lock);
  4528. if (!__is_session_valid(device, session, func))
  4529. rc = -ECONNRESET;
  4530. mutex_unlock(&device->lock);
  4531. return rc;
  4532. }
  4533. static void iris_init_hfi_callbacks(struct cvp_hfi_device *hdev)
  4534. {
  4535. hdev->core_init = iris_hfi_core_init;
  4536. hdev->core_release = iris_hfi_core_release;
  4537. hdev->core_trigger_ssr = iris_hfi_core_trigger_ssr;
  4538. hdev->session_init = iris_hfi_session_init;
  4539. hdev->session_end = iris_hfi_session_end;
  4540. hdev->session_start = iris_hfi_session_start;
  4541. hdev->session_stop = iris_hfi_session_stop;
  4542. hdev->session_abort = iris_hfi_session_abort;
  4543. hdev->session_clean = iris_hfi_session_clean;
  4544. hdev->session_set_buffers = iris_hfi_session_set_buffers;
  4545. hdev->session_release_buffers = iris_hfi_session_release_buffers;
  4546. hdev->session_send = iris_hfi_session_send;
  4547. hdev->session_flush = iris_hfi_session_flush;
  4548. hdev->scale_clocks = iris_hfi_scale_clocks;
  4549. hdev->vote_bus = iris_hfi_vote_buses;
  4550. hdev->get_fw_info = iris_hfi_get_fw_info;
  4551. hdev->get_core_capabilities = iris_hfi_get_core_capabilities;
  4552. hdev->suspend = iris_hfi_suspend;
  4553. hdev->resume = iris_hfi_resume;
  4554. hdev->flush_debug_queue = iris_hfi_flush_debug_queue;
  4555. hdev->noc_error_info = iris_hfi_noc_error_info;
  4556. hdev->validate_session = iris_hfi_validate_session;
  4557. hdev->pm_qos_update = iris_pm_qos_update;
  4558. hdev->debug_hook = iris_debug_hook;
  4559. }
  4560. int cvp_iris_hfi_initialize(struct cvp_hfi_device *hdev, u32 device_id,
  4561. struct msm_cvp_platform_resources *res,
  4562. hfi_cmd_response_callback callback)
  4563. {
  4564. int rc = 0;
  4565. if (!hdev || !res || !callback) {
  4566. dprintk(CVP_ERR, "Invalid params: %pK %pK %pK\n",
  4567. hdev, res, callback);
  4568. rc = -EINVAL;
  4569. goto err_iris_hfi_init;
  4570. }
  4571. hdev->hfi_device_data = __get_device(device_id, res, callback);
  4572. if (IS_ERR_OR_NULL(hdev->hfi_device_data)) {
  4573. rc = PTR_ERR(hdev->hfi_device_data) ?: -EINVAL;
  4574. goto err_iris_hfi_init;
  4575. }
  4576. iris_init_hfi_callbacks(hdev);
  4577. err_iris_hfi_init:
  4578. return rc;
  4579. }