1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823 |
- // SPDX-License-Identifier: GPL-2.0-only
- /*
- * QTI Crypto Engine driver.
- *
- * Copyright (c) 2012-2021, The Linux Foundation. All rights reserved.
- * Copyright (c) 2023 Qualcomm Innovation Center, Inc. All rights reserved.
- */
- #define pr_fmt(fmt) "QCE50: %s: " fmt, __func__
- #include <linux/types.h>
- #include <linux/kernel.h>
- #include <linux/module.h>
- #include <linux/mod_devicetable.h>
- #include <linux/device.h>
- #include <linux/clk.h>
- #include <linux/err.h>
- #include <linux/dma-mapping.h>
- #include <linux/io.h>
- #include <linux/platform_device.h>
- #include <linux/spinlock.h>
- #include <linux/delay.h>
- #include <linux/crypto.h>
- #include <linux/bitops.h>
- #include <crypto/hash.h>
- #include <crypto/sha1.h>
- #include <soc/qcom/socinfo.h>
- #include <linux/iommu.h>
- #include "qcrypto.h"
- #include "qce.h"
- #include "qce50.h"
- #include "qcryptohw_50.h"
- #include "qce_ota.h"
- #define CRYPTO_SMMU_IOVA_START 0x10000000
- #define CRYPTO_SMMU_IOVA_SIZE 0x40000000
- #define CRYPTO_CONFIG_RESET 0xE01EF
- #define MAX_SPS_DESC_FIFO_SIZE 0xfff0
- #define QCE_MAX_NUM_DSCR 0x200
- #define QCE_SECTOR_SIZE 0x200
- #define CE_CLK_100MHZ 100000000
- #define CE_CLK_DIV 1000000
- #define CRYPTO_CORE_MAJOR_VER_NUM 0x05
- #define CRYPTO_CORE_MINOR_VER_NUM 0x03
- #define CRYPTO_CORE_STEP_VER_NUM 0x1
- #define CRYPTO_REQ_USER_PAT 0xdead0000
- static DEFINE_MUTEX(bam_register_lock);
- static DEFINE_MUTEX(qce_iomap_mutex);
- struct bam_registration_info {
- struct list_head qlist;
- unsigned long handle;
- uint32_t cnt;
- uint32_t bam_mem;
- void __iomem *bam_iobase;
- bool support_cmd_dscr;
- };
- static LIST_HEAD(qce50_bam_list);
- /* Used to determine the mode */
- #define MAX_BUNCH_MODE_REQ 2
- /* Max number of request supported */
- #define MAX_QCE_BAM_REQ 8
- /* Interrupt flag will be set for every SET_INTR_AT_REQ request */
- #define SET_INTR_AT_REQ (MAX_QCE_BAM_REQ / 2)
- /* To create extra request space to hold dummy request */
- #define MAX_QCE_BAM_REQ_WITH_DUMMY_REQ (MAX_QCE_BAM_REQ + 1)
- /* Allocate the memory for MAX_QCE_BAM_REQ + 1 (for dummy request) */
- #define MAX_QCE_ALLOC_BAM_REQ MAX_QCE_BAM_REQ_WITH_DUMMY_REQ
- /* QCE driver modes */
- #define IN_INTERRUPT_MODE 0
- #define IN_BUNCH_MODE 1
- /* Dummy request data length */
- #define DUMMY_REQ_DATA_LEN 64
- /* Delay timer to expire when in bunch mode */
- #define DELAY_IN_JIFFIES 5
- /* Index to point the dummy request */
- #define DUMMY_REQ_INDEX MAX_QCE_BAM_REQ
- #define TOTAL_IOVEC_SPACE_PER_PIPE (QCE_MAX_NUM_DSCR * sizeof(struct sps_iovec))
- #define AES_CTR_IV_CTR_SIZE 64
- #define QCE_NO_ERROR_VAL1 0x2000006
- #define QCE_NO_ERROR_VAL2 0x2000004
- // Crypto Engines 5.7 and below
- // Key timer expiry for pipes 1-15 (Status3)
- #define CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS3 0x0000FF00
- // Key timer expiry for pipes 16-19 (Status6)
- #define CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS6 0x00000300
- // Key pause for pipes 1-15 (Status3)
- #define CRYPTO5_LEGACY_KEY_PAUSE_STATUS3 0xFF000000
- // Key pause for pipes 16-19 (Status6)
- #define CRYPTO5_LEGACY_KEY_PAUSE_STATUS6 0x3000000
- // Crypto Engines 5.8 and above
- // Key timer expiry for all pipes (Status3)
- #define CRYPTO58_TIMER_EXPIRED 0x00000010
- // Key pause for all pipes (Status3)
- #define CRYPTO58_KEY_PAUSE 0x00001000
- // Key index for Status3 (Timer and Key Pause)
- #define KEY_INDEX_SHIFT 16
- enum qce_owner {
- QCE_OWNER_NONE = 0,
- QCE_OWNER_CLIENT = 1,
- QCE_OWNER_TIMEOUT = 2
- };
- struct dummy_request {
- struct qce_sha_req sreq;
- struct scatterlist sg;
- struct ahash_request areq;
- };
- /*
- * CE HW device structure.
- * Each engine has an instance of the structure.
- * Each engine can only handle one crypto operation at one time. It is up to
- * the sw above to ensure single threading of operation on an engine.
- */
- struct qce_device {
- struct device *pdev; /* Handle to platform_device structure */
- struct bam_registration_info *pbam;
- unsigned char *coh_vmem; /* Allocated coherent virtual memory */
- dma_addr_t coh_pmem; /* Allocated coherent physical memory */
- int memsize; /* Memory allocated */
- unsigned char *iovec_vmem; /* Allocate iovec virtual memory */
- int iovec_memsize; /* Memory allocated */
- uint32_t bam_mem; /* bam physical address, from DT */
- uint32_t bam_mem_size; /* bam io size, from DT */
- int is_shared; /* CE HW is shared */
- bool support_cmd_dscr;
- bool support_hw_key;
- bool support_clk_mgmt_sus_res;
- bool support_only_core_src_clk;
- bool request_bw_before_clk;
- void __iomem *iobase; /* Virtual io base of CE HW */
- unsigned int phy_iobase; /* Physical io base of CE HW */
- struct clk *ce_core_src_clk; /* Handle to CE src clk*/
- struct clk *ce_core_clk; /* Handle to CE clk */
- struct clk *ce_clk; /* Handle to CE clk */
- struct clk *ce_bus_clk; /* Handle to CE AXI clk*/
- bool no_get_around;
- bool no_ccm_mac_status_get_around;
- unsigned int ce_opp_freq_hz;
- bool use_sw_aes_cbc_ecb_ctr_algo;
- bool use_sw_aead_algo;
- bool use_sw_aes_xts_algo;
- bool use_sw_ahash_algo;
- bool use_sw_hmac_algo;
- bool use_sw_aes_ccm_algo;
- uint32_t engines_avail;
- struct qce_ce_cfg_reg_setting reg;
- struct ce_bam_info ce_bam_info;
- struct ce_request_info ce_request_info[MAX_QCE_ALLOC_BAM_REQ];
- unsigned int ce_request_index;
- enum qce_owner owner;
- atomic_t no_of_queued_req;
- struct timer_list timer;
- struct dummy_request dummyreq;
- unsigned int mode;
- unsigned int intr_cadence;
- unsigned int dev_no;
- struct qce_driver_stats qce_stats;
- atomic_t bunch_cmd_seq;
- atomic_t last_intr_seq;
- bool cadence_flag;
- uint8_t *dummyreq_in_buf;
- struct dma_iommu_mapping *smmu_mapping;
- bool enable_s1_smmu;
- bool no_clock_support;
- bool kernel_pipes_support;
- bool offload_pipes_support;
- };
- static void print_notify_debug(struct sps_event_notify *notify);
- static void _sps_producer_callback(struct sps_event_notify *notify);
- static int qce_dummy_req(struct qce_device *pce_dev);
- static int _qce50_disp_stats;
- /* Standard initialization vector for SHA-1, source: FIPS 180-2 */
- static uint32_t _std_init_vector_sha1[] = {
- 0x67452301, 0xEFCDAB89, 0x98BADCFE, 0x10325476, 0xC3D2E1F0
- };
- /* Standard initialization vector for SHA-256, source: FIPS 180-2 */
- static uint32_t _std_init_vector_sha256[] = {
- 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
- 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19
- };
- /*
- * Requests for offload operations do not require explicit dma operations
- * as they already have SMMU mapped source/destination buffers.
- */
- static bool is_offload_op(int op)
- {
- return (op == QCE_OFFLOAD_HLOS_HLOS || op == QCE_OFFLOAD_HLOS_HLOS_1 ||
- op == QCE_OFFLOAD_CPB_HLOS || op == QCE_OFFLOAD_HLOS_CPB ||
- op == QCE_OFFLOAD_HLOS_CPB_1);
- }
- static uint32_t qce_get_config_be(struct qce_device *pce_dev,
- uint32_t pipe_pair)
- {
- uint32_t beats = (pce_dev->ce_bam_info.ce_burst_size >> 3) - 1;
- return (beats << CRYPTO_REQ_SIZE |
- BIT(CRYPTO_MASK_DOUT_INTR) | BIT(CRYPTO_MASK_DIN_INTR) |
- BIT(CRYPTO_MASK_OP_DONE_INTR) | 0 << CRYPTO_HIGH_SPD_EN_N |
- pipe_pair << CRYPTO_PIPE_SET_SELECT);
- }
- static void dump_status_regs(unsigned int *status)
- {
- pr_info("%s: CRYPTO_STATUS_REG = 0x%x\n", __func__, status[0]);
- pr_info("%s: CRYPTO_STATUS2_REG = 0x%x\n", __func__, status[1]);
- pr_info("%s: CRYPTO_STATUS3_REG = 0x%x\n", __func__, status[2]);
- pr_info("%s: CRYPTO_STATUS4_REG = 0x%x\n", __func__, status[3]);
- pr_info("%s: CRYPTO_STATUS5_REG = 0x%x\n", __func__, status[4]);
- pr_info("%s: CRYPTO_STATUS6_REG = 0x%x\n", __func__, status[5]);
- }
- void qce_get_crypto_status(void *handle, struct qce_error *error)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- unsigned int status[6] = {0};
- status[0] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG);
- status[1] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS2_REG);
- status[2] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS3_REG);
- status[3] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS4_REG);
- status[4] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS5_REG);
- status[5] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS6_REG);
- #ifdef QCE_DEBUG
- dump_status_regs(status);
- #endif
- if (status[0] != QCE_NO_ERROR_VAL1 && status[0] != QCE_NO_ERROR_VAL2) {
- if (pce_dev->ce_bam_info.minor_version >= 8) {
- if (status[2] & CRYPTO58_TIMER_EXPIRED) {
- error->timer_error = true;
- pr_err("%s: timer expired, index = 0x%x\n",
- __func__, (status[2] >> KEY_INDEX_SHIFT));
- } else if (status[2] & CRYPTO58_KEY_PAUSE) {
- error->key_paused = true;
- pr_err("%s: key paused, index = 0x%x\n",
- __func__, (status[2] >> KEY_INDEX_SHIFT));
- } else {
- pr_err("%s: generic error, refer all status\n",
- __func__);
- error->generic_error = true;
- }
- } else {
- if ((status[2] & CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS3) ||
- (status[5] & CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS6)) {
- error->timer_error = true;
- pr_err("%s: timer expired, refer status 3 and 6\n",
- __func__);
- }
- else if ((status[2] & CRYPTO5_LEGACY_KEY_PAUSE_STATUS3) ||
- (status[5] & CRYPTO5_LEGACY_KEY_PAUSE_STATUS6)) {
- error->key_paused = true;
- pr_err("%s: key paused, reder status 3 and 6\n",
- __func__);
- } else {
- pr_err("%s: generic error, refer all status\n",
- __func__);
- error->generic_error = true;
- }
- }
- dump_status_regs(status);
- return;
- }
- error->no_error = true;
- pr_info("%s: No crypto error, status1 = 0x%x\n",
- __func__, status[0]);
- return;
- }
- EXPORT_SYMBOL(qce_get_crypto_status);
- static int qce_crypto_config(struct qce_device *pce_dev,
- enum qce_offload_op_enum offload_op)
- {
- uint32_t config_be = 0;
- config_be = qce_get_config_be(pce_dev,
- pce_dev->ce_bam_info.pipe_pair_index[offload_op]);
- pce_dev->reg.crypto_cfg_be = config_be;
- pce_dev->reg.crypto_cfg_le = (config_be |
- CRYPTO_LITTLE_ENDIAN_MASK);
- return 0;
- }
- static void qce_enable_clock_gating(struct qce_device *pce_dev)
- {
- /* This feature might cause some HW issues, noop till resolved. */
- return;
- }
- /*
- * IV counter mask is be set based on the values sent through the offload ioctl
- * calls. Currently for offload operations, it is 64 bytes of mask for AES CTR,
- * and 128 bytes of mask for AES CBC.
- */
- static void qce_set_iv_ctr_mask(struct qce_device *pce_dev,
- struct qce_req *creq)
- {
- if (creq->iv_ctr_size == AES_CTR_IV_CTR_SIZE) {
- pce_dev->reg.encr_cntr_mask_0 = 0x0;
- pce_dev->reg.encr_cntr_mask_1 = 0x0;
- pce_dev->reg.encr_cntr_mask_2 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_3 = 0xFFFFFFFF;
- } else {
- pce_dev->reg.encr_cntr_mask_0 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_1 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_2 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_3 = 0xFFFFFFFF;
- }
- return;
- }
- static void _byte_stream_to_net_words(uint32_t *iv, unsigned char *b,
- unsigned int len)
- {
- unsigned int n;
- n = len / sizeof(uint32_t);
- for (; n > 0; n--) {
- *iv = ((*b << 24) & 0xff000000) |
- (((*(b+1)) << 16) & 0xff0000) |
- (((*(b+2)) << 8) & 0xff00) |
- (*(b+3) & 0xff);
- b += sizeof(uint32_t);
- iv++;
- }
- n = len % sizeof(uint32_t);
- if (n == 3) {
- *iv = ((*b << 24) & 0xff000000) |
- (((*(b+1)) << 16) & 0xff0000) |
- (((*(b+2)) << 8) & 0xff00);
- } else if (n == 2) {
- *iv = ((*b << 24) & 0xff000000) |
- (((*(b+1)) << 16) & 0xff0000);
- } else if (n == 1) {
- *iv = ((*b << 24) & 0xff000000);
- }
- }
- static void _byte_stream_swap_to_net_words(uint32_t *iv, unsigned char *b,
- unsigned int len)
- {
- unsigned int i, j;
- unsigned char swap_iv[AES_IV_LENGTH];
- memset(swap_iv, 0, AES_IV_LENGTH);
- for (i = (AES_IV_LENGTH-len), j = len-1; i < AES_IV_LENGTH; i++, j--)
- swap_iv[i] = b[j];
- _byte_stream_to_net_words(iv, swap_iv, AES_IV_LENGTH);
- }
- static int count_sg(struct scatterlist *sg, int nbytes)
- {
- int i;
- for (i = 0; nbytes > 0; i++, sg = sg_next(sg))
- nbytes -= sg->length;
- return i;
- }
- static int qce_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
- enum dma_data_direction direction)
- {
- int i;
- for (i = 0; i < nents; ++i) {
- dma_map_sg(dev, sg, 1, direction);
- sg = sg_next(sg);
- }
- return nents;
- }
- static int qce_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
- int nents, enum dma_data_direction direction)
- {
- int i;
- for (i = 0; i < nents; ++i) {
- dma_unmap_sg(dev, sg, 1, direction);
- sg = sg_next(sg);
- }
- return nents;
- }
- static int _probe_ce_engine(struct qce_device *pce_dev)
- {
- unsigned int rev;
- unsigned int maj_rev, min_rev, step_rev;
- int i = 0;
- rev = readl_relaxed(pce_dev->iobase + CRYPTO_VERSION_REG);
- /*
- * Ensure previous instructions (setting the GO register)
- * was completed before checking the version.
- */
- mb();
- maj_rev = (rev & CRYPTO_CORE_MAJOR_REV_MASK) >> CRYPTO_CORE_MAJOR_REV;
- min_rev = (rev & CRYPTO_CORE_MINOR_REV_MASK) >> CRYPTO_CORE_MINOR_REV;
- step_rev = (rev & CRYPTO_CORE_STEP_REV_MASK) >> CRYPTO_CORE_STEP_REV;
- if (maj_rev != CRYPTO_CORE_MAJOR_VER_NUM) {
- pr_err("Unsupported QTI crypto device at 0x%x, rev %d.%d.%d\n",
- pce_dev->phy_iobase, maj_rev, min_rev, step_rev);
- return -EIO;
- }
- /*
- * The majority of crypto HW bugs have been fixed in 5.3.0 and
- * above. That allows a single sps transfer of consumer
- * pipe, and a single sps transfer of producer pipe
- * for a crypto request. no_get_around flag indicates this.
- *
- * In 5.3.1, the CCM MAC_FAILED in result dump issue is
- * fixed. no_ccm_mac_status_get_around flag indicates this.
- */
- pce_dev->no_get_around = (min_rev >=
- CRYPTO_CORE_MINOR_VER_NUM) ? true : false;
- if (min_rev > CRYPTO_CORE_MINOR_VER_NUM)
- pce_dev->no_ccm_mac_status_get_around = true;
- else if ((min_rev == CRYPTO_CORE_MINOR_VER_NUM) &&
- (step_rev >= CRYPTO_CORE_STEP_VER_NUM))
- pce_dev->no_ccm_mac_status_get_around = true;
- else
- pce_dev->no_ccm_mac_status_get_around = false;
- pce_dev->ce_bam_info.minor_version = min_rev;
- pce_dev->ce_bam_info.major_version = maj_rev;
- pce_dev->engines_avail = readl_relaxed(pce_dev->iobase +
- CRYPTO_ENGINES_AVAIL);
- dev_info(pce_dev->pdev, "QTI Crypto %d.%d.%d device found @0x%x\n",
- maj_rev, min_rev, step_rev, pce_dev->phy_iobase);
- pce_dev->ce_bam_info.ce_burst_size = MAX_CE_BAM_BURST_SIZE;
- dev_dbg(pce_dev->pdev, "CE device = %#x IO base, CE = %pK, IO base BAM = %pK\nBAM IRQ %d Engines Availability = %#x\n",
- pce_dev->ce_bam_info.ce_device, pce_dev->iobase,
- pce_dev->ce_bam_info.bam_iobase,
- pce_dev->ce_bam_info.bam_irq, pce_dev->engines_avail);
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++) {
- dev_dbg(pce_dev->pdev, "Consumer pipe IN [%d] = %d, Producer Pipe OUT [%d] = %d\n",
- i, pce_dev->ce_bam_info.src_pipe_index[i],
- i, pce_dev->ce_bam_info.dest_pipe_index[i]);
- }
- return 0;
- };
- static struct qce_cmdlist_info *_ce_get_hash_cmdlistinfo(
- struct qce_device *pce_dev,
- int req_info, struct qce_sha_req *sreq)
- {
- struct ce_sps_data *pce_sps_data;
- struct qce_cmdlistptr_ops *cmdlistptr;
- pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps;
- cmdlistptr = &pce_sps_data->cmdlistptr;
- switch (sreq->alg) {
- case QCE_HASH_SHA1:
- return &cmdlistptr->auth_sha1;
- case QCE_HASH_SHA256:
- return &cmdlistptr->auth_sha256;
- case QCE_HASH_SHA1_HMAC:
- return &cmdlistptr->auth_sha1_hmac;
- case QCE_HASH_SHA256_HMAC:
- return &cmdlistptr->auth_sha256_hmac;
- case QCE_HASH_AES_CMAC:
- if (sreq->authklen == AES128_KEY_SIZE)
- return &cmdlistptr->auth_aes_128_cmac;
- return &cmdlistptr->auth_aes_256_cmac;
- default:
- return NULL;
- }
- return NULL;
- }
- static int _ce_setup_hash(struct qce_device *pce_dev,
- struct qce_sha_req *sreq,
- struct qce_cmdlist_info *cmdlistinfo)
- {
- uint32_t auth32[SHA256_DIGEST_SIZE / sizeof(uint32_t)];
- uint32_t diglen;
- int i;
- uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- bool sha1 = false;
- struct sps_command_element *pce = NULL;
- bool use_hw_key = false;
- bool use_pipe_key = false;
- uint32_t authk_size_in_word = sreq->authklen/sizeof(uint32_t);
- uint32_t auth_cfg;
- if (qce_crypto_config(pce_dev, QCE_OFFLOAD_NONE))
- return -EINVAL;
- pce = cmdlistinfo->crypto_cfg;
- pce->data = pce_dev->reg.crypto_cfg_be;
- pce = cmdlistinfo->crypto_cfg_le;
- pce->data = pce_dev->reg.crypto_cfg_le;
- if ((sreq->alg == QCE_HASH_SHA1_HMAC) ||
- (sreq->alg == QCE_HASH_SHA256_HMAC) ||
- (sreq->alg == QCE_HASH_AES_CMAC)) {
- /* no more check for null key. use flag */
- if ((sreq->flags & QCRYPTO_CTX_USE_HW_KEY)
- == QCRYPTO_CTX_USE_HW_KEY)
- use_hw_key = true;
- else if ((sreq->flags & QCRYPTO_CTX_USE_PIPE_KEY) ==
- QCRYPTO_CTX_USE_PIPE_KEY)
- use_pipe_key = true;
- pce = cmdlistinfo->go_proc;
- if (use_hw_key) {
- pce->addr = (uint32_t)(CRYPTO_GOPROC_QC_KEY_REG +
- pce_dev->phy_iobase);
- } else {
- pce->addr = (uint32_t)(CRYPTO_GOPROC_REG +
- pce_dev->phy_iobase);
- pce = cmdlistinfo->auth_key;
- if (!use_pipe_key) {
- _byte_stream_to_net_words(mackey32,
- sreq->authkey,
- sreq->authklen);
- for (i = 0; i < authk_size_in_word; i++, pce++)
- pce->data = mackey32[i];
- }
- }
- }
- if (sreq->alg == QCE_HASH_AES_CMAC)
- goto go_proc;
- /* if not the last, the size has to be on the block boundary */
- if (!sreq->last_blk && (sreq->size % SHA256_BLOCK_SIZE))
- return -EIO;
- switch (sreq->alg) {
- case QCE_HASH_SHA1:
- case QCE_HASH_SHA1_HMAC:
- diglen = SHA1_DIGEST_SIZE;
- sha1 = true;
- break;
- case QCE_HASH_SHA256:
- case QCE_HASH_SHA256_HMAC:
- diglen = SHA256_DIGEST_SIZE;
- break;
- default:
- return -EINVAL;
- }
- /* write 20/32 bytes, 5/8 words into auth_iv for SHA1/SHA256 */
- if (sreq->first_blk) {
- if (sha1) {
- for (i = 0; i < 5; i++)
- auth32[i] = _std_init_vector_sha1[i];
- } else {
- for (i = 0; i < 8; i++)
- auth32[i] = _std_init_vector_sha256[i];
- }
- } else {
- _byte_stream_to_net_words(auth32, sreq->digest, diglen);
- }
- pce = cmdlistinfo->auth_iv;
- for (i = 0; i < 5; i++, pce++)
- pce->data = auth32[i];
- if ((sreq->alg == QCE_HASH_SHA256) ||
- (sreq->alg == QCE_HASH_SHA256_HMAC)) {
- for (i = 5; i < 8; i++, pce++)
- pce->data = auth32[i];
- }
- /* write auth_bytecnt 0/1, start with 0 */
- pce = cmdlistinfo->auth_bytecount;
- for (i = 0; i < 2; i++, pce++)
- pce->data = sreq->auth_data[i];
- /* Set/reset last bit in CFG register */
- pce = cmdlistinfo->auth_seg_cfg;
- auth_cfg = pce->data & ~(1 << CRYPTO_LAST |
- 1 << CRYPTO_FIRST |
- 1 << CRYPTO_USE_PIPE_KEY_AUTH |
- 1 << CRYPTO_USE_HW_KEY_AUTH);
- if (sreq->last_blk)
- auth_cfg |= 1 << CRYPTO_LAST;
- if (sreq->first_blk)
- auth_cfg |= 1 << CRYPTO_FIRST;
- if (use_hw_key)
- auth_cfg |= 1 << CRYPTO_USE_HW_KEY_AUTH;
- if (use_pipe_key)
- auth_cfg |= 1 << CRYPTO_USE_PIPE_KEY_AUTH;
- pce->data = auth_cfg;
- go_proc:
- /* write auth seg size */
- pce = cmdlistinfo->auth_seg_size;
- pce->data = sreq->size;
- pce = cmdlistinfo->encr_seg_cfg;
- pce->data = 0;
- /* write auth seg size start*/
- pce = cmdlistinfo->auth_seg_start;
- pce->data = 0;
- /* write seg size */
- pce = cmdlistinfo->seg_size;
- /* always ensure there is input data. ZLT does not work for bam-ndp */
- if (sreq->size)
- pce->data = sreq->size;
- else
- pce->data = pce_dev->ce_bam_info.ce_burst_size;
- return 0;
- }
- static struct qce_cmdlist_info *_ce_get_aead_cmdlistinfo(
- struct qce_device *pce_dev,
- int req_info, struct qce_req *creq)
- {
- struct ce_sps_data *pce_sps_data;
- struct qce_cmdlistptr_ops *cmdlistptr;
- pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps;
- cmdlistptr = &pce_sps_data->cmdlistptr;
- switch (creq->alg) {
- case CIPHER_ALG_DES:
- switch (creq->mode) {
- case QCE_MODE_CBC:
- if (creq->auth_alg == QCE_HASH_SHA1_HMAC)
- return &cmdlistptr->aead_hmac_sha1_cbc_des;
- else if (creq->auth_alg == QCE_HASH_SHA256_HMAC)
- return &cmdlistptr->aead_hmac_sha256_cbc_des;
- else
- return NULL;
- break;
- default:
- return NULL;
- }
- break;
- case CIPHER_ALG_3DES:
- switch (creq->mode) {
- case QCE_MODE_CBC:
- if (creq->auth_alg == QCE_HASH_SHA1_HMAC)
- return &cmdlistptr->aead_hmac_sha1_cbc_3des;
- else if (creq->auth_alg == QCE_HASH_SHA256_HMAC)
- return &cmdlistptr->aead_hmac_sha256_cbc_3des;
- else
- return NULL;
- break;
- default:
- return NULL;
- }
- break;
- case CIPHER_ALG_AES:
- switch (creq->mode) {
- case QCE_MODE_CBC:
- if (creq->encklen == AES128_KEY_SIZE) {
- if (creq->auth_alg == QCE_HASH_SHA1_HMAC)
- return
- &cmdlistptr->aead_hmac_sha1_cbc_aes_128;
- else if (creq->auth_alg == QCE_HASH_SHA256_HMAC)
- return
- &cmdlistptr->aead_hmac_sha256_cbc_aes_128;
- else
- return NULL;
- } else if (creq->encklen == AES256_KEY_SIZE) {
- if (creq->auth_alg == QCE_HASH_SHA1_HMAC)
- return &cmdlistptr->aead_hmac_sha1_cbc_aes_256;
- else if (creq->auth_alg == QCE_HASH_SHA256_HMAC)
- return
- &cmdlistptr->aead_hmac_sha256_cbc_aes_256;
- else
- return NULL;
- } else
- return NULL;
- break;
- default:
- return NULL;
- }
- break;
- default:
- return NULL;
- }
- return NULL;
- }
- static int _ce_setup_aead(struct qce_device *pce_dev, struct qce_req *q_req,
- uint32_t totallen_in, uint32_t coffset,
- struct qce_cmdlist_info *cmdlistinfo)
- {
- int32_t authk_size_in_word = SHA_HMAC_KEY_SIZE/sizeof(uint32_t);
- int i;
- uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {0};
- struct sps_command_element *pce;
- uint32_t a_cfg;
- uint32_t enckey32[(MAX_CIPHER_KEY_SIZE*2)/sizeof(uint32_t)] = {0};
- uint32_t enciv32[MAX_IV_LENGTH/sizeof(uint32_t)] = {0};
- uint32_t enck_size_in_word = 0;
- uint32_t enciv_in_word;
- uint32_t key_size;
- uint32_t encr_cfg = 0;
- uint32_t ivsize = q_req->ivsize;
- key_size = q_req->encklen;
- enck_size_in_word = key_size/sizeof(uint32_t);
- if (qce_crypto_config(pce_dev, q_req->offload_op))
- return -EINVAL;
- pce = cmdlistinfo->crypto_cfg;
- pce->data = pce_dev->reg.crypto_cfg_be;
- pce = cmdlistinfo->crypto_cfg_le;
- pce->data = pce_dev->reg.crypto_cfg_le;
- switch (q_req->alg) {
- case CIPHER_ALG_DES:
- enciv_in_word = 2;
- break;
- case CIPHER_ALG_3DES:
- enciv_in_word = 2;
- break;
- case CIPHER_ALG_AES:
- if ((key_size != AES128_KEY_SIZE) &&
- (key_size != AES256_KEY_SIZE))
- return -EINVAL;
- enciv_in_word = 4;
- break;
- default:
- return -EINVAL;
- }
- /* only support cbc mode */
- if (q_req->mode != QCE_MODE_CBC)
- return -EINVAL;
- _byte_stream_to_net_words(enciv32, q_req->iv, ivsize);
- pce = cmdlistinfo->encr_cntr_iv;
- for (i = 0; i < enciv_in_word; i++, pce++)
- pce->data = enciv32[i];
- /*
- * write encr key
- * do not use hw key or pipe key
- */
- _byte_stream_to_net_words(enckey32, q_req->enckey, key_size);
- pce = cmdlistinfo->encr_key;
- for (i = 0; i < enck_size_in_word; i++, pce++)
- pce->data = enckey32[i];
- /* write encr seg cfg */
- pce = cmdlistinfo->encr_seg_cfg;
- encr_cfg = pce->data;
- if (q_req->dir == QCE_ENCRYPT)
- encr_cfg |= (1 << CRYPTO_ENCODE);
- else
- encr_cfg &= ~(1 << CRYPTO_ENCODE);
- pce->data = encr_cfg;
- /* we only support sha1-hmac and sha256-hmac at this point */
- _byte_stream_to_net_words(mackey32, q_req->authkey,
- q_req->authklen);
- pce = cmdlistinfo->auth_key;
- for (i = 0; i < authk_size_in_word; i++, pce++)
- pce->data = mackey32[i];
- pce = cmdlistinfo->auth_iv;
- if (q_req->auth_alg == QCE_HASH_SHA1_HMAC)
- for (i = 0; i < 5; i++, pce++)
- pce->data = _std_init_vector_sha1[i];
- else
- for (i = 0; i < 8; i++, pce++)
- pce->data = _std_init_vector_sha256[i];
- /* write auth_bytecnt 0/1, start with 0 */
- pce = cmdlistinfo->auth_bytecount;
- for (i = 0; i < 2; i++, pce++)
- pce->data = 0;
- pce = cmdlistinfo->auth_seg_cfg;
- a_cfg = pce->data;
- a_cfg &= ~(CRYPTO_AUTH_POS_MASK);
- if (q_req->dir == QCE_ENCRYPT)
- a_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS);
- else
- a_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- pce->data = a_cfg;
- /* write auth seg size */
- pce = cmdlistinfo->auth_seg_size;
- pce->data = totallen_in;
- /* write auth seg size start*/
- pce = cmdlistinfo->auth_seg_start;
- pce->data = 0;
- /* write seg size */
- pce = cmdlistinfo->seg_size;
- pce->data = totallen_in;
- /* write encr seg size */
- pce = cmdlistinfo->encr_seg_size;
- pce->data = q_req->cryptlen;
- /* write encr seg start */
- pce = cmdlistinfo->encr_seg_start;
- pce->data = (coffset & 0xffff);
- return 0;
- }
- static struct qce_cmdlist_info *_ce_get_cipher_cmdlistinfo(
- struct qce_device *pce_dev,
- int req_info, struct qce_req *creq)
- {
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- struct qce_cmdlistptr_ops *cmdlistptr;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- cmdlistptr = &pce_sps_data->cmdlistptr;
- if (creq->alg != CIPHER_ALG_AES) {
- switch (creq->alg) {
- case CIPHER_ALG_DES:
- if (creq->mode == QCE_MODE_ECB)
- return &cmdlistptr->cipher_des_ecb;
- return &cmdlistptr->cipher_des_cbc;
- case CIPHER_ALG_3DES:
- if (creq->mode == QCE_MODE_ECB)
- return &cmdlistptr->cipher_3des_ecb;
- return &cmdlistptr->cipher_3des_cbc;
- default:
- return NULL;
- }
- } else {
- switch (creq->mode) {
- case QCE_MODE_ECB:
- if (creq->encklen == AES128_KEY_SIZE)
- return &cmdlistptr->cipher_aes_128_ecb;
- return &cmdlistptr->cipher_aes_256_ecb;
- case QCE_MODE_CBC:
- case QCE_MODE_CTR:
- if (creq->encklen == AES128_KEY_SIZE)
- return &cmdlistptr->cipher_aes_128_cbc_ctr;
- return &cmdlistptr->cipher_aes_256_cbc_ctr;
- case QCE_MODE_XTS:
- if (creq->encklen/2 == AES128_KEY_SIZE)
- return &cmdlistptr->cipher_aes_128_xts;
- return &cmdlistptr->cipher_aes_256_xts;
- case QCE_MODE_CCM:
- if (creq->encklen == AES128_KEY_SIZE)
- return &cmdlistptr->aead_aes_128_ccm;
- return &cmdlistptr->aead_aes_256_ccm;
- default:
- return NULL;
- }
- }
- return NULL;
- }
- static int _ce_setup_cipher(struct qce_device *pce_dev, struct qce_req *creq,
- uint32_t totallen_in, uint32_t coffset,
- struct qce_cmdlist_info *cmdlistinfo)
- {
- uint32_t enckey32[(MAX_CIPHER_KEY_SIZE * 2)/sizeof(uint32_t)] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- uint32_t enciv32[MAX_IV_LENGTH / sizeof(uint32_t)] = {
- 0, 0, 0, 0};
- uint32_t enck_size_in_word = 0;
- uint32_t key_size;
- bool use_hw_key = false;
- bool use_pipe_key = false;
- uint32_t encr_cfg = 0;
- uint32_t ivsize = creq->ivsize;
- int i;
- struct sps_command_element *pce = NULL;
- bool is_des_cipher = false;
- if (creq->mode == QCE_MODE_XTS)
- key_size = creq->encklen/2;
- else
- key_size = creq->encklen;
- if (qce_crypto_config(pce_dev, creq->offload_op))
- return -EINVAL;
- pce = cmdlistinfo->crypto_cfg;
- pce->data = pce_dev->reg.crypto_cfg_be;
- pce = cmdlistinfo->crypto_cfg_le;
- pce->data = pce_dev->reg.crypto_cfg_le;
- pce = cmdlistinfo->go_proc;
- if ((creq->flags & QCRYPTO_CTX_USE_HW_KEY) == QCRYPTO_CTX_USE_HW_KEY) {
- use_hw_key = true;
- } else {
- if ((creq->flags & QCRYPTO_CTX_USE_PIPE_KEY) ==
- QCRYPTO_CTX_USE_PIPE_KEY)
- use_pipe_key = true;
- }
- if (use_hw_key)
- pce->addr = (uint32_t)(CRYPTO_GOPROC_QC_KEY_REG +
- pce_dev->phy_iobase);
- else
- pce->addr = (uint32_t)(CRYPTO_GOPROC_REG +
- pce_dev->phy_iobase);
- if (!use_pipe_key && !use_hw_key) {
- _byte_stream_to_net_words(enckey32, creq->enckey, key_size);
- enck_size_in_word = key_size/sizeof(uint32_t);
- }
- if ((creq->op == QCE_REQ_AEAD) && (creq->mode == QCE_MODE_CCM)) {
- uint32_t authklen32 = creq->encklen/sizeof(uint32_t);
- uint32_t noncelen32 = MAX_NONCE/sizeof(uint32_t);
- uint32_t nonce32[MAX_NONCE/sizeof(uint32_t)] = {0, 0, 0, 0};
- uint32_t auth_cfg = 0;
- /* write nonce */
- _byte_stream_to_net_words(nonce32, creq->nonce, MAX_NONCE);
- pce = cmdlistinfo->auth_nonce_info;
- for (i = 0; i < noncelen32; i++, pce++)
- pce->data = nonce32[i];
- if (creq->authklen == AES128_KEY_SIZE)
- auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_128;
- else {
- if (creq->authklen == AES256_KEY_SIZE)
- auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_256;
- }
- if (creq->dir == QCE_ENCRYPT)
- auth_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- else
- auth_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS);
- auth_cfg |= ((creq->authsize - 1) << CRYPTO_AUTH_SIZE);
- if (use_hw_key) {
- auth_cfg |= (1 << CRYPTO_USE_HW_KEY_AUTH);
- } else {
- auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
- /* write auth key */
- pce = cmdlistinfo->auth_key;
- for (i = 0; i < authklen32; i++, pce++)
- pce->data = enckey32[i];
- }
- pce = cmdlistinfo->auth_seg_cfg;
- pce->data = auth_cfg;
- pce = cmdlistinfo->auth_seg_size;
- if (creq->dir == QCE_ENCRYPT)
- pce->data = totallen_in;
- else
- pce->data = totallen_in - creq->authsize;
- pce = cmdlistinfo->auth_seg_start;
- pce->data = 0;
- } else {
- if (creq->op != QCE_REQ_AEAD) {
- pce = cmdlistinfo->auth_seg_cfg;
- pce->data = 0;
- }
- }
- switch (creq->mode) {
- case QCE_MODE_ECB:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_256;
- break;
- case QCE_MODE_CBC:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_256;
- break;
- case QCE_MODE_XTS:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_xts_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_xts_256;
- break;
- case QCE_MODE_CCM:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_256;
- encr_cfg |= (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE) |
- (CRYPTO_LAST_CCM_XFR << CRYPTO_LAST_CCM);
- break;
- case QCE_MODE_CTR:
- default:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_256;
- break;
- }
- switch (creq->alg) {
- case CIPHER_ALG_DES:
- if (creq->mode != QCE_MODE_ECB) {
- if (ivsize > MAX_IV_LENGTH) {
- pr_err("%s: error: Invalid length parameter\n",
- __func__);
- return -EINVAL;
- }
- _byte_stream_to_net_words(enciv32, creq->iv, ivsize);
- pce = cmdlistinfo->encr_cntr_iv;
- pce->data = enciv32[0];
- pce++;
- pce->data = enciv32[1];
- }
- if (!use_hw_key) {
- pce = cmdlistinfo->encr_key;
- pce->data = enckey32[0];
- pce++;
- pce->data = enckey32[1];
- }
- is_des_cipher = true;
- break;
- case CIPHER_ALG_3DES:
- if (creq->mode != QCE_MODE_ECB) {
- if (ivsize > MAX_IV_LENGTH) {
- pr_err("%s: error: Invalid length parameter\n",
- __func__);
- return -EINVAL;
- }
- _byte_stream_to_net_words(enciv32, creq->iv, ivsize);
- pce = cmdlistinfo->encr_cntr_iv;
- pce->data = enciv32[0];
- pce++;
- pce->data = enciv32[1];
- }
- if (!use_hw_key) {
- /* write encr key */
- pce = cmdlistinfo->encr_key;
- for (i = 0; i < 6; i++, pce++)
- pce->data = enckey32[i];
- }
- is_des_cipher = true;
- break;
- case CIPHER_ALG_AES:
- default:
- if (creq->mode == QCE_MODE_XTS) {
- uint32_t xtskey32[MAX_CIPHER_KEY_SIZE/sizeof(uint32_t)]
- = {0, 0, 0, 0, 0, 0, 0, 0};
- uint32_t xtsklen =
- creq->encklen/(2 * sizeof(uint32_t));
- if (!use_hw_key && !use_pipe_key) {
- _byte_stream_to_net_words(xtskey32,
- (creq->enckey + creq->encklen/2),
- creq->encklen/2);
- /* write xts encr key */
- pce = cmdlistinfo->encr_xts_key;
- for (i = 0; i < xtsklen; i++, pce++)
- pce->data = xtskey32[i];
- }
- /* write xts du size */
- pce = cmdlistinfo->encr_xts_du_size;
- switch (creq->flags & QCRYPTO_CTX_XTS_MASK) {
- case QCRYPTO_CTX_XTS_DU_SIZE_512B:
- pce->data = min((unsigned int)QCE_SECTOR_SIZE,
- creq->cryptlen);
- break;
- case QCRYPTO_CTX_XTS_DU_SIZE_1KB:
- pce->data =
- min((unsigned int)QCE_SECTOR_SIZE * 2,
- creq->cryptlen);
- break;
- default:
- pce->data = creq->cryptlen;
- break;
- }
- }
- if (creq->mode != QCE_MODE_ECB) {
- if (ivsize > MAX_IV_LENGTH) {
- pr_err("%s: error: Invalid length parameter\n",
- __func__);
- return -EINVAL;
- }
- if (creq->mode == QCE_MODE_XTS)
- _byte_stream_swap_to_net_words(enciv32,
- creq->iv, ivsize);
- else
- _byte_stream_to_net_words(enciv32, creq->iv,
- ivsize);
- /* write encr cntr iv */
- pce = cmdlistinfo->encr_cntr_iv;
- for (i = 0; i < 4; i++, pce++)
- pce->data = enciv32[i];
- if (creq->mode == QCE_MODE_CCM) {
- /* write cntr iv for ccm */
- pce = cmdlistinfo->encr_ccm_cntr_iv;
- for (i = 0; i < 4; i++, pce++)
- pce->data = enciv32[i];
- /* update cntr_iv[3] by one */
- pce = cmdlistinfo->encr_cntr_iv;
- pce += 3;
- pce->data += 1;
- }
- }
- if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) {
- encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 <<
- CRYPTO_ENCR_KEY_SZ);
- } else {
- if (!use_hw_key) {
- /* write encr key */
- pce = cmdlistinfo->encr_key;
- for (i = 0; i < enck_size_in_word; i++, pce++)
- pce->data = enckey32[i];
- }
- } /* else of if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) */
- break;
- } /* end of switch (creq->mode) */
- if (use_pipe_key)
- encr_cfg |= (CRYPTO_USE_PIPE_KEY_ENCR_ENABLED
- << CRYPTO_USE_PIPE_KEY_ENCR);
- /* write encr seg cfg */
- pce = cmdlistinfo->encr_seg_cfg;
- if ((creq->alg == CIPHER_ALG_DES) || (creq->alg == CIPHER_ALG_3DES)) {
- if (creq->dir == QCE_ENCRYPT)
- pce->data |= (1 << CRYPTO_ENCODE);
- else
- pce->data &= ~(1 << CRYPTO_ENCODE);
- encr_cfg = pce->data;
- } else {
- encr_cfg |=
- ((creq->dir == QCE_ENCRYPT) ? 1 : 0) << CRYPTO_ENCODE;
- }
- if (use_hw_key)
- encr_cfg |= (CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR);
- else
- encr_cfg &= ~(CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR);
- pce->data = encr_cfg;
- /* write encr seg size */
- pce = cmdlistinfo->encr_seg_size;
- if (creq->is_copy_op) {
- pce->data = 0;
- } else {
- if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT))
- pce->data = (creq->cryptlen + creq->authsize);
- else
- pce->data = creq->cryptlen;
- }
- /* write encr seg start */
- pce = cmdlistinfo->encr_seg_start;
- pce->data = (coffset & 0xffff);
- /* write seg size */
- pce = cmdlistinfo->seg_size;
- pce->data = totallen_in;
- if (!is_des_cipher) {
- /* pattern info */
- pce = cmdlistinfo->pattern_info;
- pce->data = creq->pattern_info;
- /* block offset */
- pce = cmdlistinfo->block_offset;
- pce->data = (creq->block_offset << 4) |
- (creq->block_offset ? 1: 0);
- /* IV counter size */
- qce_set_iv_ctr_mask(pce_dev, creq);
- pce = cmdlistinfo->encr_mask_3;
- pce->data = pce_dev->reg.encr_cntr_mask_3;
- pce = cmdlistinfo->encr_mask_2;
- pce->data = pce_dev->reg.encr_cntr_mask_2;
- pce = cmdlistinfo->encr_mask_1;
- pce->data = pce_dev->reg.encr_cntr_mask_1;
- pce = cmdlistinfo->encr_mask_0;
- pce->data = pce_dev->reg.encr_cntr_mask_0;
- }
- pce = cmdlistinfo->go_proc;
- pce->data = 0;
- if (is_offload_op(creq->offload_op))
- pce->data = ((1 << CRYPTO_GO) | (1 << CRYPTO_CLR_CNTXT));
- else
- pce->data = ((1 << CRYPTO_GO) | (1 << CRYPTO_CLR_CNTXT) |
- (1 << CRYPTO_RESULTS_DUMP));
- return 0;
- }
- static int _ce_f9_setup(struct qce_device *pce_dev, struct qce_f9_req *req,
- struct qce_cmdlist_info *cmdlistinfo)
- {
- uint32_t ikey32[OTA_KEY_SIZE/sizeof(uint32_t)];
- uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t);
- uint32_t cfg;
- struct sps_command_element *pce;
- int i;
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- cfg = pce_dev->reg.auth_cfg_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- default:
- cfg = pce_dev->reg.auth_cfg_snow3g;
- break;
- }
- if (qce_crypto_config(pce_dev, QCE_OFFLOAD_NONE))
- return -EINVAL;
- pce = cmdlistinfo->crypto_cfg;
- pce->data = pce_dev->reg.crypto_cfg_be;
- pce = cmdlistinfo->crypto_cfg_le;
- pce->data = pce_dev->reg.crypto_cfg_le;
- /* write key in CRYPTO_AUTH_IV0-3_REG */
- _byte_stream_to_net_words(ikey32, &req->ikey[0], OTA_KEY_SIZE);
- pce = cmdlistinfo->auth_iv;
- for (i = 0; i < key_size_in_word; i++, pce++)
- pce->data = ikey32[i];
- /* write last bits in CRYPTO_AUTH_IV4_REG */
- pce->data = req->last_bits;
- /* write fresh to CRYPTO_AUTH_BYTECNT0_REG */
- pce = cmdlistinfo->auth_bytecount;
- pce->data = req->fresh;
- /* write count-i to CRYPTO_AUTH_BYTECNT1_REG */
- pce++;
- pce->data = req->count_i;
- /* write auth seg cfg */
- pce = cmdlistinfo->auth_seg_cfg;
- if (req->direction == QCE_OTA_DIR_DOWNLINK)
- cfg |= BIT(CRYPTO_F9_DIRECTION);
- pce->data = cfg;
- /* write auth seg size */
- pce = cmdlistinfo->auth_seg_size;
- pce->data = req->msize;
- /* write auth seg start*/
- pce = cmdlistinfo->auth_seg_start;
- pce->data = 0;
- /* write seg size */
- pce = cmdlistinfo->seg_size;
- pce->data = req->msize;
- /* write go */
- pce = cmdlistinfo->go_proc;
- pce->addr = (uint32_t)(CRYPTO_GOPROC_REG + pce_dev->phy_iobase);
- return 0;
- }
- static int _ce_f8_setup(struct qce_device *pce_dev, struct qce_f8_req *req,
- bool key_stream_mode, uint16_t npkts, uint16_t cipher_offset,
- uint16_t cipher_size,
- struct qce_cmdlist_info *cmdlistinfo)
- {
- uint32_t ckey32[OTA_KEY_SIZE/sizeof(uint32_t)];
- uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t);
- uint32_t cfg;
- struct sps_command_element *pce;
- int i;
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- cfg = pce_dev->reg.encr_cfg_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- default:
- cfg = pce_dev->reg.encr_cfg_snow3g;
- break;
- }
- if (qce_crypto_config(pce_dev, QCE_OFFLOAD_NONE))
- return -EINVAL;
- pce = cmdlistinfo->crypto_cfg;
- pce->data = pce_dev->reg.crypto_cfg_be;
- pce = cmdlistinfo->crypto_cfg_le;
- pce->data = pce_dev->reg.crypto_cfg_le;
- /* write key */
- _byte_stream_to_net_words(ckey32, &req->ckey[0], OTA_KEY_SIZE);
- pce = cmdlistinfo->encr_key;
- for (i = 0; i < key_size_in_word; i++, pce++)
- pce->data = ckey32[i];
- /* write encr seg cfg */
- pce = cmdlistinfo->encr_seg_cfg;
- if (key_stream_mode)
- cfg |= BIT(CRYPTO_F8_KEYSTREAM_ENABLE);
- if (req->direction == QCE_OTA_DIR_DOWNLINK)
- cfg |= BIT(CRYPTO_F8_DIRECTION);
- pce->data = cfg;
- /* write encr seg start */
- pce = cmdlistinfo->encr_seg_start;
- pce->data = (cipher_offset & 0xffff);
- /* write encr seg size */
- pce = cmdlistinfo->encr_seg_size;
- pce->data = cipher_size;
- /* write seg size */
- pce = cmdlistinfo->seg_size;
- pce->data = req->data_len;
- /* write cntr0_iv0 for countC */
- pce = cmdlistinfo->encr_cntr_iv;
- pce->data = req->count_c;
- /* write cntr1_iv1 for nPkts, and bearer */
- pce++;
- if (npkts == 1)
- npkts = 0;
- pce->data = req->bearer << CRYPTO_CNTR1_IV1_REG_F8_BEARER |
- npkts << CRYPTO_CNTR1_IV1_REG_F8_PKT_CNT;
- /* write go */
- pce = cmdlistinfo->go_proc;
- pce->addr = (uint32_t)(CRYPTO_GOPROC_REG + pce_dev->phy_iobase);
- return 0;
- }
- static void _qce_dump_descr_fifos(struct qce_device *pce_dev, int req_info)
- {
- int i, j, ents;
- struct ce_sps_data *pce_sps_data;
- struct sps_iovec *iovec;
- uint32_t cmd_flags = SPS_IOVEC_FLAG_CMD;
- pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps;
- iovec = pce_sps_data->in_transfer.iovec;
- pr_info("==============================================\n");
- pr_info("CONSUMER (TX/IN/DEST) PIPE DESCRIPTOR\n");
- pr_info("==============================================\n");
- for (i = 0; i < pce_sps_data->in_transfer.iovec_count; i++) {
- pr_info(" [%d] addr=0x%x size=0x%x flags=0x%x\n", i,
- iovec->addr, iovec->size, iovec->flags);
- if (iovec->flags & cmd_flags) {
- struct sps_command_element *pced;
- pced = (struct sps_command_element *)
- (GET_VIRT_ADDR(iovec->addr));
- ents = iovec->size/(sizeof(struct sps_command_element));
- for (j = 0; j < ents; j++) {
- pr_info(" [%d] [0x%x] 0x%x\n", j,
- pced->addr, pced->data);
- pced++;
- }
- }
- iovec++;
- }
- pr_info("==============================================\n");
- pr_info("PRODUCER (RX/OUT/SRC) PIPE DESCRIPTOR\n");
- pr_info("==============================================\n");
- iovec = pce_sps_data->out_transfer.iovec;
- for (i = 0; i < pce_sps_data->out_transfer.iovec_count; i++) {
- pr_info(" [%d] addr=0x%x size=0x%x flags=0x%x\n", i,
- iovec->addr, iovec->size, iovec->flags);
- iovec++;
- }
- }
- #ifdef QCE_DEBUG
- static void _qce_dump_descr_fifos_dbg(struct qce_device *pce_dev, int req_info)
- {
- _qce_dump_descr_fifos(pce_dev, req_info);
- }
- #define QCE_WRITE_REG(val, addr) \
- { \
- pr_info(" [0x%pK] 0x%x\n", addr, (uint32_t)val); \
- writel_relaxed(val, addr); \
- }
- #else
- static void _qce_dump_descr_fifos_dbg(struct qce_device *pce_dev, int req_info)
- {
- }
- #define QCE_WRITE_REG(val, addr) \
- writel_relaxed(val, addr)
- #endif
- static int _ce_setup_hash_direct(struct qce_device *pce_dev,
- struct qce_sha_req *sreq)
- {
- uint32_t auth32[SHA256_DIGEST_SIZE / sizeof(uint32_t)];
- uint32_t diglen;
- bool use_hw_key = false;
- bool use_pipe_key = false;
- int i;
- uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- uint32_t authk_size_in_word = sreq->authklen/sizeof(uint32_t);
- bool sha1 = false;
- uint32_t auth_cfg = 0;
- /* clear status */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG);
- if (qce_crypto_config(pce_dev, QCE_OFFLOAD_NONE))
- return -EINVAL;
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /*
- * Ensure previous instructions (setting the CONFIG register)
- * was completed before issuing starting to set other config register
- * This is to ensure the configurations are done in correct endian-ness
- * as set in the CONFIG registers
- */
- mb();
- if (sreq->alg == QCE_HASH_AES_CMAC) {
- /* write seg_cfg */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG);
- /* write seg_cfg */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG);
- /* write seg_cfg */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG);
- /* Clear auth_ivn, auth_keyn registers */
- for (i = 0; i < 16; i++) {
- QCE_WRITE_REG(0, (pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t))));
- QCE_WRITE_REG(0, (pce_dev->iobase +
- (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t))));
- }
- /* write auth_bytecnt 0/1/2/3, start with 0 */
- for (i = 0; i < 4; i++)
- QCE_WRITE_REG(0, pce_dev->iobase +
- CRYPTO_AUTH_BYTECNT0_REG +
- i * sizeof(uint32_t));
- if (sreq->authklen == AES128_KEY_SIZE)
- auth_cfg = pce_dev->reg.auth_cfg_cmac_128;
- else
- auth_cfg = pce_dev->reg.auth_cfg_cmac_256;
- }
- if ((sreq->alg == QCE_HASH_SHA1_HMAC) ||
- (sreq->alg == QCE_HASH_SHA256_HMAC) ||
- (sreq->alg == QCE_HASH_AES_CMAC)) {
- _byte_stream_to_net_words(mackey32, sreq->authkey,
- sreq->authklen);
- /* no more check for null key. use flag to check*/
- if ((sreq->flags & QCRYPTO_CTX_USE_HW_KEY) ==
- QCRYPTO_CTX_USE_HW_KEY) {
- use_hw_key = true;
- } else if ((sreq->flags & QCRYPTO_CTX_USE_PIPE_KEY) ==
- QCRYPTO_CTX_USE_PIPE_KEY) {
- use_pipe_key = true;
- } else {
- /* setup key */
- for (i = 0; i < authk_size_in_word; i++)
- QCE_WRITE_REG(mackey32[i], (pce_dev->iobase +
- (CRYPTO_AUTH_KEY0_REG +
- i*sizeof(uint32_t))));
- }
- }
- if (sreq->alg == QCE_HASH_AES_CMAC)
- goto go_proc;
- /* if not the last, the size has to be on the block boundary */
- if (!sreq->last_blk && (sreq->size % SHA256_BLOCK_SIZE))
- return -EIO;
- switch (sreq->alg) {
- case QCE_HASH_SHA1:
- auth_cfg = pce_dev->reg.auth_cfg_sha1;
- diglen = SHA1_DIGEST_SIZE;
- sha1 = true;
- break;
- case QCE_HASH_SHA1_HMAC:
- auth_cfg = pce_dev->reg.auth_cfg_hmac_sha1;
- diglen = SHA1_DIGEST_SIZE;
- sha1 = true;
- break;
- case QCE_HASH_SHA256:
- auth_cfg = pce_dev->reg.auth_cfg_sha256;
- diglen = SHA256_DIGEST_SIZE;
- break;
- case QCE_HASH_SHA256_HMAC:
- auth_cfg = pce_dev->reg.auth_cfg_hmac_sha256;
- diglen = SHA256_DIGEST_SIZE;
- break;
- default:
- return -EINVAL;
- }
- /* write 20/32 bytes, 5/8 words into auth_iv for SHA1/SHA256 */
- if (sreq->first_blk) {
- if (sha1) {
- for (i = 0; i < 5; i++)
- auth32[i] = _std_init_vector_sha1[i];
- } else {
- for (i = 0; i < 8; i++)
- auth32[i] = _std_init_vector_sha256[i];
- }
- } else {
- _byte_stream_to_net_words(auth32, sreq->digest, diglen);
- }
- /* Set auth_ivn, auth_keyn registers */
- for (i = 0; i < 5; i++)
- QCE_WRITE_REG(auth32[i], (pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t))));
- if ((sreq->alg == QCE_HASH_SHA256) ||
- (sreq->alg == QCE_HASH_SHA256_HMAC)) {
- for (i = 5; i < 8; i++)
- QCE_WRITE_REG(auth32[i], (pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t))));
- }
- /* write auth_bytecnt 0/1/2/3, start with 0 */
- for (i = 0; i < 2; i++)
- QCE_WRITE_REG(sreq->auth_data[i], pce_dev->iobase +
- CRYPTO_AUTH_BYTECNT0_REG +
- i * sizeof(uint32_t));
- /* Set/reset last bit in CFG register */
- if (sreq->last_blk)
- auth_cfg |= 1 << CRYPTO_LAST;
- else
- auth_cfg &= ~(1 << CRYPTO_LAST);
- if (sreq->first_blk)
- auth_cfg |= 1 << CRYPTO_FIRST;
- else
- auth_cfg &= ~(1 << CRYPTO_FIRST);
- if (use_hw_key)
- auth_cfg |= 1 << CRYPTO_USE_HW_KEY_AUTH;
- if (use_pipe_key)
- auth_cfg |= 1 << CRYPTO_USE_PIPE_KEY_AUTH;
- go_proc:
- /* write seg_cfg */
- QCE_WRITE_REG(auth_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG);
- /* write auth seg_size */
- QCE_WRITE_REG(sreq->size, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG);
- /* write auth_seg_start */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG);
- /* reset encr seg_cfg */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG);
- /* write seg_size */
- QCE_WRITE_REG(sreq->size, pce_dev->iobase + CRYPTO_SEG_SIZE_REG);
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /* issue go to crypto */
- if (!use_hw_key) {
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)),
- pce_dev->iobase + CRYPTO_GOPROC_REG);
- } else {
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)),
- pce_dev->iobase + CRYPTO_GOPROC_QC_KEY_REG);
- }
- /*
- * Ensure previous instructions (setting the GO register)
- * was completed before issuing a DMA transfer request
- */
- mb();
- return 0;
- }
- static int _ce_setup_aead_direct(struct qce_device *pce_dev,
- struct qce_req *q_req, uint32_t totallen_in, uint32_t coffset)
- {
- int32_t authk_size_in_word = SHA_HMAC_KEY_SIZE/sizeof(uint32_t);
- int i;
- uint32_t mackey32[SHA_HMAC_KEY_SIZE/sizeof(uint32_t)] = {0};
- uint32_t a_cfg;
- uint32_t enckey32[(MAX_CIPHER_KEY_SIZE*2)/sizeof(uint32_t)] = {0};
- uint32_t enciv32[MAX_IV_LENGTH/sizeof(uint32_t)] = {0};
- uint32_t enck_size_in_word = 0;
- uint32_t enciv_in_word;
- uint32_t key_size;
- uint32_t ivsize = q_req->ivsize;
- uint32_t encr_cfg;
- /* clear status */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG);
- if (qce_crypto_config(pce_dev, q_req->offload_op))
- return -EINVAL;
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /*
- * Ensure previous instructions (setting the CONFIG register)
- * was completed before issuing starting to set other config register
- * This is to ensure the configurations are done in correct endian-ness
- * as set in the CONFIG registers
- */
- mb();
- key_size = q_req->encklen;
- enck_size_in_word = key_size/sizeof(uint32_t);
- switch (q_req->alg) {
- case CIPHER_ALG_DES:
- switch (q_req->mode) {
- case QCE_MODE_CBC:
- encr_cfg = pce_dev->reg.encr_cfg_des_cbc;
- break;
- default:
- return -EINVAL;
- }
- enciv_in_word = 2;
- break;
- case CIPHER_ALG_3DES:
- switch (q_req->mode) {
- case QCE_MODE_CBC:
- encr_cfg = pce_dev->reg.encr_cfg_3des_cbc;
- break;
- default:
- return -EINVAL;
- }
- enciv_in_word = 2;
- break;
- case CIPHER_ALG_AES:
- switch (q_req->mode) {
- case QCE_MODE_CBC:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_128;
- else if (key_size == AES256_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_256;
- else
- return -EINVAL;
- break;
- default:
- return -EINVAL;
- }
- enciv_in_word = 4;
- break;
- default:
- return -EINVAL;
- }
- /* write CNTR0_IV0_REG */
- if (q_req->mode != QCE_MODE_ECB) {
- _byte_stream_to_net_words(enciv32, q_req->iv, ivsize);
- for (i = 0; i < enciv_in_word; i++)
- QCE_WRITE_REG(enciv32[i], pce_dev->iobase +
- (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)));
- }
- /*
- * write encr key
- * do not use hw key or pipe key
- */
- _byte_stream_to_net_words(enckey32, q_req->enckey, key_size);
- for (i = 0; i < enck_size_in_word; i++)
- QCE_WRITE_REG(enckey32[i], pce_dev->iobase +
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)));
- /* write encr seg cfg */
- if (q_req->dir == QCE_ENCRYPT)
- encr_cfg |= (1 << CRYPTO_ENCODE);
- QCE_WRITE_REG(encr_cfg, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG);
- /* we only support sha1-hmac and sha256-hmac at this point */
- _byte_stream_to_net_words(mackey32, q_req->authkey,
- q_req->authklen);
- for (i = 0; i < authk_size_in_word; i++)
- QCE_WRITE_REG(mackey32[i], pce_dev->iobase +
- (CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)));
- if (q_req->auth_alg == QCE_HASH_SHA1_HMAC) {
- for (i = 0; i < 5; i++)
- QCE_WRITE_REG(_std_init_vector_sha1[i],
- pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)));
- } else {
- for (i = 0; i < 8; i++)
- QCE_WRITE_REG(_std_init_vector_sha256[i],
- pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)));
- }
- /* write auth_bytecnt 0/1, start with 0 */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT0_REG);
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_BYTECNT1_REG);
- /* write encr seg size */
- QCE_WRITE_REG(q_req->cryptlen, pce_dev->iobase +
- CRYPTO_ENCR_SEG_SIZE_REG);
- /* write encr start */
- QCE_WRITE_REG(coffset & 0xffff, pce_dev->iobase +
- CRYPTO_ENCR_SEG_START_REG);
- if (q_req->auth_alg == QCE_HASH_SHA1_HMAC)
- a_cfg = pce_dev->reg.auth_cfg_aead_sha1_hmac;
- else
- a_cfg = pce_dev->reg.auth_cfg_aead_sha256_hmac;
- if (q_req->dir == QCE_ENCRYPT)
- a_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS);
- else
- a_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- /* write auth seg_cfg */
- QCE_WRITE_REG(a_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG);
- /* write auth seg_size */
- QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG);
- /* write auth_seg_start */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG);
- /* write seg_size */
- QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_SEG_SIZE_REG);
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /* issue go to crypto */
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)),
- pce_dev->iobase + CRYPTO_GOPROC_REG);
- /*
- * Ensure previous instructions (setting the GO register)
- * was completed before issuing a DMA transfer request
- */
- mb();
- return 0;
- }
- static int _ce_setup_cipher_direct(struct qce_device *pce_dev,
- struct qce_req *creq, uint32_t totallen_in, uint32_t coffset)
- {
- uint32_t enckey32[(MAX_CIPHER_KEY_SIZE * 2)/sizeof(uint32_t)] = {
- 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
- uint32_t enciv32[MAX_IV_LENGTH / sizeof(uint32_t)] = {
- 0, 0, 0, 0};
- uint32_t enck_size_in_word = 0;
- uint32_t key_size;
- bool use_hw_key = false;
- bool use_pipe_key = false;
- uint32_t encr_cfg = 0;
- uint32_t ivsize = creq->ivsize;
- int i;
- /* clear status */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG);
- if (qce_crypto_config(pce_dev, creq->offload_op))
- return -EINVAL;
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be,
- (pce_dev->iobase + CRYPTO_CONFIG_REG));
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le,
- (pce_dev->iobase + CRYPTO_CONFIG_REG));
- /*
- * Ensure previous instructions (setting the CONFIG register)
- * was completed before issuing starting to set other config register
- * This is to ensure the configurations are done in correct endian-ness
- * as set in the CONFIG registers
- */
- mb();
- if (creq->mode == QCE_MODE_XTS)
- key_size = creq->encklen/2;
- else
- key_size = creq->encklen;
- if ((creq->flags & QCRYPTO_CTX_USE_HW_KEY) == QCRYPTO_CTX_USE_HW_KEY) {
- use_hw_key = true;
- } else {
- if ((creq->flags & QCRYPTO_CTX_USE_PIPE_KEY) ==
- QCRYPTO_CTX_USE_PIPE_KEY)
- use_pipe_key = true;
- }
- if (!use_pipe_key && !use_hw_key) {
- _byte_stream_to_net_words(enckey32, creq->enckey, key_size);
- enck_size_in_word = key_size/sizeof(uint32_t);
- }
- if ((creq->op == QCE_REQ_AEAD) && (creq->mode == QCE_MODE_CCM)) {
- uint32_t authklen32 = creq->encklen/sizeof(uint32_t);
- uint32_t noncelen32 = MAX_NONCE/sizeof(uint32_t);
- uint32_t nonce32[MAX_NONCE/sizeof(uint32_t)] = {0, 0, 0, 0};
- uint32_t auth_cfg = 0;
- /* Clear auth_ivn, auth_keyn registers */
- for (i = 0; i < 16; i++) {
- QCE_WRITE_REG(0, (pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t))));
- QCE_WRITE_REG(0, (pce_dev->iobase +
- (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t))));
- }
- /* write auth_bytecnt 0/1/2/3, start with 0 */
- for (i = 0; i < 4; i++)
- QCE_WRITE_REG(0, pce_dev->iobase +
- CRYPTO_AUTH_BYTECNT0_REG +
- i * sizeof(uint32_t));
- /* write nonce */
- _byte_stream_to_net_words(nonce32, creq->nonce, MAX_NONCE);
- for (i = 0; i < noncelen32; i++)
- QCE_WRITE_REG(nonce32[i], pce_dev->iobase +
- CRYPTO_AUTH_INFO_NONCE0_REG +
- (i*sizeof(uint32_t)));
- if (creq->authklen == AES128_KEY_SIZE)
- auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_128;
- else {
- if (creq->authklen == AES256_KEY_SIZE)
- auth_cfg = pce_dev->reg.auth_cfg_aes_ccm_256;
- }
- if (creq->dir == QCE_ENCRYPT)
- auth_cfg |= (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- else
- auth_cfg |= (CRYPTO_AUTH_POS_AFTER << CRYPTO_AUTH_POS);
- auth_cfg |= ((creq->authsize - 1) << CRYPTO_AUTH_SIZE);
- if (use_hw_key) {
- auth_cfg |= (1 << CRYPTO_USE_HW_KEY_AUTH);
- } else {
- auth_cfg &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
- /* write auth key */
- for (i = 0; i < authklen32; i++)
- QCE_WRITE_REG(enckey32[i], pce_dev->iobase +
- CRYPTO_AUTH_KEY0_REG + (i*sizeof(uint32_t)));
- }
- QCE_WRITE_REG(auth_cfg, pce_dev->iobase +
- CRYPTO_AUTH_SEG_CFG_REG);
- if (creq->dir == QCE_ENCRYPT) {
- QCE_WRITE_REG(totallen_in, pce_dev->iobase +
- CRYPTO_AUTH_SEG_SIZE_REG);
- } else {
- QCE_WRITE_REG((totallen_in - creq->authsize),
- pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG);
- }
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG);
- } else {
- if (creq->op != QCE_REQ_AEAD)
- QCE_WRITE_REG(0, pce_dev->iobase +
- CRYPTO_AUTH_SEG_CFG_REG);
- }
- /*
- * Ensure previous instructions (write to all AUTH registers)
- * was completed before accessing a register that is not in
- * in the same 1K range.
- */
- mb();
- switch (creq->mode) {
- case QCE_MODE_ECB:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_ecb_256;
- break;
- case QCE_MODE_CBC:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_cbc_256;
- break;
- case QCE_MODE_XTS:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_xts_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_xts_256;
- break;
- case QCE_MODE_CCM:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_ccm_256;
- break;
- case QCE_MODE_CTR:
- default:
- if (key_size == AES128_KEY_SIZE)
- encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_128;
- else
- encr_cfg = pce_dev->reg.encr_cfg_aes_ctr_256;
- break;
- }
- switch (creq->alg) {
- case CIPHER_ALG_DES:
- if (creq->mode != QCE_MODE_ECB) {
- encr_cfg = pce_dev->reg.encr_cfg_des_cbc;
- _byte_stream_to_net_words(enciv32, creq->iv, ivsize);
- QCE_WRITE_REG(enciv32[0], pce_dev->iobase +
- CRYPTO_CNTR0_IV0_REG);
- QCE_WRITE_REG(enciv32[1], pce_dev->iobase +
- CRYPTO_CNTR1_IV1_REG);
- } else {
- encr_cfg = pce_dev->reg.encr_cfg_des_ecb;
- }
- if (!use_hw_key) {
- QCE_WRITE_REG(enckey32[0], pce_dev->iobase +
- CRYPTO_ENCR_KEY0_REG);
- QCE_WRITE_REG(enckey32[1], pce_dev->iobase +
- CRYPTO_ENCR_KEY1_REG);
- }
- break;
- case CIPHER_ALG_3DES:
- if (creq->mode != QCE_MODE_ECB) {
- _byte_stream_to_net_words(enciv32, creq->iv, ivsize);
- QCE_WRITE_REG(enciv32[0], pce_dev->iobase +
- CRYPTO_CNTR0_IV0_REG);
- QCE_WRITE_REG(enciv32[1], pce_dev->iobase +
- CRYPTO_CNTR1_IV1_REG);
- encr_cfg = pce_dev->reg.encr_cfg_3des_cbc;
- } else {
- encr_cfg = pce_dev->reg.encr_cfg_3des_ecb;
- }
- if (!use_hw_key) {
- /* write encr key */
- for (i = 0; i < 6; i++)
- QCE_WRITE_REG(enckey32[0], (pce_dev->iobase +
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t))));
- }
- break;
- case CIPHER_ALG_AES:
- default:
- if (creq->mode == QCE_MODE_XTS) {
- uint32_t xtskey32[MAX_CIPHER_KEY_SIZE/sizeof(uint32_t)]
- = {0, 0, 0, 0, 0, 0, 0, 0};
- uint32_t xtsklen =
- creq->encklen/(2 * sizeof(uint32_t));
- if (!use_hw_key && !use_pipe_key) {
- _byte_stream_to_net_words(xtskey32,
- (creq->enckey + creq->encklen/2),
- creq->encklen/2);
- /* write xts encr key */
- for (i = 0; i < xtsklen; i++)
- QCE_WRITE_REG(xtskey32[i],
- pce_dev->iobase +
- CRYPTO_ENCR_XTS_KEY0_REG +
- (i * sizeof(uint32_t)));
- }
- /* write xts du size */
- switch (creq->flags & QCRYPTO_CTX_XTS_MASK) {
- case QCRYPTO_CTX_XTS_DU_SIZE_512B:
- QCE_WRITE_REG(
- min((uint32_t)QCE_SECTOR_SIZE,
- creq->cryptlen), pce_dev->iobase +
- CRYPTO_ENCR_XTS_DU_SIZE_REG);
- break;
- case QCRYPTO_CTX_XTS_DU_SIZE_1KB:
- QCE_WRITE_REG(
- min((uint32_t)(QCE_SECTOR_SIZE * 2),
- creq->cryptlen), pce_dev->iobase +
- CRYPTO_ENCR_XTS_DU_SIZE_REG);
- break;
- default:
- QCE_WRITE_REG(creq->cryptlen,
- pce_dev->iobase +
- CRYPTO_ENCR_XTS_DU_SIZE_REG);
- break;
- }
- }
- if (creq->mode != QCE_MODE_ECB) {
- if (creq->mode == QCE_MODE_XTS)
- _byte_stream_swap_to_net_words(enciv32,
- creq->iv, ivsize);
- else
- _byte_stream_to_net_words(enciv32, creq->iv,
- ivsize);
- /* write encr cntr iv */
- for (i = 0; i <= 3; i++)
- QCE_WRITE_REG(enciv32[i], pce_dev->iobase +
- CRYPTO_CNTR0_IV0_REG +
- (i * sizeof(uint32_t)));
- if (creq->mode == QCE_MODE_CCM) {
- /* write cntr iv for ccm */
- for (i = 0; i <= 3; i++)
- QCE_WRITE_REG(enciv32[i],
- pce_dev->iobase +
- CRYPTO_ENCR_CCM_INT_CNTR0_REG +
- (i * sizeof(uint32_t)));
- /* update cntr_iv[3] by one */
- QCE_WRITE_REG((enciv32[3] + 1),
- pce_dev->iobase +
- CRYPTO_CNTR0_IV0_REG +
- (3 * sizeof(uint32_t)));
- }
- }
- if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) {
- encr_cfg |= (CRYPTO_ENCR_KEY_SZ_AES128 <<
- CRYPTO_ENCR_KEY_SZ);
- } else {
- if (!use_hw_key && !use_pipe_key) {
- for (i = 0; i < enck_size_in_word; i++)
- QCE_WRITE_REG(enckey32[i],
- pce_dev->iobase +
- CRYPTO_ENCR_KEY0_REG +
- (i * sizeof(uint32_t)));
- }
- } /* else of if (creq->op == QCE_REQ_ABLK_CIPHER_NO_KEY) */
- break;
- } /* end of switch (creq->mode) */
- if (use_pipe_key)
- encr_cfg |= (CRYPTO_USE_PIPE_KEY_ENCR_ENABLED
- << CRYPTO_USE_PIPE_KEY_ENCR);
- /* write encr seg cfg */
- encr_cfg |= ((creq->dir == QCE_ENCRYPT) ? 1 : 0) << CRYPTO_ENCODE;
- if (use_hw_key)
- encr_cfg |= (CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR);
- else
- encr_cfg &= ~(CRYPTO_USE_HW_KEY << CRYPTO_USE_HW_KEY_ENCR);
- /* write encr seg cfg */
- QCE_WRITE_REG(encr_cfg, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG);
- /* write encr seg size */
- if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT)) {
- QCE_WRITE_REG((creq->cryptlen + creq->authsize),
- pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG);
- } else {
- QCE_WRITE_REG(creq->cryptlen,
- pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG);
- }
- /* write pattern */
- if (creq->is_pattern_valid)
- QCE_WRITE_REG(creq->pattern_info, pce_dev->iobase +
- CRYPTO_DATA_PATT_PROC_CFG_REG);
- /* write block offset to CRYPTO_DATA_PARTIAL_BLOCK_PROC_CFG? */
- QCE_WRITE_REG(((creq->block_offset << 4) |
- (creq->block_offset ? 1 : 0)),
- pce_dev->iobase + CRYPTO_DATA_PARTIAL_BLOCK_PROC_CFG_REG);
- /* write encr seg start */
- QCE_WRITE_REG((coffset & 0xffff),
- pce_dev->iobase + CRYPTO_ENCR_SEG_START_REG);
- /* write encr counter mask */
- qce_set_iv_ctr_mask(pce_dev, creq);
- QCE_WRITE_REG(pce_dev->reg.encr_cntr_mask_3,
- pce_dev->iobase + CRYPTO_CNTR_MASK_REG);
- QCE_WRITE_REG(pce_dev->reg.encr_cntr_mask_2,
- pce_dev->iobase + CRYPTO_CNTR_MASK_REG2);
- QCE_WRITE_REG(pce_dev->reg.encr_cntr_mask_1,
- pce_dev->iobase + CRYPTO_CNTR_MASK_REG1);
- QCE_WRITE_REG(pce_dev->reg.encr_cntr_mask_0,
- pce_dev->iobase + CRYPTO_CNTR_MASK_REG0);
- /* write seg size */
- QCE_WRITE_REG(totallen_in, pce_dev->iobase + CRYPTO_SEG_SIZE_REG);
- /* issue go to crypto */
- if (!use_hw_key) {
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)),
- pce_dev->iobase + CRYPTO_GOPROC_REG);
- } else {
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP)),
- pce_dev->iobase + CRYPTO_GOPROC_QC_KEY_REG);
- }
- /*
- * Ensure previous instructions (setting the GO register)
- * was completed before issuing a DMA transfer request
- */
- mb();
- return 0;
- }
- static int _ce_f9_setup_direct(struct qce_device *pce_dev,
- struct qce_f9_req *req)
- {
- uint32_t ikey32[OTA_KEY_SIZE/sizeof(uint32_t)];
- uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t);
- uint32_t auth_cfg;
- int i;
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- auth_cfg = pce_dev->reg.auth_cfg_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- default:
- auth_cfg = pce_dev->reg.auth_cfg_snow3g;
- break;
- }
- if (qce_crypto_config(pce_dev, QCE_OFFLOAD_NONE))
- return -EINVAL;
- /* clear status */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG);
- /* set big endian configuration */
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /*
- * Ensure previous instructions (setting the CONFIG register)
- * was completed before issuing starting to set other config register
- * This is to ensure the configurations are done in correct endian-ness
- * as set in the CONFIG registers
- */
- mb();
- /* write enc_seg_cfg */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_CFG_REG);
- /* write ecn_seg_size */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_ENCR_SEG_SIZE_REG);
- /* write key in CRYPTO_AUTH_IV0-3_REG */
- _byte_stream_to_net_words(ikey32, &req->ikey[0], OTA_KEY_SIZE);
- for (i = 0; i < key_size_in_word; i++)
- QCE_WRITE_REG(ikey32[i], (pce_dev->iobase +
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t))));
- /* write last bits in CRYPTO_AUTH_IV4_REG */
- QCE_WRITE_REG(req->last_bits, (pce_dev->iobase +
- CRYPTO_AUTH_IV4_REG));
- /* write fresh to CRYPTO_AUTH_BYTECNT0_REG */
- QCE_WRITE_REG(req->fresh, (pce_dev->iobase +
- CRYPTO_AUTH_BYTECNT0_REG));
- /* write count-i to CRYPTO_AUTH_BYTECNT1_REG */
- QCE_WRITE_REG(req->count_i, (pce_dev->iobase +
- CRYPTO_AUTH_BYTECNT1_REG));
- /* write auth seg cfg */
- if (req->direction == QCE_OTA_DIR_DOWNLINK)
- auth_cfg |= BIT(CRYPTO_F9_DIRECTION);
- QCE_WRITE_REG(auth_cfg, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG);
- /* write auth seg size */
- QCE_WRITE_REG(req->msize, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG);
- /* write auth seg start*/
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_START_REG);
- /* write seg size */
- QCE_WRITE_REG(req->msize, pce_dev->iobase + CRYPTO_SEG_SIZE_REG);
- /* set little endian configuration before go*/
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /* write go */
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)),
- pce_dev->iobase + CRYPTO_GOPROC_REG);
- /*
- * Ensure previous instructions (setting the GO register)
- * was completed before issuing a DMA transfer request
- */
- mb();
- return 0;
- }
- static int _ce_f8_setup_direct(struct qce_device *pce_dev,
- struct qce_f8_req *req, bool key_stream_mode,
- uint16_t npkts, uint16_t cipher_offset, uint16_t cipher_size)
- {
- int i = 0;
- uint32_t encr_cfg = 0;
- uint32_t ckey32[OTA_KEY_SIZE/sizeof(uint32_t)];
- uint32_t key_size_in_word = OTA_KEY_SIZE/sizeof(uint32_t);
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- encr_cfg = pce_dev->reg.encr_cfg_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- default:
- encr_cfg = pce_dev->reg.encr_cfg_snow3g;
- break;
- }
- /* clear status */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_STATUS_REG);
- /* set big endian configuration */
- if (qce_crypto_config(pce_dev, QCE_OFFLOAD_NONE))
- return -EINVAL;
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_be, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /* write auth seg configuration */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_CFG_REG);
- /* write auth seg size */
- QCE_WRITE_REG(0, pce_dev->iobase + CRYPTO_AUTH_SEG_SIZE_REG);
- /* write key */
- _byte_stream_to_net_words(ckey32, &req->ckey[0], OTA_KEY_SIZE);
- for (i = 0; i < key_size_in_word; i++)
- QCE_WRITE_REG(ckey32[i], (pce_dev->iobase +
- (CRYPTO_ENCR_KEY0_REG + i*sizeof(uint32_t))));
- /* write encr seg cfg */
- if (key_stream_mode)
- encr_cfg |= BIT(CRYPTO_F8_KEYSTREAM_ENABLE);
- if (req->direction == QCE_OTA_DIR_DOWNLINK)
- encr_cfg |= BIT(CRYPTO_F8_DIRECTION);
- QCE_WRITE_REG(encr_cfg, pce_dev->iobase +
- CRYPTO_ENCR_SEG_CFG_REG);
- /* write encr seg start */
- QCE_WRITE_REG((cipher_offset & 0xffff), pce_dev->iobase +
- CRYPTO_ENCR_SEG_START_REG);
- /* write encr seg size */
- QCE_WRITE_REG(cipher_size, pce_dev->iobase +
- CRYPTO_ENCR_SEG_SIZE_REG);
- /* write seg size */
- QCE_WRITE_REG(req->data_len, pce_dev->iobase +
- CRYPTO_SEG_SIZE_REG);
- /* write cntr0_iv0 for countC */
- QCE_WRITE_REG(req->count_c, pce_dev->iobase +
- CRYPTO_CNTR0_IV0_REG);
- /* write cntr1_iv1 for nPkts, and bearer */
- if (npkts == 1)
- npkts = 0;
- QCE_WRITE_REG(req->bearer << CRYPTO_CNTR1_IV1_REG_F8_BEARER |
- npkts << CRYPTO_CNTR1_IV1_REG_F8_PKT_CNT,
- pce_dev->iobase + CRYPTO_CNTR1_IV1_REG);
- /* set little endian configuration before go*/
- QCE_WRITE_REG(pce_dev->reg.crypto_cfg_le, (pce_dev->iobase +
- CRYPTO_CONFIG_REG));
- /* write go */
- QCE_WRITE_REG(((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)),
- pce_dev->iobase + CRYPTO_GOPROC_REG);
- /*
- * Ensure previous instructions (setting the GO register)
- * was completed before issuing a DMA transfer request
- */
- mb();
- return 0;
- }
- static int _qce_unlock_other_pipes(struct qce_device *pce_dev, int req_info)
- {
- int rc = 0;
- struct ce_sps_data *pce_sps_data = &pce_dev->ce_request_info
- [req_info].ce_sps;
- uint16_t op = pce_dev->ce_request_info[req_info].offload_op;
- if (pce_dev->no_get_around || !pce_dev->support_cmd_dscr)
- return rc;
- rc = sps_transfer_one(pce_dev->ce_bam_info.consumer[op].pipe,
- GET_PHYS_ADDR(
- pce_sps_data->cmdlistptr.unlock_all_pipes.cmdlist),
- 0, NULL, (SPS_IOVEC_FLAG_CMD | SPS_IOVEC_FLAG_UNLOCK));
- if (rc) {
- pr_err("sps_xfr_one() fail rc=%d\n", rc);
- rc = -EINVAL;
- }
- return rc;
- }
- static int qce_sps_set_irqs(struct qce_device *pce_dev, bool enable)
- {
- if (enable)
- return sps_bam_enable_irqs(pce_dev->ce_bam_info.bam_handle);
- else
- return sps_bam_disable_irqs(pce_dev->ce_bam_info.bam_handle);
- }
- int qce_set_irqs(void *handle, bool enable)
- {
- return qce_sps_set_irqs(handle, enable);
- }
- EXPORT_SYMBOL(qce_set_irqs);
- static inline void qce_free_req_info(struct qce_device *pce_dev, int req_info,
- bool is_complete);
- static int qce_sps_pipe_reset(struct qce_device *pce_dev, int op)
- {
- int rc = -1;
- struct sps_pipe *sps_pipe_info = NULL;
- struct sps_connect *sps_connect_info = NULL;
- /* Reset both the pipe sets in the pipe group */
- sps_pipe_reset(pce_dev->ce_bam_info.bam_handle,
- pce_dev->ce_bam_info.dest_pipe_index[op]);
- sps_pipe_reset(pce_dev->ce_bam_info.bam_handle,
- pce_dev->ce_bam_info.src_pipe_index[op]);
- /* Reconnect to consumer pipe */
- sps_pipe_info = pce_dev->ce_bam_info.consumer[op].pipe;
- sps_connect_info = &pce_dev->ce_bam_info.consumer[op].connect;
- rc = sps_disconnect(sps_pipe_info);
- if (rc) {
- pr_err("sps_disconnect() fail pipe=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- goto exit;
- }
- memset(sps_connect_info->desc.base, 0x00,
- sps_connect_info->desc.size);
- rc = sps_connect(sps_pipe_info, sps_connect_info);
- if (rc) {
- pr_err("sps_connect() fail pipe=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- goto exit;
- }
- /* Reconnect to producer pipe */
- sps_pipe_info = pce_dev->ce_bam_info.producer[op].pipe;
- sps_connect_info = &pce_dev->ce_bam_info.producer[op].connect;
- rc = sps_disconnect(sps_pipe_info);
- if (rc) {
- pr_err("sps_connect() fail pipe=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- goto exit;
- }
- memset(sps_connect_info->desc.base, 0x00,
- sps_connect_info->desc.size);
- rc = sps_connect(sps_pipe_info, sps_connect_info);
- if (rc) {
- pr_err("sps_connect() fail pipe=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- goto exit;
- }
- /* Register producer callback */
- rc = sps_register_event(sps_pipe_info,
- &pce_dev->ce_bam_info.producer[op].event);
- if (rc)
- pr_err("Producer cb registration failed rc = %d\n",
- rc);
- exit:
- return rc;
- }
- #define MAX_RESET_TIME_RETRIES 1000
- int qce_manage_timeout(void *handle, int req_info)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- struct skcipher_request *areq;
- struct ce_request_info *preq_info;
- qce_comp_func_ptr_t qce_callback;
- uint16_t op = pce_dev->ce_request_info[req_info].offload_op;
- struct qce_error error = {0};
- int retries = 0;
- preq_info = &pce_dev->ce_request_info[req_info];
- qce_callback = preq_info->qce_cb;
- areq = (struct skcipher_request *) preq_info->areq;
- pr_info("%s: req info = %d, offload op = %d\n", __func__, req_info, op);
- if (qce_sps_pipe_reset(pce_dev, op))
- pr_err("%s: pipe reset failed\n", __func__);
- qce_get_crypto_status(pce_dev, &error);
- while (!error.no_error && retries < MAX_RESET_TIME_RETRIES) {
- usleep_range(3000, 5000);
- retries++;
- qce_get_crypto_status(pce_dev, &error);
- pr_info("%s: waiting for reset to complete\n", __func__);
- }
- // Write memory barrier
- wmb();
- if (_qce_unlock_other_pipes(pce_dev, req_info))
- pr_err("%s: fail unlock other pipes\n", __func__);
- qce_enable_clock_gating(pce_dev);
- if (!atomic_read(&preq_info->in_use)) {
- pr_err("request information %d already done\n", req_info);
- return -ENXIO;
- }
- qce_free_req_info(pce_dev, req_info, true);
- return 0;
- }
- EXPORT_SYMBOL(qce_manage_timeout);
- static int _aead_complete(struct qce_device *pce_dev, int req_info)
- {
- struct aead_request *areq;
- unsigned char mac[SHA256_DIGEST_SIZE];
- uint32_t ccm_fail_status = 0;
- uint32_t result_dump_status = 0;
- int32_t result_status = 0;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- qce_comp_func_ptr_t qce_callback;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- qce_callback = preq_info->qce_cb;
- areq = (struct aead_request *) preq_info->areq;
- if (areq->src != areq->dst) {
- qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents,
- DMA_FROM_DEVICE);
- }
- qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- if (preq_info->asg)
- qce_dma_unmap_sg(pce_dev->pdev, preq_info->asg,
- preq_info->assoc_nents, DMA_TO_DEVICE);
- /* check MAC */
- memcpy(mac, (char *)(&pce_sps_data->result->auth_iv[0]),
- SHA256_DIGEST_SIZE);
- /* read status before unlock */
- if (preq_info->dir == QCE_DECRYPT) {
- if (pce_dev->no_get_around)
- if (pce_dev->no_ccm_mac_status_get_around)
- ccm_fail_status =
- be32_to_cpu(pce_sps_data->result->status);
- else
- ccm_fail_status =
- be32_to_cpu(pce_sps_data->result_null->status);
- else
- ccm_fail_status = readl_relaxed(pce_dev->iobase +
- CRYPTO_STATUS_REG);
- }
- if (_qce_unlock_other_pipes(pce_dev, req_info)) {
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, mac, NULL, -ENXIO);
- return -ENXIO;
- }
- result_dump_status = be32_to_cpu(pce_sps_data->result->status);
- pce_sps_data->result->status = 0;
- if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR)
- | (1 << CRYPTO_HSD_ERR))) {
- pr_err("aead operation error. Status %x\n", result_dump_status);
- result_status = -ENXIO;
- } else if (pce_sps_data->consumer_status |
- pce_sps_data->producer_status) {
- pr_err("aead sps operation error. sps status %x %x\n",
- pce_sps_data->consumer_status,
- pce_sps_data->producer_status);
- result_status = -ENXIO;
- }
- if (!atomic_read(&preq_info->in_use)) {
- pr_err("request information %d already done\n", req_info);
- return -ENXIO;
- }
- if (preq_info->mode == QCE_MODE_CCM) {
- /*
- * Not from result dump, instead, use the status we just
- * read of device for MAC_FAILED.
- */
- if (result_status == 0 && (preq_info->dir == QCE_DECRYPT) &&
- (ccm_fail_status & (1 << CRYPTO_MAC_FAILED)))
- result_status = -EBADMSG;
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, mac, NULL, result_status);
- } else {
- uint32_t ivsize = 0;
- struct crypto_aead *aead;
- unsigned char iv[NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE];
- aead = crypto_aead_reqtfm(areq);
- ivsize = crypto_aead_ivsize(aead);
- memcpy(iv, (char *)(pce_sps_data->result->encr_cntr_iv),
- sizeof(iv));
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, mac, iv, result_status);
- }
- return 0;
- }
- static int _sha_complete(struct qce_device *pce_dev, int req_info)
- {
- struct ahash_request *areq;
- unsigned char digest[SHA256_DIGEST_SIZE];
- uint32_t bytecount32[2];
- int32_t result_status = 0;
- uint32_t result_dump_status;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- qce_comp_func_ptr_t qce_callback;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- qce_callback = preq_info->qce_cb;
- areq = (struct ahash_request *) preq_info->areq;
- if (!areq) {
- pr_err("sha operation error. areq is NULL\n");
- return -ENXIO;
- }
- qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents,
- DMA_TO_DEVICE);
- memcpy(digest, (char *)(&pce_sps_data->result->auth_iv[0]),
- SHA256_DIGEST_SIZE);
- _byte_stream_to_net_words(bytecount32,
- (unsigned char *)pce_sps_data->result->auth_byte_count,
- 2 * CRYPTO_REG_SIZE);
- if (_qce_unlock_other_pipes(pce_dev, req_info)) {
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, digest, (char *)bytecount32,
- -ENXIO);
- return -ENXIO;
- }
- result_dump_status = be32_to_cpu(pce_sps_data->result->status);
- pce_sps_data->result->status = 0;
- if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR)
- | (1 << CRYPTO_HSD_ERR))) {
- pr_err("sha operation error. Status %x\n", result_dump_status);
- result_status = -ENXIO;
- } else if (pce_sps_data->consumer_status) {
- pr_err("sha sps operation error. sps status %x\n",
- pce_sps_data->consumer_status);
- result_status = -ENXIO;
- }
- if (!atomic_read(&preq_info->in_use)) {
- pr_err("request information %d already done\n", req_info);
- return -ENXIO;
- }
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, digest, (char *)bytecount32, result_status);
- return 0;
- }
- static int _f9_complete(struct qce_device *pce_dev, int req_info)
- {
- uint32_t mac_i;
- int32_t result_status = 0;
- uint32_t result_dump_status;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- qce_comp_func_ptr_t qce_callback;
- void *areq;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- qce_callback = preq_info->qce_cb;
- areq = preq_info->areq;
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src,
- preq_info->ota_size, DMA_TO_DEVICE);
- _byte_stream_to_net_words(&mac_i,
- (char *)(&pce_sps_data->result->auth_iv[0]),
- CRYPTO_REG_SIZE);
- if (_qce_unlock_other_pipes(pce_dev, req_info)) {
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, NULL, NULL, -ENXIO);
- return -ENXIO;
- }
- result_dump_status = be32_to_cpu(pce_sps_data->result->status);
- pce_sps_data->result->status = 0;
- if (result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR)
- | (1 << CRYPTO_HSD_ERR))) {
- pr_err("f9 operation error. Status %x\n", result_dump_status);
- result_status = -ENXIO;
- } else if (pce_sps_data->consumer_status |
- pce_sps_data->producer_status) {
- pr_err("f9 sps operation error. sps status %x %x\n",
- pce_sps_data->consumer_status,
- pce_sps_data->producer_status);
- result_status = -ENXIO;
- }
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, (char *)&mac_i, NULL, result_status);
- return 0;
- }
- static int _ablk_cipher_complete(struct qce_device *pce_dev, int req_info)
- {
- struct skcipher_request *areq;
- unsigned char iv[NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE];
- int32_t result_status = 0;
- uint32_t result_dump_status;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- qce_comp_func_ptr_t qce_callback;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- qce_callback = preq_info->qce_cb;
- areq = (struct skcipher_request *) preq_info->areq;
- if (!is_offload_op(preq_info->offload_op)) {
- if (areq->src != areq->dst)
- qce_dma_unmap_sg(pce_dev->pdev, areq->dst,
- preq_info->dst_nents, DMA_FROM_DEVICE);
- qce_dma_unmap_sg(pce_dev->pdev, areq->src,
- preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- }
- if (_qce_unlock_other_pipes(pce_dev, req_info)) {
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, NULL, NULL, -ENXIO);
- return -ENXIO;
- }
- result_dump_status = be32_to_cpu(pce_sps_data->result->status);
- pce_sps_data->result->status = 0;
- if (!is_offload_op(preq_info->offload_op)) {
- if (result_dump_status & ((1 << CRYPTO_SW_ERR) |
- (1 << CRYPTO_AXI_ERR) | (1 << CRYPTO_HSD_ERR))) {
- pr_err("ablk_cipher operation error. Status %x\n",
- result_dump_status);
- result_status = -ENXIO;
- }
- }
- if (pce_sps_data->consumer_status |
- pce_sps_data->producer_status) {
- pr_err("ablk_cipher sps operation error. sps status %x %x\n",
- pce_sps_data->consumer_status,
- pce_sps_data->producer_status);
- result_status = -ENXIO;
- }
- if (preq_info->mode == QCE_MODE_ECB) {
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, NULL, NULL, pce_sps_data->consumer_status |
- result_status);
- } else {
- if (pce_dev->ce_bam_info.minor_version == 0) {
- if (preq_info->mode == QCE_MODE_CBC) {
- if (preq_info->dir == QCE_DECRYPT)
- memcpy(iv, (char *)preq_info->dec_iv,
- sizeof(iv));
- else
- memcpy(iv, (unsigned char *)
- (sg_virt(areq->src) +
- areq->src->length - 16),
- sizeof(iv));
- }
- if ((preq_info->mode == QCE_MODE_CTR) ||
- (preq_info->mode == QCE_MODE_XTS)) {
- uint32_t num_blk = 0;
- uint32_t cntr_iv3 = 0;
- unsigned long long cntr_iv64 = 0;
- unsigned char *b = (unsigned char *)(&cntr_iv3);
- memcpy(iv, areq->iv, sizeof(iv));
- if (preq_info->mode != QCE_MODE_XTS)
- num_blk = areq->cryptlen/16;
- else
- num_blk = 1;
- cntr_iv3 = ((*(iv + 12) << 24) & 0xff000000) |
- (((*(iv + 13)) << 16) & 0xff0000) |
- (((*(iv + 14)) << 8) & 0xff00) |
- (*(iv + 15) & 0xff);
- cntr_iv64 =
- (((unsigned long long)cntr_iv3 &
- 0xFFFFFFFFULL) +
- (unsigned long long)num_blk) %
- (unsigned long long)(0x100000000ULL);
- cntr_iv3 = (u32)(cntr_iv64 & 0xFFFFFFFF);
- *(iv + 15) = (char)(*b);
- *(iv + 14) = (char)(*(b + 1));
- *(iv + 13) = (char)(*(b + 2));
- *(iv + 12) = (char)(*(b + 3));
- }
- } else {
- memcpy(iv,
- (char *)(pce_sps_data->result->encr_cntr_iv),
- sizeof(iv));
- }
- if (!atomic_read(&preq_info->in_use)) {
- pr_err("request information %d already done\n", req_info);
- return -ENXIO;
- }
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, NULL, iv, result_status);
- }
- return 0;
- }
- static int _f8_complete(struct qce_device *pce_dev, int req_info)
- {
- int32_t result_status = 0;
- uint32_t result_dump_status;
- uint32_t result_dump_status2;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- qce_comp_func_ptr_t qce_callback;
- void *areq;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- qce_callback = preq_info->qce_cb;
- areq = preq_info->areq;
- if (preq_info->phy_ota_dst)
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_dst,
- preq_info->ota_size, DMA_FROM_DEVICE);
- if (preq_info->phy_ota_src)
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src,
- preq_info->ota_size, (preq_info->phy_ota_dst) ?
- DMA_TO_DEVICE : DMA_BIDIRECTIONAL);
- if (_qce_unlock_other_pipes(pce_dev, req_info)) {
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, NULL, NULL, -ENXIO);
- return -ENXIO;
- }
- result_dump_status = be32_to_cpu(pce_sps_data->result->status);
- result_dump_status2 = be32_to_cpu(pce_sps_data->result->status2);
- if ((result_dump_status & ((1 << CRYPTO_SW_ERR) | (1 << CRYPTO_AXI_ERR)
- | (1 << CRYPTO_HSD_ERR)))) {
- pr_err(
- "f8 oper error. Dump Sta %x Sta2 %x req %d\n",
- result_dump_status, result_dump_status2, req_info);
- result_status = -ENXIO;
- } else if (pce_sps_data->consumer_status |
- pce_sps_data->producer_status) {
- pr_err("f8 sps operation error. sps status %x %x\n",
- pce_sps_data->consumer_status,
- pce_sps_data->producer_status);
- result_status = -ENXIO;
- }
- pce_sps_data->result->status = 0;
- pce_sps_data->result->status2 = 0;
- qce_free_req_info(pce_dev, req_info, true);
- qce_callback(areq, NULL, NULL, result_status);
- return 0;
- }
- static void _qce_sps_iovec_count_init(struct qce_device *pce_dev, int req_info)
- {
- struct ce_sps_data *pce_sps_data = &pce_dev->ce_request_info[req_info]
- .ce_sps;
- pce_sps_data->in_transfer.iovec_count = 0;
- pce_sps_data->out_transfer.iovec_count = 0;
- }
- static void _qce_set_flag(struct sps_transfer *sps_bam_pipe, uint32_t flag)
- {
- struct sps_iovec *iovec;
- if (sps_bam_pipe->iovec_count == 0)
- return;
- iovec = sps_bam_pipe->iovec + (sps_bam_pipe->iovec_count - 1);
- iovec->flags |= flag;
- }
- static int _qce_sps_add_data(dma_addr_t paddr, uint32_t len,
- struct sps_transfer *sps_bam_pipe)
- {
- struct sps_iovec *iovec = sps_bam_pipe->iovec +
- sps_bam_pipe->iovec_count;
- uint32_t data_cnt;
- while (len > 0) {
- if (sps_bam_pipe->iovec_count == QCE_MAX_NUM_DSCR) {
- pr_err("Num of descrptor %d exceed max (%d)\n",
- sps_bam_pipe->iovec_count,
- (uint32_t)QCE_MAX_NUM_DSCR);
- return -ENOMEM;
- }
- if (len > SPS_MAX_PKT_SIZE)
- data_cnt = SPS_MAX_PKT_SIZE;
- else
- data_cnt = len;
- iovec->size = data_cnt;
- iovec->addr = SPS_GET_LOWER_ADDR(paddr);
- iovec->flags = SPS_GET_UPPER_ADDR(paddr);
- sps_bam_pipe->iovec_count++;
- iovec++;
- paddr += data_cnt;
- len -= data_cnt;
- }
- return 0;
- }
- static int _qce_sps_add_sg_data(struct qce_device *pce_dev,
- struct scatterlist *sg_src, uint32_t nbytes,
- struct sps_transfer *sps_bam_pipe)
- {
- uint32_t data_cnt, len;
- dma_addr_t addr;
- struct sps_iovec *iovec = sps_bam_pipe->iovec +
- sps_bam_pipe->iovec_count;
- while (nbytes > 0 && sg_src) {
- len = min(nbytes, sg_dma_len(sg_src));
- nbytes -= len;
- addr = sg_dma_address(sg_src);
- if (pce_dev->ce_bam_info.minor_version == 0)
- len = ALIGN(len, pce_dev->ce_bam_info.ce_burst_size);
- while (len > 0) {
- if (sps_bam_pipe->iovec_count == QCE_MAX_NUM_DSCR) {
- pr_err("Num of descrptor %d exceed max (%d)\n",
- sps_bam_pipe->iovec_count,
- (uint32_t)QCE_MAX_NUM_DSCR);
- return -ENOMEM;
- }
- if (len > SPS_MAX_PKT_SIZE) {
- data_cnt = SPS_MAX_PKT_SIZE;
- iovec->size = data_cnt;
- iovec->addr = SPS_GET_LOWER_ADDR(addr);
- iovec->flags = SPS_GET_UPPER_ADDR(addr);
- } else {
- data_cnt = len;
- iovec->size = data_cnt;
- iovec->addr = SPS_GET_LOWER_ADDR(addr);
- iovec->flags = SPS_GET_UPPER_ADDR(addr);
- }
- iovec++;
- sps_bam_pipe->iovec_count++;
- addr += data_cnt;
- len -= data_cnt;
- }
- sg_src = sg_next(sg_src);
- }
- return 0;
- }
- static int _qce_sps_add_sg_data_off(struct qce_device *pce_dev,
- struct scatterlist *sg_src, uint32_t nbytes, uint32_t off,
- struct sps_transfer *sps_bam_pipe)
- {
- uint32_t data_cnt, len;
- dma_addr_t addr;
- struct sps_iovec *iovec = sps_bam_pipe->iovec +
- sps_bam_pipe->iovec_count;
- unsigned int res_within_sg;
- if (!sg_src)
- return -ENOENT;
- res_within_sg = sg_dma_len(sg_src);
- while (off > 0) {
- if (!sg_src) {
- pr_err("broken sg list off %d nbytes %d\n",
- off, nbytes);
- return -ENOENT;
- }
- len = sg_dma_len(sg_src);
- if (off < len) {
- res_within_sg = len - off;
- break;
- }
- off -= len;
- sg_src = sg_next(sg_src);
- if (sg_src)
- res_within_sg = sg_dma_len(sg_src);
- }
- while (nbytes > 0 && sg_src) {
- len = min(nbytes, res_within_sg);
- nbytes -= len;
- addr = sg_dma_address(sg_src) + off;
- if (pce_dev->ce_bam_info.minor_version == 0)
- len = ALIGN(len, pce_dev->ce_bam_info.ce_burst_size);
- while (len > 0) {
- if (sps_bam_pipe->iovec_count == QCE_MAX_NUM_DSCR) {
- pr_err("Num of descrptor %d exceed max (%d)\n",
- sps_bam_pipe->iovec_count,
- (uint32_t)QCE_MAX_NUM_DSCR);
- return -ENOMEM;
- }
- if (len > SPS_MAX_PKT_SIZE) {
- data_cnt = SPS_MAX_PKT_SIZE;
- iovec->size = data_cnt;
- iovec->addr = SPS_GET_LOWER_ADDR(addr);
- iovec->flags = SPS_GET_UPPER_ADDR(addr);
- } else {
- data_cnt = len;
- iovec->size = data_cnt;
- iovec->addr = SPS_GET_LOWER_ADDR(addr);
- iovec->flags = SPS_GET_UPPER_ADDR(addr);
- }
- iovec++;
- sps_bam_pipe->iovec_count++;
- addr += data_cnt;
- len -= data_cnt;
- }
- if (nbytes) {
- sg_src = sg_next(sg_src);
- if (!sg_src) {
- pr_err("more data bytes %d\n", nbytes);
- return -ENOMEM;
- }
- res_within_sg = sg_dma_len(sg_src);
- off = 0;
- }
- }
- return 0;
- }
- static int _qce_sps_add_cmd(struct qce_device *pce_dev, uint32_t flag,
- struct qce_cmdlist_info *cmdptr,
- struct sps_transfer *sps_bam_pipe)
- {
- dma_addr_t paddr = GET_PHYS_ADDR(cmdptr->cmdlist);
- struct sps_iovec *iovec = sps_bam_pipe->iovec +
- sps_bam_pipe->iovec_count;
- iovec->size = cmdptr->size;
- iovec->addr = SPS_GET_LOWER_ADDR(paddr);
- iovec->flags = SPS_GET_UPPER_ADDR(paddr) | SPS_IOVEC_FLAG_CMD | flag;
- sps_bam_pipe->iovec_count++;
- if (sps_bam_pipe->iovec_count >= QCE_MAX_NUM_DSCR) {
- pr_err("Num of descrptor %d exceed max (%d)\n",
- sps_bam_pipe->iovec_count, (uint32_t)QCE_MAX_NUM_DSCR);
- return -ENOMEM;
- }
- return 0;
- }
- static int _qce_sps_transfer(struct qce_device *pce_dev, int req_info)
- {
- int rc = 0;
- struct ce_sps_data *pce_sps_data;
- uint16_t op = pce_dev->ce_request_info[req_info].offload_op;
- pce_sps_data = &pce_dev->ce_request_info[req_info].ce_sps;
- pce_sps_data->out_transfer.user =
- (void *)((uintptr_t)(CRYPTO_REQ_USER_PAT |
- (unsigned int) req_info));
- pce_sps_data->in_transfer.user =
- (void *)((uintptr_t)(CRYPTO_REQ_USER_PAT |
- (unsigned int) req_info));
- _qce_dump_descr_fifos_dbg(pce_dev, req_info);
- if (pce_sps_data->in_transfer.iovec_count) {
- rc = sps_transfer(pce_dev->ce_bam_info.consumer[op].pipe,
- &pce_sps_data->in_transfer);
- if (rc) {
- pr_err("sps_xfr() fail (cons pipe=0x%lx) rc = %d\n",
- (uintptr_t)pce_dev->ce_bam_info.consumer[op].pipe,
- rc);
- goto ret;
- }
- }
- rc = sps_transfer(pce_dev->ce_bam_info.producer[op].pipe,
- &pce_sps_data->out_transfer);
- if (rc)
- pr_err("sps_xfr() fail (producer pipe=0x%lx) rc = %d\n",
- (uintptr_t)pce_dev->ce_bam_info.producer[op].pipe, rc);
- ret:
- return rc;
- }
- /**
- * Allocate and Connect a CE peripheral's SPS endpoint
- *
- * This function allocates endpoint context and
- * connect it with memory endpoint by calling
- * appropriate SPS driver APIs.
- *
- * Also registers a SPS callback function with
- * SPS driver
- *
- * This function should only be called once typically
- * during driver probe.
- *
- * @pce_dev - Pointer to qce_device structure
- * @ep - Pointer to sps endpoint data structure
- * @index - Points to crypto use case
- * @is_produce - 1 means Producer endpoint
- * 0 means Consumer endpoint
- *
- * @return - 0 if successful else negative value.
- *
- */
- static int qce_sps_init_ep_conn(struct qce_device *pce_dev,
- struct qce_sps_ep_conn_data *ep,
- int index,
- bool is_producer)
- {
- int rc = 0;
- struct sps_pipe *sps_pipe_info;
- struct sps_connect *sps_connect_info = &ep->connect;
- struct sps_register_event *sps_event = &ep->event;
- /* Allocate endpoint context */
- sps_pipe_info = sps_alloc_endpoint();
- if (!sps_pipe_info) {
- pr_err("sps_alloc_endpoint() failed!!! is_producer=%d\n",
- is_producer);
- rc = -ENOMEM;
- goto out;
- }
- /* Now save the sps pipe handle */
- ep->pipe = sps_pipe_info;
- /* Get default connection configuration for an endpoint */
- rc = sps_get_config(sps_pipe_info, sps_connect_info);
- if (rc) {
- pr_err("sps_get_config() fail pipe_handle=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- goto get_config_err;
- }
- /* Modify the default connection configuration */
- if (is_producer) {
- /*
- * For CE producer transfer, source should be
- * CE peripheral where as destination should
- * be system memory.
- */
- sps_connect_info->source = pce_dev->ce_bam_info.bam_handle;
- sps_connect_info->destination = SPS_DEV_HANDLE_MEM;
- /* Producer pipe will handle this connection */
- sps_connect_info->mode = SPS_MODE_SRC;
- sps_connect_info->options =
- SPS_O_AUTO_ENABLE | SPS_O_DESC_DONE;
- } else {
- /* For CE consumer transfer, source should be
- * system memory where as destination should
- * CE peripheral
- */
- sps_connect_info->source = SPS_DEV_HANDLE_MEM;
- sps_connect_info->destination = pce_dev->ce_bam_info.bam_handle;
- sps_connect_info->mode = SPS_MODE_DEST;
- sps_connect_info->options =
- SPS_O_AUTO_ENABLE;
- }
- /* Producer pipe index */
- sps_connect_info->src_pipe_index =
- pce_dev->ce_bam_info.src_pipe_index[index];
- /* Consumer pipe index */
- sps_connect_info->dest_pipe_index =
- pce_dev->ce_bam_info.dest_pipe_index[index];
- /* Set pipe group */
- sps_connect_info->lock_group =
- pce_dev->ce_bam_info.pipe_pair_index[index];
- sps_connect_info->event_thresh = 0x10;
- /*
- * Max. no of scatter/gather buffers that can
- * be passed by block layer = 32 (NR_SG).
- * Each BAM descritor needs 64 bits (8 bytes).
- * One BAM descriptor is required per buffer transfer.
- * So we would require total 256 (32 * 8) bytes of descriptor FIFO.
- * But due to HW limitation we need to allocate atleast one extra
- * descriptor memory (256 bytes + 8 bytes). But in order to be
- * in power of 2, we are allocating 512 bytes of memory.
- */
- sps_connect_info->desc.size = QCE_MAX_NUM_DSCR * MAX_QCE_ALLOC_BAM_REQ *
- sizeof(struct sps_iovec);
- if (sps_connect_info->desc.size > MAX_SPS_DESC_FIFO_SIZE)
- sps_connect_info->desc.size = MAX_SPS_DESC_FIFO_SIZE;
- sps_connect_info->desc.base = dma_alloc_coherent(pce_dev->pdev,
- sps_connect_info->desc.size,
- &sps_connect_info->desc.phys_base,
- GFP_KERNEL | __GFP_ZERO);
- if (sps_connect_info->desc.base == NULL) {
- rc = -ENOMEM;
- pr_err("Can not allocate coherent memory for sps data\n");
- goto get_config_err;
- }
- /* Establish connection between peripheral and memory endpoint */
- rc = sps_connect(sps_pipe_info, sps_connect_info);
- if (rc) {
- pr_err("sps_connect() fail pipe_handle=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- goto sps_connect_err;
- }
- sps_event->mode = SPS_TRIGGER_CALLBACK;
- sps_event->xfer_done = NULL;
- sps_event->user = (void *)pce_dev;
- if (is_producer) {
- sps_event->options = SPS_O_EOT | SPS_O_DESC_DONE;
- sps_event->callback = _sps_producer_callback;
- rc = sps_register_event(ep->pipe, sps_event);
- if (rc) {
- pr_err("Producer callback registration failed rc=%d\n",
- rc);
- goto sps_connect_err;
- }
- } else {
- sps_event->options = SPS_O_EOT;
- sps_event->callback = NULL;
- }
- pr_debug("success, %s : pipe_handle=0x%lx, desc fifo base (phy) = 0x%pK\n",
- is_producer ? "PRODUCER(RX/OUT)" : "CONSUMER(TX/IN)",
- (uintptr_t)sps_pipe_info, &sps_connect_info->desc.phys_base);
- goto out;
- sps_connect_err:
- dma_free_coherent(pce_dev->pdev,
- sps_connect_info->desc.size,
- sps_connect_info->desc.base,
- sps_connect_info->desc.phys_base);
- get_config_err:
- sps_free_endpoint(sps_pipe_info);
- out:
- return rc;
- }
- /**
- * Disconnect and Deallocate a CE peripheral's SPS endpoint
- *
- * This function disconnect endpoint and deallocates
- * endpoint context.
- *
- * This function should only be called once typically
- * during driver remove.
- *
- * @pce_dev - Pointer to qce_device structure
- * @ep - Pointer to sps endpoint data structure
- *
- */
- static void qce_sps_exit_ep_conn(struct qce_device *pce_dev,
- struct qce_sps_ep_conn_data *ep)
- {
- struct sps_pipe *sps_pipe_info = ep->pipe;
- struct sps_connect *sps_connect_info = &ep->connect;
- sps_disconnect(sps_pipe_info);
- dma_free_coherent(pce_dev->pdev,
- sps_connect_info->desc.size,
- sps_connect_info->desc.base,
- sps_connect_info->desc.phys_base);
- sps_free_endpoint(sps_pipe_info);
- }
- static void qce_sps_release_bam(struct qce_device *pce_dev)
- {
- struct bam_registration_info *pbam;
- mutex_lock(&bam_register_lock);
- pbam = pce_dev->pbam;
- if (pbam == NULL)
- goto ret;
- pbam->cnt--;
- if (pbam->cnt > 0)
- goto ret;
- if (pce_dev->ce_bam_info.bam_handle) {
- sps_deregister_bam_device(pce_dev->ce_bam_info.bam_handle);
- pr_debug("deregister bam handle 0x%lx\n",
- pce_dev->ce_bam_info.bam_handle);
- pce_dev->ce_bam_info.bam_handle = 0;
- }
- iounmap(pbam->bam_iobase);
- pr_debug("delete bam 0x%x\n", pbam->bam_mem);
- list_del(&pbam->qlist);
- kfree(pbam);
- ret:
- pce_dev->pbam = NULL;
- mutex_unlock(&bam_register_lock);
- }
- static int qce_sps_get_bam(struct qce_device *pce_dev)
- {
- int rc = 0;
- struct sps_bam_props bam = {0};
- struct bam_registration_info *pbam = NULL;
- struct bam_registration_info *p;
- uint32_t bam_cfg = 0;
- mutex_lock(&bam_register_lock);
- list_for_each_entry(p, &qce50_bam_list, qlist) {
- if (p->bam_mem == pce_dev->bam_mem) {
- pbam = p; /* found */
- break;
- }
- }
- if (pbam) {
- pr_debug("found bam 0x%x\n", pbam->bam_mem);
- pbam->cnt++;
- pce_dev->ce_bam_info.bam_handle = pbam->handle;
- pce_dev->ce_bam_info.bam_mem = pbam->bam_mem;
- pce_dev->ce_bam_info.bam_iobase = pbam->bam_iobase;
- pce_dev->pbam = pbam;
- pce_dev->support_cmd_dscr = pbam->support_cmd_dscr;
- goto ret;
- }
- pbam = kzalloc(sizeof(struct bam_registration_info), GFP_KERNEL);
- if (!pbam) {
- rc = -ENOMEM;
- goto ret;
- }
- pbam->cnt = 1;
- pbam->bam_mem = pce_dev->bam_mem;
- pbam->bam_iobase = ioremap(pce_dev->bam_mem,
- pce_dev->bam_mem_size);
- if (!pbam->bam_iobase) {
- kfree(pbam);
- rc = -ENOMEM;
- pr_err("Can not map BAM io memory\n");
- goto ret;
- }
- pce_dev->ce_bam_info.bam_mem = pbam->bam_mem;
- pce_dev->ce_bam_info.bam_iobase = pbam->bam_iobase;
- pbam->handle = 0;
- pr_debug("allocate bam 0x%x\n", pbam->bam_mem);
- bam_cfg = readl_relaxed(pce_dev->ce_bam_info.bam_iobase +
- CRYPTO_BAM_CNFG_BITS_REG);
- pbam->support_cmd_dscr = (bam_cfg & CRYPTO_BAM_CD_ENABLE_MASK) ?
- true : false;
- if (!pbam->support_cmd_dscr) {
- pr_info("qce50 don't support command descriptor. bam_cfg%x\n",
- bam_cfg);
- pce_dev->no_get_around = false;
- }
- pce_dev->support_cmd_dscr = pbam->support_cmd_dscr;
- bam.phys_addr = pce_dev->ce_bam_info.bam_mem;
- bam.virt_addr = pce_dev->ce_bam_info.bam_iobase;
- /*
- * This event threshold value is only significant for BAM-to-BAM
- * transfer. It's ignored for BAM-to-System mode transfer.
- */
- bam.event_threshold = 0x10; /* Pipe event threshold */
- /*
- * This threshold controls when the BAM publish
- * the descriptor size on the sideband interface.
- * SPS HW will only be used when
- * data transfer size > 64 bytes.
- */
- bam.summing_threshold = 64;
- /* SPS driver wll handle the crypto BAM IRQ */
- bam.irq = (u32)pce_dev->ce_bam_info.bam_irq;
- /*
- * Set flag to indicate BAM global device control is managed
- * remotely.
- */
- if (!pce_dev->support_cmd_dscr || pce_dev->is_shared)
- bam.manage = SPS_BAM_MGR_DEVICE_REMOTE;
- else
- bam.manage = SPS_BAM_MGR_LOCAL;
- bam.ee = pce_dev->ce_bam_info.bam_ee;
- bam.ipc_loglevel = QCE_BAM_DEFAULT_IPC_LOGLVL;
- bam.options |= SPS_BAM_CACHED_WP;
- pr_debug("bam physical base=0x%lx\n", (uintptr_t)bam.phys_addr);
- pr_debug("bam virtual base=0x%pK\n", bam.virt_addr);
- /* Register CE Peripheral BAM device to SPS driver */
- rc = sps_register_bam_device(&bam, &pbam->handle);
- if (rc) {
- pr_err("sps_register_bam_device() failed! err=%d\n", rc);
- rc = -EIO;
- iounmap(pbam->bam_iobase);
- kfree(pbam);
- goto ret;
- }
- pce_dev->pbam = pbam;
- list_add_tail(&pbam->qlist, &qce50_bam_list);
- pce_dev->ce_bam_info.bam_handle = pbam->handle;
- ret:
- mutex_unlock(&bam_register_lock);
- return rc;
- }
- /**
- * Initialize SPS HW connected with CE core
- *
- * This function register BAM HW resources with
- * SPS driver and then initialize 2 SPS endpoints
- *
- * This function should only be called once typically
- * during driver probe.
- *
- * @pce_dev - Pointer to qce_device structure
- *
- * @return - 0 if successful else negative value.
- *
- */
- static int qce_sps_init(struct qce_device *pce_dev)
- {
- int rc = 0, i = 0;
- rc = qce_sps_get_bam(pce_dev);
- if (rc)
- return rc;
- pr_debug("BAM device registered. bam_handle=0x%lx\n",
- pce_dev->ce_bam_info.bam_handle);
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++) {
- if (i == QCE_OFFLOAD_NONE && !(pce_dev->kernel_pipes_support))
- continue;
- else if ((i > 0) && !(pce_dev->offload_pipes_support))
- break;
- if (!pce_dev->ce_bam_info.pipe_pair_index[i])
- continue;
- rc = qce_sps_init_ep_conn(pce_dev,
- &pce_dev->ce_bam_info.producer[i], i, true);
- if (rc)
- goto sps_connect_producer_err;
- rc = qce_sps_init_ep_conn(pce_dev,
- &pce_dev->ce_bam_info.consumer[i], i, false);
- if (rc)
- goto sps_connect_consumer_err;
- }
- pr_info(" QTI MSM CE-BAM at 0x%016llx irq %d\n",
- (unsigned long long)pce_dev->ce_bam_info.bam_mem,
- (unsigned int)pce_dev->ce_bam_info.bam_irq);
- return rc;
- sps_connect_consumer_err:
- qce_sps_exit_ep_conn(pce_dev, &pce_dev->ce_bam_info.producer[i]);
- sps_connect_producer_err:
- qce_sps_release_bam(pce_dev);
- return rc;
- }
- static inline int qce_alloc_req_info(struct qce_device *pce_dev)
- {
- int i;
- int request_index = pce_dev->ce_request_index;
- for (i = 0; i < MAX_QCE_BAM_REQ; i++) {
- request_index++;
- if (request_index >= MAX_QCE_BAM_REQ)
- request_index = 0;
- if (!atomic_xchg(
- &pce_dev->ce_request_info[request_index].in_use,
- true)) {
- pce_dev->ce_request_index = request_index;
- return request_index;
- }
- }
- pr_warn("pcedev %d no reqs available no_of_queued_req %d\n",
- pce_dev->dev_no, atomic_read(
- &pce_dev->no_of_queued_req));
- return -EBUSY;
- }
- static inline void qce_free_req_info(struct qce_device *pce_dev, int req_info,
- bool is_complete)
- {
- pce_dev->ce_request_info[req_info].xfer_type = QCE_XFER_TYPE_LAST;
- if (atomic_xchg(&pce_dev->ce_request_info[req_info].in_use,
- false)) {
- if (req_info < MAX_QCE_BAM_REQ && is_complete)
- atomic_dec(&pce_dev->no_of_queued_req);
- } else
- pr_warn("request info %d free already\n", req_info);
- }
- static void print_notify_debug(struct sps_event_notify *notify)
- {
- phys_addr_t addr =
- DESC_FULL_ADDR((phys_addr_t) notify->data.transfer.iovec.flags,
- notify->data.transfer.iovec.addr);
- pr_debug("sps ev_id=%d, addr=0x%pa, size=0x%x, flags=0x%x user=0x%pK\n",
- notify->event_id, &addr,
- notify->data.transfer.iovec.size,
- notify->data.transfer.iovec.flags,
- notify->data.transfer.user);
- }
- static void _qce_req_complete(struct qce_device *pce_dev, unsigned int req_info)
- {
- struct ce_request_info *preq_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- switch (preq_info->xfer_type) {
- case QCE_XFER_CIPHERING:
- _ablk_cipher_complete(pce_dev, req_info);
- break;
- case QCE_XFER_HASHING:
- _sha_complete(pce_dev, req_info);
- break;
- case QCE_XFER_AEAD:
- _aead_complete(pce_dev, req_info);
- break;
- case QCE_XFER_F8:
- _f8_complete(pce_dev, req_info);
- break;
- case QCE_XFER_F9:
- _f9_complete(pce_dev, req_info);
- break;
- default:
- qce_free_req_info(pce_dev, req_info, true);
- break;
- }
- }
- static void qce_multireq_timeout(struct timer_list *data)
- {
- struct qce_device *pce_dev = from_timer(pce_dev, data, timer);
- int ret = 0;
- int last_seq;
- unsigned long flags;
- last_seq = atomic_read(&pce_dev->bunch_cmd_seq);
- if (last_seq == 0 ||
- last_seq != atomic_read(&pce_dev->last_intr_seq)) {
- atomic_set(&pce_dev->last_intr_seq, last_seq);
- mod_timer(&(pce_dev->timer), (jiffies + DELAY_IN_JIFFIES));
- return;
- }
- /* last bunch mode command time out */
- /*
- * From here to dummy request finish sps request and set owner back
- * to none, we disable interrupt.
- * So it won't get preempted or interrupted. If bam inerrupts happen
- * between, and completion callback gets called from BAM, a new
- * request may be issued by the client driver. Deadlock may happen.
- */
- local_irq_save(flags);
- if (cmpxchg(&pce_dev->owner, QCE_OWNER_NONE, QCE_OWNER_TIMEOUT)
- != QCE_OWNER_NONE) {
- local_irq_restore(flags);
- mod_timer(&(pce_dev->timer), (jiffies + DELAY_IN_JIFFIES));
- return;
- }
- ret = qce_dummy_req(pce_dev);
- if (ret)
- pr_warn("pcedev %d: Failed to insert dummy req\n",
- pce_dev->dev_no);
- cmpxchg(&pce_dev->owner, QCE_OWNER_TIMEOUT, QCE_OWNER_NONE);
- pce_dev->mode = IN_INTERRUPT_MODE;
- local_irq_restore(flags);
- del_timer(&(pce_dev->timer));
- pce_dev->qce_stats.no_of_timeouts++;
- pr_debug("pcedev %d mode switch to INTR\n", pce_dev->dev_no);
- }
- void qce_get_driver_stats(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- if (!_qce50_disp_stats)
- return;
- pr_info("Engine %d timeout occuured %d\n", pce_dev->dev_no,
- pce_dev->qce_stats.no_of_timeouts);
- pr_info("Engine %d dummy request inserted %d\n", pce_dev->dev_no,
- pce_dev->qce_stats.no_of_dummy_reqs);
- if (pce_dev->mode)
- pr_info("Engine %d is in BUNCH MODE\n", pce_dev->dev_no);
- else
- pr_info("Engine %d is in INTERRUPT MODE\n", pce_dev->dev_no);
- pr_info("Engine %d outstanding request %d\n", pce_dev->dev_no,
- atomic_read(&pce_dev->no_of_queued_req));
- }
- EXPORT_SYMBOL(qce_get_driver_stats);
- void qce_clear_driver_stats(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- pce_dev->qce_stats.no_of_timeouts = 0;
- pce_dev->qce_stats.no_of_dummy_reqs = 0;
- }
- EXPORT_SYMBOL(qce_clear_driver_stats);
- static void _sps_producer_callback(struct sps_event_notify *notify)
- {
- struct qce_device *pce_dev = (struct qce_device *)
- ((struct sps_event_notify *)notify)->user;
- int rc = 0;
- unsigned int req_info;
- struct ce_sps_data *pce_sps_data;
- struct ce_request_info *preq_info;
- uint16_t op;
- print_notify_debug(notify);
- req_info = (unsigned int)((uintptr_t)notify->data.transfer.user);
- if ((req_info & 0xffff0000) != CRYPTO_REQ_USER_PAT) {
- pr_warn("request information %d out of range\n", req_info);
- return;
- }
- req_info = req_info & 0x00ff;
- if (req_info < 0 || req_info >= MAX_QCE_ALLOC_BAM_REQ) {
- pr_warn("request information %d out of range\n", req_info);
- return;
- }
- preq_info = &pce_dev->ce_request_info[req_info];
- if (!atomic_read(&preq_info->in_use)) {
- pr_err("request information %d already done\n", req_info);
- return;
- }
- op = pce_dev->ce_request_info[req_info].offload_op;
- pce_sps_data = &preq_info->ce_sps;
- if ((preq_info->xfer_type == QCE_XFER_CIPHERING ||
- preq_info->xfer_type == QCE_XFER_AEAD) &&
- pce_sps_data->producer_state == QCE_PIPE_STATE_IDLE) {
- pce_sps_data->producer_state = QCE_PIPE_STATE_COMP;
- if (!is_offload_op(op) && (op < QCE_OFFLOAD_OPER_LAST)) {
- pce_sps_data->out_transfer.iovec_count = 0;
- _qce_sps_add_data(GET_PHYS_ADDR(
- pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- _qce_set_flag(&pce_sps_data->out_transfer,
- SPS_IOVEC_FLAG_INT);
- rc = sps_transfer(
- pce_dev->ce_bam_info.producer[op].pipe,
- &pce_sps_data->out_transfer);
- if (rc) {
- pr_err("sps_xfr fail (prod pipe=0x%lx) rc = %d\n",
- (uintptr_t)pce_dev->ce_bam_info.producer[op].pipe,
- rc);
- }
- }
- return;
- }
- _qce_req_complete(pce_dev, req_info);
- }
- /**
- * De-initialize SPS HW connected with CE core
- *
- * This function deinitialize SPS endpoints and then
- * deregisters BAM resources from SPS driver.
- *
- * This function should only be called once typically
- * during driver remove.
- *
- * @pce_dev - Pointer to qce_device structure
- *
- */
- static void qce_sps_exit(struct qce_device *pce_dev)
- {
- int i = 0;
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++) {
- if (i == QCE_OFFLOAD_NONE && !(pce_dev->kernel_pipes_support))
- continue;
- else if ((i > 0) && !(pce_dev->offload_pipes_support))
- break;
- if (!pce_dev->ce_bam_info.pipe_pair_index[i])
- continue;
- qce_sps_exit_ep_conn(pce_dev,
- &pce_dev->ce_bam_info.consumer[i]);
- qce_sps_exit_ep_conn(pce_dev,
- &pce_dev->ce_bam_info.producer[i]);
- }
- qce_sps_release_bam(pce_dev);
- }
- static void qce_add_cmd_element(struct qce_device *pdev,
- struct sps_command_element **cmd_ptr, u32 addr,
- u32 data, struct sps_command_element **populate)
- {
- (*cmd_ptr)->addr = (uint32_t)(addr + pdev->phy_iobase);
- (*cmd_ptr)->command = 0;
- (*cmd_ptr)->data = data;
- (*cmd_ptr)->mask = 0xFFFFFFFF;
- (*cmd_ptr)->reserved = 0;
- if (populate != NULL)
- *populate = *cmd_ptr;
- (*cmd_ptr)++;
- }
- static int _setup_cipher_aes_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr, enum qce_cipher_mode_enum mode,
- bool key_128)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- int i = 0;
- uint32_t encr_cfg = 0;
- uint32_t key_reg = 0;
- uint32_t xts_key_reg = 0;
- uint32_t iv_reg = 0;
- cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to AES cipher operations defined
- * in ce_cmdlistptrs_ops structure.
- */
- switch (mode) {
- case QCE_MODE_CBC:
- case QCE_MODE_CTR:
- if (key_128) {
- cmdlistptr->cipher_aes_128_cbc_ctr.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_aes_128_cbc_ctr);
- if (mode == QCE_MODE_CBC)
- encr_cfg = pdev->reg.encr_cfg_aes_cbc_128;
- else
- encr_cfg = pdev->reg.encr_cfg_aes_ctr_128;
- iv_reg = 4;
- key_reg = 4;
- xts_key_reg = 0;
- } else {
- cmdlistptr->cipher_aes_256_cbc_ctr.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_aes_256_cbc_ctr);
- if (mode == QCE_MODE_CBC)
- encr_cfg = pdev->reg.encr_cfg_aes_cbc_256;
- else
- encr_cfg = pdev->reg.encr_cfg_aes_ctr_256;
- iv_reg = 4;
- key_reg = 8;
- xts_key_reg = 0;
- }
- break;
- case QCE_MODE_ECB:
- if (key_128) {
- cmdlistptr->cipher_aes_128_ecb.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_aes_128_ecb);
- encr_cfg = pdev->reg.encr_cfg_aes_ecb_128;
- iv_reg = 0;
- key_reg = 4;
- xts_key_reg = 0;
- } else {
- cmdlistptr->cipher_aes_256_ecb.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_aes_256_ecb);
- encr_cfg = pdev->reg.encr_cfg_aes_ecb_256;
- iv_reg = 0;
- key_reg = 8;
- xts_key_reg = 0;
- }
- break;
- case QCE_MODE_XTS:
- if (key_128) {
- cmdlistptr->cipher_aes_128_xts.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_aes_128_xts);
- encr_cfg = pdev->reg.encr_cfg_aes_xts_128;
- iv_reg = 4;
- key_reg = 4;
- xts_key_reg = 4;
- } else {
- cmdlistptr->cipher_aes_256_xts.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_aes_256_xts);
- encr_cfg = pdev->reg.encr_cfg_aes_xts_256;
- iv_reg = 4;
- key_reg = 8;
- xts_key_reg = 8;
- }
- break;
- default:
- pr_err("Unknown mode of operation %d received, exiting now\n",
- mode);
- return -EINVAL;
- break;
- }
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS2_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS3_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS4_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS5_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS6_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg,
- &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
- &pcl_info->encr_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- &pcl_info->encr_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG,
- pdev->reg.encr_cntr_mask_3, &pcl_info->encr_mask_3);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG2,
- pdev->reg.encr_cntr_mask_2, &pcl_info->encr_mask_2);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG1,
- pdev->reg.encr_cntr_mask_1, &pcl_info->encr_mask_1);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG0,
- pdev->reg.encr_cntr_mask_0, &pcl_info->encr_mask_0);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0,
- &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_DATA_PATT_PROC_CFG_REG, 0,
- &pcl_info->pattern_info);
- qce_add_cmd_element(pdev, &ce_vaddr,
- CRYPTO_DATA_PARTIAL_BLOCK_PROC_CFG_REG, 0,
- &pcl_info->block_offset);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
- &pcl_info->encr_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- if (xts_key_reg) {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_XTS_KEY0_REG,
- 0, &pcl_info->encr_xts_key);
- for (i = 1; i < xts_key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_XTS_KEY0_REG +
- i * sizeof(uint32_t)), 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr,
- CRYPTO_ENCR_XTS_DU_SIZE_REG, 0,
- &pcl_info->encr_xts_du_size);
- }
- if (iv_reg) {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
- &pcl_info->encr_cntr_iv);
- for (i = 1; i < iv_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- }
- /* Add dummy to align size to burst-size multiple */
- if (mode == QCE_MODE_XTS) {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG,
- 0, &pcl_info->auth_seg_size);
- } else {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG,
- 0, &pcl_info->auth_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG,
- 0, &pcl_info->auth_seg_size);
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_cipher_des_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr, enum qce_cipher_alg_enum alg,
- bool mode_cbc)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- int i = 0;
- uint32_t encr_cfg = 0;
- uint32_t key_reg = 0;
- uint32_t iv_reg = 0;
- cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to cipher operations defined
- * in ce_cmdlistptrs_ops structure.
- */
- switch (alg) {
- case CIPHER_ALG_DES:
- if (mode_cbc) {
- cmdlistptr->cipher_des_cbc.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_des_cbc);
- encr_cfg = pdev->reg.encr_cfg_des_cbc;
- iv_reg = 2;
- key_reg = 2;
- } else {
- cmdlistptr->cipher_des_ecb.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_des_ecb);
- encr_cfg = pdev->reg.encr_cfg_des_ecb;
- iv_reg = 0;
- key_reg = 2;
- }
- break;
- case CIPHER_ALG_3DES:
- if (mode_cbc) {
- cmdlistptr->cipher_3des_cbc.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_3des_cbc);
- encr_cfg = pdev->reg.encr_cfg_3des_cbc;
- iv_reg = 2;
- key_reg = 6;
- } else {
- cmdlistptr->cipher_3des_ecb.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_3des_ecb);
- encr_cfg = pdev->reg.encr_cfg_3des_ecb;
- iv_reg = 0;
- key_reg = 6;
- }
- break;
- default:
- pr_err("Unknown algorithms %d received, exiting now\n", alg);
- return -EINVAL;
- break;
- }
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg,
- &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
- &pcl_info->encr_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- &pcl_info->encr_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0,
- &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
- &pcl_info->encr_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- if (iv_reg) {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
- &pcl_info->encr_cntr_iv);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR1_IV1_REG, 0,
- NULL);
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_cipher_null_cmdlistptrs(struct qce_device *pdev,
- int cri_index, unsigned char **pvaddr)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_request_info
- [cri_index].ce_sps.cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- cmdlistptr->cipher_null.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->cipher_null);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG,
- pdev->ce_bam_info.ce_burst_size, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG,
- pdev->reg.encr_cfg_aes_ecb_128, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG,
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_auth_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr, enum qce_hash_alg_enum alg,
- bool key_128)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- int i = 0;
- uint32_t key_reg = 0;
- uint32_t auth_cfg = 0;
- uint32_t iv_reg = 0;
- cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to authentication operations
- * defined in ce_cmdlistptrs_ops structure.
- */
- switch (alg) {
- case QCE_HASH_SHA1:
- cmdlistptr->auth_sha1.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->auth_sha1);
- auth_cfg = pdev->reg.auth_cfg_sha1;
- iv_reg = 5;
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG,
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- break;
- case QCE_HASH_SHA256:
- cmdlistptr->auth_sha256.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->auth_sha256);
- auth_cfg = pdev->reg.auth_cfg_sha256;
- iv_reg = 8;
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG,
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- /* 1 dummy write */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
- 0, NULL);
- break;
- case QCE_HASH_SHA1_HMAC:
- cmdlistptr->auth_sha1_hmac.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->auth_sha1_hmac);
- auth_cfg = pdev->reg.auth_cfg_hmac_sha1;
- key_reg = 16;
- iv_reg = 5;
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG,
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- break;
- case QCE_HASH_SHA256_HMAC:
- cmdlistptr->auth_sha256_hmac.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->auth_sha256_hmac);
- auth_cfg = pdev->reg.auth_cfg_hmac_sha256;
- key_reg = 16;
- iv_reg = 8;
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- /* 1 dummy write */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
- 0, NULL);
- break;
- case QCE_HASH_AES_CMAC:
- if (key_128) {
- cmdlistptr->auth_aes_128_cmac.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->auth_aes_128_cmac);
- auth_cfg = pdev->reg.auth_cfg_cmac_128;
- key_reg = 4;
- } else {
- cmdlistptr->auth_aes_256_cmac.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->auth_aes_256_cmac);
- auth_cfg = pdev->reg.auth_cfg_cmac_256;
- key_reg = 8;
- }
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- /* 1 dummy write */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG,
- 0, NULL);
- break;
- default:
- pr_err("Unknown algorithms %d received, exiting now\n", alg);
- return -EINVAL;
- break;
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0,
- &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
- auth_cfg, &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0,
- &pcl_info->auth_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
- &pcl_info->auth_seg_start);
- if (alg == QCE_HASH_AES_CMAC) {
- /* reset auth iv, bytecount and key registers */
- for (i = 0; i < 16; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- for (i = 0; i < 16; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
- 0, NULL);
- } else {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0,
- &pcl_info->auth_iv);
- for (i = 1; i < iv_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
- 0, &pcl_info->auth_bytecount);
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL);
- if (key_reg) {
- qce_add_cmd_element(pdev, &ce_vaddr,
- CRYPTO_AUTH_KEY0_REG, 0, &pcl_info->auth_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)),
- 0, NULL);
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_aead_cmdlistptrs(struct qce_device *pdev,
- int cri_index,
- unsigned char **pvaddr,
- uint32_t alg,
- uint32_t mode,
- uint32_t key_size,
- bool sha1)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmd;
- struct qce_cmdlist_info *pcl_info = NULL;
- uint32_t key_reg;
- uint32_t iv_reg;
- uint32_t i;
- uint32_t enciv_in_word;
- uint32_t encr_cfg;
- cmd = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- switch (alg) {
- case CIPHER_ALG_DES:
- switch (mode) {
- case QCE_MODE_CBC:
- if (sha1) {
- cmd->aead_hmac_sha1_cbc_des.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha1_cbc_des);
- } else {
- cmd->aead_hmac_sha256_cbc_des.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha256_cbc_des);
- }
- encr_cfg = pdev->reg.encr_cfg_des_cbc;
- break;
- default:
- return -EINVAL;
- }
- enciv_in_word = 2;
- break;
- case CIPHER_ALG_3DES:
- switch (mode) {
- case QCE_MODE_CBC:
- if (sha1) {
- cmd->aead_hmac_sha1_cbc_3des.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha1_cbc_3des);
- } else {
- cmd->aead_hmac_sha256_cbc_3des.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha256_cbc_3des);
- }
- encr_cfg = pdev->reg.encr_cfg_3des_cbc;
- break;
- default:
- return -EINVAL;
- }
- enciv_in_word = 2;
- break;
- case CIPHER_ALG_AES:
- switch (mode) {
- case QCE_MODE_CBC:
- if (key_size == AES128_KEY_SIZE) {
- if (sha1) {
- cmd->aead_hmac_sha1_cbc_aes_128.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha1_cbc_aes_128);
- } else {
- cmd->aead_hmac_sha256_cbc_aes_128.cmdlist
- = (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha256_cbc_aes_128);
- }
- encr_cfg = pdev->reg.encr_cfg_aes_cbc_128;
- } else if (key_size == AES256_KEY_SIZE) {
- if (sha1) {
- cmd->aead_hmac_sha1_cbc_aes_256.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha1_cbc_aes_256);
- } else {
- cmd->aead_hmac_sha256_cbc_aes_256.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info =
- &(cmd->aead_hmac_sha256_cbc_aes_256);
- }
- encr_cfg = pdev->reg.encr_cfg_aes_cbc_256;
- } else {
- return -EINVAL;
- }
- break;
- default:
- return -EINVAL;
- }
- enciv_in_word = 4;
- break;
- default:
- return -EINVAL;
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- key_reg = key_size/sizeof(uint32_t);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
- &pcl_info->encr_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- if (mode != QCE_MODE_ECB) {
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
- &pcl_info->encr_cntr_iv);
- for (i = 1; i < enciv_in_word; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- }
- if (sha1)
- iv_reg = 5;
- else
- iv_reg = 8;
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0,
- &pcl_info->auth_iv);
- for (i = 1; i < iv_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
- 0, &pcl_info->auth_bytecount);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL);
- key_reg = SHA_HMAC_KEY_SIZE/sizeof(uint32_t);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_KEY0_REG, 0,
- &pcl_info->auth_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_KEY0_REG + i*sizeof(uint32_t)), 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg,
- &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
- &pcl_info->encr_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- &pcl_info->encr_seg_start);
- if (sha1)
- qce_add_cmd_element(
- pdev,
- &ce_vaddr,
- CRYPTO_AUTH_SEG_CFG_REG,
- pdev->reg.auth_cfg_aead_sha1_hmac,
- &pcl_info->auth_seg_cfg);
- else
- qce_add_cmd_element(
- pdev,
- &ce_vaddr,
- CRYPTO_AUTH_SEG_CFG_REG,
- pdev->reg.auth_cfg_aead_sha256_hmac,
- &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0,
- &pcl_info->auth_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
- &pcl_info->auth_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_aead_ccm_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr, bool key_128)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr = &pdev->ce_request_info
- [cri_index].ce_sps.cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- int i = 0;
- uint32_t encr_cfg = 0;
- uint32_t auth_cfg = 0;
- uint32_t key_reg = 0;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to aead operations
- * defined in ce_cmdlistptrs_ops structure.
- */
- if (key_128) {
- cmdlistptr->aead_aes_128_ccm.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->aead_aes_128_ccm);
- auth_cfg = pdev->reg.auth_cfg_aes_ccm_128;
- encr_cfg = pdev->reg.encr_cfg_aes_ccm_128;
- key_reg = 4;
- } else {
- cmdlistptr->aead_aes_256_ccm.cmdlist =
- (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->aead_aes_256_ccm);
- auth_cfg = pdev->reg.auth_cfg_aes_ccm_256;
- encr_cfg = pdev->reg.encr_cfg_aes_ccm_256;
- key_reg = 8;
- }
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG,
- encr_cfg, &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
- &pcl_info->encr_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- &pcl_info->encr_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG,
- pdev->reg.encr_cntr_mask_3, &pcl_info->encr_mask_3);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG0,
- pdev->reg.encr_cntr_mask_2, &pcl_info->encr_mask_2);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG1,
- pdev->reg.encr_cntr_mask_1, &pcl_info->encr_mask_1);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR_MASK_REG2,
- pdev->reg.encr_cntr_mask_0, &pcl_info->encr_mask_0);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
- auth_cfg, &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0,
- &pcl_info->auth_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
- &pcl_info->auth_seg_start);
- /* reset auth iv, bytecount and key registers */
- for (i = 0; i < 8; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_IV0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG,
- 0, NULL);
- for (i = 0; i < 16; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- /* set auth key */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_KEY0_REG, 0,
- &pcl_info->auth_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- /* set NONCE info */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_INFO_NONCE0_REG, 0,
- &pcl_info->auth_nonce_info);
- for (i = 1; i < 4; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_INFO_NONCE0_REG +
- i * sizeof(uint32_t)), 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
- &pcl_info->encr_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
- &pcl_info->encr_cntr_iv);
- for (i = 1; i < 4; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_CNTR0_IV0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_CCM_INT_CNTR0_REG, 0,
- &pcl_info->encr_ccm_cntr_iv);
- for (i = 1; i < 4; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_CCM_INT_CNTR0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_f8_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr, enum qce_ota_algo_enum alg)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- int i = 0;
- uint32_t encr_cfg = 0;
- uint32_t key_reg = 4;
- cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to f8 cipher algorithm defined
- * in ce_cmdlistptrs_ops structure.
- */
- switch (alg) {
- case QCE_OTA_ALGO_KASUMI:
- cmdlistptr->f8_kasumi.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->f8_kasumi);
- encr_cfg = pdev->reg.encr_cfg_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- default:
- cmdlistptr->f8_snow3g.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->f8_snow3g);
- encr_cfg = pdev->reg.encr_cfg_snow3g;
- break;
- }
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG,
- 0, NULL);
- /* set config to big endian */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, encr_cfg,
- &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_SIZE_REG, 0,
- &pcl_info->encr_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_START_REG, 0,
- &pcl_info->encr_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG, 0,
- &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG,
- 0, &pcl_info->auth_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG,
- 0, &pcl_info->auth_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_KEY0_REG, 0,
- &pcl_info->encr_key);
- for (i = 1; i < key_reg; i++)
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_ENCR_KEY0_REG + i * sizeof(uint32_t)),
- 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR0_IV0_REG, 0,
- &pcl_info->encr_cntr_iv);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CNTR1_IV1_REG, 0,
- NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_f9_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr, enum qce_ota_algo_enum alg)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start;
- struct qce_cmdlistptr_ops *cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- int i = 0;
- uint32_t auth_cfg = 0;
- uint32_t iv_reg = 0;
- cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr_start = (uintptr_t)(*pvaddr);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to authentication operations
- * defined in ce_cmdlistptrs_ops structure.
- */
- switch (alg) {
- case QCE_OTA_ALGO_KASUMI:
- cmdlistptr->f9_kasumi.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->f9_kasumi);
- auth_cfg = pdev->reg.auth_cfg_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- default:
- cmdlistptr->f9_snow3g.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->f9_snow3g);
- auth_cfg = pdev->reg.auth_cfg_snow3g;
- }
- /* clear status register */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_STATUS_REG,
- 0, NULL);
- /* set config to big endian */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_be, &pcl_info->crypto_cfg);
- iv_reg = 5;
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_SEG_SIZE_REG, 0,
- &pcl_info->seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_ENCR_SEG_CFG_REG, 0,
- &pcl_info->encr_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_CFG_REG,
- auth_cfg, &pcl_info->auth_seg_cfg);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_SIZE_REG, 0,
- &pcl_info->auth_seg_size);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_SEG_START_REG, 0,
- &pcl_info->auth_seg_start);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_IV0_REG, 0,
- &pcl_info->auth_iv);
- for (i = 1; i < iv_reg; i++) {
- qce_add_cmd_element(pdev, &ce_vaddr,
- (CRYPTO_AUTH_IV0_REG + i*sizeof(uint32_t)),
- 0, NULL);
- }
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT0_REG,
- 0, &pcl_info->auth_bytecount);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_AUTH_BYTECNT1_REG, 0, NULL);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- pdev->reg.crypto_cfg_le, &pcl_info->crypto_cfg_le);
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_GOPROC_REG,
- ((1 << CRYPTO_GO) | (1 << CRYPTO_RESULTS_DUMP) |
- (1 << CRYPTO_CLR_CNTXT)), &pcl_info->go_proc);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int _setup_unlock_pipe_cmdlistptrs(struct qce_device *pdev,
- int cri_index, unsigned char **pvaddr)
- {
- struct sps_command_element *ce_vaddr;
- uintptr_t ce_vaddr_start = (uintptr_t)(*pvaddr);
- struct qce_cmdlistptr_ops *cmdlistptr;
- struct qce_cmdlist_info *pcl_info = NULL;
- cmdlistptr = &pdev->ce_request_info[cri_index].ce_sps.cmdlistptr;
- *pvaddr = (unsigned char *)ALIGN(((uintptr_t)(*pvaddr)),
- pdev->ce_bam_info.ce_burst_size);
- ce_vaddr = (struct sps_command_element *)(*pvaddr);
- cmdlistptr->unlock_all_pipes.cmdlist = (uintptr_t)ce_vaddr;
- pcl_info = &(cmdlistptr->unlock_all_pipes);
- /*
- * Designate chunks of the allocated memory to command list
- * to unlock pipes.
- */
- qce_add_cmd_element(pdev, &ce_vaddr, CRYPTO_CONFIG_REG,
- CRYPTO_CONFIG_RESET, NULL);
- pcl_info->size = (uintptr_t)ce_vaddr - (uintptr_t)ce_vaddr_start;
- *pvaddr = (unsigned char *) ce_vaddr;
- return 0;
- }
- static int qce_setup_cmdlistptrs(struct qce_device *pdev, int cri_index,
- unsigned char **pvaddr)
- {
- struct sps_command_element *ce_vaddr =
- (struct sps_command_element *)(*pvaddr);
- /*
- * Designate chunks of the allocated memory to various
- * command list pointers related to operations defined
- * in ce_cmdlistptrs_ops structure.
- */
- ce_vaddr =
- (struct sps_command_element *)ALIGN(((uintptr_t) ce_vaddr),
- pdev->ce_bam_info.ce_burst_size);
- *pvaddr = (unsigned char *) ce_vaddr;
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CBC,
- true);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CTR,
- true);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_ECB,
- true);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_XTS,
- true);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CBC,
- false);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_CTR,
- false);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_ECB,
- false);
- _setup_cipher_aes_cmdlistptrs(pdev, cri_index, pvaddr, QCE_MODE_XTS,
- false);
- _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES,
- true);
- _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES,
- false);
- _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES,
- true);
- _setup_cipher_des_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES,
- false);
- _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA1,
- false);
- _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA256,
- false);
- _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA1_HMAC,
- false);
- _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_SHA256_HMAC,
- false);
- _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_AES_CMAC,
- true);
- _setup_auth_cmdlistptrs(pdev, cri_index, pvaddr, QCE_HASH_AES_CMAC,
- false);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES,
- QCE_MODE_CBC, DES_KEY_SIZE, true);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES,
- QCE_MODE_CBC, DES3_EDE_KEY_SIZE, true);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES,
- QCE_MODE_CBC, AES128_KEY_SIZE, true);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES,
- QCE_MODE_CBC, AES256_KEY_SIZE, true);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_DES,
- QCE_MODE_CBC, DES_KEY_SIZE, false);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_3DES,
- QCE_MODE_CBC, DES3_EDE_KEY_SIZE, false);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES,
- QCE_MODE_CBC, AES128_KEY_SIZE, false);
- _setup_aead_cmdlistptrs(pdev, cri_index, pvaddr, CIPHER_ALG_AES,
- QCE_MODE_CBC, AES256_KEY_SIZE, false);
- _setup_cipher_null_cmdlistptrs(pdev, cri_index, pvaddr);
- _setup_aead_ccm_cmdlistptrs(pdev, cri_index, pvaddr, true);
- _setup_aead_ccm_cmdlistptrs(pdev, cri_index, pvaddr, false);
- _setup_f8_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_KASUMI);
- _setup_f8_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_SNOW3G);
- _setup_f9_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_KASUMI);
- _setup_f9_cmdlistptrs(pdev, cri_index, pvaddr, QCE_OTA_ALGO_SNOW3G);
- _setup_unlock_pipe_cmdlistptrs(pdev, cri_index, pvaddr);
- return 0;
- }
- static int qce_setup_ce_sps_data(struct qce_device *pce_dev)
- {
- unsigned char *vaddr;
- int i;
- unsigned char *iovec_vaddr;
- int iovec_memsize;
- vaddr = pce_dev->coh_vmem;
- vaddr = (unsigned char *)ALIGN(((uintptr_t)vaddr),
- pce_dev->ce_bam_info.ce_burst_size);
- iovec_vaddr = pce_dev->iovec_vmem;
- iovec_memsize = pce_dev->iovec_memsize;
- for (i = 0; i < MAX_QCE_ALLOC_BAM_REQ; i++) {
- /* Allow for 256 descriptor (cmd and data) entries per pipe */
- pce_dev->ce_request_info[i].ce_sps.in_transfer.iovec =
- (struct sps_iovec *)iovec_vaddr;
- pce_dev->ce_request_info[i].ce_sps.in_transfer.iovec_phys =
- virt_to_phys(
- pce_dev->ce_request_info[i].ce_sps.in_transfer.iovec);
- iovec_vaddr += TOTAL_IOVEC_SPACE_PER_PIPE;
- iovec_memsize -= TOTAL_IOVEC_SPACE_PER_PIPE;
- pce_dev->ce_request_info[i].ce_sps.out_transfer.iovec =
- (struct sps_iovec *)iovec_vaddr;
- pce_dev->ce_request_info[i].ce_sps.out_transfer.iovec_phys =
- virt_to_phys(
- pce_dev->ce_request_info[i].ce_sps.out_transfer.iovec);
- iovec_vaddr += TOTAL_IOVEC_SPACE_PER_PIPE;
- iovec_memsize -= TOTAL_IOVEC_SPACE_PER_PIPE;
- if (pce_dev->support_cmd_dscr)
- qce_setup_cmdlistptrs(pce_dev, i, &vaddr);
- vaddr = (unsigned char *)ALIGN(((uintptr_t)vaddr),
- pce_dev->ce_bam_info.ce_burst_size);
- pce_dev->ce_request_info[i].ce_sps.result_dump =
- (uintptr_t)vaddr;
- pce_dev->ce_request_info[i].ce_sps.result_dump_phy =
- GET_PHYS_ADDR((uintptr_t)vaddr);
- pce_dev->ce_request_info[i].ce_sps.result =
- (struct ce_result_dump_format *)vaddr;
- vaddr += CRYPTO_RESULT_DUMP_SIZE;
- pce_dev->ce_request_info[i].ce_sps.result_dump_null =
- (uintptr_t)vaddr;
- pce_dev->ce_request_info[i].ce_sps.result_dump_null_phy =
- GET_PHYS_ADDR((uintptr_t)vaddr);
- pce_dev->ce_request_info[i].ce_sps.result_null =
- (struct ce_result_dump_format *)vaddr;
- vaddr += CRYPTO_RESULT_DUMP_SIZE;
- pce_dev->ce_request_info[i].ce_sps.ignore_buffer =
- (uintptr_t)vaddr;
- vaddr += pce_dev->ce_bam_info.ce_burst_size * 2;
- }
- if ((vaddr - pce_dev->coh_vmem) > pce_dev->memsize ||
- iovec_memsize < 0)
- panic("qce50: Not enough coherent memory. Allocate %x , need %lx\n",
- pce_dev->memsize, (uintptr_t)vaddr -
- (uintptr_t)pce_dev->coh_vmem);
- return 0;
- }
- static int qce_init_ce_cfg_val(struct qce_device *pce_dev)
- {
- uint32_t pipe_pair =
- pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_NONE];
- pce_dev->reg.crypto_cfg_be = qce_get_config_be(pce_dev, pipe_pair);
- pce_dev->reg.crypto_cfg_le =
- (pce_dev->reg.crypto_cfg_be | CRYPTO_LITTLE_ENDIAN_MASK);
- /* Initialize encr_cfg register for AES alg */
- pce_dev->reg.encr_cfg_aes_cbc_128 =
- (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_cbc_256 =
- (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_ctr_128 =
- (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CTR << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_ctr_256 =
- (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CTR << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_xts_128 =
- (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_XTS << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_xts_256 =
- (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_XTS << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_ecb_128 =
- (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_ecb_256 =
- (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_aes_ccm_128 =
- (CRYPTO_ENCR_KEY_SZ_AES128 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE)|
- (CRYPTO_LAST_CCM_XFR << CRYPTO_LAST_CCM);
- pce_dev->reg.encr_cfg_aes_ccm_256 =
- (CRYPTO_ENCR_KEY_SZ_AES256 << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_AES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CCM << CRYPTO_ENCR_MODE) |
- (CRYPTO_LAST_CCM_XFR << CRYPTO_LAST_CCM);
- /* Initialize encr_cfg register for DES alg */
- pce_dev->reg.encr_cfg_des_ecb =
- (CRYPTO_ENCR_KEY_SZ_DES << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_des_cbc =
- (CRYPTO_ENCR_KEY_SZ_DES << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_3des_ecb =
- (CRYPTO_ENCR_KEY_SZ_3DES << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_ECB << CRYPTO_ENCR_MODE);
- pce_dev->reg.encr_cfg_3des_cbc =
- (CRYPTO_ENCR_KEY_SZ_3DES << CRYPTO_ENCR_KEY_SZ) |
- (CRYPTO_ENCR_ALG_DES << CRYPTO_ENCR_ALG) |
- (CRYPTO_ENCR_MODE_CBC << CRYPTO_ENCR_MODE);
- /* Initialize encr_cfg register for kasumi/snow3g alg */
- pce_dev->reg.encr_cfg_kasumi =
- (CRYPTO_ENCR_ALG_KASUMI << CRYPTO_ENCR_ALG);
- pce_dev->reg.encr_cfg_snow3g =
- (CRYPTO_ENCR_ALG_SNOW_3G << CRYPTO_ENCR_ALG);
- /* Initialize auth_cfg register for CMAC alg */
- pce_dev->reg.auth_cfg_cmac_128 =
- (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
- (CRYPTO_AUTH_MODE_CMAC << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_ENUM_16_BYTES << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_KEY_SZ_AES128 << CRYPTO_AUTH_KEY_SIZE);
- pce_dev->reg.auth_cfg_cmac_256 =
- (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
- (CRYPTO_AUTH_MODE_CMAC << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_ENUM_16_BYTES << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_KEY_SZ_AES256 << CRYPTO_AUTH_KEY_SIZE);
- /* Initialize auth_cfg register for HMAC alg */
- pce_dev->reg.auth_cfg_hmac_sha1 =
- (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- pce_dev->reg.auth_cfg_hmac_sha256 =
- (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- /* Initialize auth_cfg register for SHA1/256 alg */
- pce_dev->reg.auth_cfg_sha1 =
- (CRYPTO_AUTH_MODE_HASH << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- pce_dev->reg.auth_cfg_sha256 =
- (CRYPTO_AUTH_MODE_HASH << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_POS_BEFORE << CRYPTO_AUTH_POS);
- /* Initialize auth_cfg register for AEAD alg */
- pce_dev->reg.auth_cfg_aead_sha1_hmac =
- (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_SHA1 << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
- (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST);
- pce_dev->reg.auth_cfg_aead_sha256_hmac =
- (CRYPTO_AUTH_MODE_HMAC << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_SIZE_SHA256 << CRYPTO_AUTH_SIZE) |
- (CRYPTO_AUTH_ALG_SHA << CRYPTO_AUTH_ALG) |
- (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST);
- pce_dev->reg.auth_cfg_aes_ccm_128 =
- (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
- (CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_KEY_SZ_AES128 << CRYPTO_AUTH_KEY_SIZE) |
- ((MAX_NONCE/sizeof(uint32_t)) << CRYPTO_AUTH_NONCE_NUM_WORDS);
- pce_dev->reg.auth_cfg_aes_ccm_128 &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
- pce_dev->reg.auth_cfg_aes_ccm_256 =
- (1 << CRYPTO_LAST) | (1 << CRYPTO_FIRST) |
- (CRYPTO_AUTH_MODE_CCM << CRYPTO_AUTH_MODE)|
- (CRYPTO_AUTH_ALG_AES << CRYPTO_AUTH_ALG) |
- (CRYPTO_AUTH_KEY_SZ_AES256 << CRYPTO_AUTH_KEY_SIZE) |
- ((MAX_NONCE/sizeof(uint32_t)) << CRYPTO_AUTH_NONCE_NUM_WORDS);
- pce_dev->reg.auth_cfg_aes_ccm_256 &= ~(1 << CRYPTO_USE_HW_KEY_AUTH);
- /* Initialize auth_cfg register for kasumi/snow3g */
- pce_dev->reg.auth_cfg_kasumi =
- (CRYPTO_AUTH_ALG_KASUMI << CRYPTO_AUTH_ALG) |
- BIT(CRYPTO_FIRST) | BIT(CRYPTO_LAST);
- pce_dev->reg.auth_cfg_snow3g =
- (CRYPTO_AUTH_ALG_SNOW3G << CRYPTO_AUTH_ALG) |
- BIT(CRYPTO_FIRST) | BIT(CRYPTO_LAST);
- /* Initialize IV counter mask values */
- pce_dev->reg.encr_cntr_mask_3 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_2 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_1 = 0xFFFFFFFF;
- pce_dev->reg.encr_cntr_mask_0 = 0xFFFFFFFF;
- return 0;
- }
- static void _qce_ccm_get_around_input(struct qce_device *pce_dev,
- struct ce_request_info *preq_info, enum qce_cipher_dir_enum dir)
- {
- struct qce_cmdlist_info *cmdlistinfo;
- struct ce_sps_data *pce_sps_data;
- pce_sps_data = &preq_info->ce_sps;
- if ((dir == QCE_DECRYPT) && pce_dev->no_get_around &&
- !(pce_dev->no_ccm_mac_status_get_around)) {
- cmdlistinfo = &pce_sps_data->cmdlistptr.cipher_null;
- _qce_sps_add_cmd(pce_dev, 0, cmdlistinfo,
- &pce_sps_data->in_transfer);
- _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->ignore_buffer),
- pce_dev->ce_bam_info.ce_burst_size,
- &pce_sps_data->in_transfer);
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT | SPS_IOVEC_FLAG_NWD);
- }
- }
- static void _qce_ccm_get_around_output(struct qce_device *pce_dev,
- struct ce_request_info *preq_info, enum qce_cipher_dir_enum dir)
- {
- struct ce_sps_data *pce_sps_data;
- pce_sps_data = &preq_info->ce_sps;
- if ((dir == QCE_DECRYPT) && pce_dev->no_get_around &&
- !(pce_dev->no_ccm_mac_status_get_around)) {
- _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->ignore_buffer),
- pce_dev->ce_bam_info.ce_burst_size,
- &pce_sps_data->out_transfer);
- _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump_null),
- CRYPTO_RESULT_DUMP_SIZE, &pce_sps_data->out_transfer);
- }
- }
- /* QCE_DUMMY_REQ */
- static void qce_dummy_complete(void *cookie, unsigned char *digest,
- unsigned char *authdata, int ret)
- {
- if (!cookie)
- pr_err("invalid cookie\n");
- }
- static int qce_dummy_req(struct qce_device *pce_dev)
- {
- int ret = 0;
- if (atomic_xchg(
- &pce_dev->ce_request_info[DUMMY_REQ_INDEX].in_use, true))
- return -EBUSY;
- ret = qce_process_sha_req(pce_dev, NULL);
- pce_dev->qce_stats.no_of_dummy_reqs++;
- return ret;
- }
- static int select_mode(struct qce_device *pce_dev,
- struct ce_request_info *preq_info)
- {
- struct ce_sps_data *pce_sps_data = &preq_info->ce_sps;
- unsigned int no_of_queued_req;
- unsigned int cadence;
- if (!pce_dev->no_get_around) {
- _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT);
- return 0;
- }
- /*
- * claim ownership of device
- */
- again:
- if (cmpxchg(&pce_dev->owner, QCE_OWNER_NONE, QCE_OWNER_CLIENT)
- != QCE_OWNER_NONE) {
- ndelay(40);
- goto again;
- }
- no_of_queued_req = atomic_inc_return(&pce_dev->no_of_queued_req);
- if (pce_dev->mode == IN_INTERRUPT_MODE) {
- if (no_of_queued_req >= MAX_BUNCH_MODE_REQ) {
- pce_dev->mode = IN_BUNCH_MODE;
- pr_debug("pcedev %d mode switch to BUNCH\n",
- pce_dev->dev_no);
- _qce_set_flag(&pce_sps_data->out_transfer,
- SPS_IOVEC_FLAG_INT);
- pce_dev->intr_cadence = 0;
- atomic_set(&pce_dev->bunch_cmd_seq, 1);
- atomic_set(&pce_dev->last_intr_seq, 1);
- mod_timer(&(pce_dev->timer),
- (jiffies + DELAY_IN_JIFFIES));
- } else {
- _qce_set_flag(&pce_sps_data->out_transfer,
- SPS_IOVEC_FLAG_INT);
- }
- } else {
- pce_dev->intr_cadence++;
- cadence = (preq_info->req_len >> 7) + 1;
- if (cadence > SET_INTR_AT_REQ)
- cadence = SET_INTR_AT_REQ;
- if (pce_dev->intr_cadence < cadence || ((pce_dev->intr_cadence
- == cadence) && pce_dev->cadence_flag))
- atomic_inc(&pce_dev->bunch_cmd_seq);
- else {
- _qce_set_flag(&pce_sps_data->out_transfer,
- SPS_IOVEC_FLAG_INT);
- pce_dev->intr_cadence = 0;
- atomic_set(&pce_dev->bunch_cmd_seq, 0);
- atomic_set(&pce_dev->last_intr_seq, 0);
- pce_dev->cadence_flag = !pce_dev->cadence_flag;
- }
- }
- return 0;
- }
- static int _qce_aead_ccm_req(void *handle, struct qce_req *q_req)
- {
- int rc = 0;
- struct qce_device *pce_dev = (struct qce_device *) handle;
- struct aead_request *areq = (struct aead_request *) q_req->areq;
- uint32_t authsize = q_req->authsize;
- uint32_t totallen_in, out_len;
- uint32_t hw_pad_out = 0;
- int ce_burst_size;
- struct qce_cmdlist_info *cmdlistinfo = NULL;
- int req_info = -1;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- q_req->current_req_info = req_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- ce_burst_size = pce_dev->ce_bam_info.ce_burst_size;
- totallen_in = areq->cryptlen + q_req->assoclen;
- if (q_req->dir == QCE_ENCRYPT) {
- q_req->cryptlen = areq->cryptlen;
- out_len = areq->cryptlen + authsize;
- hw_pad_out = ALIGN(authsize, ce_burst_size) - authsize;
- } else {
- q_req->cryptlen = areq->cryptlen - authsize;
- out_len = q_req->cryptlen;
- hw_pad_out = authsize;
- }
- /*
- * For crypto 5.0 that has burst size alignment requirement
- * for data descritpor,
- * the agent above(qcrypto) prepares the src scatter list with
- * memory starting with associated data, followed by
- * data stream to be ciphered.
- * The destination scatter list is pointing to the same
- * data area as source.
- */
- if (pce_dev->ce_bam_info.minor_version == 0)
- preq_info->src_nents = count_sg(areq->src, totallen_in);
- else
- preq_info->src_nents = count_sg(areq->src, areq->cryptlen +
- areq->assoclen);
- if (q_req->assoclen) {
- preq_info->assoc_nents = count_sg(q_req->asg, q_req->assoclen);
- /* formatted associated data input */
- qce_dma_map_sg(pce_dev->pdev, q_req->asg,
- preq_info->assoc_nents, DMA_TO_DEVICE);
- preq_info->asg = q_req->asg;
- } else {
- preq_info->assoc_nents = 0;
- preq_info->asg = NULL;
- }
- /* cipher input */
- qce_dma_map_sg(pce_dev->pdev, areq->src, preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- /* cipher + mac output for encryption */
- if (areq->src != areq->dst) {
- /*
- * The destination scatter list is pointing to the same
- * data area as src.
- * Note, the associated data will be pass-through
- * at the beginning of destination area.
- */
- preq_info->dst_nents = count_sg(areq->dst,
- out_len + areq->assoclen);
- qce_dma_map_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents,
- DMA_FROM_DEVICE);
- } else {
- preq_info->dst_nents = preq_info->src_nents;
- }
- if (pce_dev->support_cmd_dscr) {
- cmdlistinfo = _ce_get_cipher_cmdlistinfo(pce_dev, req_info,
- q_req);
- if (cmdlistinfo == NULL) {
- pr_err("Unsupported cipher algorithm %d, mode %d\n",
- q_req->alg, q_req->mode);
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- /* set up crypto device */
- rc = _ce_setup_cipher(pce_dev, q_req, totallen_in,
- q_req->assoclen, cmdlistinfo);
- } else {
- /* set up crypto device */
- rc = _ce_setup_cipher_direct(pce_dev, q_req, totallen_in,
- q_req->assoclen);
- }
- if (rc < 0)
- goto bad;
- preq_info->mode = q_req->mode;
- /* setup for callback, and issue command to bam */
- preq_info->areq = q_req->areq;
- preq_info->qce_cb = q_req->qce_cb;
- preq_info->dir = q_req->dir;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_AEAD;
- preq_info->req_len = totallen_in;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr && cmdlistinfo) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- if (pce_dev->ce_bam_info.minor_version == 0) {
- goto bad;
- } else {
- if (q_req->assoclen) {
- rc = _qce_sps_add_sg_data(pce_dev, q_req->asg,
- q_req->assoclen, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_sg_data_off(pce_dev, areq->src, areq->cryptlen,
- areq->assoclen,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- _qce_ccm_get_around_input(pce_dev, preq_info, q_req->dir);
- if (pce_dev->no_get_around) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- /* Pass through to ignore associated data*/
- rc = _qce_sps_add_data(
- GET_PHYS_ADDR(pce_sps_data->ignore_buffer),
- q_req->assoclen,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- rc = _qce_sps_add_sg_data_off(pce_dev, areq->dst, out_len,
- areq->assoclen,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- /* Pass through to ignore hw_pad (padding of the MAC data) */
- rc = _qce_sps_add_data(
- GET_PHYS_ADDR(pce_sps_data->ignore_buffer),
- hw_pad_out, &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- if (pce_dev->no_get_around ||
- totallen_in <= SPS_MAX_PKT_SIZE) {
- rc = _qce_sps_add_data(
- GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- pce_sps_data->producer_state = QCE_PIPE_STATE_COMP;
- } else {
- pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE;
- }
- _qce_ccm_get_around_output(pce_dev, preq_info, q_req->dir);
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- }
- if (rc)
- goto bad;
- return 0;
- bad:
- if (preq_info->assoc_nents) {
- qce_dma_unmap_sg(pce_dev->pdev, q_req->asg,
- preq_info->assoc_nents, DMA_TO_DEVICE);
- }
- if (preq_info->src_nents) {
- qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- }
- if (areq->src != areq->dst) {
- qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents,
- DMA_FROM_DEVICE);
- }
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- static int _qce_suspend(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *)handle;
- struct sps_pipe *sps_pipe_info;
- int i = 0;
- if (handle == NULL)
- return -ENODEV;
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++) {
- if (i == QCE_OFFLOAD_NONE && !(pce_dev->kernel_pipes_support))
- continue;
- else if ((i > 0) && !(pce_dev->offload_pipes_support))
- break;
- if (!pce_dev->ce_bam_info.pipe_pair_index[i])
- continue;
- sps_pipe_info = pce_dev->ce_bam_info.consumer[i].pipe;
- sps_disconnect(sps_pipe_info);
- sps_pipe_info = pce_dev->ce_bam_info.producer[i].pipe;
- sps_disconnect(sps_pipe_info);
- }
- return 0;
- }
- static int _qce_resume(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *)handle;
- struct sps_pipe *sps_pipe_info;
- struct sps_connect *sps_connect_info;
- int rc, i;
- rc = -ENODEV;
- if (handle == NULL)
- return rc;
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++) {
- if (i == QCE_OFFLOAD_NONE && !(pce_dev->kernel_pipes_support))
- continue;
- else if ((i > 0) && !(pce_dev->offload_pipes_support))
- break;
- if (!pce_dev->ce_bam_info.pipe_pair_index[i])
- continue;
- sps_pipe_info = pce_dev->ce_bam_info.consumer[i].pipe;
- sps_connect_info = &pce_dev->ce_bam_info.consumer[i].connect;
- memset(sps_connect_info->desc.base, 0x00,
- sps_connect_info->desc.size);
- rc = sps_connect(sps_pipe_info, sps_connect_info);
- if (rc) {
- pr_err("sps_connect() fail pipe=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- return rc;
- }
- sps_pipe_info = pce_dev->ce_bam_info.producer[i].pipe;
- sps_connect_info = &pce_dev->ce_bam_info.producer[i].connect;
- memset(sps_connect_info->desc.base, 0x00,
- sps_connect_info->desc.size);
- rc = sps_connect(sps_pipe_info, sps_connect_info);
- if (rc)
- pr_err("sps_connect() fail pipe=0x%lx, rc = %d\n",
- (uintptr_t)sps_pipe_info, rc);
- rc = sps_register_event(sps_pipe_info,
- &pce_dev->ce_bam_info.producer[i].event);
- if (rc)
- pr_err("Producer cb registration failed rc = %d\n",
- rc);
- }
- qce_enable_clock_gating(pce_dev);
- return rc;
- }
- struct qce_pm_table qce_pm_table = {_qce_suspend, _qce_resume};
- EXPORT_SYMBOL(qce_pm_table);
- int qce_aead_req(void *handle, struct qce_req *q_req)
- {
- struct qce_device *pce_dev = (struct qce_device *)handle;
- struct aead_request *areq;
- uint32_t authsize;
- struct crypto_aead *aead;
- uint32_t ivsize;
- uint32_t totallen;
- int rc = 0;
- struct qce_cmdlist_info *cmdlistinfo = NULL;
- int req_info = -1;
- struct ce_sps_data *pce_sps_data;
- struct ce_request_info *preq_info;
- if (q_req->mode == QCE_MODE_CCM)
- return _qce_aead_ccm_req(handle, q_req);
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- q_req->current_req_info = req_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- areq = (struct aead_request *) q_req->areq;
- aead = crypto_aead_reqtfm(areq);
- ivsize = crypto_aead_ivsize(aead);
- q_req->ivsize = ivsize;
- authsize = q_req->authsize;
- if (q_req->dir == QCE_ENCRYPT)
- q_req->cryptlen = areq->cryptlen;
- else
- q_req->cryptlen = areq->cryptlen - authsize;
- if (q_req->cryptlen > UINT_MAX - areq->assoclen) {
- pr_err("Integer overflow on total aead req length.\n");
- return -EINVAL;
- }
- totallen = q_req->cryptlen + areq->assoclen;
- if (pce_dev->support_cmd_dscr) {
- cmdlistinfo = _ce_get_aead_cmdlistinfo(pce_dev,
- req_info, q_req);
- if (cmdlistinfo == NULL) {
- pr_err("Unsupported aead ciphering algorithm %d, mode %d, ciphering key length %d, auth digest size %d\n",
- q_req->alg, q_req->mode, q_req->encklen,
- q_req->authsize);
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- /* set up crypto device */
- rc = _ce_setup_aead(pce_dev, q_req, totallen,
- areq->assoclen, cmdlistinfo);
- if (rc < 0) {
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- }
- /*
- * For crypto 5.0 that has burst size alignment requirement
- * for data descritpor,
- * the agent above(qcrypto) prepares the src scatter list with
- * memory starting with associated data, followed by
- * iv, and data stream to be ciphered.
- */
- preq_info->src_nents = count_sg(areq->src, totallen);
- /* cipher input */
- qce_dma_map_sg(pce_dev->pdev, areq->src, preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- /* cipher output for encryption */
- if (areq->src != areq->dst) {
- preq_info->dst_nents = count_sg(areq->dst, totallen);
- qce_dma_map_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents,
- DMA_FROM_DEVICE);
- }
- /* setup for callback, and issue command to bam */
- preq_info->areq = q_req->areq;
- preq_info->qce_cb = q_req->qce_cb;
- preq_info->dir = q_req->dir;
- preq_info->asg = NULL;
- preq_info->offload_op = QCE_OFFLOAD_NONE;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_AEAD;
- preq_info->req_len = totallen;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- } else {
- rc = _ce_setup_aead_direct(pce_dev, q_req, totallen,
- areq->assoclen);
- if (rc)
- goto bad;
- }
- preq_info->mode = q_req->mode;
- if (pce_dev->ce_bam_info.minor_version == 0) {
- rc = _qce_sps_add_sg_data(pce_dev, areq->src, totallen,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- rc = _qce_sps_add_sg_data(pce_dev, areq->dst, totallen,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- if (totallen > SPS_MAX_PKT_SIZE) {
- _qce_set_flag(&pce_sps_data->out_transfer,
- SPS_IOVEC_FLAG_INT);
- pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE;
- } else {
- rc = _qce_sps_add_data(GET_PHYS_ADDR(
- pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->out_transfer,
- SPS_IOVEC_FLAG_INT);
- pce_sps_data->producer_state = QCE_PIPE_STATE_COMP;
- }
- rc = _qce_sps_transfer(pce_dev, req_info);
- } else {
- rc = _qce_sps_add_sg_data(pce_dev, areq->src, totallen,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- if (pce_dev->no_get_around) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_sg_data(pce_dev, areq->dst, totallen,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- if (pce_dev->no_get_around || totallen <= SPS_MAX_PKT_SIZE) {
- rc = _qce_sps_add_data(
- GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- pce_sps_data->producer_state = QCE_PIPE_STATE_COMP;
- } else {
- pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE;
- }
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- }
- if (rc)
- goto bad;
- return 0;
- bad:
- if (preq_info->src_nents)
- qce_dma_unmap_sg(pce_dev->pdev, areq->src, preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- if (areq->src != areq->dst)
- qce_dma_unmap_sg(pce_dev->pdev, areq->dst, preq_info->dst_nents,
- DMA_FROM_DEVICE);
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- EXPORT_SYMBOL(qce_aead_req);
- int qce_ablk_cipher_req(void *handle, struct qce_req *c_req)
- {
- int rc = 0;
- struct qce_device *pce_dev = (struct qce_device *) handle;
- struct skcipher_request *areq = (struct skcipher_request *)
- c_req->areq;
- struct qce_cmdlist_info *cmdlistinfo = NULL;
- int req_info = -1;
- struct ce_sps_data *pce_sps_data;
- struct ce_request_info *preq_info;
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- c_req->current_req_info = req_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- preq_info->src_nents = 0;
- preq_info->dst_nents = 0;
- /* cipher input */
- preq_info->src_nents = count_sg(areq->src, areq->cryptlen);
- if (!is_offload_op(c_req->offload_op))
- qce_dma_map_sg(pce_dev->pdev, areq->src,
- preq_info->src_nents,
- (areq->src == areq->dst) ? DMA_BIDIRECTIONAL :
- DMA_TO_DEVICE);
- /* cipher output */
- if (areq->src != areq->dst) {
- preq_info->dst_nents = count_sg(areq->dst, areq->cryptlen);
- if (!is_offload_op(c_req->offload_op))
- qce_dma_map_sg(pce_dev->pdev, areq->dst,
- preq_info->dst_nents, DMA_FROM_DEVICE);
- } else {
- preq_info->dst_nents = preq_info->src_nents;
- }
- preq_info->dir = c_req->dir;
- if ((pce_dev->ce_bam_info.minor_version == 0) &&
- (preq_info->dir == QCE_DECRYPT) &&
- (c_req->mode == QCE_MODE_CBC)) {
- memcpy(preq_info->dec_iv, (unsigned char *)
- sg_virt(areq->src) + areq->src->length - 16,
- NUM_OF_CRYPTO_CNTR_IV_REG * CRYPTO_REG_SIZE);
- }
- /* set up crypto device */
- if (pce_dev->support_cmd_dscr) {
- cmdlistinfo = _ce_get_cipher_cmdlistinfo(pce_dev,
- req_info, c_req);
- if (cmdlistinfo == NULL) {
- pr_err("Unsupported cipher algorithm %d, mode %d\n",
- c_req->alg, c_req->mode);
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- rc = _ce_setup_cipher(pce_dev, c_req, areq->cryptlen, 0,
- cmdlistinfo);
- } else {
- rc = _ce_setup_cipher_direct(pce_dev, c_req, areq->cryptlen, 0);
- }
- if (rc < 0)
- goto bad;
- preq_info->mode = c_req->mode;
- preq_info->offload_op = c_req->offload_op;
- /* setup for client callback, and issue command to BAM */
- preq_info->areq = areq;
- preq_info->qce_cb = c_req->qce_cb;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_CIPHERING;
- preq_info->req_len = areq->cryptlen;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr && cmdlistinfo) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_data(areq->src->dma_address, areq->cryptlen,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- if (pce_dev->no_get_around) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_data(areq->dst->dma_address, areq->cryptlen,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- if (pce_dev->no_get_around || areq->cryptlen <= SPS_MAX_PKT_SIZE) {
- pce_sps_data->producer_state = QCE_PIPE_STATE_COMP;
- if (!is_offload_op(c_req->offload_op)) {
- rc = _qce_sps_add_data(
- GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- }
- } else {
- pce_sps_data->producer_state = QCE_PIPE_STATE_IDLE;
- }
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- if (rc)
- goto bad;
- return 0;
- bad:
- if (!is_offload_op(c_req->offload_op)) {
- if (areq->src != areq->dst)
- if (preq_info->dst_nents)
- qce_dma_unmap_sg(pce_dev->pdev, areq->dst,
- preq_info->dst_nents, DMA_FROM_DEVICE);
- if (preq_info->src_nents)
- qce_dma_unmap_sg(pce_dev->pdev, areq->src,
- preq_info->src_nents,
- (areq->src == areq->dst) ?
- DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
- }
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- EXPORT_SYMBOL(qce_ablk_cipher_req);
- int qce_process_sha_req(void *handle, struct qce_sha_req *sreq)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- int rc;
- struct ahash_request *areq;
- struct qce_cmdlist_info *cmdlistinfo = NULL;
- int req_info = -1;
- struct ce_sps_data *pce_sps_data;
- struct ce_request_info *preq_info;
- bool is_dummy = false;
- if (!sreq) {
- sreq = &(pce_dev->dummyreq.sreq);
- req_info = DUMMY_REQ_INDEX;
- is_dummy = true;
- } else {
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- }
- sreq->current_req_info = req_info;
- areq = (struct ahash_request *)sreq->areq;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- preq_info->src_nents = count_sg(sreq->src, sreq->size);
- qce_dma_map_sg(pce_dev->pdev, sreq->src, preq_info->src_nents,
- DMA_TO_DEVICE);
- if (pce_dev->support_cmd_dscr) {
- cmdlistinfo = _ce_get_hash_cmdlistinfo(pce_dev, req_info, sreq);
- if (cmdlistinfo == NULL) {
- pr_err("Unsupported hash algorithm %d\n", sreq->alg);
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- rc = _ce_setup_hash(pce_dev, sreq, cmdlistinfo);
- } else {
- rc = _ce_setup_hash_direct(pce_dev, sreq);
- }
- if (rc < 0)
- goto bad;
- preq_info->areq = areq;
- preq_info->qce_cb = sreq->qce_cb;
- preq_info->offload_op = QCE_OFFLOAD_NONE;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_HASHING;
- preq_info->req_len = sreq->size;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr && cmdlistinfo) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_sg_data(pce_dev, areq->src, areq->nbytes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- /* always ensure there is input data. ZLT does not work for bam-ndp */
- if (!areq->nbytes) {
- rc = _qce_sps_add_data(
- GET_PHYS_ADDR(pce_sps_data->ignore_buffer),
- pce_dev->ce_bam_info.ce_burst_size,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- if (pce_dev->no_get_around) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- if (is_dummy) {
- _qce_set_flag(&pce_sps_data->out_transfer, SPS_IOVEC_FLAG_INT);
- rc = _qce_sps_transfer(pce_dev, req_info);
- } else {
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- }
- if (rc)
- goto bad;
- return 0;
- bad:
- if (preq_info->src_nents) {
- qce_dma_unmap_sg(pce_dev->pdev, sreq->src,
- preq_info->src_nents, DMA_TO_DEVICE);
- }
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- EXPORT_SYMBOL(qce_process_sha_req);
- int qce_f8_req(void *handle, struct qce_f8_req *req,
- void *cookie, qce_comp_func_ptr_t qce_cb)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- bool key_stream_mode;
- dma_addr_t dst;
- int rc;
- struct qce_cmdlist_info *cmdlistinfo;
- int req_info = -1;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- req->current_req_info = req_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- cmdlistinfo = &pce_sps_data->cmdlistptr.f8_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- cmdlistinfo = &pce_sps_data->cmdlistptr.f8_snow3g;
- break;
- default:
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- key_stream_mode = (req->data_in == NULL);
- /* don't support key stream mode */
- if (key_stream_mode || (req->bearer >= QCE_OTA_MAX_BEARER)) {
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- /* F8 cipher input */
- preq_info->phy_ota_src = dma_map_single(pce_dev->pdev,
- req->data_in, req->data_len,
- (req->data_in == req->data_out) ?
- DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
- /* F8 cipher output */
- if (req->data_in != req->data_out) {
- dst = dma_map_single(pce_dev->pdev, req->data_out,
- req->data_len, DMA_FROM_DEVICE);
- preq_info->phy_ota_dst = dst;
- } else {
- /* in place ciphering */
- dst = preq_info->phy_ota_src;
- preq_info->phy_ota_dst = 0;
- }
- preq_info->ota_size = req->data_len;
- /* set up crypto device */
- if (pce_dev->support_cmd_dscr)
- rc = _ce_f8_setup(pce_dev, req, key_stream_mode, 1, 0,
- req->data_len, cmdlistinfo);
- else
- rc = _ce_f8_setup_direct(pce_dev, req, key_stream_mode, 1, 0,
- req->data_len);
- if (rc < 0)
- goto bad;
- /* setup for callback, and issue command to sps */
- preq_info->areq = cookie;
- preq_info->qce_cb = qce_cb;
- preq_info->offload_op = QCE_OFFLOAD_NONE;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_F8;
- preq_info->req_len = req->data_len;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_data((uint32_t)preq_info->phy_ota_src, req->data_len,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- rc = _qce_sps_add_data((uint32_t)dst, req->data_len,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- rc = _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- if (rc)
- goto bad;
- return 0;
- bad:
- if (preq_info->phy_ota_dst != 0)
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_dst,
- req->data_len, DMA_FROM_DEVICE);
- if (preq_info->phy_ota_src != 0)
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src,
- req->data_len,
- (req->data_in == req->data_out) ?
- DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- EXPORT_SYMBOL(qce_f8_req);
- int qce_f8_multi_pkt_req(void *handle, struct qce_f8_multi_pkt_req *mreq,
- void *cookie, qce_comp_func_ptr_t qce_cb)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- uint16_t num_pkt = mreq->num_pkt;
- uint16_t cipher_start = mreq->cipher_start;
- uint16_t cipher_size = mreq->cipher_size;
- struct qce_f8_req *req = &mreq->qce_f8_req;
- uint32_t total;
- dma_addr_t dst = 0;
- int rc = 0;
- struct qce_cmdlist_info *cmdlistinfo;
- int req_info = -1;
- struct ce_request_info *preq_info;
- struct ce_sps_data *pce_sps_data;
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- req->current_req_info = req_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- cmdlistinfo = &pce_sps_data->cmdlistptr.f8_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- cmdlistinfo = &pce_sps_data->cmdlistptr.f8_snow3g;
- break;
- default:
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- total = num_pkt * req->data_len;
- /* F8 cipher input */
- preq_info->phy_ota_src = dma_map_single(pce_dev->pdev,
- req->data_in, total,
- (req->data_in == req->data_out) ?
- DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
- /* F8 cipher output */
- if (req->data_in != req->data_out) {
- dst = dma_map_single(pce_dev->pdev, req->data_out, total,
- DMA_FROM_DEVICE);
- preq_info->phy_ota_dst = dst;
- } else {
- /* in place ciphering */
- dst = preq_info->phy_ota_src;
- preq_info->phy_ota_dst = 0;
- }
- preq_info->ota_size = total;
- /* set up crypto device */
- if (pce_dev->support_cmd_dscr)
- rc = _ce_f8_setup(pce_dev, req, false, num_pkt, cipher_start,
- cipher_size, cmdlistinfo);
- else
- rc = _ce_f8_setup_direct(pce_dev, req, false, num_pkt,
- cipher_start, cipher_size);
- if (rc)
- goto bad;
- /* setup for callback, and issue command to sps */
- preq_info->areq = cookie;
- preq_info->qce_cb = qce_cb;
- preq_info->offload_op = QCE_OFFLOAD_NONE;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_F8;
- preq_info->req_len = total;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- goto bad;
- }
- rc = _qce_sps_add_data((uint32_t)preq_info->phy_ota_src, total,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- rc = _qce_sps_add_data((uint32_t)dst, total,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- rc = _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- if (rc == 0)
- return 0;
- bad:
- if (preq_info->phy_ota_dst)
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_dst, total,
- DMA_FROM_DEVICE);
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src, total,
- (req->data_in == req->data_out) ?
- DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- EXPORT_SYMBOL(qce_f8_multi_pkt_req);
- int qce_f9_req(void *handle, struct qce_f9_req *req, void *cookie,
- qce_comp_func_ptr_t qce_cb)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- int rc;
- struct qce_cmdlist_info *cmdlistinfo;
- int req_info = -1;
- struct ce_sps_data *pce_sps_data;
- struct ce_request_info *preq_info;
- req_info = qce_alloc_req_info(pce_dev);
- if (req_info < 0)
- return -EBUSY;
- req->current_req_info = req_info;
- preq_info = &pce_dev->ce_request_info[req_info];
- pce_sps_data = &preq_info->ce_sps;
- switch (req->algorithm) {
- case QCE_OTA_ALGO_KASUMI:
- cmdlistinfo = &pce_sps_data->cmdlistptr.f9_kasumi;
- break;
- case QCE_OTA_ALGO_SNOW3G:
- cmdlistinfo = &pce_sps_data->cmdlistptr.f9_snow3g;
- break;
- default:
- qce_free_req_info(pce_dev, req_info, false);
- return -EINVAL;
- }
- preq_info->phy_ota_src = dma_map_single(pce_dev->pdev, req->message,
- req->msize, DMA_TO_DEVICE);
- preq_info->ota_size = req->msize;
- if (pce_dev->support_cmd_dscr)
- rc = _ce_f9_setup(pce_dev, req, cmdlistinfo);
- else
- rc = _ce_f9_setup_direct(pce_dev, req);
- if (rc < 0)
- goto bad;
- /* setup for callback, and issue command to sps */
- preq_info->areq = cookie;
- preq_info->qce_cb = qce_cb;
- preq_info->offload_op = QCE_OFFLOAD_NONE;
- /* setup xfer type for producer callback handling */
- preq_info->xfer_type = QCE_XFER_F9;
- preq_info->req_len = req->msize;
- _qce_sps_iovec_count_init(pce_dev, req_info);
- if (pce_dev->support_cmd_dscr) {
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_LOCK,
- cmdlistinfo, &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- }
- rc = _qce_sps_add_data((uint32_t)preq_info->phy_ota_src, req->msize,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- _qce_set_flag(&pce_sps_data->in_transfer,
- SPS_IOVEC_FLAG_EOT|SPS_IOVEC_FLAG_NWD);
- rc = _qce_sps_add_cmd(pce_dev, SPS_IOVEC_FLAG_UNLOCK,
- &pce_sps_data->cmdlistptr.unlock_all_pipes,
- &pce_sps_data->in_transfer);
- if (rc)
- goto bad;
- rc = _qce_sps_add_data(GET_PHYS_ADDR(pce_sps_data->result_dump),
- CRYPTO_RESULT_DUMP_SIZE,
- &pce_sps_data->out_transfer);
- if (rc)
- goto bad;
- select_mode(pce_dev, preq_info);
- rc = _qce_sps_transfer(pce_dev, req_info);
- cmpxchg(&pce_dev->owner, QCE_OWNER_CLIENT, QCE_OWNER_NONE);
- if (rc)
- goto bad;
- return 0;
- bad:
- dma_unmap_single(pce_dev->pdev, preq_info->phy_ota_src,
- req->msize, DMA_TO_DEVICE);
- qce_free_req_info(pce_dev, req_info, false);
- return rc;
- }
- EXPORT_SYMBOL(qce_f9_req);
- static int __qce_get_device_tree_data(struct platform_device *pdev,
- struct qce_device *pce_dev)
- {
- struct resource *resource;
- int rc = 0, i = 0;
- pce_dev->is_shared = of_property_read_bool((&pdev->dev)->of_node,
- "qcom,ce-hw-shared");
- pce_dev->support_hw_key = of_property_read_bool((&pdev->dev)->of_node,
- "qcom,ce-hw-key");
- pce_dev->use_sw_aes_cbc_ecb_ctr_algo =
- of_property_read_bool((&pdev->dev)->of_node,
- "qcom,use-sw-aes-cbc-ecb-ctr-algo");
- pce_dev->use_sw_aead_algo =
- of_property_read_bool((&pdev->dev)->of_node,
- "qcom,use-sw-aead-algo");
- pce_dev->use_sw_aes_xts_algo =
- of_property_read_bool((&pdev->dev)->of_node,
- "qcom,use-sw-aes-xts-algo");
- pce_dev->use_sw_ahash_algo =
- of_property_read_bool((&pdev->dev)->of_node,
- "qcom,use-sw-ahash-algo");
- pce_dev->use_sw_hmac_algo =
- of_property_read_bool((&pdev->dev)->of_node,
- "qcom,use-sw-hmac-algo");
- pce_dev->use_sw_aes_ccm_algo =
- of_property_read_bool((&pdev->dev)->of_node,
- "qcom,use-sw-aes-ccm-algo");
- pce_dev->support_clk_mgmt_sus_res = of_property_read_bool(
- (&pdev->dev)->of_node, "qcom,clk-mgmt-sus-res");
- pce_dev->support_only_core_src_clk = of_property_read_bool(
- (&pdev->dev)->of_node, "qcom,support-core-clk-only");
- pce_dev->request_bw_before_clk = of_property_read_bool(
- (&pdev->dev)->of_node, "qcom,request-bw-before-clk");
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++)
- pce_dev->ce_bam_info.pipe_pair_index[i] = 0;
- pce_dev->kernel_pipes_support = true;
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-pipe-pair",
- &pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_NONE])) {
- pr_warn("Kernel pipes not supported.\n");
- //Unused pipe, just as failsafe.
- pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_NONE] = 2;
- pce_dev->kernel_pipes_support = false;
- }
- if (of_property_read_bool((&pdev->dev)->of_node,
- "qcom,offload-ops-support")) {
- pce_dev->offload_pipes_support = true;
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-pipe-offload-cpb-hlos",
- &pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_CPB_HLOS])) {
- pr_err("Fail to get bam offload cpb-hlos pipe pair info.\n");
- return -EINVAL;
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-pipe-offload-hlos-hlos",
- &pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_HLOS_HLOS])) {
- pr_err("Fail to get bam offload hlos-hlos info.\n");
- return -EINVAL;
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-pipe-offload-hlos-hlos-1",
- &pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_HLOS_HLOS_1])) {
- pr_info("No bam offload hlos-hlos-1 info.\n");
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-pipe-offload-hlos-cpb",
- &pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_HLOS_CPB])) {
- pr_err("Fail to get bam offload hlos-cpb info\n");
- return -EINVAL;
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-pipe-offload-hlos-cpb-1",
- &pce_dev->ce_bam_info.pipe_pair_index[QCE_OFFLOAD_HLOS_CPB_1])) {
- pr_info("No bam offload hlos-cpb-1 info\n");
- }
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,ce-device",
- &pce_dev->ce_bam_info.ce_device)) {
- pr_err("Fail to get CE device information.\n");
- return -EINVAL;
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,ce-hw-instance",
- &pce_dev->ce_bam_info.ce_hw_instance)) {
- pr_err("Fail to get CE hw instance information.\n");
- return -EINVAL;
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,bam-ee",
- &pce_dev->ce_bam_info.bam_ee)) {
- pr_info("BAM Apps EE is not defined, setting to default 1\n");
- pce_dev->ce_bam_info.bam_ee = 1;
- }
- if (of_property_read_u32((&pdev->dev)->of_node,
- "qcom,ce-opp-freq",
- &pce_dev->ce_opp_freq_hz)) {
- pr_info("CE operating frequency is not defined, setting to default 100MHZ\n");
- pce_dev->ce_opp_freq_hz = CE_CLK_100MHZ;
- }
- if (of_property_read_bool((&pdev->dev)->of_node, "qcom,smmu-s1-enable"))
- pce_dev->enable_s1_smmu = true;
- pce_dev->no_clock_support = of_property_read_bool((&pdev->dev)->of_node,
- "qcom,no-clock-support");
- for (i = 0; i < QCE_OFFLOAD_OPER_LAST; i++) {
- /* Source/destination pipes for all usecases */
- pce_dev->ce_bam_info.dest_pipe_index[i] =
- 2 * pce_dev->ce_bam_info.pipe_pair_index[i];
- pce_dev->ce_bam_info.src_pipe_index[i] =
- pce_dev->ce_bam_info.dest_pipe_index[i] + 1;
- }
- resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
- "crypto-base");
- if (resource) {
- pce_dev->phy_iobase = resource->start;
- pce_dev->iobase = ioremap(resource->start,
- resource_size(resource));
- if (!pce_dev->iobase) {
- pr_err("Can not map CRYPTO io memory\n");
- return -ENOMEM;
- }
- } else {
- pr_err("CRYPTO HW mem unavailable.\n");
- return -ENODEV;
- }
- resource = platform_get_resource_byname(pdev, IORESOURCE_MEM,
- "crypto-bam-base");
- if (resource) {
- pce_dev->bam_mem = resource->start;
- pce_dev->bam_mem_size = resource_size(resource);
- } else {
- pr_err("CRYPTO BAM mem unavailable.\n");
- rc = -ENODEV;
- goto err_getting_bam_info;
- }
- pce_dev->ce_bam_info.bam_irq = platform_get_irq(pdev,0);
- if (pce_dev->ce_bam_info.bam_irq < 0) {
- pr_err("CRYPTO BAM IRQ unavailable.\n");
- goto err_dev;
- }
- return rc;
- err_dev:
- if (pce_dev->ce_bam_info.bam_iobase)
- iounmap(pce_dev->ce_bam_info.bam_iobase);
- err_getting_bam_info:
- if (pce_dev->iobase)
- iounmap(pce_dev->iobase);
- return rc;
- }
- static int __qce_init_clk(struct qce_device *pce_dev)
- {
- int rc = 0;
- if (pce_dev->no_clock_support) {
- pr_debug("No clock support defined in dts\n");
- return rc;
- }
- pce_dev->ce_core_src_clk = clk_get(pce_dev->pdev, "core_clk_src");
- if (!IS_ERR(pce_dev->ce_core_src_clk)) {
- if (pce_dev->request_bw_before_clk)
- goto skip_set_rate;
- rc = clk_set_rate(pce_dev->ce_core_src_clk,
- pce_dev->ce_opp_freq_hz);
- if (rc) {
- pr_err("Unable to set the core src clk @%uMhz.\n",
- pce_dev->ce_opp_freq_hz/CE_CLK_DIV);
- goto exit_put_core_src_clk;
- }
- } else {
- if (pce_dev->support_only_core_src_clk) {
- rc = PTR_ERR(pce_dev->ce_core_src_clk);
- pce_dev->ce_core_src_clk = NULL;
- pr_err("Unable to get CE core src clk\n");
- return rc;
- }
- pr_warn("Unable to get CE core src clk, set to NULL\n");
- pce_dev->ce_core_src_clk = NULL;
- }
- skip_set_rate:
- if (pce_dev->support_only_core_src_clk) {
- pce_dev->ce_core_clk = NULL;
- pce_dev->ce_clk = NULL;
- pce_dev->ce_bus_clk = NULL;
- } else {
- pce_dev->ce_core_clk = clk_get(pce_dev->pdev, "core_clk");
- if (IS_ERR(pce_dev->ce_core_clk)) {
- rc = PTR_ERR(pce_dev->ce_core_clk);
- pr_err("Unable to get CE core clk\n");
- goto exit_put_core_src_clk;
- }
- pce_dev->ce_clk = clk_get(pce_dev->pdev, "iface_clk");
- if (IS_ERR(pce_dev->ce_clk)) {
- rc = PTR_ERR(pce_dev->ce_clk);
- pr_err("Unable to get CE interface clk\n");
- goto exit_put_core_clk;
- }
- pce_dev->ce_bus_clk = clk_get(pce_dev->pdev, "bus_clk");
- if (IS_ERR(pce_dev->ce_bus_clk)) {
- rc = PTR_ERR(pce_dev->ce_bus_clk);
- pr_err("Unable to get CE BUS interface clk\n");
- goto exit_put_iface_clk;
- }
- }
- return rc;
- exit_put_iface_clk:
- if (pce_dev->ce_clk)
- clk_put(pce_dev->ce_clk);
- exit_put_core_clk:
- if (pce_dev->ce_core_clk)
- clk_put(pce_dev->ce_core_clk);
- exit_put_core_src_clk:
- if (pce_dev->ce_core_src_clk)
- clk_put(pce_dev->ce_core_src_clk);
- pr_err("Unable to init CE clks, rc = %d\n", rc);
- return rc;
- }
- static void __qce_deinit_clk(struct qce_device *pce_dev)
- {
- if (pce_dev->no_clock_support) {
- pr_debug("No clock support defined in dts\n");
- return;
- }
- if (pce_dev->ce_bus_clk)
- clk_put(pce_dev->ce_bus_clk);
- if (pce_dev->ce_clk)
- clk_put(pce_dev->ce_clk);
- if (pce_dev->ce_core_clk)
- clk_put(pce_dev->ce_core_clk);
- if (pce_dev->ce_core_src_clk)
- clk_put(pce_dev->ce_core_src_clk);
- }
- int qce_enable_clk(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *)handle;
- int rc = 0;
- if (pce_dev->no_clock_support) {
- pr_debug("No clock support defined in dts\n");
- return rc;
- }
- if (pce_dev->ce_core_src_clk) {
- rc = clk_prepare_enable(pce_dev->ce_core_src_clk);
- if (rc) {
- pr_err("Unable to enable/prepare CE core src clk\n");
- return rc;
- }
- }
- if (pce_dev->support_only_core_src_clk)
- return rc;
- if (pce_dev->ce_core_clk) {
- rc = clk_prepare_enable(pce_dev->ce_core_clk);
- if (rc) {
- pr_err("Unable to enable/prepare CE core clk\n");
- goto exit_disable_core_src_clk;
- }
- }
- if (pce_dev->ce_clk) {
- rc = clk_prepare_enable(pce_dev->ce_clk);
- if (rc) {
- pr_err("Unable to enable/prepare CE iface clk\n");
- goto exit_disable_core_clk;
- }
- }
- if (pce_dev->ce_bus_clk) {
- rc = clk_prepare_enable(pce_dev->ce_bus_clk);
- if (rc) {
- pr_err("Unable to enable/prepare CE BUS clk\n");
- goto exit_disable_ce_clk;
- }
- }
- return rc;
- exit_disable_ce_clk:
- if (pce_dev->ce_clk)
- clk_disable_unprepare(pce_dev->ce_clk);
- exit_disable_core_clk:
- if (pce_dev->ce_core_clk)
- clk_disable_unprepare(pce_dev->ce_core_clk);
- exit_disable_core_src_clk:
- if (pce_dev->ce_core_src_clk)
- clk_disable_unprepare(pce_dev->ce_core_src_clk);
- return rc;
- }
- EXPORT_SYMBOL(qce_enable_clk);
- int qce_disable_clk(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- if (pce_dev->no_clock_support) {
- pr_debug("No clock support defined in dts\n");
- return 0;
- }
- if (pce_dev->ce_bus_clk)
- clk_disable_unprepare(pce_dev->ce_bus_clk);
- if (pce_dev->ce_clk)
- clk_disable_unprepare(pce_dev->ce_clk);
- if (pce_dev->ce_core_clk)
- clk_disable_unprepare(pce_dev->ce_core_clk);
- if (pce_dev->ce_core_src_clk)
- clk_disable_unprepare(pce_dev->ce_core_src_clk);
- return 0;
- }
- EXPORT_SYMBOL(qce_disable_clk);
- /* dummy req setup */
- static int setup_dummy_req(struct qce_device *pce_dev)
- {
- char *input =
- "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopqopqrpqrs";
- int len = DUMMY_REQ_DATA_LEN;
- memcpy(pce_dev->dummyreq_in_buf, input, len);
- sg_init_one(&pce_dev->dummyreq.sg, pce_dev->dummyreq_in_buf, len);
- pce_dev->dummyreq.sreq.alg = QCE_HASH_SHA1;
- pce_dev->dummyreq.sreq.qce_cb = qce_dummy_complete;
- pce_dev->dummyreq.sreq.src = &pce_dev->dummyreq.sg;
- pce_dev->dummyreq.sreq.auth_data[0] = 0;
- pce_dev->dummyreq.sreq.auth_data[1] = 0;
- pce_dev->dummyreq.sreq.auth_data[2] = 0;
- pce_dev->dummyreq.sreq.auth_data[3] = 0;
- pce_dev->dummyreq.sreq.first_blk = true;
- pce_dev->dummyreq.sreq.last_blk = true;
- pce_dev->dummyreq.sreq.size = len;
- pce_dev->dummyreq.sreq.areq = &pce_dev->dummyreq.areq;
- pce_dev->dummyreq.sreq.flags = 0;
- pce_dev->dummyreq.sreq.authkey = NULL;
- pce_dev->dummyreq.areq.src = pce_dev->dummyreq.sreq.src;
- pce_dev->dummyreq.areq.nbytes = pce_dev->dummyreq.sreq.size;
- return 0;
- }
- static int qce_smmu_init(struct qce_device *pce_dev)
- {
- struct device *dev = pce_dev->pdev;
- if (!dev->dma_parms) {
- dev->dma_parms = devm_kzalloc(dev,
- sizeof(*dev->dma_parms), GFP_KERNEL);
- if (!dev->dma_parms)
- return -ENOMEM;
- }
- dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
- dma_set_seg_boundary(dev, (unsigned long)DMA_BIT_MASK(64));
- return 0;
- }
- /* crypto engine open function. */
- void *qce_open(struct platform_device *pdev, int *rc)
- {
- struct qce_device *pce_dev;
- int i;
- static int pcedev_no = 1;
- pce_dev = kzalloc(sizeof(struct qce_device), GFP_KERNEL);
- if (!pce_dev) {
- *rc = -ENOMEM;
- pr_err("Can not allocate memory: %d\n", *rc);
- return NULL;
- }
- pce_dev->pdev = &pdev->dev;
- mutex_lock(&qce_iomap_mutex);
- if (pdev->dev.of_node) {
- *rc = __qce_get_device_tree_data(pdev, pce_dev);
- if (*rc)
- goto err_pce_dev;
- } else {
- *rc = -EINVAL;
- pr_err("Device Node not found.\n");
- goto err_pce_dev;
- }
- if (pce_dev->enable_s1_smmu) {
- if (qce_smmu_init(pce_dev)) {
- *rc = -EIO;
- goto err_pce_dev;
- }
- }
- for (i = 0; i < MAX_QCE_ALLOC_BAM_REQ; i++)
- atomic_set(&pce_dev->ce_request_info[i].in_use, false);
- pce_dev->ce_request_index = 0;
- pce_dev->memsize = 10 * PAGE_SIZE * MAX_QCE_ALLOC_BAM_REQ;
- pce_dev->coh_vmem = dma_alloc_coherent(pce_dev->pdev,
- pce_dev->memsize, &pce_dev->coh_pmem, GFP_KERNEL);
- if (pce_dev->coh_vmem == NULL) {
- *rc = -ENOMEM;
- pr_err("Can not allocate coherent memory for sps data\n");
- goto err_iobase;
- }
- pce_dev->iovec_memsize = TOTAL_IOVEC_SPACE_PER_PIPE *
- MAX_QCE_ALLOC_BAM_REQ * 2;
- pce_dev->iovec_vmem = kzalloc(pce_dev->iovec_memsize, GFP_KERNEL);
- if (pce_dev->iovec_vmem == NULL)
- goto err_mem;
- pce_dev->dummyreq_in_buf = kzalloc(DUMMY_REQ_DATA_LEN, GFP_KERNEL);
- if (pce_dev->dummyreq_in_buf == NULL)
- goto err_mem;
- *rc = __qce_init_clk(pce_dev);
- if (*rc)
- goto err_mem;
- *rc = qce_enable_clk(pce_dev);
- if (*rc)
- goto err_enable_clk;
- if (_probe_ce_engine(pce_dev)) {
- *rc = -ENXIO;
- goto err;
- }
- *rc = 0;
- qce_init_ce_cfg_val(pce_dev);
- *rc = qce_sps_init(pce_dev);
- if (*rc)
- goto err;
- qce_setup_ce_sps_data(pce_dev);
- qce_disable_clk(pce_dev);
- setup_dummy_req(pce_dev);
- atomic_set(&pce_dev->no_of_queued_req, 0);
- pce_dev->mode = IN_INTERRUPT_MODE;
- timer_setup(&(pce_dev->timer), qce_multireq_timeout, 0);
- //pce_dev->timer.function = qce_multireq_timeout;
- //pce_dev->timer.data = (unsigned long)pce_dev;
- pce_dev->timer.expires = jiffies + DELAY_IN_JIFFIES;
- pce_dev->intr_cadence = 0;
- pce_dev->dev_no = pcedev_no;
- pcedev_no++;
- pce_dev->owner = QCE_OWNER_NONE;
- qce_enable_clock_gating(pce_dev);
- mutex_unlock(&qce_iomap_mutex);
- return pce_dev;
- err:
- qce_disable_clk(pce_dev);
- err_enable_clk:
- __qce_deinit_clk(pce_dev);
- err_mem:
- kfree(pce_dev->dummyreq_in_buf);
- kfree(pce_dev->iovec_vmem);
- if (pce_dev->coh_vmem)
- dma_free_coherent(pce_dev->pdev, pce_dev->memsize,
- pce_dev->coh_vmem, pce_dev->coh_pmem);
- err_iobase:
- if (pce_dev->iobase)
- iounmap(pce_dev->iobase);
- err_pce_dev:
- mutex_unlock(&qce_iomap_mutex);
- kfree(pce_dev);
- return NULL;
- }
- EXPORT_SYMBOL(qce_open);
- /* crypto engine close function. */
- int qce_close(void *handle)
- {
- struct qce_device *pce_dev = (struct qce_device *) handle;
- if (handle == NULL)
- return -ENODEV;
- mutex_lock(&qce_iomap_mutex);
- qce_enable_clk(pce_dev);
- qce_sps_exit(pce_dev);
- if (pce_dev->iobase)
- iounmap(pce_dev->iobase);
- if (pce_dev->coh_vmem)
- dma_free_coherent(pce_dev->pdev, pce_dev->memsize,
- pce_dev->coh_vmem, pce_dev->coh_pmem);
- kfree(pce_dev->dummyreq_in_buf);
- kfree(pce_dev->iovec_vmem);
- qce_disable_clk(pce_dev);
- __qce_deinit_clk(pce_dev);
- mutex_unlock(&qce_iomap_mutex);
- kfree(handle);
- return 0;
- }
- EXPORT_SYMBOL(qce_close);
- #define OTA_SUPPORT_MASK (1 << CRYPTO_ENCR_SNOW3G_SEL |\
- 1 << CRYPTO_ENCR_KASUMI_SEL |\
- 1 << CRYPTO_AUTH_SNOW3G_SEL |\
- 1 << CRYPTO_AUTH_KASUMI_SEL)
- int qce_hw_support(void *handle, struct ce_hw_support *ce_support)
- {
- struct qce_device *pce_dev = (struct qce_device *)handle;
- if (ce_support == NULL)
- return -EINVAL;
- ce_support->sha1_hmac_20 = false;
- ce_support->sha1_hmac = false;
- ce_support->sha256_hmac = false;
- ce_support->sha_hmac = true;
- ce_support->cmac = true;
- ce_support->aes_key_192 = false;
- ce_support->aes_xts = true;
- if ((pce_dev->engines_avail & OTA_SUPPORT_MASK) == OTA_SUPPORT_MASK)
- ce_support->ota = true;
- else
- ce_support->ota = false;
- ce_support->bam = true;
- ce_support->is_shared = (pce_dev->is_shared == 1) ? true : false;
- ce_support->hw_key = pce_dev->support_hw_key;
- ce_support->aes_ccm = true;
- ce_support->clk_mgmt_sus_res = pce_dev->support_clk_mgmt_sus_res;
- ce_support->req_bw_before_clk = pce_dev->request_bw_before_clk;
- if (pce_dev->ce_bam_info.minor_version)
- ce_support->aligned_only = false;
- else
- ce_support->aligned_only = true;
- ce_support->use_sw_aes_cbc_ecb_ctr_algo =
- pce_dev->use_sw_aes_cbc_ecb_ctr_algo;
- ce_support->use_sw_aead_algo =
- pce_dev->use_sw_aead_algo;
- ce_support->use_sw_aes_xts_algo =
- pce_dev->use_sw_aes_xts_algo;
- ce_support->use_sw_ahash_algo =
- pce_dev->use_sw_ahash_algo;
- ce_support->use_sw_hmac_algo =
- pce_dev->use_sw_hmac_algo;
- ce_support->use_sw_aes_ccm_algo =
- pce_dev->use_sw_aes_ccm_algo;
- ce_support->ce_device = pce_dev->ce_bam_info.ce_device;
- ce_support->ce_hw_instance = pce_dev->ce_bam_info.ce_hw_instance;
- if (pce_dev->no_get_around)
- ce_support->max_request = MAX_QCE_BAM_REQ;
- else
- ce_support->max_request = 1;
- return 0;
- }
- EXPORT_SYMBOL(qce_hw_support);
- void qce_dump_req(void *handle)
- {
- int i;
- bool req_in_use;
- struct qce_device *pce_dev = (struct qce_device *)handle;
- for (i = 0; i < MAX_QCE_BAM_REQ; i++) {
- req_in_use = atomic_read(&pce_dev->ce_request_info[i].in_use);
- pr_info("%s: %d %d\n", __func__, i, req_in_use);
- if (req_in_use)
- _qce_dump_descr_fifos(pce_dev, i);
- }
- }
- EXPORT_SYMBOL(qce_dump_req);
- MODULE_LICENSE("GPL v2");
- MODULE_DESCRIPTION("Crypto Engine driver");
|