hifn_795x.c 74 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * 2007+ Copyright (c) Evgeniy Polyakov <[email protected]>
  4. * All rights reserved.
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/module.h>
  8. #include <linux/moduleparam.h>
  9. #include <linux/mod_devicetable.h>
  10. #include <linux/interrupt.h>
  11. #include <linux/pci.h>
  12. #include <linux/slab.h>
  13. #include <linux/delay.h>
  14. #include <linux/mm.h>
  15. #include <linux/dma-mapping.h>
  16. #include <linux/scatterlist.h>
  17. #include <linux/highmem.h>
  18. #include <linux/crypto.h>
  19. #include <linux/hw_random.h>
  20. #include <linux/ktime.h>
  21. #include <crypto/algapi.h>
  22. #include <crypto/internal/des.h>
  23. #include <crypto/internal/skcipher.h>
  24. static char hifn_pll_ref[sizeof("extNNN")] = "ext";
  25. module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
  26. MODULE_PARM_DESC(hifn_pll_ref,
  27. "PLL reference clock (pci[freq] or ext[freq], default ext)");
  28. static atomic_t hifn_dev_number;
  29. #define ACRYPTO_OP_DECRYPT 0
  30. #define ACRYPTO_OP_ENCRYPT 1
  31. #define ACRYPTO_OP_HMAC 2
  32. #define ACRYPTO_OP_RNG 3
  33. #define ACRYPTO_MODE_ECB 0
  34. #define ACRYPTO_MODE_CBC 1
  35. #define ACRYPTO_MODE_CFB 2
  36. #define ACRYPTO_MODE_OFB 3
  37. #define ACRYPTO_TYPE_AES_128 0
  38. #define ACRYPTO_TYPE_AES_192 1
  39. #define ACRYPTO_TYPE_AES_256 2
  40. #define ACRYPTO_TYPE_3DES 3
  41. #define ACRYPTO_TYPE_DES 4
  42. #define PCI_VENDOR_ID_HIFN 0x13A3
  43. #define PCI_DEVICE_ID_HIFN_7955 0x0020
  44. #define PCI_DEVICE_ID_HIFN_7956 0x001d
  45. /* I/O region sizes */
  46. #define HIFN_BAR0_SIZE 0x1000
  47. #define HIFN_BAR1_SIZE 0x2000
  48. #define HIFN_BAR2_SIZE 0x8000
  49. /* DMA registres */
  50. #define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */
  51. #define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */
  52. #define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */
  53. #define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */
  54. #define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */
  55. #define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */
  56. #define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */
  57. #define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */
  58. #define HIFN_CHIP_ID 0x98 /* Chip ID */
  59. /*
  60. * Processing Unit Registers (offset from BASEREG0)
  61. */
  62. #define HIFN_0_PUDATA 0x00 /* Processing Unit Data */
  63. #define HIFN_0_PUCTRL 0x04 /* Processing Unit Control */
  64. #define HIFN_0_PUISR 0x08 /* Processing Unit Interrupt Status */
  65. #define HIFN_0_PUCNFG 0x0c /* Processing Unit Configuration */
  66. #define HIFN_0_PUIER 0x10 /* Processing Unit Interrupt Enable */
  67. #define HIFN_0_PUSTAT 0x14 /* Processing Unit Status/Chip ID */
  68. #define HIFN_0_FIFOSTAT 0x18 /* FIFO Status */
  69. #define HIFN_0_FIFOCNFG 0x1c /* FIFO Configuration */
  70. #define HIFN_0_SPACESIZE 0x20 /* Register space size */
  71. /* Processing Unit Control Register (HIFN_0_PUCTRL) */
  72. #define HIFN_PUCTRL_CLRSRCFIFO 0x0010 /* clear source fifo */
  73. #define HIFN_PUCTRL_STOP 0x0008 /* stop pu */
  74. #define HIFN_PUCTRL_LOCKRAM 0x0004 /* lock ram */
  75. #define HIFN_PUCTRL_DMAENA 0x0002 /* enable dma */
  76. #define HIFN_PUCTRL_RESET 0x0001 /* Reset processing unit */
  77. /* Processing Unit Interrupt Status Register (HIFN_0_PUISR) */
  78. #define HIFN_PUISR_CMDINVAL 0x8000 /* Invalid command interrupt */
  79. #define HIFN_PUISR_DATAERR 0x4000 /* Data error interrupt */
  80. #define HIFN_PUISR_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
  81. #define HIFN_PUISR_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
  82. #define HIFN_PUISR_DSTOVER 0x0200 /* Destination overrun interrupt */
  83. #define HIFN_PUISR_SRCCMD 0x0080 /* Source command interrupt */
  84. #define HIFN_PUISR_SRCCTX 0x0040 /* Source context interrupt */
  85. #define HIFN_PUISR_SRCDATA 0x0020 /* Source data interrupt */
  86. #define HIFN_PUISR_DSTDATA 0x0010 /* Destination data interrupt */
  87. #define HIFN_PUISR_DSTRESULT 0x0004 /* Destination result interrupt */
  88. /* Processing Unit Configuration Register (HIFN_0_PUCNFG) */
  89. #define HIFN_PUCNFG_DRAMMASK 0xe000 /* DRAM size mask */
  90. #define HIFN_PUCNFG_DSZ_256K 0x0000 /* 256k dram */
  91. #define HIFN_PUCNFG_DSZ_512K 0x2000 /* 512k dram */
  92. #define HIFN_PUCNFG_DSZ_1M 0x4000 /* 1m dram */
  93. #define HIFN_PUCNFG_DSZ_2M 0x6000 /* 2m dram */
  94. #define HIFN_PUCNFG_DSZ_4M 0x8000 /* 4m dram */
  95. #define HIFN_PUCNFG_DSZ_8M 0xa000 /* 8m dram */
  96. #define HIFN_PUNCFG_DSZ_16M 0xc000 /* 16m dram */
  97. #define HIFN_PUCNFG_DSZ_32M 0xe000 /* 32m dram */
  98. #define HIFN_PUCNFG_DRAMREFRESH 0x1800 /* DRAM refresh rate mask */
  99. #define HIFN_PUCNFG_DRFR_512 0x0000 /* 512 divisor of ECLK */
  100. #define HIFN_PUCNFG_DRFR_256 0x0800 /* 256 divisor of ECLK */
  101. #define HIFN_PUCNFG_DRFR_128 0x1000 /* 128 divisor of ECLK */
  102. #define HIFN_PUCNFG_TCALLPHASES 0x0200 /* your guess is as good as mine... */
  103. #define HIFN_PUCNFG_TCDRVTOTEM 0x0100 /* your guess is as good as mine... */
  104. #define HIFN_PUCNFG_BIGENDIAN 0x0080 /* DMA big endian mode */
  105. #define HIFN_PUCNFG_BUS32 0x0040 /* Bus width 32bits */
  106. #define HIFN_PUCNFG_BUS16 0x0000 /* Bus width 16 bits */
  107. #define HIFN_PUCNFG_CHIPID 0x0020 /* Allow chipid from PUSTAT */
  108. #define HIFN_PUCNFG_DRAM 0x0010 /* Context RAM is DRAM */
  109. #define HIFN_PUCNFG_SRAM 0x0000 /* Context RAM is SRAM */
  110. #define HIFN_PUCNFG_COMPSING 0x0004 /* Enable single compression context */
  111. #define HIFN_PUCNFG_ENCCNFG 0x0002 /* Encryption configuration */
  112. /* Processing Unit Interrupt Enable Register (HIFN_0_PUIER) */
  113. #define HIFN_PUIER_CMDINVAL 0x8000 /* Invalid command interrupt */
  114. #define HIFN_PUIER_DATAERR 0x4000 /* Data error interrupt */
  115. #define HIFN_PUIER_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
  116. #define HIFN_PUIER_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
  117. #define HIFN_PUIER_DSTOVER 0x0200 /* Destination overrun interrupt */
  118. #define HIFN_PUIER_SRCCMD 0x0080 /* Source command interrupt */
  119. #define HIFN_PUIER_SRCCTX 0x0040 /* Source context interrupt */
  120. #define HIFN_PUIER_SRCDATA 0x0020 /* Source data interrupt */
  121. #define HIFN_PUIER_DSTDATA 0x0010 /* Destination data interrupt */
  122. #define HIFN_PUIER_DSTRESULT 0x0004 /* Destination result interrupt */
  123. /* Processing Unit Status Register/Chip ID (HIFN_0_PUSTAT) */
  124. #define HIFN_PUSTAT_CMDINVAL 0x8000 /* Invalid command interrupt */
  125. #define HIFN_PUSTAT_DATAERR 0x4000 /* Data error interrupt */
  126. #define HIFN_PUSTAT_SRCFIFO 0x2000 /* Source FIFO ready interrupt */
  127. #define HIFN_PUSTAT_DSTFIFO 0x1000 /* Destination FIFO ready interrupt */
  128. #define HIFN_PUSTAT_DSTOVER 0x0200 /* Destination overrun interrupt */
  129. #define HIFN_PUSTAT_SRCCMD 0x0080 /* Source command interrupt */
  130. #define HIFN_PUSTAT_SRCCTX 0x0040 /* Source context interrupt */
  131. #define HIFN_PUSTAT_SRCDATA 0x0020 /* Source data interrupt */
  132. #define HIFN_PUSTAT_DSTDATA 0x0010 /* Destination data interrupt */
  133. #define HIFN_PUSTAT_DSTRESULT 0x0004 /* Destination result interrupt */
  134. #define HIFN_PUSTAT_CHIPREV 0x00ff /* Chip revision mask */
  135. #define HIFN_PUSTAT_CHIPENA 0xff00 /* Chip enabled mask */
  136. #define HIFN_PUSTAT_ENA_2 0x1100 /* Level 2 enabled */
  137. #define HIFN_PUSTAT_ENA_1 0x1000 /* Level 1 enabled */
  138. #define HIFN_PUSTAT_ENA_0 0x3000 /* Level 0 enabled */
  139. #define HIFN_PUSTAT_REV_2 0x0020 /* 7751 PT6/2 */
  140. #define HIFN_PUSTAT_REV_3 0x0030 /* 7751 PT6/3 */
  141. /* FIFO Status Register (HIFN_0_FIFOSTAT) */
  142. #define HIFN_FIFOSTAT_SRC 0x7f00 /* Source FIFO available */
  143. #define HIFN_FIFOSTAT_DST 0x007f /* Destination FIFO available */
  144. /* FIFO Configuration Register (HIFN_0_FIFOCNFG) */
  145. #define HIFN_FIFOCNFG_THRESHOLD 0x0400 /* must be written as 1 */
  146. /*
  147. * DMA Interface Registers (offset from BASEREG1)
  148. */
  149. #define HIFN_1_DMA_CRAR 0x0c /* DMA Command Ring Address */
  150. #define HIFN_1_DMA_SRAR 0x1c /* DMA Source Ring Address */
  151. #define HIFN_1_DMA_RRAR 0x2c /* DMA Result Ring Address */
  152. #define HIFN_1_DMA_DRAR 0x3c /* DMA Destination Ring Address */
  153. #define HIFN_1_DMA_CSR 0x40 /* DMA Status and Control */
  154. #define HIFN_1_DMA_IER 0x44 /* DMA Interrupt Enable */
  155. #define HIFN_1_DMA_CNFG 0x48 /* DMA Configuration */
  156. #define HIFN_1_PLL 0x4c /* 795x: PLL config */
  157. #define HIFN_1_7811_RNGENA 0x60 /* 7811: rng enable */
  158. #define HIFN_1_7811_RNGCFG 0x64 /* 7811: rng config */
  159. #define HIFN_1_7811_RNGDAT 0x68 /* 7811: rng data */
  160. #define HIFN_1_7811_RNGSTS 0x6c /* 7811: rng status */
  161. #define HIFN_1_7811_MIPSRST 0x94 /* 7811: MIPS reset */
  162. #define HIFN_1_REVID 0x98 /* Revision ID */
  163. #define HIFN_1_UNLOCK_SECRET1 0xf4
  164. #define HIFN_1_UNLOCK_SECRET2 0xfc
  165. #define HIFN_1_PUB_RESET 0x204 /* Public/RNG Reset */
  166. #define HIFN_1_PUB_BASE 0x300 /* Public Base Address */
  167. #define HIFN_1_PUB_OPLEN 0x304 /* Public Operand Length */
  168. #define HIFN_1_PUB_OP 0x308 /* Public Operand */
  169. #define HIFN_1_PUB_STATUS 0x30c /* Public Status */
  170. #define HIFN_1_PUB_IEN 0x310 /* Public Interrupt enable */
  171. #define HIFN_1_RNG_CONFIG 0x314 /* RNG config */
  172. #define HIFN_1_RNG_DATA 0x318 /* RNG data */
  173. #define HIFN_1_PUB_MEM 0x400 /* start of Public key memory */
  174. #define HIFN_1_PUB_MEMEND 0xbff /* end of Public key memory */
  175. /* DMA Status and Control Register (HIFN_1_DMA_CSR) */
  176. #define HIFN_DMACSR_D_CTRLMASK 0xc0000000 /* Destinition Ring Control */
  177. #define HIFN_DMACSR_D_CTRL_NOP 0x00000000 /* Dest. Control: no-op */
  178. #define HIFN_DMACSR_D_CTRL_DIS 0x40000000 /* Dest. Control: disable */
  179. #define HIFN_DMACSR_D_CTRL_ENA 0x80000000 /* Dest. Control: enable */
  180. #define HIFN_DMACSR_D_ABORT 0x20000000 /* Destinition Ring PCIAbort */
  181. #define HIFN_DMACSR_D_DONE 0x10000000 /* Destinition Ring Done */
  182. #define HIFN_DMACSR_D_LAST 0x08000000 /* Destinition Ring Last */
  183. #define HIFN_DMACSR_D_WAIT 0x04000000 /* Destinition Ring Waiting */
  184. #define HIFN_DMACSR_D_OVER 0x02000000 /* Destinition Ring Overflow */
  185. #define HIFN_DMACSR_R_CTRL 0x00c00000 /* Result Ring Control */
  186. #define HIFN_DMACSR_R_CTRL_NOP 0x00000000 /* Result Control: no-op */
  187. #define HIFN_DMACSR_R_CTRL_DIS 0x00400000 /* Result Control: disable */
  188. #define HIFN_DMACSR_R_CTRL_ENA 0x00800000 /* Result Control: enable */
  189. #define HIFN_DMACSR_R_ABORT 0x00200000 /* Result Ring PCI Abort */
  190. #define HIFN_DMACSR_R_DONE 0x00100000 /* Result Ring Done */
  191. #define HIFN_DMACSR_R_LAST 0x00080000 /* Result Ring Last */
  192. #define HIFN_DMACSR_R_WAIT 0x00040000 /* Result Ring Waiting */
  193. #define HIFN_DMACSR_R_OVER 0x00020000 /* Result Ring Overflow */
  194. #define HIFN_DMACSR_S_CTRL 0x0000c000 /* Source Ring Control */
  195. #define HIFN_DMACSR_S_CTRL_NOP 0x00000000 /* Source Control: no-op */
  196. #define HIFN_DMACSR_S_CTRL_DIS 0x00004000 /* Source Control: disable */
  197. #define HIFN_DMACSR_S_CTRL_ENA 0x00008000 /* Source Control: enable */
  198. #define HIFN_DMACSR_S_ABORT 0x00002000 /* Source Ring PCI Abort */
  199. #define HIFN_DMACSR_S_DONE 0x00001000 /* Source Ring Done */
  200. #define HIFN_DMACSR_S_LAST 0x00000800 /* Source Ring Last */
  201. #define HIFN_DMACSR_S_WAIT 0x00000400 /* Source Ring Waiting */
  202. #define HIFN_DMACSR_ILLW 0x00000200 /* Illegal write (7811 only) */
  203. #define HIFN_DMACSR_ILLR 0x00000100 /* Illegal read (7811 only) */
  204. #define HIFN_DMACSR_C_CTRL 0x000000c0 /* Command Ring Control */
  205. #define HIFN_DMACSR_C_CTRL_NOP 0x00000000 /* Command Control: no-op */
  206. #define HIFN_DMACSR_C_CTRL_DIS 0x00000040 /* Command Control: disable */
  207. #define HIFN_DMACSR_C_CTRL_ENA 0x00000080 /* Command Control: enable */
  208. #define HIFN_DMACSR_C_ABORT 0x00000020 /* Command Ring PCI Abort */
  209. #define HIFN_DMACSR_C_DONE 0x00000010 /* Command Ring Done */
  210. #define HIFN_DMACSR_C_LAST 0x00000008 /* Command Ring Last */
  211. #define HIFN_DMACSR_C_WAIT 0x00000004 /* Command Ring Waiting */
  212. #define HIFN_DMACSR_PUBDONE 0x00000002 /* Public op done (7951 only) */
  213. #define HIFN_DMACSR_ENGINE 0x00000001 /* Command Ring Engine IRQ */
  214. /* DMA Interrupt Enable Register (HIFN_1_DMA_IER) */
  215. #define HIFN_DMAIER_D_ABORT 0x20000000 /* Destination Ring PCIAbort */
  216. #define HIFN_DMAIER_D_DONE 0x10000000 /* Destination Ring Done */
  217. #define HIFN_DMAIER_D_LAST 0x08000000 /* Destination Ring Last */
  218. #define HIFN_DMAIER_D_WAIT 0x04000000 /* Destination Ring Waiting */
  219. #define HIFN_DMAIER_D_OVER 0x02000000 /* Destination Ring Overflow */
  220. #define HIFN_DMAIER_R_ABORT 0x00200000 /* Result Ring PCI Abort */
  221. #define HIFN_DMAIER_R_DONE 0x00100000 /* Result Ring Done */
  222. #define HIFN_DMAIER_R_LAST 0x00080000 /* Result Ring Last */
  223. #define HIFN_DMAIER_R_WAIT 0x00040000 /* Result Ring Waiting */
  224. #define HIFN_DMAIER_R_OVER 0x00020000 /* Result Ring Overflow */
  225. #define HIFN_DMAIER_S_ABORT 0x00002000 /* Source Ring PCI Abort */
  226. #define HIFN_DMAIER_S_DONE 0x00001000 /* Source Ring Done */
  227. #define HIFN_DMAIER_S_LAST 0x00000800 /* Source Ring Last */
  228. #define HIFN_DMAIER_S_WAIT 0x00000400 /* Source Ring Waiting */
  229. #define HIFN_DMAIER_ILLW 0x00000200 /* Illegal write (7811 only) */
  230. #define HIFN_DMAIER_ILLR 0x00000100 /* Illegal read (7811 only) */
  231. #define HIFN_DMAIER_C_ABORT 0x00000020 /* Command Ring PCI Abort */
  232. #define HIFN_DMAIER_C_DONE 0x00000010 /* Command Ring Done */
  233. #define HIFN_DMAIER_C_LAST 0x00000008 /* Command Ring Last */
  234. #define HIFN_DMAIER_C_WAIT 0x00000004 /* Command Ring Waiting */
  235. #define HIFN_DMAIER_PUBDONE 0x00000002 /* public op done (7951 only) */
  236. #define HIFN_DMAIER_ENGINE 0x00000001 /* Engine IRQ */
  237. /* DMA Configuration Register (HIFN_1_DMA_CNFG) */
  238. #define HIFN_DMACNFG_BIGENDIAN 0x10000000 /* big endian mode */
  239. #define HIFN_DMACNFG_POLLFREQ 0x00ff0000 /* Poll frequency mask */
  240. #define HIFN_DMACNFG_UNLOCK 0x00000800
  241. #define HIFN_DMACNFG_POLLINVAL 0x00000700 /* Invalid Poll Scalar */
  242. #define HIFN_DMACNFG_LAST 0x00000010 /* Host control LAST bit */
  243. #define HIFN_DMACNFG_MODE 0x00000004 /* DMA mode */
  244. #define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */
  245. #define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */
  246. /* PLL configuration register */
  247. #define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
  248. #define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
  249. #define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
  250. #define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
  251. #define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
  252. #define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
  253. #define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
  254. #define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */
  255. #define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
  256. #define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
  257. #define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
  258. #define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
  259. #define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
  260. #define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
  261. #define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
  262. #define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
  263. #define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
  264. #define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */
  265. /* Public key reset register (HIFN_1_PUB_RESET) */
  266. #define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */
  267. /* Public base address register (HIFN_1_PUB_BASE) */
  268. #define HIFN_PUBBASE_ADDR 0x00003fff /* base address */
  269. /* Public operand length register (HIFN_1_PUB_OPLEN) */
  270. #define HIFN_PUBOPLEN_MOD_M 0x0000007f /* modulus length mask */
  271. #define HIFN_PUBOPLEN_MOD_S 0 /* modulus length shift */
  272. #define HIFN_PUBOPLEN_EXP_M 0x0003ff80 /* exponent length mask */
  273. #define HIFN_PUBOPLEN_EXP_S 7 /* exponent length shift */
  274. #define HIFN_PUBOPLEN_RED_M 0x003c0000 /* reducend length mask */
  275. #define HIFN_PUBOPLEN_RED_S 18 /* reducend length shift */
  276. /* Public operation register (HIFN_1_PUB_OP) */
  277. #define HIFN_PUBOP_AOFFSET_M 0x0000007f /* A offset mask */
  278. #define HIFN_PUBOP_AOFFSET_S 0 /* A offset shift */
  279. #define HIFN_PUBOP_BOFFSET_M 0x00000f80 /* B offset mask */
  280. #define HIFN_PUBOP_BOFFSET_S 7 /* B offset shift */
  281. #define HIFN_PUBOP_MOFFSET_M 0x0003f000 /* M offset mask */
  282. #define HIFN_PUBOP_MOFFSET_S 12 /* M offset shift */
  283. #define HIFN_PUBOP_OP_MASK 0x003c0000 /* Opcode: */
  284. #define HIFN_PUBOP_OP_NOP 0x00000000 /* NOP */
  285. #define HIFN_PUBOP_OP_ADD 0x00040000 /* ADD */
  286. #define HIFN_PUBOP_OP_ADDC 0x00080000 /* ADD w/carry */
  287. #define HIFN_PUBOP_OP_SUB 0x000c0000 /* SUB */
  288. #define HIFN_PUBOP_OP_SUBC 0x00100000 /* SUB w/carry */
  289. #define HIFN_PUBOP_OP_MODADD 0x00140000 /* Modular ADD */
  290. #define HIFN_PUBOP_OP_MODSUB 0x00180000 /* Modular SUB */
  291. #define HIFN_PUBOP_OP_INCA 0x001c0000 /* INC A */
  292. #define HIFN_PUBOP_OP_DECA 0x00200000 /* DEC A */
  293. #define HIFN_PUBOP_OP_MULT 0x00240000 /* MULT */
  294. #define HIFN_PUBOP_OP_MODMULT 0x00280000 /* Modular MULT */
  295. #define HIFN_PUBOP_OP_MODRED 0x002c0000 /* Modular RED */
  296. #define HIFN_PUBOP_OP_MODEXP 0x00300000 /* Modular EXP */
  297. /* Public status register (HIFN_1_PUB_STATUS) */
  298. #define HIFN_PUBSTS_DONE 0x00000001 /* operation done */
  299. #define HIFN_PUBSTS_CARRY 0x00000002 /* carry */
  300. /* Public interrupt enable register (HIFN_1_PUB_IEN) */
  301. #define HIFN_PUBIEN_DONE 0x00000001 /* operation done interrupt */
  302. /* Random number generator config register (HIFN_1_RNG_CONFIG) */
  303. #define HIFN_RNGCFG_ENA 0x00000001 /* enable rng */
  304. #define HIFN_NAMESIZE 32
  305. #define HIFN_MAX_RESULT_ORDER 5
  306. #define HIFN_D_CMD_RSIZE (24 * 1)
  307. #define HIFN_D_SRC_RSIZE (80 * 1)
  308. #define HIFN_D_DST_RSIZE (80 * 1)
  309. #define HIFN_D_RES_RSIZE (24 * 1)
  310. #define HIFN_D_DST_DALIGN 4
  311. #define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1)
  312. #define AES_MIN_KEY_SIZE 16
  313. #define AES_MAX_KEY_SIZE 32
  314. #define HIFN_DES_KEY_LENGTH 8
  315. #define HIFN_3DES_KEY_LENGTH 24
  316. #define HIFN_MAX_CRYPT_KEY_LENGTH AES_MAX_KEY_SIZE
  317. #define HIFN_IV_LENGTH 8
  318. #define HIFN_AES_IV_LENGTH 16
  319. #define HIFN_MAX_IV_LENGTH HIFN_AES_IV_LENGTH
  320. #define HIFN_MAC_KEY_LENGTH 64
  321. #define HIFN_MD5_LENGTH 16
  322. #define HIFN_SHA1_LENGTH 20
  323. #define HIFN_MAC_TRUNC_LENGTH 12
  324. #define HIFN_MAX_COMMAND (8 + 8 + 8 + 64 + 260)
  325. #define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4)
  326. #define HIFN_USED_RESULT 12
  327. struct hifn_desc {
  328. volatile __le32 l;
  329. volatile __le32 p;
  330. };
  331. struct hifn_dma {
  332. struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1];
  333. struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1];
  334. struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1];
  335. struct hifn_desc resr[HIFN_D_RES_RSIZE + 1];
  336. u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
  337. u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
  338. /*
  339. * Our current positions for insertion and removal from the descriptor
  340. * rings.
  341. */
  342. volatile int cmdi, srci, dsti, resi;
  343. volatile int cmdu, srcu, dstu, resu;
  344. int cmdk, srck, dstk, resk;
  345. };
  346. #define HIFN_FLAG_CMD_BUSY (1 << 0)
  347. #define HIFN_FLAG_SRC_BUSY (1 << 1)
  348. #define HIFN_FLAG_DST_BUSY (1 << 2)
  349. #define HIFN_FLAG_RES_BUSY (1 << 3)
  350. #define HIFN_FLAG_OLD_KEY (1 << 4)
  351. #define HIFN_DEFAULT_ACTIVE_NUM 5
  352. struct hifn_device {
  353. char name[HIFN_NAMESIZE];
  354. int irq;
  355. struct pci_dev *pdev;
  356. void __iomem *bar[3];
  357. void *desc_virt;
  358. dma_addr_t desc_dma;
  359. u32 dmareg;
  360. void *sa[HIFN_D_RES_RSIZE];
  361. spinlock_t lock;
  362. u32 flags;
  363. int active, started;
  364. struct delayed_work work;
  365. unsigned long reset;
  366. unsigned long success;
  367. unsigned long prev_success;
  368. u8 snum;
  369. struct tasklet_struct tasklet;
  370. struct crypto_queue queue;
  371. struct list_head alg_list;
  372. unsigned int pk_clk_freq;
  373. #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
  374. unsigned int rng_wait_time;
  375. ktime_t rngtime;
  376. struct hwrng rng;
  377. #endif
  378. };
  379. #define HIFN_D_LENGTH 0x0000ffff
  380. #define HIFN_D_NOINVALID 0x01000000
  381. #define HIFN_D_MASKDONEIRQ 0x02000000
  382. #define HIFN_D_DESTOVER 0x04000000
  383. #define HIFN_D_OVER 0x08000000
  384. #define HIFN_D_LAST 0x20000000
  385. #define HIFN_D_JUMP 0x40000000
  386. #define HIFN_D_VALID 0x80000000
  387. struct hifn_base_command {
  388. volatile __le16 masks;
  389. volatile __le16 session_num;
  390. volatile __le16 total_source_count;
  391. volatile __le16 total_dest_count;
  392. };
  393. #define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */
  394. #define HIFN_BASE_CMD_PAD 0x0200 /* enable padding engine */
  395. #define HIFN_BASE_CMD_MAC 0x0400 /* enable MAC engine */
  396. #define HIFN_BASE_CMD_CRYPT 0x0800 /* enable crypt engine */
  397. #define HIFN_BASE_CMD_DECODE 0x2000
  398. #define HIFN_BASE_CMD_SRCLEN_M 0xc000
  399. #define HIFN_BASE_CMD_SRCLEN_S 14
  400. #define HIFN_BASE_CMD_DSTLEN_M 0x3000
  401. #define HIFN_BASE_CMD_DSTLEN_S 12
  402. #define HIFN_BASE_CMD_LENMASK_HI 0x30000
  403. #define HIFN_BASE_CMD_LENMASK_LO 0x0ffff
  404. /*
  405. * Structure to help build up the command data structure.
  406. */
  407. struct hifn_crypt_command {
  408. volatile __le16 masks;
  409. volatile __le16 header_skip;
  410. volatile __le16 source_count;
  411. volatile __le16 reserved;
  412. };
  413. #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */
  414. #define HIFN_CRYPT_CMD_ALG_DES 0x0000 /* DES */
  415. #define HIFN_CRYPT_CMD_ALG_3DES 0x0001 /* 3DES */
  416. #define HIFN_CRYPT_CMD_ALG_RC4 0x0002 /* RC4 */
  417. #define HIFN_CRYPT_CMD_ALG_AES 0x0003 /* AES */
  418. #define HIFN_CRYPT_CMD_MODE_MASK 0x0018 /* Encrypt mode: */
  419. #define HIFN_CRYPT_CMD_MODE_ECB 0x0000 /* ECB */
  420. #define HIFN_CRYPT_CMD_MODE_CBC 0x0008 /* CBC */
  421. #define HIFN_CRYPT_CMD_MODE_CFB 0x0010 /* CFB */
  422. #define HIFN_CRYPT_CMD_MODE_OFB 0x0018 /* OFB */
  423. #define HIFN_CRYPT_CMD_CLR_CTX 0x0040 /* clear context */
  424. #define HIFN_CRYPT_CMD_KSZ_MASK 0x0600 /* AES key size: */
  425. #define HIFN_CRYPT_CMD_KSZ_128 0x0000 /* 128 bit */
  426. #define HIFN_CRYPT_CMD_KSZ_192 0x0200 /* 192 bit */
  427. #define HIFN_CRYPT_CMD_KSZ_256 0x0400 /* 256 bit */
  428. #define HIFN_CRYPT_CMD_NEW_KEY 0x0800 /* expect new key */
  429. #define HIFN_CRYPT_CMD_NEW_IV 0x1000 /* expect new iv */
  430. #define HIFN_CRYPT_CMD_SRCLEN_M 0xc000
  431. #define HIFN_CRYPT_CMD_SRCLEN_S 14
  432. /*
  433. * Structure to help build up the command data structure.
  434. */
  435. struct hifn_mac_command {
  436. volatile __le16 masks;
  437. volatile __le16 header_skip;
  438. volatile __le16 source_count;
  439. volatile __le16 reserved;
  440. };
  441. #define HIFN_MAC_CMD_ALG_MASK 0x0001
  442. #define HIFN_MAC_CMD_ALG_SHA1 0x0000
  443. #define HIFN_MAC_CMD_ALG_MD5 0x0001
  444. #define HIFN_MAC_CMD_MODE_MASK 0x000c
  445. #define HIFN_MAC_CMD_MODE_HMAC 0x0000
  446. #define HIFN_MAC_CMD_MODE_SSL_MAC 0x0004
  447. #define HIFN_MAC_CMD_MODE_HASH 0x0008
  448. #define HIFN_MAC_CMD_MODE_FULL 0x0004
  449. #define HIFN_MAC_CMD_TRUNC 0x0010
  450. #define HIFN_MAC_CMD_RESULT 0x0020
  451. #define HIFN_MAC_CMD_APPEND 0x0040
  452. #define HIFN_MAC_CMD_SRCLEN_M 0xc000
  453. #define HIFN_MAC_CMD_SRCLEN_S 14
  454. /*
  455. * MAC POS IPsec initiates authentication after encryption on encodes
  456. * and before decryption on decodes.
  457. */
  458. #define HIFN_MAC_CMD_POS_IPSEC 0x0200
  459. #define HIFN_MAC_CMD_NEW_KEY 0x0800
  460. struct hifn_comp_command {
  461. volatile __le16 masks;
  462. volatile __le16 header_skip;
  463. volatile __le16 source_count;
  464. volatile __le16 reserved;
  465. };
  466. #define HIFN_COMP_CMD_SRCLEN_M 0xc000
  467. #define HIFN_COMP_CMD_SRCLEN_S 14
  468. #define HIFN_COMP_CMD_ONE 0x0100 /* must be one */
  469. #define HIFN_COMP_CMD_CLEARHIST 0x0010 /* clear history */
  470. #define HIFN_COMP_CMD_UPDATEHIST 0x0008 /* update history */
  471. #define HIFN_COMP_CMD_LZS_STRIP0 0x0004 /* LZS: strip zero */
  472. #define HIFN_COMP_CMD_MPPC_RESTART 0x0004 /* MPPC: restart */
  473. #define HIFN_COMP_CMD_ALG_MASK 0x0001 /* compression mode: */
  474. #define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */
  475. #define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */
  476. struct hifn_base_result {
  477. volatile __le16 flags;
  478. volatile __le16 session;
  479. volatile __le16 src_cnt; /* 15:0 of source count */
  480. volatile __le16 dst_cnt; /* 15:0 of dest count */
  481. };
  482. #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
  483. #define HIFN_BASE_RES_SRCLEN_M 0xc000 /* 17:16 of source count */
  484. #define HIFN_BASE_RES_SRCLEN_S 14
  485. #define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */
  486. #define HIFN_BASE_RES_DSTLEN_S 12
  487. struct hifn_comp_result {
  488. volatile __le16 flags;
  489. volatile __le16 crc;
  490. };
  491. #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
  492. #define HIFN_COMP_RES_LCB_S 8
  493. #define HIFN_COMP_RES_RESTART 0x0004 /* MPPC: restart */
  494. #define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */
  495. #define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */
  496. struct hifn_mac_result {
  497. volatile __le16 flags;
  498. volatile __le16 reserved;
  499. /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
  500. };
  501. #define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */
  502. #define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */
  503. struct hifn_crypt_result {
  504. volatile __le16 flags;
  505. volatile __le16 reserved;
  506. };
  507. #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
  508. #ifndef HIFN_POLL_FREQUENCY
  509. #define HIFN_POLL_FREQUENCY 0x1
  510. #endif
  511. #ifndef HIFN_POLL_SCALAR
  512. #define HIFN_POLL_SCALAR 0x0
  513. #endif
  514. #define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */
  515. #define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */
  516. struct hifn_crypto_alg {
  517. struct list_head entry;
  518. struct skcipher_alg alg;
  519. struct hifn_device *dev;
  520. };
  521. #define ASYNC_SCATTERLIST_CACHE 16
  522. #define ASYNC_FLAGS_MISALIGNED (1 << 0)
  523. struct hifn_cipher_walk {
  524. struct scatterlist cache[ASYNC_SCATTERLIST_CACHE];
  525. u32 flags;
  526. int num;
  527. };
  528. struct hifn_context {
  529. u8 key[HIFN_MAX_CRYPT_KEY_LENGTH];
  530. struct hifn_device *dev;
  531. unsigned int keysize;
  532. };
  533. struct hifn_request_context {
  534. u8 *iv;
  535. unsigned int ivsize;
  536. u8 op, type, mode, unused;
  537. struct hifn_cipher_walk walk;
  538. };
  539. #define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
  540. static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
  541. {
  542. return readl(dev->bar[0] + reg);
  543. }
  544. static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg)
  545. {
  546. return readl(dev->bar[1] + reg);
  547. }
  548. static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
  549. {
  550. writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
  551. }
  552. static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
  553. {
  554. writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
  555. }
  556. static void hifn_wait_puc(struct hifn_device *dev)
  557. {
  558. int i;
  559. u32 ret;
  560. for (i = 10000; i > 0; --i) {
  561. ret = hifn_read_0(dev, HIFN_0_PUCTRL);
  562. if (!(ret & HIFN_PUCTRL_RESET))
  563. break;
  564. udelay(1);
  565. }
  566. if (!i)
  567. dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n");
  568. }
  569. static void hifn_reset_puc(struct hifn_device *dev)
  570. {
  571. hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
  572. hifn_wait_puc(dev);
  573. }
  574. static void hifn_stop_device(struct hifn_device *dev)
  575. {
  576. hifn_write_1(dev, HIFN_1_DMA_CSR,
  577. HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
  578. HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS);
  579. hifn_write_0(dev, HIFN_0_PUIER, 0);
  580. hifn_write_1(dev, HIFN_1_DMA_IER, 0);
  581. }
  582. static void hifn_reset_dma(struct hifn_device *dev, int full)
  583. {
  584. hifn_stop_device(dev);
  585. /*
  586. * Setting poll frequency and others to 0.
  587. */
  588. hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
  589. HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
  590. mdelay(1);
  591. /*
  592. * Reset DMA.
  593. */
  594. if (full) {
  595. hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE);
  596. mdelay(1);
  597. } else {
  598. hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MODE |
  599. HIFN_DMACNFG_MSTRESET);
  600. hifn_reset_puc(dev);
  601. }
  602. hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
  603. HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
  604. hifn_reset_puc(dev);
  605. }
  606. static u32 hifn_next_signature(u32 a, u_int cnt)
  607. {
  608. int i;
  609. u32 v;
  610. for (i = 0; i < cnt; i++) {
  611. /* get the parity */
  612. v = a & 0x80080125;
  613. v ^= v >> 16;
  614. v ^= v >> 8;
  615. v ^= v >> 4;
  616. v ^= v >> 2;
  617. v ^= v >> 1;
  618. a = (v & 1) ^ (a << 1);
  619. }
  620. return a;
  621. }
  622. static struct pci2id {
  623. u_short pci_vendor;
  624. u_short pci_prod;
  625. char card_id[13];
  626. } pci2id[] = {
  627. {
  628. PCI_VENDOR_ID_HIFN,
  629. PCI_DEVICE_ID_HIFN_7955,
  630. { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  631. 0x00, 0x00, 0x00, 0x00, 0x00 }
  632. },
  633. {
  634. PCI_VENDOR_ID_HIFN,
  635. PCI_DEVICE_ID_HIFN_7956,
  636. { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  637. 0x00, 0x00, 0x00, 0x00, 0x00 }
  638. }
  639. };
  640. #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
  641. static int hifn_rng_data_present(struct hwrng *rng, int wait)
  642. {
  643. struct hifn_device *dev = (struct hifn_device *)rng->priv;
  644. s64 nsec;
  645. nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
  646. nsec -= dev->rng_wait_time;
  647. if (nsec <= 0)
  648. return 1;
  649. if (!wait)
  650. return 0;
  651. ndelay(nsec);
  652. return 1;
  653. }
  654. static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
  655. {
  656. struct hifn_device *dev = (struct hifn_device *)rng->priv;
  657. *data = hifn_read_1(dev, HIFN_1_RNG_DATA);
  658. dev->rngtime = ktime_get();
  659. return 4;
  660. }
  661. static int hifn_register_rng(struct hifn_device *dev)
  662. {
  663. /*
  664. * We must wait at least 256 Pk_clk cycles between two reads of the rng.
  665. */
  666. dev->rng_wait_time = DIV_ROUND_UP_ULL(NSEC_PER_SEC,
  667. dev->pk_clk_freq) * 256;
  668. dev->rng.name = dev->name;
  669. dev->rng.data_present = hifn_rng_data_present;
  670. dev->rng.data_read = hifn_rng_data_read;
  671. dev->rng.priv = (unsigned long)dev;
  672. return hwrng_register(&dev->rng);
  673. }
  674. static void hifn_unregister_rng(struct hifn_device *dev)
  675. {
  676. hwrng_unregister(&dev->rng);
  677. }
  678. #else
  679. #define hifn_register_rng(dev) 0
  680. #define hifn_unregister_rng(dev)
  681. #endif
  682. static int hifn_init_pubrng(struct hifn_device *dev)
  683. {
  684. int i;
  685. hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) |
  686. HIFN_PUBRST_RESET);
  687. for (i = 100; i > 0; --i) {
  688. mdelay(1);
  689. if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0)
  690. break;
  691. }
  692. if (!i) {
  693. dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n");
  694. } else {
  695. hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE);
  696. dev->dmareg |= HIFN_DMAIER_PUBDONE;
  697. hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
  698. dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n");
  699. }
  700. /* Enable RNG engine. */
  701. hifn_write_1(dev, HIFN_1_RNG_CONFIG,
  702. hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA);
  703. dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n");
  704. #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
  705. /* First value must be discarded */
  706. hifn_read_1(dev, HIFN_1_RNG_DATA);
  707. dev->rngtime = ktime_get();
  708. #endif
  709. return 0;
  710. }
  711. static int hifn_enable_crypto(struct hifn_device *dev)
  712. {
  713. u32 dmacfg, addr;
  714. char *offtbl = NULL;
  715. int i;
  716. for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
  717. if (pci2id[i].pci_vendor == dev->pdev->vendor &&
  718. pci2id[i].pci_prod == dev->pdev->device) {
  719. offtbl = pci2id[i].card_id;
  720. break;
  721. }
  722. }
  723. if (!offtbl) {
  724. dev_err(&dev->pdev->dev, "Unknown card!\n");
  725. return -ENODEV;
  726. }
  727. dmacfg = hifn_read_1(dev, HIFN_1_DMA_CNFG);
  728. hifn_write_1(dev, HIFN_1_DMA_CNFG,
  729. HIFN_DMACNFG_UNLOCK | HIFN_DMACNFG_MSTRESET |
  730. HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE);
  731. mdelay(1);
  732. addr = hifn_read_1(dev, HIFN_1_UNLOCK_SECRET1);
  733. mdelay(1);
  734. hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0);
  735. mdelay(1);
  736. for (i = 0; i < 12; ++i) {
  737. addr = hifn_next_signature(addr, offtbl[i] + 0x101);
  738. hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr);
  739. mdelay(1);
  740. }
  741. hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg);
  742. dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev));
  743. return 0;
  744. }
  745. static void hifn_init_dma(struct hifn_device *dev)
  746. {
  747. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  748. u32 dptr = dev->desc_dma;
  749. int i;
  750. for (i = 0; i < HIFN_D_CMD_RSIZE; ++i)
  751. dma->cmdr[i].p = __cpu_to_le32(dptr +
  752. offsetof(struct hifn_dma, command_bufs[i][0]));
  753. for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
  754. dma->resr[i].p = __cpu_to_le32(dptr +
  755. offsetof(struct hifn_dma, result_bufs[i][0]));
  756. /* Setup LAST descriptors. */
  757. dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr +
  758. offsetof(struct hifn_dma, cmdr[0]));
  759. dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr +
  760. offsetof(struct hifn_dma, srcr[0]));
  761. dma->dstr[HIFN_D_DST_RSIZE].p = __cpu_to_le32(dptr +
  762. offsetof(struct hifn_dma, dstr[0]));
  763. dma->resr[HIFN_D_RES_RSIZE].p = __cpu_to_le32(dptr +
  764. offsetof(struct hifn_dma, resr[0]));
  765. dma->cmdu = dma->srcu = dma->dstu = dma->resu = 0;
  766. dma->cmdi = dma->srci = dma->dsti = dma->resi = 0;
  767. dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
  768. }
  769. /*
  770. * Initialize the PLL. We need to know the frequency of the reference clock
  771. * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
  772. * allows us to operate without the risk of overclocking the chip. If it
  773. * actually uses 33MHz, the chip will operate at half the speed, this can be
  774. * overridden by specifying the frequency as module parameter (pci33).
  775. *
  776. * Unfortunately the PCI clock is not very suitable since the HIFN needs a
  777. * stable clock and the PCI clock frequency may vary, so the default is the
  778. * external clock. There is no way to find out its frequency, we default to
  779. * 66MHz since according to Mike Ham of HiFn, almost every board in existence
  780. * has an external crystal populated at 66MHz.
  781. */
  782. static void hifn_init_pll(struct hifn_device *dev)
  783. {
  784. unsigned int freq, m;
  785. u32 pllcfg;
  786. pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
  787. if (strncmp(hifn_pll_ref, "ext", 3) == 0)
  788. pllcfg |= HIFN_PLL_REF_CLK_PLL;
  789. else
  790. pllcfg |= HIFN_PLL_REF_CLK_HBI;
  791. if (hifn_pll_ref[3] != '\0')
  792. freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
  793. else {
  794. freq = 66;
  795. dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n",
  796. freq, hifn_pll_ref);
  797. }
  798. m = HIFN_PLL_FCK_MAX / freq;
  799. pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
  800. if (m <= 8)
  801. pllcfg |= HIFN_PLL_IS_1_8;
  802. else
  803. pllcfg |= HIFN_PLL_IS_9_12;
  804. /* Select clock source and enable clock bypass */
  805. hifn_write_1(dev, HIFN_1_PLL, pllcfg |
  806. HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
  807. /* Let the chip lock to the input clock */
  808. mdelay(10);
  809. /* Disable clock bypass */
  810. hifn_write_1(dev, HIFN_1_PLL, pllcfg |
  811. HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
  812. /* Switch the engines to the PLL */
  813. hifn_write_1(dev, HIFN_1_PLL, pllcfg |
  814. HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
  815. /*
  816. * The Fpk_clk runs at half the total speed. Its frequency is needed to
  817. * calculate the minimum time between two reads of the rng. Since 33MHz
  818. * is actually 33.333... we overestimate the frequency here, resulting
  819. * in slightly larger intervals.
  820. */
  821. dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
  822. }
  823. static void hifn_init_registers(struct hifn_device *dev)
  824. {
  825. u32 dptr = dev->desc_dma;
  826. /* Initialization magic... */
  827. hifn_write_0(dev, HIFN_0_PUCTRL, HIFN_PUCTRL_DMAENA);
  828. hifn_write_0(dev, HIFN_0_FIFOCNFG, HIFN_FIFOCNFG_THRESHOLD);
  829. hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
  830. /* write all 4 ring address registers */
  831. hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
  832. offsetof(struct hifn_dma, cmdr[0]));
  833. hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
  834. offsetof(struct hifn_dma, srcr[0]));
  835. hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
  836. offsetof(struct hifn_dma, dstr[0]));
  837. hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
  838. offsetof(struct hifn_dma, resr[0]));
  839. mdelay(2);
  840. #if 0
  841. hifn_write_1(dev, HIFN_1_DMA_CSR,
  842. HIFN_DMACSR_D_CTRL_DIS | HIFN_DMACSR_R_CTRL_DIS |
  843. HIFN_DMACSR_S_CTRL_DIS | HIFN_DMACSR_C_CTRL_DIS |
  844. HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
  845. HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
  846. HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
  847. HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
  848. HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
  849. HIFN_DMACSR_S_WAIT |
  850. HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
  851. HIFN_DMACSR_C_WAIT |
  852. HIFN_DMACSR_ENGINE |
  853. HIFN_DMACSR_PUBDONE);
  854. #else
  855. hifn_write_1(dev, HIFN_1_DMA_CSR,
  856. HIFN_DMACSR_C_CTRL_ENA | HIFN_DMACSR_S_CTRL_ENA |
  857. HIFN_DMACSR_D_CTRL_ENA | HIFN_DMACSR_R_CTRL_ENA |
  858. HIFN_DMACSR_D_ABORT | HIFN_DMACSR_D_DONE | HIFN_DMACSR_D_LAST |
  859. HIFN_DMACSR_D_WAIT | HIFN_DMACSR_D_OVER |
  860. HIFN_DMACSR_R_ABORT | HIFN_DMACSR_R_DONE | HIFN_DMACSR_R_LAST |
  861. HIFN_DMACSR_R_WAIT | HIFN_DMACSR_R_OVER |
  862. HIFN_DMACSR_S_ABORT | HIFN_DMACSR_S_DONE | HIFN_DMACSR_S_LAST |
  863. HIFN_DMACSR_S_WAIT |
  864. HIFN_DMACSR_C_ABORT | HIFN_DMACSR_C_DONE | HIFN_DMACSR_C_LAST |
  865. HIFN_DMACSR_C_WAIT |
  866. HIFN_DMACSR_ENGINE |
  867. HIFN_DMACSR_PUBDONE);
  868. #endif
  869. hifn_read_1(dev, HIFN_1_DMA_CSR);
  870. dev->dmareg |= HIFN_DMAIER_R_DONE | HIFN_DMAIER_C_ABORT |
  871. HIFN_DMAIER_D_OVER | HIFN_DMAIER_R_OVER |
  872. HIFN_DMAIER_S_ABORT | HIFN_DMAIER_D_ABORT | HIFN_DMAIER_R_ABORT |
  873. HIFN_DMAIER_ENGINE;
  874. dev->dmareg &= ~HIFN_DMAIER_C_WAIT;
  875. hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
  876. hifn_read_1(dev, HIFN_1_DMA_IER);
  877. #if 0
  878. hifn_write_0(dev, HIFN_0_PUCNFG, HIFN_PUCNFG_ENCCNFG |
  879. HIFN_PUCNFG_DRFR_128 | HIFN_PUCNFG_TCALLPHASES |
  880. HIFN_PUCNFG_TCDRVTOTEM | HIFN_PUCNFG_BUS32 |
  881. HIFN_PUCNFG_DRAM);
  882. #else
  883. hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
  884. #endif
  885. hifn_init_pll(dev);
  886. hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
  887. hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
  888. HIFN_DMACNFG_DMARESET | HIFN_DMACNFG_MODE | HIFN_DMACNFG_LAST |
  889. ((HIFN_POLL_FREQUENCY << 16 ) & HIFN_DMACNFG_POLLFREQ) |
  890. ((HIFN_POLL_SCALAR << 8) & HIFN_DMACNFG_POLLINVAL));
  891. }
  892. static int hifn_setup_base_command(struct hifn_device *dev, u8 *buf,
  893. unsigned dlen, unsigned slen, u16 mask, u8 snum)
  894. {
  895. struct hifn_base_command *base_cmd;
  896. u8 *buf_pos = buf;
  897. base_cmd = (struct hifn_base_command *)buf_pos;
  898. base_cmd->masks = __cpu_to_le16(mask);
  899. base_cmd->total_source_count =
  900. __cpu_to_le16(slen & HIFN_BASE_CMD_LENMASK_LO);
  901. base_cmd->total_dest_count =
  902. __cpu_to_le16(dlen & HIFN_BASE_CMD_LENMASK_LO);
  903. dlen >>= 16;
  904. slen >>= 16;
  905. base_cmd->session_num = __cpu_to_le16(snum |
  906. ((slen << HIFN_BASE_CMD_SRCLEN_S) & HIFN_BASE_CMD_SRCLEN_M) |
  907. ((dlen << HIFN_BASE_CMD_DSTLEN_S) & HIFN_BASE_CMD_DSTLEN_M));
  908. return sizeof(struct hifn_base_command);
  909. }
  910. static int hifn_setup_crypto_command(struct hifn_device *dev,
  911. u8 *buf, unsigned dlen, unsigned slen,
  912. u8 *key, int keylen, u8 *iv, int ivsize, u16 mode)
  913. {
  914. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  915. struct hifn_crypt_command *cry_cmd;
  916. u8 *buf_pos = buf;
  917. u16 cmd_len;
  918. cry_cmd = (struct hifn_crypt_command *)buf_pos;
  919. cry_cmd->source_count = __cpu_to_le16(dlen & 0xffff);
  920. dlen >>= 16;
  921. cry_cmd->masks = __cpu_to_le16(mode |
  922. ((dlen << HIFN_CRYPT_CMD_SRCLEN_S) &
  923. HIFN_CRYPT_CMD_SRCLEN_M));
  924. cry_cmd->header_skip = 0;
  925. cry_cmd->reserved = 0;
  926. buf_pos += sizeof(struct hifn_crypt_command);
  927. dma->cmdu++;
  928. if (dma->cmdu > 1) {
  929. dev->dmareg |= HIFN_DMAIER_C_WAIT;
  930. hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
  931. }
  932. if (keylen) {
  933. memcpy(buf_pos, key, keylen);
  934. buf_pos += keylen;
  935. }
  936. if (ivsize) {
  937. memcpy(buf_pos, iv, ivsize);
  938. buf_pos += ivsize;
  939. }
  940. cmd_len = buf_pos - buf;
  941. return cmd_len;
  942. }
  943. static int hifn_setup_cmd_desc(struct hifn_device *dev,
  944. struct hifn_context *ctx, struct hifn_request_context *rctx,
  945. void *priv, unsigned int nbytes)
  946. {
  947. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  948. int cmd_len, sa_idx;
  949. u8 *buf, *buf_pos;
  950. u16 mask;
  951. sa_idx = dma->cmdi;
  952. buf_pos = buf = dma->command_bufs[dma->cmdi];
  953. mask = 0;
  954. switch (rctx->op) {
  955. case ACRYPTO_OP_DECRYPT:
  956. mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
  957. break;
  958. case ACRYPTO_OP_ENCRYPT:
  959. mask = HIFN_BASE_CMD_CRYPT;
  960. break;
  961. case ACRYPTO_OP_HMAC:
  962. mask = HIFN_BASE_CMD_MAC;
  963. break;
  964. default:
  965. goto err_out;
  966. }
  967. buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
  968. nbytes, mask, dev->snum);
  969. if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
  970. u16 md = 0;
  971. if (ctx->keysize)
  972. md |= HIFN_CRYPT_CMD_NEW_KEY;
  973. if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
  974. md |= HIFN_CRYPT_CMD_NEW_IV;
  975. switch (rctx->mode) {
  976. case ACRYPTO_MODE_ECB:
  977. md |= HIFN_CRYPT_CMD_MODE_ECB;
  978. break;
  979. case ACRYPTO_MODE_CBC:
  980. md |= HIFN_CRYPT_CMD_MODE_CBC;
  981. break;
  982. case ACRYPTO_MODE_CFB:
  983. md |= HIFN_CRYPT_CMD_MODE_CFB;
  984. break;
  985. case ACRYPTO_MODE_OFB:
  986. md |= HIFN_CRYPT_CMD_MODE_OFB;
  987. break;
  988. default:
  989. goto err_out;
  990. }
  991. switch (rctx->type) {
  992. case ACRYPTO_TYPE_AES_128:
  993. if (ctx->keysize != 16)
  994. goto err_out;
  995. md |= HIFN_CRYPT_CMD_KSZ_128 |
  996. HIFN_CRYPT_CMD_ALG_AES;
  997. break;
  998. case ACRYPTO_TYPE_AES_192:
  999. if (ctx->keysize != 24)
  1000. goto err_out;
  1001. md |= HIFN_CRYPT_CMD_KSZ_192 |
  1002. HIFN_CRYPT_CMD_ALG_AES;
  1003. break;
  1004. case ACRYPTO_TYPE_AES_256:
  1005. if (ctx->keysize != 32)
  1006. goto err_out;
  1007. md |= HIFN_CRYPT_CMD_KSZ_256 |
  1008. HIFN_CRYPT_CMD_ALG_AES;
  1009. break;
  1010. case ACRYPTO_TYPE_3DES:
  1011. if (ctx->keysize != 24)
  1012. goto err_out;
  1013. md |= HIFN_CRYPT_CMD_ALG_3DES;
  1014. break;
  1015. case ACRYPTO_TYPE_DES:
  1016. if (ctx->keysize != 8)
  1017. goto err_out;
  1018. md |= HIFN_CRYPT_CMD_ALG_DES;
  1019. break;
  1020. default:
  1021. goto err_out;
  1022. }
  1023. buf_pos += hifn_setup_crypto_command(dev, buf_pos,
  1024. nbytes, nbytes, ctx->key, ctx->keysize,
  1025. rctx->iv, rctx->ivsize, md);
  1026. }
  1027. dev->sa[sa_idx] = priv;
  1028. dev->started++;
  1029. cmd_len = buf_pos - buf;
  1030. dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
  1031. HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
  1032. if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
  1033. dma->cmdr[dma->cmdi].l = __cpu_to_le32(
  1034. HIFN_D_VALID | HIFN_D_LAST |
  1035. HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
  1036. dma->cmdi = 0;
  1037. } else {
  1038. dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID);
  1039. }
  1040. if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) {
  1041. hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
  1042. dev->flags |= HIFN_FLAG_CMD_BUSY;
  1043. }
  1044. return 0;
  1045. err_out:
  1046. return -EINVAL;
  1047. }
  1048. static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
  1049. unsigned int offset, unsigned int size, int last)
  1050. {
  1051. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1052. int idx;
  1053. dma_addr_t addr;
  1054. addr = dma_map_page(&dev->pdev->dev, page, offset, size,
  1055. DMA_TO_DEVICE);
  1056. idx = dma->srci;
  1057. dma->srcr[idx].p = __cpu_to_le32(addr);
  1058. dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
  1059. HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
  1060. if (++idx == HIFN_D_SRC_RSIZE) {
  1061. dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
  1062. HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
  1063. (last ? HIFN_D_LAST : 0));
  1064. idx = 0;
  1065. }
  1066. dma->srci = idx;
  1067. dma->srcu++;
  1068. if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
  1069. hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
  1070. dev->flags |= HIFN_FLAG_SRC_BUSY;
  1071. }
  1072. return size;
  1073. }
  1074. static void hifn_setup_res_desc(struct hifn_device *dev)
  1075. {
  1076. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1077. dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
  1078. HIFN_D_VALID | HIFN_D_LAST);
  1079. /*
  1080. * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
  1081. * HIFN_D_LAST);
  1082. */
  1083. if (++dma->resi == HIFN_D_RES_RSIZE) {
  1084. dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
  1085. HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
  1086. dma->resi = 0;
  1087. }
  1088. dma->resu++;
  1089. if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
  1090. hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
  1091. dev->flags |= HIFN_FLAG_RES_BUSY;
  1092. }
  1093. }
  1094. static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
  1095. unsigned offset, unsigned size, int last)
  1096. {
  1097. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1098. int idx;
  1099. dma_addr_t addr;
  1100. addr = dma_map_page(&dev->pdev->dev, page, offset, size,
  1101. DMA_FROM_DEVICE);
  1102. idx = dma->dsti;
  1103. dma->dstr[idx].p = __cpu_to_le32(addr);
  1104. dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
  1105. HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
  1106. if (++idx == HIFN_D_DST_RSIZE) {
  1107. dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
  1108. HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
  1109. (last ? HIFN_D_LAST : 0));
  1110. idx = 0;
  1111. }
  1112. dma->dsti = idx;
  1113. dma->dstu++;
  1114. if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
  1115. hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
  1116. dev->flags |= HIFN_FLAG_DST_BUSY;
  1117. }
  1118. }
  1119. static int hifn_setup_dma(struct hifn_device *dev,
  1120. struct hifn_context *ctx, struct hifn_request_context *rctx,
  1121. struct scatterlist *src, struct scatterlist *dst,
  1122. unsigned int nbytes, void *priv)
  1123. {
  1124. struct scatterlist *t;
  1125. struct page *spage, *dpage;
  1126. unsigned int soff, doff;
  1127. unsigned int n, len;
  1128. n = nbytes;
  1129. while (n) {
  1130. spage = sg_page(src);
  1131. soff = src->offset;
  1132. len = min(src->length, n);
  1133. hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
  1134. src++;
  1135. n -= len;
  1136. }
  1137. t = &rctx->walk.cache[0];
  1138. n = nbytes;
  1139. while (n) {
  1140. if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
  1141. BUG_ON(!sg_page(t));
  1142. dpage = sg_page(t);
  1143. doff = 0;
  1144. len = t->length;
  1145. } else {
  1146. BUG_ON(!sg_page(dst));
  1147. dpage = sg_page(dst);
  1148. doff = dst->offset;
  1149. len = dst->length;
  1150. }
  1151. len = min(len, n);
  1152. hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
  1153. dst++;
  1154. t++;
  1155. n -= len;
  1156. }
  1157. hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
  1158. hifn_setup_res_desc(dev);
  1159. return 0;
  1160. }
  1161. static int hifn_cipher_walk_init(struct hifn_cipher_walk *w,
  1162. int num, gfp_t gfp_flags)
  1163. {
  1164. int i;
  1165. num = min(ASYNC_SCATTERLIST_CACHE, num);
  1166. sg_init_table(w->cache, num);
  1167. w->num = 0;
  1168. for (i = 0; i < num; ++i) {
  1169. struct page *page = alloc_page(gfp_flags);
  1170. struct scatterlist *s;
  1171. if (!page)
  1172. break;
  1173. s = &w->cache[i];
  1174. sg_set_page(s, page, PAGE_SIZE, 0);
  1175. w->num++;
  1176. }
  1177. return i;
  1178. }
  1179. static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w)
  1180. {
  1181. int i;
  1182. for (i = 0; i < w->num; ++i) {
  1183. struct scatterlist *s = &w->cache[i];
  1184. __free_page(sg_page(s));
  1185. s->length = 0;
  1186. }
  1187. w->num = 0;
  1188. }
  1189. static int skcipher_add(unsigned int *drestp, struct scatterlist *dst,
  1190. unsigned int size, unsigned int *nbytesp)
  1191. {
  1192. unsigned int copy, drest = *drestp, nbytes = *nbytesp;
  1193. int idx = 0;
  1194. if (drest < size || size > nbytes)
  1195. return -EINVAL;
  1196. while (size) {
  1197. copy = min3(drest, size, dst->length);
  1198. size -= copy;
  1199. drest -= copy;
  1200. nbytes -= copy;
  1201. pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
  1202. __func__, copy, size, drest, nbytes);
  1203. dst++;
  1204. idx++;
  1205. }
  1206. *nbytesp = nbytes;
  1207. *drestp = drest;
  1208. return idx;
  1209. }
  1210. static int hifn_cipher_walk(struct skcipher_request *req,
  1211. struct hifn_cipher_walk *w)
  1212. {
  1213. struct scatterlist *dst, *t;
  1214. unsigned int nbytes = req->cryptlen, offset, copy, diff;
  1215. int idx, tidx, err;
  1216. tidx = idx = 0;
  1217. offset = 0;
  1218. while (nbytes) {
  1219. if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
  1220. return -EINVAL;
  1221. dst = &req->dst[idx];
  1222. pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
  1223. __func__, dst->length, dst->offset, offset, nbytes);
  1224. if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
  1225. !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
  1226. offset) {
  1227. unsigned slen = min(dst->length - offset, nbytes);
  1228. unsigned dlen = PAGE_SIZE;
  1229. t = &w->cache[idx];
  1230. err = skcipher_add(&dlen, dst, slen, &nbytes);
  1231. if (err < 0)
  1232. return err;
  1233. idx += err;
  1234. copy = slen & ~(HIFN_D_DST_DALIGN - 1);
  1235. diff = slen & (HIFN_D_DST_DALIGN - 1);
  1236. if (dlen < nbytes) {
  1237. /*
  1238. * Destination page does not have enough space
  1239. * to put there additional blocksized chunk,
  1240. * so we mark that page as containing only
  1241. * blocksize aligned chunks:
  1242. * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
  1243. * and increase number of bytes to be processed
  1244. * in next chunk:
  1245. * nbytes += diff;
  1246. */
  1247. nbytes += diff;
  1248. /*
  1249. * Temporary of course...
  1250. * Kick author if you will catch this one.
  1251. */
  1252. pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n",
  1253. __func__, dlen, nbytes, slen, offset);
  1254. pr_err("%s: please contact author to fix this "
  1255. "issue, generally you should not catch "
  1256. "this path under any condition but who "
  1257. "knows how did you use crypto code.\n"
  1258. "Thank you.\n", __func__);
  1259. BUG();
  1260. } else {
  1261. copy += diff + nbytes;
  1262. dst = &req->dst[idx];
  1263. err = skcipher_add(&dlen, dst, nbytes, &nbytes);
  1264. if (err < 0)
  1265. return err;
  1266. idx += err;
  1267. }
  1268. t->length = copy;
  1269. t->offset = offset;
  1270. } else {
  1271. nbytes -= min(dst->length, nbytes);
  1272. idx++;
  1273. }
  1274. tidx++;
  1275. }
  1276. return tidx;
  1277. }
  1278. static int hifn_setup_session(struct skcipher_request *req)
  1279. {
  1280. struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
  1281. struct hifn_request_context *rctx = skcipher_request_ctx(req);
  1282. struct hifn_device *dev = ctx->dev;
  1283. unsigned long dlen, flags;
  1284. unsigned int nbytes = req->cryptlen, idx = 0;
  1285. int err = -EINVAL, sg_num;
  1286. struct scatterlist *dst;
  1287. if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
  1288. goto err_out_exit;
  1289. rctx->walk.flags = 0;
  1290. while (nbytes) {
  1291. dst = &req->dst[idx];
  1292. dlen = min(dst->length, nbytes);
  1293. if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
  1294. !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
  1295. rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
  1296. nbytes -= dlen;
  1297. idx++;
  1298. }
  1299. if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
  1300. err = hifn_cipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
  1301. if (err < 0)
  1302. return err;
  1303. }
  1304. sg_num = hifn_cipher_walk(req, &rctx->walk);
  1305. if (sg_num < 0) {
  1306. err = sg_num;
  1307. goto err_out_exit;
  1308. }
  1309. spin_lock_irqsave(&dev->lock, flags);
  1310. if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
  1311. err = -EAGAIN;
  1312. goto err_out;
  1313. }
  1314. err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->cryptlen, req);
  1315. if (err)
  1316. goto err_out;
  1317. dev->snum++;
  1318. dev->active = HIFN_DEFAULT_ACTIVE_NUM;
  1319. spin_unlock_irqrestore(&dev->lock, flags);
  1320. return 0;
  1321. err_out:
  1322. spin_unlock_irqrestore(&dev->lock, flags);
  1323. err_out_exit:
  1324. if (err) {
  1325. dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
  1326. "type: %u, err: %d.\n",
  1327. rctx->iv, rctx->ivsize,
  1328. ctx->key, ctx->keysize,
  1329. rctx->mode, rctx->op, rctx->type, err);
  1330. }
  1331. return err;
  1332. }
  1333. static int hifn_start_device(struct hifn_device *dev)
  1334. {
  1335. int err;
  1336. dev->started = dev->active = 0;
  1337. hifn_reset_dma(dev, 1);
  1338. err = hifn_enable_crypto(dev);
  1339. if (err)
  1340. return err;
  1341. hifn_reset_puc(dev);
  1342. hifn_init_dma(dev);
  1343. hifn_init_registers(dev);
  1344. hifn_init_pubrng(dev);
  1345. return 0;
  1346. }
  1347. static int skcipher_get(void *saddr, unsigned int *srestp, unsigned int offset,
  1348. struct scatterlist *dst, unsigned int size, unsigned int *nbytesp)
  1349. {
  1350. unsigned int srest = *srestp, nbytes = *nbytesp, copy;
  1351. void *daddr;
  1352. int idx = 0;
  1353. if (srest < size || size > nbytes)
  1354. return -EINVAL;
  1355. while (size) {
  1356. copy = min3(srest, dst->length, size);
  1357. daddr = kmap_atomic(sg_page(dst));
  1358. memcpy(daddr + dst->offset + offset, saddr, copy);
  1359. kunmap_atomic(daddr);
  1360. nbytes -= copy;
  1361. size -= copy;
  1362. srest -= copy;
  1363. saddr += copy;
  1364. offset = 0;
  1365. pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n",
  1366. __func__, copy, size, srest, nbytes);
  1367. dst++;
  1368. idx++;
  1369. }
  1370. *nbytesp = nbytes;
  1371. *srestp = srest;
  1372. return idx;
  1373. }
  1374. static inline void hifn_complete_sa(struct hifn_device *dev, int i)
  1375. {
  1376. unsigned long flags;
  1377. spin_lock_irqsave(&dev->lock, flags);
  1378. dev->sa[i] = NULL;
  1379. dev->started--;
  1380. if (dev->started < 0)
  1381. dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__,
  1382. dev->started);
  1383. spin_unlock_irqrestore(&dev->lock, flags);
  1384. BUG_ON(dev->started < 0);
  1385. }
  1386. static void hifn_process_ready(struct skcipher_request *req, int error)
  1387. {
  1388. struct hifn_request_context *rctx = skcipher_request_ctx(req);
  1389. if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
  1390. unsigned int nbytes = req->cryptlen;
  1391. int idx = 0, err;
  1392. struct scatterlist *dst, *t;
  1393. void *saddr;
  1394. while (nbytes) {
  1395. t = &rctx->walk.cache[idx];
  1396. dst = &req->dst[idx];
  1397. pr_debug("\n%s: sg_page(t): %p, t->length: %u, "
  1398. "sg_page(dst): %p, dst->length: %u, "
  1399. "nbytes: %u.\n",
  1400. __func__, sg_page(t), t->length,
  1401. sg_page(dst), dst->length, nbytes);
  1402. if (!t->length) {
  1403. nbytes -= min(dst->length, nbytes);
  1404. idx++;
  1405. continue;
  1406. }
  1407. saddr = kmap_atomic(sg_page(t));
  1408. err = skcipher_get(saddr, &t->length, t->offset,
  1409. dst, nbytes, &nbytes);
  1410. if (err < 0) {
  1411. kunmap_atomic(saddr);
  1412. break;
  1413. }
  1414. idx += err;
  1415. kunmap_atomic(saddr);
  1416. }
  1417. hifn_cipher_walk_exit(&rctx->walk);
  1418. }
  1419. req->base.complete(&req->base, error);
  1420. }
  1421. static void hifn_clear_rings(struct hifn_device *dev, int error)
  1422. {
  1423. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1424. int i, u;
  1425. dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
  1426. "k: %d.%d.%d.%d.\n",
  1427. dma->cmdi, dma->srci, dma->dsti, dma->resi,
  1428. dma->cmdu, dma->srcu, dma->dstu, dma->resu,
  1429. dma->cmdk, dma->srck, dma->dstk, dma->resk);
  1430. i = dma->resk; u = dma->resu;
  1431. while (u != 0) {
  1432. if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
  1433. break;
  1434. if (dev->sa[i]) {
  1435. dev->success++;
  1436. dev->reset = 0;
  1437. hifn_process_ready(dev->sa[i], error);
  1438. hifn_complete_sa(dev, i);
  1439. }
  1440. if (++i == HIFN_D_RES_RSIZE)
  1441. i = 0;
  1442. u--;
  1443. }
  1444. dma->resk = i; dma->resu = u;
  1445. i = dma->srck; u = dma->srcu;
  1446. while (u != 0) {
  1447. if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
  1448. break;
  1449. if (++i == HIFN_D_SRC_RSIZE)
  1450. i = 0;
  1451. u--;
  1452. }
  1453. dma->srck = i; dma->srcu = u;
  1454. i = dma->cmdk; u = dma->cmdu;
  1455. while (u != 0) {
  1456. if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
  1457. break;
  1458. if (++i == HIFN_D_CMD_RSIZE)
  1459. i = 0;
  1460. u--;
  1461. }
  1462. dma->cmdk = i; dma->cmdu = u;
  1463. i = dma->dstk; u = dma->dstu;
  1464. while (u != 0) {
  1465. if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
  1466. break;
  1467. if (++i == HIFN_D_DST_RSIZE)
  1468. i = 0;
  1469. u--;
  1470. }
  1471. dma->dstk = i; dma->dstu = u;
  1472. dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, "
  1473. "k: %d.%d.%d.%d.\n",
  1474. dma->cmdi, dma->srci, dma->dsti, dma->resi,
  1475. dma->cmdu, dma->srcu, dma->dstu, dma->resu,
  1476. dma->cmdk, dma->srck, dma->dstk, dma->resk);
  1477. }
  1478. static void hifn_work(struct work_struct *work)
  1479. {
  1480. struct delayed_work *dw = to_delayed_work(work);
  1481. struct hifn_device *dev = container_of(dw, struct hifn_device, work);
  1482. unsigned long flags;
  1483. int reset = 0;
  1484. u32 r = 0;
  1485. spin_lock_irqsave(&dev->lock, flags);
  1486. if (dev->active == 0) {
  1487. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1488. if (dma->cmdu == 0 && (dev->flags & HIFN_FLAG_CMD_BUSY)) {
  1489. dev->flags &= ~HIFN_FLAG_CMD_BUSY;
  1490. r |= HIFN_DMACSR_C_CTRL_DIS;
  1491. }
  1492. if (dma->srcu == 0 && (dev->flags & HIFN_FLAG_SRC_BUSY)) {
  1493. dev->flags &= ~HIFN_FLAG_SRC_BUSY;
  1494. r |= HIFN_DMACSR_S_CTRL_DIS;
  1495. }
  1496. if (dma->dstu == 0 && (dev->flags & HIFN_FLAG_DST_BUSY)) {
  1497. dev->flags &= ~HIFN_FLAG_DST_BUSY;
  1498. r |= HIFN_DMACSR_D_CTRL_DIS;
  1499. }
  1500. if (dma->resu == 0 && (dev->flags & HIFN_FLAG_RES_BUSY)) {
  1501. dev->flags &= ~HIFN_FLAG_RES_BUSY;
  1502. r |= HIFN_DMACSR_R_CTRL_DIS;
  1503. }
  1504. if (r)
  1505. hifn_write_1(dev, HIFN_1_DMA_CSR, r);
  1506. } else
  1507. dev->active--;
  1508. if ((dev->prev_success == dev->success) && dev->started)
  1509. reset = 1;
  1510. dev->prev_success = dev->success;
  1511. spin_unlock_irqrestore(&dev->lock, flags);
  1512. if (reset) {
  1513. if (++dev->reset >= 5) {
  1514. int i;
  1515. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1516. dev_info(&dev->pdev->dev,
  1517. "r: %08x, active: %d, started: %d, "
  1518. "success: %lu: qlen: %u/%u, reset: %d.\n",
  1519. r, dev->active, dev->started,
  1520. dev->success, dev->queue.qlen, dev->queue.max_qlen,
  1521. reset);
  1522. dev_info(&dev->pdev->dev, "%s: res: ", __func__);
  1523. for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
  1524. pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]);
  1525. if (dev->sa[i]) {
  1526. hifn_process_ready(dev->sa[i], -ENODEV);
  1527. hifn_complete_sa(dev, i);
  1528. }
  1529. }
  1530. pr_info("\n");
  1531. hifn_reset_dma(dev, 1);
  1532. hifn_stop_device(dev);
  1533. hifn_start_device(dev);
  1534. dev->reset = 0;
  1535. }
  1536. tasklet_schedule(&dev->tasklet);
  1537. }
  1538. schedule_delayed_work(&dev->work, HZ);
  1539. }
  1540. static irqreturn_t hifn_interrupt(int irq, void *data)
  1541. {
  1542. struct hifn_device *dev = (struct hifn_device *)data;
  1543. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1544. u32 dmacsr, restart;
  1545. dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR);
  1546. dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
  1547. "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
  1548. dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
  1549. dma->cmdi, dma->srci, dma->dsti, dma->resi,
  1550. dma->cmdu, dma->srcu, dma->dstu, dma->resu);
  1551. if ((dmacsr & dev->dmareg) == 0)
  1552. return IRQ_NONE;
  1553. hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & dev->dmareg);
  1554. if (dmacsr & HIFN_DMACSR_ENGINE)
  1555. hifn_write_0(dev, HIFN_0_PUISR, hifn_read_0(dev, HIFN_0_PUISR));
  1556. if (dmacsr & HIFN_DMACSR_PUBDONE)
  1557. hifn_write_1(dev, HIFN_1_PUB_STATUS,
  1558. hifn_read_1(dev, HIFN_1_PUB_STATUS) | HIFN_PUBSTS_DONE);
  1559. restart = dmacsr & (HIFN_DMACSR_R_OVER | HIFN_DMACSR_D_OVER);
  1560. if (restart) {
  1561. u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
  1562. dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
  1563. !!(dmacsr & HIFN_DMACSR_R_OVER),
  1564. !!(dmacsr & HIFN_DMACSR_D_OVER),
  1565. puisr, !!(puisr & HIFN_PUISR_DSTOVER));
  1566. if (!!(puisr & HIFN_PUISR_DSTOVER))
  1567. hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
  1568. hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
  1569. HIFN_DMACSR_D_OVER));
  1570. }
  1571. restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
  1572. HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
  1573. if (restart) {
  1574. dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n",
  1575. !!(dmacsr & HIFN_DMACSR_C_ABORT),
  1576. !!(dmacsr & HIFN_DMACSR_S_ABORT),
  1577. !!(dmacsr & HIFN_DMACSR_D_ABORT),
  1578. !!(dmacsr & HIFN_DMACSR_R_ABORT));
  1579. hifn_reset_dma(dev, 1);
  1580. hifn_init_dma(dev);
  1581. hifn_init_registers(dev);
  1582. }
  1583. if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) {
  1584. dev_dbg(&dev->pdev->dev, "wait on command.\n");
  1585. dev->dmareg &= ~(HIFN_DMAIER_C_WAIT);
  1586. hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
  1587. }
  1588. tasklet_schedule(&dev->tasklet);
  1589. return IRQ_HANDLED;
  1590. }
  1591. static void hifn_flush(struct hifn_device *dev)
  1592. {
  1593. unsigned long flags;
  1594. struct crypto_async_request *async_req;
  1595. struct skcipher_request *req;
  1596. struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
  1597. int i;
  1598. for (i = 0; i < HIFN_D_RES_RSIZE; ++i) {
  1599. struct hifn_desc *d = &dma->resr[i];
  1600. if (dev->sa[i]) {
  1601. hifn_process_ready(dev->sa[i],
  1602. (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0);
  1603. hifn_complete_sa(dev, i);
  1604. }
  1605. }
  1606. spin_lock_irqsave(&dev->lock, flags);
  1607. while ((async_req = crypto_dequeue_request(&dev->queue))) {
  1608. req = skcipher_request_cast(async_req);
  1609. spin_unlock_irqrestore(&dev->lock, flags);
  1610. hifn_process_ready(req, -ENODEV);
  1611. spin_lock_irqsave(&dev->lock, flags);
  1612. }
  1613. spin_unlock_irqrestore(&dev->lock, flags);
  1614. }
  1615. static int hifn_setkey(struct crypto_skcipher *cipher, const u8 *key,
  1616. unsigned int len)
  1617. {
  1618. struct hifn_context *ctx = crypto_skcipher_ctx(cipher);
  1619. struct hifn_device *dev = ctx->dev;
  1620. int err;
  1621. err = verify_skcipher_des_key(cipher, key);
  1622. if (err)
  1623. return err;
  1624. dev->flags &= ~HIFN_FLAG_OLD_KEY;
  1625. memcpy(ctx->key, key, len);
  1626. ctx->keysize = len;
  1627. return 0;
  1628. }
  1629. static int hifn_des3_setkey(struct crypto_skcipher *cipher, const u8 *key,
  1630. unsigned int len)
  1631. {
  1632. struct hifn_context *ctx = crypto_skcipher_ctx(cipher);
  1633. struct hifn_device *dev = ctx->dev;
  1634. int err;
  1635. err = verify_skcipher_des3_key(cipher, key);
  1636. if (err)
  1637. return err;
  1638. dev->flags &= ~HIFN_FLAG_OLD_KEY;
  1639. memcpy(ctx->key, key, len);
  1640. ctx->keysize = len;
  1641. return 0;
  1642. }
  1643. static int hifn_handle_req(struct skcipher_request *req)
  1644. {
  1645. struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
  1646. struct hifn_device *dev = ctx->dev;
  1647. int err = -EAGAIN;
  1648. if (dev->started + DIV_ROUND_UP(req->cryptlen, PAGE_SIZE) <= HIFN_QUEUE_LENGTH)
  1649. err = hifn_setup_session(req);
  1650. if (err == -EAGAIN) {
  1651. unsigned long flags;
  1652. spin_lock_irqsave(&dev->lock, flags);
  1653. err = crypto_enqueue_request(&dev->queue, &req->base);
  1654. spin_unlock_irqrestore(&dev->lock, flags);
  1655. }
  1656. return err;
  1657. }
  1658. static int hifn_setup_crypto_req(struct skcipher_request *req, u8 op,
  1659. u8 type, u8 mode)
  1660. {
  1661. struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
  1662. struct hifn_request_context *rctx = skcipher_request_ctx(req);
  1663. unsigned ivsize;
  1664. ivsize = crypto_skcipher_ivsize(crypto_skcipher_reqtfm(req));
  1665. if (req->iv && mode != ACRYPTO_MODE_ECB) {
  1666. if (type == ACRYPTO_TYPE_AES_128)
  1667. ivsize = HIFN_AES_IV_LENGTH;
  1668. else if (type == ACRYPTO_TYPE_DES)
  1669. ivsize = HIFN_DES_KEY_LENGTH;
  1670. else if (type == ACRYPTO_TYPE_3DES)
  1671. ivsize = HIFN_3DES_KEY_LENGTH;
  1672. }
  1673. if (ctx->keysize != 16 && type == ACRYPTO_TYPE_AES_128) {
  1674. if (ctx->keysize == 24)
  1675. type = ACRYPTO_TYPE_AES_192;
  1676. else if (ctx->keysize == 32)
  1677. type = ACRYPTO_TYPE_AES_256;
  1678. }
  1679. rctx->op = op;
  1680. rctx->mode = mode;
  1681. rctx->type = type;
  1682. rctx->iv = req->iv;
  1683. rctx->ivsize = ivsize;
  1684. /*
  1685. * HEAVY TODO: needs to kick Herbert XU to write documentation.
  1686. * HEAVY TODO: needs to kick Herbert XU to write documentation.
  1687. * HEAVY TODO: needs to kick Herbert XU to write documentation.
  1688. */
  1689. return hifn_handle_req(req);
  1690. }
  1691. static int hifn_process_queue(struct hifn_device *dev)
  1692. {
  1693. struct crypto_async_request *async_req, *backlog;
  1694. struct skcipher_request *req;
  1695. unsigned long flags;
  1696. int err = 0;
  1697. while (dev->started < HIFN_QUEUE_LENGTH) {
  1698. spin_lock_irqsave(&dev->lock, flags);
  1699. backlog = crypto_get_backlog(&dev->queue);
  1700. async_req = crypto_dequeue_request(&dev->queue);
  1701. spin_unlock_irqrestore(&dev->lock, flags);
  1702. if (!async_req)
  1703. break;
  1704. if (backlog)
  1705. backlog->complete(backlog, -EINPROGRESS);
  1706. req = skcipher_request_cast(async_req);
  1707. err = hifn_handle_req(req);
  1708. if (err)
  1709. break;
  1710. }
  1711. return err;
  1712. }
  1713. static int hifn_setup_crypto(struct skcipher_request *req, u8 op,
  1714. u8 type, u8 mode)
  1715. {
  1716. int err;
  1717. struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
  1718. struct hifn_device *dev = ctx->dev;
  1719. err = hifn_setup_crypto_req(req, op, type, mode);
  1720. if (err)
  1721. return err;
  1722. if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
  1723. hifn_process_queue(dev);
  1724. return -EINPROGRESS;
  1725. }
  1726. /*
  1727. * AES ecryption functions.
  1728. */
  1729. static inline int hifn_encrypt_aes_ecb(struct skcipher_request *req)
  1730. {
  1731. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1732. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
  1733. }
  1734. static inline int hifn_encrypt_aes_cbc(struct skcipher_request *req)
  1735. {
  1736. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1737. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
  1738. }
  1739. static inline int hifn_encrypt_aes_cfb(struct skcipher_request *req)
  1740. {
  1741. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1742. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
  1743. }
  1744. static inline int hifn_encrypt_aes_ofb(struct skcipher_request *req)
  1745. {
  1746. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1747. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
  1748. }
  1749. /*
  1750. * AES decryption functions.
  1751. */
  1752. static inline int hifn_decrypt_aes_ecb(struct skcipher_request *req)
  1753. {
  1754. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1755. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_ECB);
  1756. }
  1757. static inline int hifn_decrypt_aes_cbc(struct skcipher_request *req)
  1758. {
  1759. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1760. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CBC);
  1761. }
  1762. static inline int hifn_decrypt_aes_cfb(struct skcipher_request *req)
  1763. {
  1764. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1765. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_CFB);
  1766. }
  1767. static inline int hifn_decrypt_aes_ofb(struct skcipher_request *req)
  1768. {
  1769. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1770. ACRYPTO_TYPE_AES_128, ACRYPTO_MODE_OFB);
  1771. }
  1772. /*
  1773. * DES ecryption functions.
  1774. */
  1775. static inline int hifn_encrypt_des_ecb(struct skcipher_request *req)
  1776. {
  1777. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1778. ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
  1779. }
  1780. static inline int hifn_encrypt_des_cbc(struct skcipher_request *req)
  1781. {
  1782. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1783. ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
  1784. }
  1785. static inline int hifn_encrypt_des_cfb(struct skcipher_request *req)
  1786. {
  1787. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1788. ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
  1789. }
  1790. static inline int hifn_encrypt_des_ofb(struct skcipher_request *req)
  1791. {
  1792. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1793. ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
  1794. }
  1795. /*
  1796. * DES decryption functions.
  1797. */
  1798. static inline int hifn_decrypt_des_ecb(struct skcipher_request *req)
  1799. {
  1800. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1801. ACRYPTO_TYPE_DES, ACRYPTO_MODE_ECB);
  1802. }
  1803. static inline int hifn_decrypt_des_cbc(struct skcipher_request *req)
  1804. {
  1805. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1806. ACRYPTO_TYPE_DES, ACRYPTO_MODE_CBC);
  1807. }
  1808. static inline int hifn_decrypt_des_cfb(struct skcipher_request *req)
  1809. {
  1810. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1811. ACRYPTO_TYPE_DES, ACRYPTO_MODE_CFB);
  1812. }
  1813. static inline int hifn_decrypt_des_ofb(struct skcipher_request *req)
  1814. {
  1815. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1816. ACRYPTO_TYPE_DES, ACRYPTO_MODE_OFB);
  1817. }
  1818. /*
  1819. * 3DES ecryption functions.
  1820. */
  1821. static inline int hifn_encrypt_3des_ecb(struct skcipher_request *req)
  1822. {
  1823. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1824. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
  1825. }
  1826. static inline int hifn_encrypt_3des_cbc(struct skcipher_request *req)
  1827. {
  1828. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1829. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
  1830. }
  1831. static inline int hifn_encrypt_3des_cfb(struct skcipher_request *req)
  1832. {
  1833. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1834. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
  1835. }
  1836. static inline int hifn_encrypt_3des_ofb(struct skcipher_request *req)
  1837. {
  1838. return hifn_setup_crypto(req, ACRYPTO_OP_ENCRYPT,
  1839. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
  1840. }
  1841. /* 3DES decryption functions. */
  1842. static inline int hifn_decrypt_3des_ecb(struct skcipher_request *req)
  1843. {
  1844. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1845. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_ECB);
  1846. }
  1847. static inline int hifn_decrypt_3des_cbc(struct skcipher_request *req)
  1848. {
  1849. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1850. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CBC);
  1851. }
  1852. static inline int hifn_decrypt_3des_cfb(struct skcipher_request *req)
  1853. {
  1854. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1855. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_CFB);
  1856. }
  1857. static inline int hifn_decrypt_3des_ofb(struct skcipher_request *req)
  1858. {
  1859. return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT,
  1860. ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB);
  1861. }
  1862. struct hifn_alg_template {
  1863. char name[CRYPTO_MAX_ALG_NAME];
  1864. char drv_name[CRYPTO_MAX_ALG_NAME];
  1865. unsigned int bsize;
  1866. struct skcipher_alg skcipher;
  1867. };
  1868. static const struct hifn_alg_template hifn_alg_templates[] = {
  1869. /*
  1870. * 3DES ECB, CBC, CFB and OFB modes.
  1871. */
  1872. {
  1873. .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
  1874. .skcipher = {
  1875. .min_keysize = HIFN_3DES_KEY_LENGTH,
  1876. .max_keysize = HIFN_3DES_KEY_LENGTH,
  1877. .setkey = hifn_des3_setkey,
  1878. .encrypt = hifn_encrypt_3des_cfb,
  1879. .decrypt = hifn_decrypt_3des_cfb,
  1880. },
  1881. },
  1882. {
  1883. .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
  1884. .skcipher = {
  1885. .min_keysize = HIFN_3DES_KEY_LENGTH,
  1886. .max_keysize = HIFN_3DES_KEY_LENGTH,
  1887. .setkey = hifn_des3_setkey,
  1888. .encrypt = hifn_encrypt_3des_ofb,
  1889. .decrypt = hifn_decrypt_3des_ofb,
  1890. },
  1891. },
  1892. {
  1893. .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
  1894. .skcipher = {
  1895. .ivsize = HIFN_IV_LENGTH,
  1896. .min_keysize = HIFN_3DES_KEY_LENGTH,
  1897. .max_keysize = HIFN_3DES_KEY_LENGTH,
  1898. .setkey = hifn_des3_setkey,
  1899. .encrypt = hifn_encrypt_3des_cbc,
  1900. .decrypt = hifn_decrypt_3des_cbc,
  1901. },
  1902. },
  1903. {
  1904. .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
  1905. .skcipher = {
  1906. .min_keysize = HIFN_3DES_KEY_LENGTH,
  1907. .max_keysize = HIFN_3DES_KEY_LENGTH,
  1908. .setkey = hifn_des3_setkey,
  1909. .encrypt = hifn_encrypt_3des_ecb,
  1910. .decrypt = hifn_decrypt_3des_ecb,
  1911. },
  1912. },
  1913. /*
  1914. * DES ECB, CBC, CFB and OFB modes.
  1915. */
  1916. {
  1917. .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
  1918. .skcipher = {
  1919. .min_keysize = HIFN_DES_KEY_LENGTH,
  1920. .max_keysize = HIFN_DES_KEY_LENGTH,
  1921. .setkey = hifn_setkey,
  1922. .encrypt = hifn_encrypt_des_cfb,
  1923. .decrypt = hifn_decrypt_des_cfb,
  1924. },
  1925. },
  1926. {
  1927. .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
  1928. .skcipher = {
  1929. .min_keysize = HIFN_DES_KEY_LENGTH,
  1930. .max_keysize = HIFN_DES_KEY_LENGTH,
  1931. .setkey = hifn_setkey,
  1932. .encrypt = hifn_encrypt_des_ofb,
  1933. .decrypt = hifn_decrypt_des_ofb,
  1934. },
  1935. },
  1936. {
  1937. .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
  1938. .skcipher = {
  1939. .ivsize = HIFN_IV_LENGTH,
  1940. .min_keysize = HIFN_DES_KEY_LENGTH,
  1941. .max_keysize = HIFN_DES_KEY_LENGTH,
  1942. .setkey = hifn_setkey,
  1943. .encrypt = hifn_encrypt_des_cbc,
  1944. .decrypt = hifn_decrypt_des_cbc,
  1945. },
  1946. },
  1947. {
  1948. .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
  1949. .skcipher = {
  1950. .min_keysize = HIFN_DES_KEY_LENGTH,
  1951. .max_keysize = HIFN_DES_KEY_LENGTH,
  1952. .setkey = hifn_setkey,
  1953. .encrypt = hifn_encrypt_des_ecb,
  1954. .decrypt = hifn_decrypt_des_ecb,
  1955. },
  1956. },
  1957. /*
  1958. * AES ECB, CBC, CFB and OFB modes.
  1959. */
  1960. {
  1961. .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
  1962. .skcipher = {
  1963. .min_keysize = AES_MIN_KEY_SIZE,
  1964. .max_keysize = AES_MAX_KEY_SIZE,
  1965. .setkey = hifn_setkey,
  1966. .encrypt = hifn_encrypt_aes_ecb,
  1967. .decrypt = hifn_decrypt_aes_ecb,
  1968. },
  1969. },
  1970. {
  1971. .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
  1972. .skcipher = {
  1973. .ivsize = HIFN_AES_IV_LENGTH,
  1974. .min_keysize = AES_MIN_KEY_SIZE,
  1975. .max_keysize = AES_MAX_KEY_SIZE,
  1976. .setkey = hifn_setkey,
  1977. .encrypt = hifn_encrypt_aes_cbc,
  1978. .decrypt = hifn_decrypt_aes_cbc,
  1979. },
  1980. },
  1981. {
  1982. .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
  1983. .skcipher = {
  1984. .min_keysize = AES_MIN_KEY_SIZE,
  1985. .max_keysize = AES_MAX_KEY_SIZE,
  1986. .setkey = hifn_setkey,
  1987. .encrypt = hifn_encrypt_aes_cfb,
  1988. .decrypt = hifn_decrypt_aes_cfb,
  1989. },
  1990. },
  1991. {
  1992. .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
  1993. .skcipher = {
  1994. .min_keysize = AES_MIN_KEY_SIZE,
  1995. .max_keysize = AES_MAX_KEY_SIZE,
  1996. .setkey = hifn_setkey,
  1997. .encrypt = hifn_encrypt_aes_ofb,
  1998. .decrypt = hifn_decrypt_aes_ofb,
  1999. },
  2000. },
  2001. };
  2002. static int hifn_init_tfm(struct crypto_skcipher *tfm)
  2003. {
  2004. struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
  2005. struct hifn_crypto_alg *ha = crypto_alg_to_hifn(alg);
  2006. struct hifn_context *ctx = crypto_skcipher_ctx(tfm);
  2007. ctx->dev = ha->dev;
  2008. crypto_skcipher_set_reqsize(tfm, sizeof(struct hifn_request_context));
  2009. return 0;
  2010. }
  2011. static int hifn_alg_alloc(struct hifn_device *dev, const struct hifn_alg_template *t)
  2012. {
  2013. struct hifn_crypto_alg *alg;
  2014. int err;
  2015. alg = kzalloc(sizeof(*alg), GFP_KERNEL);
  2016. if (!alg)
  2017. return -ENOMEM;
  2018. alg->alg = t->skcipher;
  2019. alg->alg.init = hifn_init_tfm;
  2020. snprintf(alg->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
  2021. snprintf(alg->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
  2022. t->drv_name, dev->name);
  2023. alg->alg.base.cra_priority = 300;
  2024. alg->alg.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC;
  2025. alg->alg.base.cra_blocksize = t->bsize;
  2026. alg->alg.base.cra_ctxsize = sizeof(struct hifn_context);
  2027. alg->alg.base.cra_alignmask = 0;
  2028. alg->alg.base.cra_module = THIS_MODULE;
  2029. alg->dev = dev;
  2030. list_add_tail(&alg->entry, &dev->alg_list);
  2031. err = crypto_register_skcipher(&alg->alg);
  2032. if (err) {
  2033. list_del(&alg->entry);
  2034. kfree(alg);
  2035. }
  2036. return err;
  2037. }
  2038. static void hifn_unregister_alg(struct hifn_device *dev)
  2039. {
  2040. struct hifn_crypto_alg *a, *n;
  2041. list_for_each_entry_safe(a, n, &dev->alg_list, entry) {
  2042. list_del(&a->entry);
  2043. crypto_unregister_skcipher(&a->alg);
  2044. kfree(a);
  2045. }
  2046. }
  2047. static int hifn_register_alg(struct hifn_device *dev)
  2048. {
  2049. int i, err;
  2050. for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) {
  2051. err = hifn_alg_alloc(dev, &hifn_alg_templates[i]);
  2052. if (err)
  2053. goto err_out_exit;
  2054. }
  2055. return 0;
  2056. err_out_exit:
  2057. hifn_unregister_alg(dev);
  2058. return err;
  2059. }
  2060. static void hifn_tasklet_callback(unsigned long data)
  2061. {
  2062. struct hifn_device *dev = (struct hifn_device *)data;
  2063. /*
  2064. * This is ok to call this without lock being held,
  2065. * althogh it modifies some parameters used in parallel,
  2066. * (like dev->success), but they are used in process
  2067. * context or update is atomic (like setting dev->sa[i] to NULL).
  2068. */
  2069. hifn_clear_rings(dev, 0);
  2070. if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
  2071. hifn_process_queue(dev);
  2072. }
  2073. static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
  2074. {
  2075. int err, i;
  2076. struct hifn_device *dev;
  2077. char name[8];
  2078. err = pci_enable_device(pdev);
  2079. if (err)
  2080. return err;
  2081. pci_set_master(pdev);
  2082. err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
  2083. if (err)
  2084. goto err_out_disable_pci_device;
  2085. snprintf(name, sizeof(name), "hifn%d",
  2086. atomic_inc_return(&hifn_dev_number) - 1);
  2087. err = pci_request_regions(pdev, name);
  2088. if (err)
  2089. goto err_out_disable_pci_device;
  2090. if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE ||
  2091. pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE ||
  2092. pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) {
  2093. dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n");
  2094. err = -ENODEV;
  2095. goto err_out_free_regions;
  2096. }
  2097. dev = kzalloc(sizeof(struct hifn_device) + sizeof(struct crypto_alg),
  2098. GFP_KERNEL);
  2099. if (!dev) {
  2100. err = -ENOMEM;
  2101. goto err_out_free_regions;
  2102. }
  2103. INIT_LIST_HEAD(&dev->alg_list);
  2104. snprintf(dev->name, sizeof(dev->name), "%s", name);
  2105. spin_lock_init(&dev->lock);
  2106. for (i = 0; i < 3; ++i) {
  2107. unsigned long addr, size;
  2108. addr = pci_resource_start(pdev, i);
  2109. size = pci_resource_len(pdev, i);
  2110. dev->bar[i] = ioremap(addr, size);
  2111. if (!dev->bar[i]) {
  2112. err = -ENOMEM;
  2113. goto err_out_unmap_bars;
  2114. }
  2115. }
  2116. dev->desc_virt = dma_alloc_coherent(&pdev->dev,
  2117. sizeof(struct hifn_dma),
  2118. &dev->desc_dma, GFP_KERNEL);
  2119. if (!dev->desc_virt) {
  2120. dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n");
  2121. err = -ENOMEM;
  2122. goto err_out_unmap_bars;
  2123. }
  2124. dev->pdev = pdev;
  2125. dev->irq = pdev->irq;
  2126. for (i = 0; i < HIFN_D_RES_RSIZE; ++i)
  2127. dev->sa[i] = NULL;
  2128. pci_set_drvdata(pdev, dev);
  2129. tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
  2130. crypto_init_queue(&dev->queue, 1);
  2131. err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
  2132. if (err) {
  2133. dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n",
  2134. dev->irq, err);
  2135. dev->irq = 0;
  2136. goto err_out_free_desc;
  2137. }
  2138. err = hifn_start_device(dev);
  2139. if (err)
  2140. goto err_out_free_irq;
  2141. err = hifn_register_rng(dev);
  2142. if (err)
  2143. goto err_out_stop_device;
  2144. err = hifn_register_alg(dev);
  2145. if (err)
  2146. goto err_out_unregister_rng;
  2147. INIT_DELAYED_WORK(&dev->work, hifn_work);
  2148. schedule_delayed_work(&dev->work, HZ);
  2149. dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been "
  2150. "successfully registered as %s.\n",
  2151. pci_name(pdev), dev->name);
  2152. return 0;
  2153. err_out_unregister_rng:
  2154. hifn_unregister_rng(dev);
  2155. err_out_stop_device:
  2156. hifn_reset_dma(dev, 1);
  2157. hifn_stop_device(dev);
  2158. err_out_free_irq:
  2159. free_irq(dev->irq, dev);
  2160. tasklet_kill(&dev->tasklet);
  2161. err_out_free_desc:
  2162. dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma), dev->desc_virt,
  2163. dev->desc_dma);
  2164. err_out_unmap_bars:
  2165. for (i = 0; i < 3; ++i)
  2166. if (dev->bar[i])
  2167. iounmap(dev->bar[i]);
  2168. kfree(dev);
  2169. err_out_free_regions:
  2170. pci_release_regions(pdev);
  2171. err_out_disable_pci_device:
  2172. pci_disable_device(pdev);
  2173. return err;
  2174. }
  2175. static void hifn_remove(struct pci_dev *pdev)
  2176. {
  2177. int i;
  2178. struct hifn_device *dev;
  2179. dev = pci_get_drvdata(pdev);
  2180. if (dev) {
  2181. cancel_delayed_work_sync(&dev->work);
  2182. hifn_unregister_rng(dev);
  2183. hifn_unregister_alg(dev);
  2184. hifn_reset_dma(dev, 1);
  2185. hifn_stop_device(dev);
  2186. free_irq(dev->irq, dev);
  2187. tasklet_kill(&dev->tasklet);
  2188. hifn_flush(dev);
  2189. dma_free_coherent(&pdev->dev, sizeof(struct hifn_dma),
  2190. dev->desc_virt, dev->desc_dma);
  2191. for (i = 0; i < 3; ++i)
  2192. if (dev->bar[i])
  2193. iounmap(dev->bar[i]);
  2194. kfree(dev);
  2195. }
  2196. pci_release_regions(pdev);
  2197. pci_disable_device(pdev);
  2198. }
  2199. static struct pci_device_id hifn_pci_tbl[] = {
  2200. { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7955) },
  2201. { PCI_DEVICE(PCI_VENDOR_ID_HIFN, PCI_DEVICE_ID_HIFN_7956) },
  2202. { 0 }
  2203. };
  2204. MODULE_DEVICE_TABLE(pci, hifn_pci_tbl);
  2205. static struct pci_driver hifn_pci_driver = {
  2206. .name = "hifn795x",
  2207. .id_table = hifn_pci_tbl,
  2208. .probe = hifn_probe,
  2209. .remove = hifn_remove,
  2210. };
  2211. static int __init hifn_init(void)
  2212. {
  2213. unsigned int freq;
  2214. int err;
  2215. if (strncmp(hifn_pll_ref, "ext", 3) &&
  2216. strncmp(hifn_pll_ref, "pci", 3)) {
  2217. pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext");
  2218. return -EINVAL;
  2219. }
  2220. /*
  2221. * For the 7955/7956 the reference clock frequency must be in the
  2222. * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
  2223. * but this chip is currently not supported.
  2224. */
  2225. if (hifn_pll_ref[3] != '\0') {
  2226. freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
  2227. if (freq < 20 || freq > 100) {
  2228. pr_err("hifn795x: invalid hifn_pll_ref frequency, must"
  2229. "be in the range of 20-100");
  2230. return -EINVAL;
  2231. }
  2232. }
  2233. err = pci_register_driver(&hifn_pci_driver);
  2234. if (err < 0) {
  2235. pr_err("Failed to register PCI driver for %s device.\n",
  2236. hifn_pci_driver.name);
  2237. return -ENODEV;
  2238. }
  2239. pr_info("Driver for HIFN 795x crypto accelerator chip "
  2240. "has been successfully registered.\n");
  2241. return 0;
  2242. }
  2243. static void __exit hifn_fini(void)
  2244. {
  2245. pci_unregister_driver(&hifn_pci_driver);
  2246. pr_info("Driver for HIFN 795x crypto accelerator chip "
  2247. "has been successfully unregistered.\n");
  2248. }
  2249. module_init(hifn_init);
  2250. module_exit(hifn_fini);
  2251. MODULE_LICENSE("GPL");
  2252. MODULE_AUTHOR("Evgeniy Polyakov <[email protected]>");
  2253. MODULE_DESCRIPTION("Driver for HIFN 795x crypto accelerator chip.");