sa2ul.c 67 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * K3 SA2UL crypto accelerator driver
  4. *
  5. * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
  6. *
  7. * Authors: Keerthy
  8. * Vitaly Andrianov
  9. * Tero Kristo
  10. */
  11. #include <linux/bitfield.h>
  12. #include <linux/clk.h>
  13. #include <linux/dma-mapping.h>
  14. #include <linux/dmaengine.h>
  15. #include <linux/dmapool.h>
  16. #include <linux/kernel.h>
  17. #include <linux/module.h>
  18. #include <linux/of_device.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/pm_runtime.h>
  21. #include <crypto/aes.h>
  22. #include <crypto/authenc.h>
  23. #include <crypto/des.h>
  24. #include <crypto/internal/aead.h>
  25. #include <crypto/internal/hash.h>
  26. #include <crypto/internal/skcipher.h>
  27. #include <crypto/scatterwalk.h>
  28. #include <crypto/sha1.h>
  29. #include <crypto/sha2.h>
  30. #include "sa2ul.h"
  31. /* Byte offset for key in encryption security context */
  32. #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
  33. /* Byte offset for Aux-1 in encryption security context */
  34. #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
  35. #define SA_CMDL_UPD_ENC 0x0001
  36. #define SA_CMDL_UPD_AUTH 0x0002
  37. #define SA_CMDL_UPD_ENC_IV 0x0004
  38. #define SA_CMDL_UPD_AUTH_IV 0x0008
  39. #define SA_CMDL_UPD_AUX_KEY 0x0010
  40. #define SA_AUTH_SUBKEY_LEN 16
  41. #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF
  42. #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000
  43. #define MODE_CONTROL_BYTES 27
  44. #define SA_HASH_PROCESSING 0
  45. #define SA_CRYPTO_PROCESSING 0
  46. #define SA_UPLOAD_HASH_TO_TLR BIT(6)
  47. #define SA_SW0_FLAGS_MASK 0xF0000
  48. #define SA_SW0_CMDL_INFO_MASK 0x1F00000
  49. #define SA_SW0_CMDL_PRESENT BIT(4)
  50. #define SA_SW0_ENG_ID_MASK 0x3E000000
  51. #define SA_SW0_DEST_INFO_PRESENT BIT(30)
  52. #define SA_SW2_EGRESS_LENGTH 0xFF000000
  53. #define SA_BASIC_HASH 0x10
  54. #define SHA256_DIGEST_WORDS 8
  55. /* Make 32-bit word from 4 bytes */
  56. #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
  57. ((b2) << 8) | (b3))
  58. /* size of SCCTL structure in bytes */
  59. #define SA_SCCTL_SZ 16
  60. /* Max Authentication tag size */
  61. #define SA_MAX_AUTH_TAG_SZ 64
  62. enum sa_algo_id {
  63. SA_ALG_CBC_AES = 0,
  64. SA_ALG_EBC_AES,
  65. SA_ALG_CBC_DES3,
  66. SA_ALG_ECB_DES3,
  67. SA_ALG_SHA1,
  68. SA_ALG_SHA256,
  69. SA_ALG_SHA512,
  70. SA_ALG_AUTHENC_SHA1_AES,
  71. SA_ALG_AUTHENC_SHA256_AES,
  72. };
  73. struct sa_match_data {
  74. u8 priv;
  75. u8 priv_id;
  76. u32 supported_algos;
  77. };
  78. static struct device *sa_k3_dev;
  79. /**
  80. * struct sa_cmdl_cfg - Command label configuration descriptor
  81. * @aalg: authentication algorithm ID
  82. * @enc_eng_id: Encryption Engine ID supported by the SA hardware
  83. * @auth_eng_id: Authentication Engine ID
  84. * @iv_size: Initialization Vector size
  85. * @akey: Authentication key
  86. * @akey_len: Authentication key length
  87. * @enc: True, if this is an encode request
  88. */
  89. struct sa_cmdl_cfg {
  90. int aalg;
  91. u8 enc_eng_id;
  92. u8 auth_eng_id;
  93. u8 iv_size;
  94. const u8 *akey;
  95. u16 akey_len;
  96. bool enc;
  97. };
  98. /**
  99. * struct algo_data - Crypto algorithm specific data
  100. * @enc_eng: Encryption engine info structure
  101. * @auth_eng: Authentication engine info structure
  102. * @auth_ctrl: Authentication control word
  103. * @hash_size: Size of digest
  104. * @iv_idx: iv index in psdata
  105. * @iv_out_size: iv out size
  106. * @ealg_id: Encryption Algorithm ID
  107. * @aalg_id: Authentication algorithm ID
  108. * @mci_enc: Mode Control Instruction for Encryption algorithm
  109. * @mci_dec: Mode Control Instruction for Decryption
  110. * @inv_key: Whether the encryption algorithm demands key inversion
  111. * @ctx: Pointer to the algorithm context
  112. * @keyed_mac: Whether the authentication algorithm has key
  113. * @prep_iopad: Function pointer to generate intermediate ipad/opad
  114. */
  115. struct algo_data {
  116. struct sa_eng_info enc_eng;
  117. struct sa_eng_info auth_eng;
  118. u8 auth_ctrl;
  119. u8 hash_size;
  120. u8 iv_idx;
  121. u8 iv_out_size;
  122. u8 ealg_id;
  123. u8 aalg_id;
  124. u8 *mci_enc;
  125. u8 *mci_dec;
  126. bool inv_key;
  127. struct sa_tfm_ctx *ctx;
  128. bool keyed_mac;
  129. void (*prep_iopad)(struct algo_data *algo, const u8 *key,
  130. u16 key_sz, __be32 *ipad, __be32 *opad);
  131. };
  132. /**
  133. * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
  134. * @type: Type of the crypto algorithm.
  135. * @alg: Union of crypto algorithm definitions.
  136. * @registered: Flag indicating if the crypto algorithm is already registered
  137. */
  138. struct sa_alg_tmpl {
  139. u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
  140. union {
  141. struct skcipher_alg skcipher;
  142. struct ahash_alg ahash;
  143. struct aead_alg aead;
  144. } alg;
  145. bool registered;
  146. };
  147. /**
  148. * struct sa_mapped_sg: scatterlist information for tx and rx
  149. * @mapped: Set to true if the @sgt is mapped
  150. * @dir: mapping direction used for @sgt
  151. * @split_sg: Set if the sg is split and needs to be freed up
  152. * @static_sg: Static scatterlist entry for overriding data
  153. * @sgt: scatterlist table for DMA API use
  154. */
  155. struct sa_mapped_sg {
  156. bool mapped;
  157. enum dma_data_direction dir;
  158. struct scatterlist static_sg;
  159. struct scatterlist *split_sg;
  160. struct sg_table sgt;
  161. };
  162. /**
  163. * struct sa_rx_data: RX Packet miscellaneous data place holder
  164. * @req: crypto request data pointer
  165. * @ddev: pointer to the DMA device
  166. * @tx_in: dma_async_tx_descriptor pointer for rx channel
  167. * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
  168. * @enc: Flag indicating either encryption or decryption
  169. * @enc_iv_size: Initialisation vector size
  170. * @iv_idx: Initialisation vector index
  171. */
  172. struct sa_rx_data {
  173. void *req;
  174. struct device *ddev;
  175. struct dma_async_tx_descriptor *tx_in;
  176. struct sa_mapped_sg mapped_sg[2];
  177. u8 enc;
  178. u8 enc_iv_size;
  179. u8 iv_idx;
  180. };
  181. /**
  182. * struct sa_req: SA request definition
  183. * @dev: device for the request
  184. * @size: total data to the xmitted via DMA
  185. * @enc_offset: offset of cipher data
  186. * @enc_size: data to be passed to cipher engine
  187. * @enc_iv: cipher IV
  188. * @auth_offset: offset of the authentication data
  189. * @auth_size: size of the authentication data
  190. * @auth_iv: authentication IV
  191. * @type: algorithm type for the request
  192. * @cmdl: command label pointer
  193. * @base: pointer to the base request
  194. * @ctx: pointer to the algorithm context data
  195. * @enc: true if this is an encode request
  196. * @src: source data
  197. * @dst: destination data
  198. * @callback: DMA callback for the request
  199. * @mdata_size: metadata size passed to DMA
  200. */
  201. struct sa_req {
  202. struct device *dev;
  203. u16 size;
  204. u8 enc_offset;
  205. u16 enc_size;
  206. u8 *enc_iv;
  207. u8 auth_offset;
  208. u16 auth_size;
  209. u8 *auth_iv;
  210. u32 type;
  211. u32 *cmdl;
  212. struct crypto_async_request *base;
  213. struct sa_tfm_ctx *ctx;
  214. bool enc;
  215. struct scatterlist *src;
  216. struct scatterlist *dst;
  217. dma_async_tx_callback callback;
  218. u16 mdata_size;
  219. };
  220. /*
  221. * Mode Control Instructions for various Key lengths 128, 192, 256
  222. * For CBC (Cipher Block Chaining) mode for encryption
  223. */
  224. static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
  225. { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
  226. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  227. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  228. { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
  229. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  230. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  231. { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
  232. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  233. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  234. };
  235. /*
  236. * Mode Control Instructions for various Key lengths 128, 192, 256
  237. * For CBC (Cipher Block Chaining) mode for decryption
  238. */
  239. static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
  240. { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  241. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  242. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  243. { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  244. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  245. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  246. { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  247. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  248. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  249. };
  250. /*
  251. * Mode Control Instructions for various Key lengths 128, 192, 256
  252. * For CBC (Cipher Block Chaining) mode for encryption
  253. */
  254. static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
  255. { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
  256. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  257. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  258. { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
  259. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  260. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  261. { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
  262. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  263. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  264. };
  265. /*
  266. * Mode Control Instructions for various Key lengths 128, 192, 256
  267. * For CBC (Cipher Block Chaining) mode for decryption
  268. */
  269. static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
  270. { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  271. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  272. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  273. { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  274. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  275. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  276. { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
  277. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  278. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  279. };
  280. /*
  281. * Mode Control Instructions for various Key lengths 128, 192, 256
  282. * For ECB (Electronic Code Book) mode for encryption
  283. */
  284. static u8 mci_ecb_enc_array[3][27] = {
  285. { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  286. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  287. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  288. { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  289. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  290. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  291. { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  292. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  293. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  294. };
  295. /*
  296. * Mode Control Instructions for various Key lengths 128, 192, 256
  297. * For ECB (Electronic Code Book) mode for decryption
  298. */
  299. static u8 mci_ecb_dec_array[3][27] = {
  300. { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  301. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  302. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  303. { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  304. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  305. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  306. { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
  307. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  308. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
  309. };
  310. /*
  311. * Mode Control Instructions for DES algorithm
  312. * For CBC (Cipher Block Chaining) mode and ECB mode
  313. * encryption and for decryption respectively
  314. */
  315. static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
  316. 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
  317. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  318. 0x00, 0x00, 0x00,
  319. };
  320. static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
  321. 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
  322. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  323. 0x00, 0x00, 0x00,
  324. };
  325. static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
  326. 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
  327. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  328. 0x00, 0x00, 0x00,
  329. };
  330. static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
  331. 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
  332. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
  333. 0x00, 0x00, 0x00,
  334. };
  335. /*
  336. * Perform 16 byte or 128 bit swizzling
  337. * The SA2UL Expects the security context to
  338. * be in little Endian and the bus width is 128 bits or 16 bytes
  339. * Hence swap 16 bytes at a time from higher to lower address
  340. */
  341. static void sa_swiz_128(u8 *in, u16 len)
  342. {
  343. u8 data[16];
  344. int i, j;
  345. for (i = 0; i < len; i += 16) {
  346. memcpy(data, &in[i], 16);
  347. for (j = 0; j < 16; j++)
  348. in[i + j] = data[15 - j];
  349. }
  350. }
  351. /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
  352. static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz)
  353. {
  354. int i;
  355. for (i = 0; i < key_sz; i++)
  356. k_ipad[i] = key[i] ^ 0x36;
  357. /* Instead of XOR with 0 */
  358. for (; i < SHA1_BLOCK_SIZE; i++)
  359. k_ipad[i] = 0x36;
  360. }
  361. static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz)
  362. {
  363. int i;
  364. for (i = 0; i < key_sz; i++)
  365. k_opad[i] = key[i] ^ 0x5c;
  366. /* Instead of XOR with 0 */
  367. for (; i < SHA1_BLOCK_SIZE; i++)
  368. k_opad[i] = 0x5c;
  369. }
  370. static void sa_export_shash(void *state, struct shash_desc *hash,
  371. int digest_size, __be32 *out)
  372. {
  373. struct sha1_state *sha1;
  374. struct sha256_state *sha256;
  375. u32 *result;
  376. switch (digest_size) {
  377. case SHA1_DIGEST_SIZE:
  378. sha1 = state;
  379. result = sha1->state;
  380. break;
  381. case SHA256_DIGEST_SIZE:
  382. sha256 = state;
  383. result = sha256->state;
  384. break;
  385. default:
  386. dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
  387. digest_size);
  388. return;
  389. }
  390. crypto_shash_export(hash, state);
  391. cpu_to_be32_array(out, result, digest_size / 4);
  392. }
  393. static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
  394. u16 key_sz, __be32 *ipad, __be32 *opad)
  395. {
  396. SHASH_DESC_ON_STACK(shash, data->ctx->shash);
  397. int block_size = crypto_shash_blocksize(data->ctx->shash);
  398. int digest_size = crypto_shash_digestsize(data->ctx->shash);
  399. union {
  400. struct sha1_state sha1;
  401. struct sha256_state sha256;
  402. u8 k_pad[SHA1_BLOCK_SIZE];
  403. } sha;
  404. shash->tfm = data->ctx->shash;
  405. prepare_kipad(sha.k_pad, key, key_sz);
  406. crypto_shash_init(shash);
  407. crypto_shash_update(shash, sha.k_pad, block_size);
  408. sa_export_shash(&sha, shash, digest_size, ipad);
  409. prepare_kopad(sha.k_pad, key, key_sz);
  410. crypto_shash_init(shash);
  411. crypto_shash_update(shash, sha.k_pad, block_size);
  412. sa_export_shash(&sha, shash, digest_size, opad);
  413. memzero_explicit(&sha, sizeof(sha));
  414. }
  415. /* Derive the inverse key used in AES-CBC decryption operation */
  416. static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
  417. {
  418. struct crypto_aes_ctx ctx;
  419. int key_pos;
  420. if (aes_expandkey(&ctx, key, key_sz)) {
  421. dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
  422. return -EINVAL;
  423. }
  424. /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
  425. if (key_sz == AES_KEYSIZE_192) {
  426. ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
  427. ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
  428. }
  429. /* Based crypto_aes_expand_key logic */
  430. switch (key_sz) {
  431. case AES_KEYSIZE_128:
  432. case AES_KEYSIZE_192:
  433. key_pos = key_sz + 24;
  434. break;
  435. case AES_KEYSIZE_256:
  436. key_pos = key_sz + 24 - 4;
  437. break;
  438. default:
  439. dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
  440. return -EINVAL;
  441. }
  442. memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
  443. return 0;
  444. }
  445. /* Set Security context for the encryption engine */
  446. static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
  447. u8 enc, u8 *sc_buf)
  448. {
  449. const u8 *mci = NULL;
  450. /* Set Encryption mode selector to crypto processing */
  451. sc_buf[0] = SA_CRYPTO_PROCESSING;
  452. if (enc)
  453. mci = ad->mci_enc;
  454. else
  455. mci = ad->mci_dec;
  456. /* Set the mode control instructions in security context */
  457. if (mci)
  458. memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
  459. /* For AES-CBC decryption get the inverse key */
  460. if (ad->inv_key && !enc) {
  461. if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
  462. return -EINVAL;
  463. /* For all other cases: key is used */
  464. } else {
  465. memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
  466. }
  467. return 0;
  468. }
  469. /* Set Security context for the authentication engine */
  470. static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
  471. u8 *sc_buf)
  472. {
  473. __be32 *ipad = (void *)(sc_buf + 32);
  474. __be32 *opad = (void *)(sc_buf + 64);
  475. /* Set Authentication mode selector to hash processing */
  476. sc_buf[0] = SA_HASH_PROCESSING;
  477. /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
  478. sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
  479. sc_buf[1] |= ad->auth_ctrl;
  480. /* Copy the keys or ipad/opad */
  481. if (ad->keyed_mac)
  482. ad->prep_iopad(ad, key, key_sz, ipad, opad);
  483. else {
  484. /* basic hash */
  485. sc_buf[1] |= SA_BASIC_HASH;
  486. }
  487. }
  488. static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
  489. {
  490. int j;
  491. for (j = 0; j < ((size16) ? 4 : 2); j++) {
  492. *out = cpu_to_be32(*((u32 *)iv));
  493. iv += 4;
  494. out++;
  495. }
  496. }
  497. /* Format general command label */
  498. static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
  499. struct sa_cmdl_upd_info *upd_info)
  500. {
  501. u8 enc_offset = 0, auth_offset = 0, total = 0;
  502. u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
  503. u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
  504. u32 *word_ptr = (u32 *)cmdl;
  505. int i;
  506. /* Clear the command label */
  507. memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
  508. /* Iniialize the command update structure */
  509. memzero_explicit(upd_info, sizeof(*upd_info));
  510. if (cfg->enc_eng_id && cfg->auth_eng_id) {
  511. if (cfg->enc) {
  512. auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
  513. enc_next_eng = cfg->auth_eng_id;
  514. if (cfg->iv_size)
  515. auth_offset += cfg->iv_size;
  516. } else {
  517. enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
  518. auth_next_eng = cfg->enc_eng_id;
  519. }
  520. }
  521. if (cfg->enc_eng_id) {
  522. upd_info->flags |= SA_CMDL_UPD_ENC;
  523. upd_info->enc_size.index = enc_offset >> 2;
  524. upd_info->enc_offset.index = upd_info->enc_size.index + 1;
  525. /* Encryption command label */
  526. cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
  527. /* Encryption modes requiring IV */
  528. if (cfg->iv_size) {
  529. upd_info->flags |= SA_CMDL_UPD_ENC_IV;
  530. upd_info->enc_iv.index =
  531. (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
  532. upd_info->enc_iv.size = cfg->iv_size;
  533. cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
  534. SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
  535. cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
  536. (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
  537. total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
  538. } else {
  539. cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
  540. SA_CMDL_HEADER_SIZE_BYTES;
  541. total += SA_CMDL_HEADER_SIZE_BYTES;
  542. }
  543. }
  544. if (cfg->auth_eng_id) {
  545. upd_info->flags |= SA_CMDL_UPD_AUTH;
  546. upd_info->auth_size.index = auth_offset >> 2;
  547. upd_info->auth_offset.index = upd_info->auth_size.index + 1;
  548. cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
  549. cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
  550. SA_CMDL_HEADER_SIZE_BYTES;
  551. total += SA_CMDL_HEADER_SIZE_BYTES;
  552. }
  553. total = roundup(total, 8);
  554. for (i = 0; i < total / 4; i++)
  555. word_ptr[i] = swab32(word_ptr[i]);
  556. return total;
  557. }
  558. /* Update Command label */
  559. static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
  560. struct sa_cmdl_upd_info *upd_info)
  561. {
  562. int i = 0, j;
  563. if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
  564. cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
  565. cmdl[upd_info->enc_size.index] |= req->enc_size;
  566. cmdl[upd_info->enc_offset.index] &=
  567. ~SA_CMDL_SOP_BYPASS_LEN_MASK;
  568. cmdl[upd_info->enc_offset.index] |=
  569. FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
  570. req->enc_offset);
  571. if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
  572. __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
  573. u32 *enc_iv = (u32 *)req->enc_iv;
  574. for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
  575. data[j] = cpu_to_be32(*enc_iv);
  576. enc_iv++;
  577. }
  578. }
  579. }
  580. if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
  581. cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
  582. cmdl[upd_info->auth_size.index] |= req->auth_size;
  583. cmdl[upd_info->auth_offset.index] &=
  584. ~SA_CMDL_SOP_BYPASS_LEN_MASK;
  585. cmdl[upd_info->auth_offset.index] |=
  586. FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
  587. req->auth_offset);
  588. if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
  589. sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
  590. req->auth_iv,
  591. (upd_info->auth_iv.size > 8));
  592. }
  593. if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
  594. int offset = (req->auth_size & 0xF) ? 4 : 0;
  595. memcpy(&cmdl[upd_info->aux_key_info.index],
  596. &upd_info->aux_key[offset], 16);
  597. }
  598. }
  599. }
  600. /* Format SWINFO words to be sent to SA */
  601. static
  602. void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
  603. u8 cmdl_present, u8 cmdl_offset, u8 flags,
  604. u8 hash_size, u32 *swinfo)
  605. {
  606. swinfo[0] = sc_id;
  607. swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags);
  608. if (likely(cmdl_present))
  609. swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK,
  610. cmdl_offset | SA_SW0_CMDL_PRESENT);
  611. swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id);
  612. swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
  613. swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
  614. swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
  615. swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size);
  616. }
  617. /* Dump the security context */
  618. static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
  619. {
  620. #ifdef DEBUG
  621. dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
  622. print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
  623. 16, 1, buf, SA_CTX_MAX_SZ, false);
  624. #endif
  625. }
  626. static
  627. int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data,
  628. const u8 *enc_key, u16 enc_key_sz,
  629. const u8 *auth_key, u16 auth_key_sz,
  630. struct algo_data *ad, u8 enc, u32 *swinfo)
  631. {
  632. int enc_sc_offset = 0;
  633. int auth_sc_offset = 0;
  634. u8 *sc_buf = ctx->sc;
  635. u16 sc_id = ctx->sc_id;
  636. u8 first_engine = 0;
  637. memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
  638. if (ad->auth_eng.eng_id) {
  639. if (enc)
  640. first_engine = ad->enc_eng.eng_id;
  641. else
  642. first_engine = ad->auth_eng.eng_id;
  643. enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
  644. auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
  645. sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
  646. if (!ad->hash_size)
  647. return -EINVAL;
  648. ad->hash_size = roundup(ad->hash_size, 8);
  649. } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
  650. enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
  651. first_engine = ad->enc_eng.eng_id;
  652. sc_buf[1] = SA_SCCTL_FE_ENC;
  653. ad->hash_size = ad->iv_out_size;
  654. }
  655. /* SCCTL Owner info: 0=host, 1=CP_ACE */
  656. sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
  657. memcpy(&sc_buf[2], &sc_id, 2);
  658. sc_buf[4] = 0x0;
  659. sc_buf[5] = match_data->priv_id;
  660. sc_buf[6] = match_data->priv;
  661. sc_buf[7] = 0x0;
  662. /* Prepare context for encryption engine */
  663. if (ad->enc_eng.sc_size) {
  664. if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
  665. &sc_buf[enc_sc_offset]))
  666. return -EINVAL;
  667. }
  668. /* Prepare context for authentication engine */
  669. if (ad->auth_eng.sc_size)
  670. sa_set_sc_auth(ad, auth_key, auth_key_sz,
  671. &sc_buf[auth_sc_offset]);
  672. /* Set the ownership of context to CP_ACE */
  673. sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
  674. /* swizzle the security context */
  675. sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
  676. sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
  677. SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
  678. sa_dump_sc(sc_buf, ctx->sc_phys);
  679. return 0;
  680. }
  681. /* Free the per direction context memory */
  682. static void sa_free_ctx_info(struct sa_ctx_info *ctx,
  683. struct sa_crypto_data *data)
  684. {
  685. unsigned long bn;
  686. bn = ctx->sc_id - data->sc_id_start;
  687. spin_lock(&data->scid_lock);
  688. __clear_bit(bn, data->ctx_bm);
  689. data->sc_id--;
  690. spin_unlock(&data->scid_lock);
  691. if (ctx->sc) {
  692. dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
  693. ctx->sc = NULL;
  694. }
  695. }
  696. static int sa_init_ctx_info(struct sa_ctx_info *ctx,
  697. struct sa_crypto_data *data)
  698. {
  699. unsigned long bn;
  700. int err;
  701. spin_lock(&data->scid_lock);
  702. bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
  703. __set_bit(bn, data->ctx_bm);
  704. data->sc_id++;
  705. spin_unlock(&data->scid_lock);
  706. ctx->sc_id = (u16)(data->sc_id_start + bn);
  707. ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
  708. if (!ctx->sc) {
  709. dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
  710. err = -ENOMEM;
  711. goto scid_rollback;
  712. }
  713. return 0;
  714. scid_rollback:
  715. spin_lock(&data->scid_lock);
  716. __clear_bit(bn, data->ctx_bm);
  717. data->sc_id--;
  718. spin_unlock(&data->scid_lock);
  719. return err;
  720. }
  721. static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
  722. {
  723. struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  724. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  725. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  726. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  727. ctx->dec.sc_id, &ctx->dec.sc_phys);
  728. sa_free_ctx_info(&ctx->enc, data);
  729. sa_free_ctx_info(&ctx->dec, data);
  730. crypto_free_skcipher(ctx->fallback.skcipher);
  731. }
  732. static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
  733. {
  734. struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  735. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  736. const char *name = crypto_tfm_alg_name(&tfm->base);
  737. struct crypto_skcipher *child;
  738. int ret;
  739. memzero_explicit(ctx, sizeof(*ctx));
  740. ctx->dev_data = data;
  741. ret = sa_init_ctx_info(&ctx->enc, data);
  742. if (ret)
  743. return ret;
  744. ret = sa_init_ctx_info(&ctx->dec, data);
  745. if (ret) {
  746. sa_free_ctx_info(&ctx->enc, data);
  747. return ret;
  748. }
  749. child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
  750. if (IS_ERR(child)) {
  751. dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
  752. return PTR_ERR(child);
  753. }
  754. ctx->fallback.skcipher = child;
  755. crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
  756. sizeof(struct skcipher_request));
  757. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  758. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  759. ctx->dec.sc_id, &ctx->dec.sc_phys);
  760. return 0;
  761. }
  762. static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
  763. unsigned int keylen, struct algo_data *ad)
  764. {
  765. struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
  766. struct crypto_skcipher *child = ctx->fallback.skcipher;
  767. int cmdl_len;
  768. struct sa_cmdl_cfg cfg;
  769. int ret;
  770. if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
  771. keylen != AES_KEYSIZE_256)
  772. return -EINVAL;
  773. ad->enc_eng.eng_id = SA_ENG_ID_EM1;
  774. ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
  775. memzero_explicit(&cfg, sizeof(cfg));
  776. cfg.enc_eng_id = ad->enc_eng.eng_id;
  777. cfg.iv_size = crypto_skcipher_ivsize(tfm);
  778. crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
  779. crypto_skcipher_set_flags(child, tfm->base.crt_flags &
  780. CRYPTO_TFM_REQ_MASK);
  781. ret = crypto_skcipher_setkey(child, key, keylen);
  782. if (ret)
  783. return ret;
  784. /* Setup Encryption Security Context & Command label template */
  785. if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0,
  786. ad, 1, &ctx->enc.epib[1]))
  787. goto badkey;
  788. cmdl_len = sa_format_cmdl_gen(&cfg,
  789. (u8 *)ctx->enc.cmdl,
  790. &ctx->enc.cmdl_upd_info);
  791. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  792. goto badkey;
  793. ctx->enc.cmdl_size = cmdl_len;
  794. /* Setup Decryption Security Context & Command label template */
  795. if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0,
  796. ad, 0, &ctx->dec.epib[1]))
  797. goto badkey;
  798. cfg.enc_eng_id = ad->enc_eng.eng_id;
  799. cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
  800. &ctx->dec.cmdl_upd_info);
  801. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  802. goto badkey;
  803. ctx->dec.cmdl_size = cmdl_len;
  804. ctx->iv_idx = ad->iv_idx;
  805. return 0;
  806. badkey:
  807. dev_err(sa_k3_dev, "%s: badkey\n", __func__);
  808. return -EINVAL;
  809. }
  810. static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
  811. unsigned int keylen)
  812. {
  813. struct algo_data ad = { 0 };
  814. /* Convert the key size (16/24/32) to the key size index (0/1/2) */
  815. int key_idx = (keylen >> 3) - 2;
  816. if (key_idx >= 3)
  817. return -EINVAL;
  818. ad.mci_enc = mci_cbc_enc_array[key_idx];
  819. ad.mci_dec = mci_cbc_dec_array[key_idx];
  820. ad.inv_key = true;
  821. ad.ealg_id = SA_EALG_ID_AES_CBC;
  822. ad.iv_idx = 4;
  823. ad.iv_out_size = 16;
  824. return sa_cipher_setkey(tfm, key, keylen, &ad);
  825. }
  826. static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
  827. unsigned int keylen)
  828. {
  829. struct algo_data ad = { 0 };
  830. /* Convert the key size (16/24/32) to the key size index (0/1/2) */
  831. int key_idx = (keylen >> 3) - 2;
  832. if (key_idx >= 3)
  833. return -EINVAL;
  834. ad.mci_enc = mci_ecb_enc_array[key_idx];
  835. ad.mci_dec = mci_ecb_dec_array[key_idx];
  836. ad.inv_key = true;
  837. ad.ealg_id = SA_EALG_ID_AES_ECB;
  838. return sa_cipher_setkey(tfm, key, keylen, &ad);
  839. }
  840. static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
  841. unsigned int keylen)
  842. {
  843. struct algo_data ad = { 0 };
  844. ad.mci_enc = mci_cbc_3des_enc_array;
  845. ad.mci_dec = mci_cbc_3des_dec_array;
  846. ad.ealg_id = SA_EALG_ID_3DES_CBC;
  847. ad.iv_idx = 6;
  848. ad.iv_out_size = 8;
  849. return sa_cipher_setkey(tfm, key, keylen, &ad);
  850. }
  851. static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
  852. unsigned int keylen)
  853. {
  854. struct algo_data ad = { 0 };
  855. ad.mci_enc = mci_ecb_3des_enc_array;
  856. ad.mci_dec = mci_ecb_3des_dec_array;
  857. return sa_cipher_setkey(tfm, key, keylen, &ad);
  858. }
  859. static void sa_sync_from_device(struct sa_rx_data *rxd)
  860. {
  861. struct sg_table *sgt;
  862. if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
  863. sgt = &rxd->mapped_sg[0].sgt;
  864. else
  865. sgt = &rxd->mapped_sg[1].sgt;
  866. dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
  867. }
  868. static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
  869. {
  870. int i;
  871. for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
  872. struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
  873. if (mapped_sg->mapped) {
  874. dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
  875. mapped_sg->dir, 0);
  876. kfree(mapped_sg->split_sg);
  877. }
  878. }
  879. kfree(rxd);
  880. }
  881. static void sa_aes_dma_in_callback(void *data)
  882. {
  883. struct sa_rx_data *rxd = (struct sa_rx_data *)data;
  884. struct skcipher_request *req;
  885. u32 *result;
  886. __be32 *mdptr;
  887. size_t ml, pl;
  888. int i;
  889. sa_sync_from_device(rxd);
  890. req = container_of(rxd->req, struct skcipher_request, base);
  891. if (req->iv) {
  892. mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
  893. &ml);
  894. result = (u32 *)req->iv;
  895. for (i = 0; i < (rxd->enc_iv_size / 4); i++)
  896. result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
  897. }
  898. sa_free_sa_rx_data(rxd);
  899. skcipher_request_complete(req, 0);
  900. }
  901. static void
  902. sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
  903. {
  904. u32 *out, *in;
  905. int i;
  906. for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
  907. *out++ = *in++;
  908. mdptr[4] = (0xFFFF << 16);
  909. for (out = &mdptr[5], in = psdata, i = 0;
  910. i < pslen / sizeof(u32); i++)
  911. *out++ = *in++;
  912. }
  913. static int sa_run(struct sa_req *req)
  914. {
  915. struct sa_rx_data *rxd;
  916. gfp_t gfp_flags;
  917. u32 cmdl[SA_MAX_CMDL_WORDS];
  918. struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
  919. struct device *ddev;
  920. struct dma_chan *dma_rx;
  921. int sg_nents, src_nents, dst_nents;
  922. struct scatterlist *src, *dst;
  923. size_t pl, ml, split_size;
  924. struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
  925. int ret;
  926. struct dma_async_tx_descriptor *tx_out;
  927. u32 *mdptr;
  928. bool diff_dst;
  929. enum dma_data_direction dir_src;
  930. struct sa_mapped_sg *mapped_sg;
  931. gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
  932. GFP_KERNEL : GFP_ATOMIC;
  933. rxd = kzalloc(sizeof(*rxd), gfp_flags);
  934. if (!rxd)
  935. return -ENOMEM;
  936. if (req->src != req->dst) {
  937. diff_dst = true;
  938. dir_src = DMA_TO_DEVICE;
  939. } else {
  940. diff_dst = false;
  941. dir_src = DMA_BIDIRECTIONAL;
  942. }
  943. /*
  944. * SA2UL has an interesting feature where the receive DMA channel
  945. * is selected based on the data passed to the engine. Within the
  946. * transition range, there is also a space where it is impossible
  947. * to determine where the data will end up, and this should be
  948. * avoided. This will be handled by the SW fallback mechanism by
  949. * the individual algorithm implementations.
  950. */
  951. if (req->size >= 256)
  952. dma_rx = pdata->dma_rx2;
  953. else
  954. dma_rx = pdata->dma_rx1;
  955. ddev = dmaengine_get_dma_device(pdata->dma_tx);
  956. rxd->ddev = ddev;
  957. memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
  958. sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
  959. if (req->type != CRYPTO_ALG_TYPE_AHASH) {
  960. if (req->enc)
  961. req->type |=
  962. (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
  963. else
  964. req->type |=
  965. (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
  966. }
  967. cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
  968. /*
  969. * Map the packets, first we check if the data fits into a single
  970. * sg entry and use that if possible. If it does not fit, we check
  971. * if we need to do sg_split to align the scatterlist data on the
  972. * actual data size being processed by the crypto engine.
  973. */
  974. src = req->src;
  975. sg_nents = sg_nents_for_len(src, req->size);
  976. split_size = req->size;
  977. mapped_sg = &rxd->mapped_sg[0];
  978. if (sg_nents == 1 && split_size <= req->src->length) {
  979. src = &mapped_sg->static_sg;
  980. src_nents = 1;
  981. sg_init_table(src, 1);
  982. sg_set_page(src, sg_page(req->src), split_size,
  983. req->src->offset);
  984. mapped_sg->sgt.sgl = src;
  985. mapped_sg->sgt.orig_nents = src_nents;
  986. ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
  987. if (ret) {
  988. kfree(rxd);
  989. return ret;
  990. }
  991. mapped_sg->dir = dir_src;
  992. mapped_sg->mapped = true;
  993. } else {
  994. mapped_sg->sgt.sgl = req->src;
  995. mapped_sg->sgt.orig_nents = sg_nents;
  996. ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
  997. if (ret) {
  998. kfree(rxd);
  999. return ret;
  1000. }
  1001. mapped_sg->dir = dir_src;
  1002. mapped_sg->mapped = true;
  1003. ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
  1004. &split_size, &src, &src_nents, gfp_flags);
  1005. if (ret) {
  1006. src_nents = mapped_sg->sgt.nents;
  1007. src = mapped_sg->sgt.sgl;
  1008. } else {
  1009. mapped_sg->split_sg = src;
  1010. }
  1011. }
  1012. dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
  1013. if (!diff_dst) {
  1014. dst_nents = src_nents;
  1015. dst = src;
  1016. } else {
  1017. dst_nents = sg_nents_for_len(req->dst, req->size);
  1018. mapped_sg = &rxd->mapped_sg[1];
  1019. if (dst_nents == 1 && split_size <= req->dst->length) {
  1020. dst = &mapped_sg->static_sg;
  1021. dst_nents = 1;
  1022. sg_init_table(dst, 1);
  1023. sg_set_page(dst, sg_page(req->dst), split_size,
  1024. req->dst->offset);
  1025. mapped_sg->sgt.sgl = dst;
  1026. mapped_sg->sgt.orig_nents = dst_nents;
  1027. ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
  1028. DMA_FROM_DEVICE, 0);
  1029. if (ret)
  1030. goto err_cleanup;
  1031. mapped_sg->dir = DMA_FROM_DEVICE;
  1032. mapped_sg->mapped = true;
  1033. } else {
  1034. mapped_sg->sgt.sgl = req->dst;
  1035. mapped_sg->sgt.orig_nents = dst_nents;
  1036. ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
  1037. DMA_FROM_DEVICE, 0);
  1038. if (ret)
  1039. goto err_cleanup;
  1040. mapped_sg->dir = DMA_FROM_DEVICE;
  1041. mapped_sg->mapped = true;
  1042. ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
  1043. 0, 1, &split_size, &dst, &dst_nents,
  1044. gfp_flags);
  1045. if (ret) {
  1046. dst_nents = mapped_sg->sgt.nents;
  1047. dst = mapped_sg->sgt.sgl;
  1048. } else {
  1049. mapped_sg->split_sg = dst;
  1050. }
  1051. }
  1052. }
  1053. rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
  1054. DMA_DEV_TO_MEM,
  1055. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  1056. if (!rxd->tx_in) {
  1057. dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
  1058. ret = -EINVAL;
  1059. goto err_cleanup;
  1060. }
  1061. rxd->req = (void *)req->base;
  1062. rxd->enc = req->enc;
  1063. rxd->iv_idx = req->ctx->iv_idx;
  1064. rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
  1065. rxd->tx_in->callback = req->callback;
  1066. rxd->tx_in->callback_param = rxd;
  1067. tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
  1068. src_nents, DMA_MEM_TO_DEV,
  1069. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  1070. if (!tx_out) {
  1071. dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
  1072. ret = -EINVAL;
  1073. goto err_cleanup;
  1074. }
  1075. /*
  1076. * Prepare metadata for DMA engine. This essentially describes the
  1077. * crypto algorithm to be used, data sizes, different keys etc.
  1078. */
  1079. mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
  1080. sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
  1081. sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
  1082. sa_ctx->epib);
  1083. ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
  1084. dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
  1085. dmaengine_submit(tx_out);
  1086. dmaengine_submit(rxd->tx_in);
  1087. dma_async_issue_pending(dma_rx);
  1088. dma_async_issue_pending(pdata->dma_tx);
  1089. return -EINPROGRESS;
  1090. err_cleanup:
  1091. sa_free_sa_rx_data(rxd);
  1092. return ret;
  1093. }
  1094. static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
  1095. {
  1096. struct sa_tfm_ctx *ctx =
  1097. crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
  1098. struct crypto_alg *alg = req->base.tfm->__crt_alg;
  1099. struct sa_req sa_req = { 0 };
  1100. if (!req->cryptlen)
  1101. return 0;
  1102. if (req->cryptlen % alg->cra_blocksize)
  1103. return -EINVAL;
  1104. /* Use SW fallback if the data size is not supported */
  1105. if (req->cryptlen > SA_MAX_DATA_SZ ||
  1106. (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
  1107. req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
  1108. struct skcipher_request *subreq = skcipher_request_ctx(req);
  1109. skcipher_request_set_tfm(subreq, ctx->fallback.skcipher);
  1110. skcipher_request_set_callback(subreq, req->base.flags,
  1111. req->base.complete,
  1112. req->base.data);
  1113. skcipher_request_set_crypt(subreq, req->src, req->dst,
  1114. req->cryptlen, req->iv);
  1115. if (enc)
  1116. return crypto_skcipher_encrypt(subreq);
  1117. else
  1118. return crypto_skcipher_decrypt(subreq);
  1119. }
  1120. sa_req.size = req->cryptlen;
  1121. sa_req.enc_size = req->cryptlen;
  1122. sa_req.src = req->src;
  1123. sa_req.dst = req->dst;
  1124. sa_req.enc_iv = iv;
  1125. sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
  1126. sa_req.enc = enc;
  1127. sa_req.callback = sa_aes_dma_in_callback;
  1128. sa_req.mdata_size = 44;
  1129. sa_req.base = &req->base;
  1130. sa_req.ctx = ctx;
  1131. return sa_run(&sa_req);
  1132. }
  1133. static int sa_encrypt(struct skcipher_request *req)
  1134. {
  1135. return sa_cipher_run(req, req->iv, 1);
  1136. }
  1137. static int sa_decrypt(struct skcipher_request *req)
  1138. {
  1139. return sa_cipher_run(req, req->iv, 0);
  1140. }
  1141. static void sa_sha_dma_in_callback(void *data)
  1142. {
  1143. struct sa_rx_data *rxd = (struct sa_rx_data *)data;
  1144. struct ahash_request *req;
  1145. struct crypto_ahash *tfm;
  1146. unsigned int authsize;
  1147. int i;
  1148. size_t ml, pl;
  1149. u32 *result;
  1150. __be32 *mdptr;
  1151. sa_sync_from_device(rxd);
  1152. req = container_of(rxd->req, struct ahash_request, base);
  1153. tfm = crypto_ahash_reqtfm(req);
  1154. authsize = crypto_ahash_digestsize(tfm);
  1155. mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
  1156. result = (u32 *)req->result;
  1157. for (i = 0; i < (authsize / 4); i++)
  1158. result[i] = be32_to_cpu(mdptr[i + 4]);
  1159. sa_free_sa_rx_data(rxd);
  1160. ahash_request_complete(req, 0);
  1161. }
  1162. static int zero_message_process(struct ahash_request *req)
  1163. {
  1164. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1165. int sa_digest_size = crypto_ahash_digestsize(tfm);
  1166. switch (sa_digest_size) {
  1167. case SHA1_DIGEST_SIZE:
  1168. memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
  1169. break;
  1170. case SHA256_DIGEST_SIZE:
  1171. memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
  1172. break;
  1173. case SHA512_DIGEST_SIZE:
  1174. memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
  1175. break;
  1176. default:
  1177. return -EINVAL;
  1178. }
  1179. return 0;
  1180. }
  1181. static int sa_sha_run(struct ahash_request *req)
  1182. {
  1183. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
  1184. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1185. struct sa_req sa_req = { 0 };
  1186. size_t auth_len;
  1187. auth_len = req->nbytes;
  1188. if (!auth_len)
  1189. return zero_message_process(req);
  1190. if (auth_len > SA_MAX_DATA_SZ ||
  1191. (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
  1192. auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
  1193. struct ahash_request *subreq = &rctx->fallback_req;
  1194. int ret = 0;
  1195. ahash_request_set_tfm(subreq, ctx->fallback.ahash);
  1196. subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1197. crypto_ahash_init(subreq);
  1198. subreq->nbytes = auth_len;
  1199. subreq->src = req->src;
  1200. subreq->result = req->result;
  1201. ret |= crypto_ahash_update(subreq);
  1202. subreq->nbytes = 0;
  1203. ret |= crypto_ahash_final(subreq);
  1204. return ret;
  1205. }
  1206. sa_req.size = auth_len;
  1207. sa_req.auth_size = auth_len;
  1208. sa_req.src = req->src;
  1209. sa_req.dst = req->src;
  1210. sa_req.enc = true;
  1211. sa_req.type = CRYPTO_ALG_TYPE_AHASH;
  1212. sa_req.callback = sa_sha_dma_in_callback;
  1213. sa_req.mdata_size = 28;
  1214. sa_req.ctx = ctx;
  1215. sa_req.base = &req->base;
  1216. return sa_run(&sa_req);
  1217. }
  1218. static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad)
  1219. {
  1220. int bs = crypto_shash_blocksize(ctx->shash);
  1221. int cmdl_len;
  1222. struct sa_cmdl_cfg cfg;
  1223. ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
  1224. ad->auth_eng.eng_id = SA_ENG_ID_AM1;
  1225. ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
  1226. memset(ctx->authkey, 0, bs);
  1227. memset(&cfg, 0, sizeof(cfg));
  1228. cfg.aalg = ad->aalg_id;
  1229. cfg.enc_eng_id = ad->enc_eng.eng_id;
  1230. cfg.auth_eng_id = ad->auth_eng.eng_id;
  1231. cfg.iv_size = 0;
  1232. cfg.akey = NULL;
  1233. cfg.akey_len = 0;
  1234. ctx->dev_data = dev_get_drvdata(sa_k3_dev);
  1235. /* Setup Encryption Security Context & Command label template */
  1236. if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0,
  1237. ad, 0, &ctx->enc.epib[1]))
  1238. goto badkey;
  1239. cmdl_len = sa_format_cmdl_gen(&cfg,
  1240. (u8 *)ctx->enc.cmdl,
  1241. &ctx->enc.cmdl_upd_info);
  1242. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  1243. goto badkey;
  1244. ctx->enc.cmdl_size = cmdl_len;
  1245. return 0;
  1246. badkey:
  1247. dev_err(sa_k3_dev, "%s: badkey\n", __func__);
  1248. return -EINVAL;
  1249. }
  1250. static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
  1251. {
  1252. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1253. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1254. int ret;
  1255. memset(ctx, 0, sizeof(*ctx));
  1256. ctx->dev_data = data;
  1257. ret = sa_init_ctx_info(&ctx->enc, data);
  1258. if (ret)
  1259. return ret;
  1260. if (alg_base) {
  1261. ctx->shash = crypto_alloc_shash(alg_base, 0,
  1262. CRYPTO_ALG_NEED_FALLBACK);
  1263. if (IS_ERR(ctx->shash)) {
  1264. dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
  1265. alg_base);
  1266. return PTR_ERR(ctx->shash);
  1267. }
  1268. /* for fallback */
  1269. ctx->fallback.ahash =
  1270. crypto_alloc_ahash(alg_base, 0,
  1271. CRYPTO_ALG_NEED_FALLBACK);
  1272. if (IS_ERR(ctx->fallback.ahash)) {
  1273. dev_err(ctx->dev_data->dev,
  1274. "Could not load fallback driver\n");
  1275. return PTR_ERR(ctx->fallback.ahash);
  1276. }
  1277. }
  1278. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  1279. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  1280. ctx->dec.sc_id, &ctx->dec.sc_phys);
  1281. crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
  1282. sizeof(struct sa_sha_req_ctx) +
  1283. crypto_ahash_reqsize(ctx->fallback.ahash));
  1284. return 0;
  1285. }
  1286. static int sa_sha_digest(struct ahash_request *req)
  1287. {
  1288. return sa_sha_run(req);
  1289. }
  1290. static int sa_sha_init(struct ahash_request *req)
  1291. {
  1292. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1293. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1294. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1295. dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
  1296. crypto_ahash_digestsize(tfm), rctx);
  1297. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1298. rctx->fallback_req.base.flags =
  1299. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1300. return crypto_ahash_init(&rctx->fallback_req);
  1301. }
  1302. static int sa_sha_update(struct ahash_request *req)
  1303. {
  1304. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1305. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1306. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1307. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1308. rctx->fallback_req.base.flags =
  1309. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1310. rctx->fallback_req.nbytes = req->nbytes;
  1311. rctx->fallback_req.src = req->src;
  1312. return crypto_ahash_update(&rctx->fallback_req);
  1313. }
  1314. static int sa_sha_final(struct ahash_request *req)
  1315. {
  1316. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1317. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1318. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1319. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1320. rctx->fallback_req.base.flags =
  1321. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1322. rctx->fallback_req.result = req->result;
  1323. return crypto_ahash_final(&rctx->fallback_req);
  1324. }
  1325. static int sa_sha_finup(struct ahash_request *req)
  1326. {
  1327. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1328. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1329. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1330. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1331. rctx->fallback_req.base.flags =
  1332. req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1333. rctx->fallback_req.nbytes = req->nbytes;
  1334. rctx->fallback_req.src = req->src;
  1335. rctx->fallback_req.result = req->result;
  1336. return crypto_ahash_finup(&rctx->fallback_req);
  1337. }
  1338. static int sa_sha_import(struct ahash_request *req, const void *in)
  1339. {
  1340. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1341. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1342. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1343. ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
  1344. rctx->fallback_req.base.flags = req->base.flags &
  1345. CRYPTO_TFM_REQ_MAY_SLEEP;
  1346. return crypto_ahash_import(&rctx->fallback_req, in);
  1347. }
  1348. static int sa_sha_export(struct ahash_request *req, void *out)
  1349. {
  1350. struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
  1351. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  1352. struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
  1353. struct ahash_request *subreq = &rctx->fallback_req;
  1354. ahash_request_set_tfm(subreq, ctx->fallback.ahash);
  1355. subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
  1356. return crypto_ahash_export(subreq, out);
  1357. }
  1358. static int sa_sha1_cra_init(struct crypto_tfm *tfm)
  1359. {
  1360. struct algo_data ad = { 0 };
  1361. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1362. sa_sha_cra_init_alg(tfm, "sha1");
  1363. ad.aalg_id = SA_AALG_ID_SHA1;
  1364. ad.hash_size = SHA1_DIGEST_SIZE;
  1365. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
  1366. sa_sha_setup(ctx, &ad);
  1367. return 0;
  1368. }
  1369. static int sa_sha256_cra_init(struct crypto_tfm *tfm)
  1370. {
  1371. struct algo_data ad = { 0 };
  1372. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1373. sa_sha_cra_init_alg(tfm, "sha256");
  1374. ad.aalg_id = SA_AALG_ID_SHA2_256;
  1375. ad.hash_size = SHA256_DIGEST_SIZE;
  1376. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
  1377. sa_sha_setup(ctx, &ad);
  1378. return 0;
  1379. }
  1380. static int sa_sha512_cra_init(struct crypto_tfm *tfm)
  1381. {
  1382. struct algo_data ad = { 0 };
  1383. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1384. sa_sha_cra_init_alg(tfm, "sha512");
  1385. ad.aalg_id = SA_AALG_ID_SHA2_512;
  1386. ad.hash_size = SHA512_DIGEST_SIZE;
  1387. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
  1388. sa_sha_setup(ctx, &ad);
  1389. return 0;
  1390. }
  1391. static void sa_sha_cra_exit(struct crypto_tfm *tfm)
  1392. {
  1393. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
  1394. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1395. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  1396. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  1397. ctx->dec.sc_id, &ctx->dec.sc_phys);
  1398. if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
  1399. sa_free_ctx_info(&ctx->enc, data);
  1400. crypto_free_shash(ctx->shash);
  1401. crypto_free_ahash(ctx->fallback.ahash);
  1402. }
  1403. static void sa_aead_dma_in_callback(void *data)
  1404. {
  1405. struct sa_rx_data *rxd = (struct sa_rx_data *)data;
  1406. struct aead_request *req;
  1407. struct crypto_aead *tfm;
  1408. unsigned int start;
  1409. unsigned int authsize;
  1410. u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
  1411. size_t pl, ml;
  1412. int i;
  1413. int err = 0;
  1414. u32 *mdptr;
  1415. sa_sync_from_device(rxd);
  1416. req = container_of(rxd->req, struct aead_request, base);
  1417. tfm = crypto_aead_reqtfm(req);
  1418. start = req->assoclen + req->cryptlen;
  1419. authsize = crypto_aead_authsize(tfm);
  1420. mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
  1421. for (i = 0; i < (authsize / 4); i++)
  1422. mdptr[i + 4] = swab32(mdptr[i + 4]);
  1423. if (rxd->enc) {
  1424. scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
  1425. 1);
  1426. } else {
  1427. start -= authsize;
  1428. scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
  1429. 0);
  1430. err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
  1431. }
  1432. sa_free_sa_rx_data(rxd);
  1433. aead_request_complete(req, err);
  1434. }
  1435. static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
  1436. const char *fallback)
  1437. {
  1438. struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
  1439. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1440. int ret;
  1441. memzero_explicit(ctx, sizeof(*ctx));
  1442. ctx->dev_data = data;
  1443. ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
  1444. if (IS_ERR(ctx->shash)) {
  1445. dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
  1446. return PTR_ERR(ctx->shash);
  1447. }
  1448. ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
  1449. CRYPTO_ALG_NEED_FALLBACK);
  1450. if (IS_ERR(ctx->fallback.aead)) {
  1451. dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
  1452. fallback);
  1453. return PTR_ERR(ctx->fallback.aead);
  1454. }
  1455. crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
  1456. crypto_aead_reqsize(ctx->fallback.aead));
  1457. ret = sa_init_ctx_info(&ctx->enc, data);
  1458. if (ret)
  1459. return ret;
  1460. ret = sa_init_ctx_info(&ctx->dec, data);
  1461. if (ret) {
  1462. sa_free_ctx_info(&ctx->enc, data);
  1463. return ret;
  1464. }
  1465. dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
  1466. __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
  1467. ctx->dec.sc_id, &ctx->dec.sc_phys);
  1468. return ret;
  1469. }
  1470. static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
  1471. {
  1472. return sa_cra_init_aead(tfm, "sha1",
  1473. "authenc(hmac(sha1-ce),cbc(aes-ce))");
  1474. }
  1475. static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
  1476. {
  1477. return sa_cra_init_aead(tfm, "sha256",
  1478. "authenc(hmac(sha256-ce),cbc(aes-ce))");
  1479. }
  1480. static void sa_exit_tfm_aead(struct crypto_aead *tfm)
  1481. {
  1482. struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
  1483. struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
  1484. crypto_free_shash(ctx->shash);
  1485. crypto_free_aead(ctx->fallback.aead);
  1486. sa_free_ctx_info(&ctx->enc, data);
  1487. sa_free_ctx_info(&ctx->dec, data);
  1488. }
  1489. /* AEAD algorithm configuration interface function */
  1490. static int sa_aead_setkey(struct crypto_aead *authenc,
  1491. const u8 *key, unsigned int keylen,
  1492. struct algo_data *ad)
  1493. {
  1494. struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
  1495. struct crypto_authenc_keys keys;
  1496. int cmdl_len;
  1497. struct sa_cmdl_cfg cfg;
  1498. int key_idx;
  1499. if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
  1500. return -EINVAL;
  1501. /* Convert the key size (16/24/32) to the key size index (0/1/2) */
  1502. key_idx = (keys.enckeylen >> 3) - 2;
  1503. if (key_idx >= 3)
  1504. return -EINVAL;
  1505. ad->ctx = ctx;
  1506. ad->enc_eng.eng_id = SA_ENG_ID_EM1;
  1507. ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
  1508. ad->auth_eng.eng_id = SA_ENG_ID_AM1;
  1509. ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
  1510. ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
  1511. ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
  1512. ad->inv_key = true;
  1513. ad->keyed_mac = true;
  1514. ad->ealg_id = SA_EALG_ID_AES_CBC;
  1515. ad->prep_iopad = sa_prepare_iopads;
  1516. memset(&cfg, 0, sizeof(cfg));
  1517. cfg.enc = true;
  1518. cfg.aalg = ad->aalg_id;
  1519. cfg.enc_eng_id = ad->enc_eng.eng_id;
  1520. cfg.auth_eng_id = ad->auth_eng.eng_id;
  1521. cfg.iv_size = crypto_aead_ivsize(authenc);
  1522. cfg.akey = keys.authkey;
  1523. cfg.akey_len = keys.authkeylen;
  1524. /* Setup Encryption Security Context & Command label template */
  1525. if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey,
  1526. keys.enckeylen, keys.authkey, keys.authkeylen,
  1527. ad, 1, &ctx->enc.epib[1]))
  1528. return -EINVAL;
  1529. cmdl_len = sa_format_cmdl_gen(&cfg,
  1530. (u8 *)ctx->enc.cmdl,
  1531. &ctx->enc.cmdl_upd_info);
  1532. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  1533. return -EINVAL;
  1534. ctx->enc.cmdl_size = cmdl_len;
  1535. /* Setup Decryption Security Context & Command label template */
  1536. if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey,
  1537. keys.enckeylen, keys.authkey, keys.authkeylen,
  1538. ad, 0, &ctx->dec.epib[1]))
  1539. return -EINVAL;
  1540. cfg.enc = false;
  1541. cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
  1542. &ctx->dec.cmdl_upd_info);
  1543. if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
  1544. return -EINVAL;
  1545. ctx->dec.cmdl_size = cmdl_len;
  1546. crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
  1547. crypto_aead_set_flags(ctx->fallback.aead,
  1548. crypto_aead_get_flags(authenc) &
  1549. CRYPTO_TFM_REQ_MASK);
  1550. crypto_aead_setkey(ctx->fallback.aead, key, keylen);
  1551. return 0;
  1552. }
  1553. static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
  1554. {
  1555. struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
  1556. return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
  1557. }
  1558. static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
  1559. const u8 *key, unsigned int keylen)
  1560. {
  1561. struct algo_data ad = { 0 };
  1562. ad.ealg_id = SA_EALG_ID_AES_CBC;
  1563. ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
  1564. ad.hash_size = SHA1_DIGEST_SIZE;
  1565. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
  1566. return sa_aead_setkey(authenc, key, keylen, &ad);
  1567. }
  1568. static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
  1569. const u8 *key, unsigned int keylen)
  1570. {
  1571. struct algo_data ad = { 0 };
  1572. ad.ealg_id = SA_EALG_ID_AES_CBC;
  1573. ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
  1574. ad.hash_size = SHA256_DIGEST_SIZE;
  1575. ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
  1576. return sa_aead_setkey(authenc, key, keylen, &ad);
  1577. }
  1578. static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
  1579. {
  1580. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1581. struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
  1582. struct sa_req sa_req = { 0 };
  1583. size_t auth_size, enc_size;
  1584. enc_size = req->cryptlen;
  1585. auth_size = req->assoclen + req->cryptlen;
  1586. if (!enc) {
  1587. enc_size -= crypto_aead_authsize(tfm);
  1588. auth_size -= crypto_aead_authsize(tfm);
  1589. }
  1590. if (auth_size > SA_MAX_DATA_SZ ||
  1591. (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
  1592. auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
  1593. struct aead_request *subreq = aead_request_ctx(req);
  1594. int ret;
  1595. aead_request_set_tfm(subreq, ctx->fallback.aead);
  1596. aead_request_set_callback(subreq, req->base.flags,
  1597. req->base.complete, req->base.data);
  1598. aead_request_set_crypt(subreq, req->src, req->dst,
  1599. req->cryptlen, req->iv);
  1600. aead_request_set_ad(subreq, req->assoclen);
  1601. ret = enc ? crypto_aead_encrypt(subreq) :
  1602. crypto_aead_decrypt(subreq);
  1603. return ret;
  1604. }
  1605. sa_req.enc_offset = req->assoclen;
  1606. sa_req.enc_size = enc_size;
  1607. sa_req.auth_size = auth_size;
  1608. sa_req.size = auth_size;
  1609. sa_req.enc_iv = iv;
  1610. sa_req.type = CRYPTO_ALG_TYPE_AEAD;
  1611. sa_req.enc = enc;
  1612. sa_req.callback = sa_aead_dma_in_callback;
  1613. sa_req.mdata_size = 52;
  1614. sa_req.base = &req->base;
  1615. sa_req.ctx = ctx;
  1616. sa_req.src = req->src;
  1617. sa_req.dst = req->dst;
  1618. return sa_run(&sa_req);
  1619. }
  1620. /* AEAD algorithm encrypt interface function */
  1621. static int sa_aead_encrypt(struct aead_request *req)
  1622. {
  1623. return sa_aead_run(req, req->iv, 1);
  1624. }
  1625. /* AEAD algorithm decrypt interface function */
  1626. static int sa_aead_decrypt(struct aead_request *req)
  1627. {
  1628. return sa_aead_run(req, req->iv, 0);
  1629. }
  1630. static struct sa_alg_tmpl sa_algs[] = {
  1631. [SA_ALG_CBC_AES] = {
  1632. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1633. .alg.skcipher = {
  1634. .base.cra_name = "cbc(aes)",
  1635. .base.cra_driver_name = "cbc-aes-sa2ul",
  1636. .base.cra_priority = 30000,
  1637. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1638. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1639. CRYPTO_ALG_ASYNC |
  1640. CRYPTO_ALG_NEED_FALLBACK,
  1641. .base.cra_blocksize = AES_BLOCK_SIZE,
  1642. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1643. .base.cra_module = THIS_MODULE,
  1644. .init = sa_cipher_cra_init,
  1645. .exit = sa_cipher_cra_exit,
  1646. .min_keysize = AES_MIN_KEY_SIZE,
  1647. .max_keysize = AES_MAX_KEY_SIZE,
  1648. .ivsize = AES_BLOCK_SIZE,
  1649. .setkey = sa_aes_cbc_setkey,
  1650. .encrypt = sa_encrypt,
  1651. .decrypt = sa_decrypt,
  1652. }
  1653. },
  1654. [SA_ALG_EBC_AES] = {
  1655. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1656. .alg.skcipher = {
  1657. .base.cra_name = "ecb(aes)",
  1658. .base.cra_driver_name = "ecb-aes-sa2ul",
  1659. .base.cra_priority = 30000,
  1660. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1661. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1662. CRYPTO_ALG_ASYNC |
  1663. CRYPTO_ALG_NEED_FALLBACK,
  1664. .base.cra_blocksize = AES_BLOCK_SIZE,
  1665. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1666. .base.cra_module = THIS_MODULE,
  1667. .init = sa_cipher_cra_init,
  1668. .exit = sa_cipher_cra_exit,
  1669. .min_keysize = AES_MIN_KEY_SIZE,
  1670. .max_keysize = AES_MAX_KEY_SIZE,
  1671. .setkey = sa_aes_ecb_setkey,
  1672. .encrypt = sa_encrypt,
  1673. .decrypt = sa_decrypt,
  1674. }
  1675. },
  1676. [SA_ALG_CBC_DES3] = {
  1677. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1678. .alg.skcipher = {
  1679. .base.cra_name = "cbc(des3_ede)",
  1680. .base.cra_driver_name = "cbc-des3-sa2ul",
  1681. .base.cra_priority = 30000,
  1682. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1683. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1684. CRYPTO_ALG_ASYNC |
  1685. CRYPTO_ALG_NEED_FALLBACK,
  1686. .base.cra_blocksize = DES_BLOCK_SIZE,
  1687. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1688. .base.cra_module = THIS_MODULE,
  1689. .init = sa_cipher_cra_init,
  1690. .exit = sa_cipher_cra_exit,
  1691. .min_keysize = 3 * DES_KEY_SIZE,
  1692. .max_keysize = 3 * DES_KEY_SIZE,
  1693. .ivsize = DES_BLOCK_SIZE,
  1694. .setkey = sa_3des_cbc_setkey,
  1695. .encrypt = sa_encrypt,
  1696. .decrypt = sa_decrypt,
  1697. }
  1698. },
  1699. [SA_ALG_ECB_DES3] = {
  1700. .type = CRYPTO_ALG_TYPE_SKCIPHER,
  1701. .alg.skcipher = {
  1702. .base.cra_name = "ecb(des3_ede)",
  1703. .base.cra_driver_name = "ecb-des3-sa2ul",
  1704. .base.cra_priority = 30000,
  1705. .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
  1706. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1707. CRYPTO_ALG_ASYNC |
  1708. CRYPTO_ALG_NEED_FALLBACK,
  1709. .base.cra_blocksize = DES_BLOCK_SIZE,
  1710. .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1711. .base.cra_module = THIS_MODULE,
  1712. .init = sa_cipher_cra_init,
  1713. .exit = sa_cipher_cra_exit,
  1714. .min_keysize = 3 * DES_KEY_SIZE,
  1715. .max_keysize = 3 * DES_KEY_SIZE,
  1716. .setkey = sa_3des_ecb_setkey,
  1717. .encrypt = sa_encrypt,
  1718. .decrypt = sa_decrypt,
  1719. }
  1720. },
  1721. [SA_ALG_SHA1] = {
  1722. .type = CRYPTO_ALG_TYPE_AHASH,
  1723. .alg.ahash = {
  1724. .halg.base = {
  1725. .cra_name = "sha1",
  1726. .cra_driver_name = "sha1-sa2ul",
  1727. .cra_priority = 400,
  1728. .cra_flags = CRYPTO_ALG_TYPE_AHASH |
  1729. CRYPTO_ALG_ASYNC |
  1730. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1731. CRYPTO_ALG_NEED_FALLBACK,
  1732. .cra_blocksize = SHA1_BLOCK_SIZE,
  1733. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1734. .cra_module = THIS_MODULE,
  1735. .cra_init = sa_sha1_cra_init,
  1736. .cra_exit = sa_sha_cra_exit,
  1737. },
  1738. .halg.digestsize = SHA1_DIGEST_SIZE,
  1739. .halg.statesize = sizeof(struct sa_sha_req_ctx) +
  1740. sizeof(struct sha1_state),
  1741. .init = sa_sha_init,
  1742. .update = sa_sha_update,
  1743. .final = sa_sha_final,
  1744. .finup = sa_sha_finup,
  1745. .digest = sa_sha_digest,
  1746. .export = sa_sha_export,
  1747. .import = sa_sha_import,
  1748. },
  1749. },
  1750. [SA_ALG_SHA256] = {
  1751. .type = CRYPTO_ALG_TYPE_AHASH,
  1752. .alg.ahash = {
  1753. .halg.base = {
  1754. .cra_name = "sha256",
  1755. .cra_driver_name = "sha256-sa2ul",
  1756. .cra_priority = 400,
  1757. .cra_flags = CRYPTO_ALG_TYPE_AHASH |
  1758. CRYPTO_ALG_ASYNC |
  1759. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1760. CRYPTO_ALG_NEED_FALLBACK,
  1761. .cra_blocksize = SHA256_BLOCK_SIZE,
  1762. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1763. .cra_module = THIS_MODULE,
  1764. .cra_init = sa_sha256_cra_init,
  1765. .cra_exit = sa_sha_cra_exit,
  1766. },
  1767. .halg.digestsize = SHA256_DIGEST_SIZE,
  1768. .halg.statesize = sizeof(struct sa_sha_req_ctx) +
  1769. sizeof(struct sha256_state),
  1770. .init = sa_sha_init,
  1771. .update = sa_sha_update,
  1772. .final = sa_sha_final,
  1773. .finup = sa_sha_finup,
  1774. .digest = sa_sha_digest,
  1775. .export = sa_sha_export,
  1776. .import = sa_sha_import,
  1777. },
  1778. },
  1779. [SA_ALG_SHA512] = {
  1780. .type = CRYPTO_ALG_TYPE_AHASH,
  1781. .alg.ahash = {
  1782. .halg.base = {
  1783. .cra_name = "sha512",
  1784. .cra_driver_name = "sha512-sa2ul",
  1785. .cra_priority = 400,
  1786. .cra_flags = CRYPTO_ALG_TYPE_AHASH |
  1787. CRYPTO_ALG_ASYNC |
  1788. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1789. CRYPTO_ALG_NEED_FALLBACK,
  1790. .cra_blocksize = SHA512_BLOCK_SIZE,
  1791. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1792. .cra_module = THIS_MODULE,
  1793. .cra_init = sa_sha512_cra_init,
  1794. .cra_exit = sa_sha_cra_exit,
  1795. },
  1796. .halg.digestsize = SHA512_DIGEST_SIZE,
  1797. .halg.statesize = sizeof(struct sa_sha_req_ctx) +
  1798. sizeof(struct sha512_state),
  1799. .init = sa_sha_init,
  1800. .update = sa_sha_update,
  1801. .final = sa_sha_final,
  1802. .finup = sa_sha_finup,
  1803. .digest = sa_sha_digest,
  1804. .export = sa_sha_export,
  1805. .import = sa_sha_import,
  1806. },
  1807. },
  1808. [SA_ALG_AUTHENC_SHA1_AES] = {
  1809. .type = CRYPTO_ALG_TYPE_AEAD,
  1810. .alg.aead = {
  1811. .base = {
  1812. .cra_name = "authenc(hmac(sha1),cbc(aes))",
  1813. .cra_driver_name =
  1814. "authenc(hmac(sha1),cbc(aes))-sa2ul",
  1815. .cra_blocksize = AES_BLOCK_SIZE,
  1816. .cra_flags = CRYPTO_ALG_TYPE_AEAD |
  1817. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1818. CRYPTO_ALG_ASYNC |
  1819. CRYPTO_ALG_NEED_FALLBACK,
  1820. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1821. .cra_module = THIS_MODULE,
  1822. .cra_priority = 3000,
  1823. },
  1824. .ivsize = AES_BLOCK_SIZE,
  1825. .maxauthsize = SHA1_DIGEST_SIZE,
  1826. .init = sa_cra_init_aead_sha1,
  1827. .exit = sa_exit_tfm_aead,
  1828. .setkey = sa_aead_cbc_sha1_setkey,
  1829. .setauthsize = sa_aead_setauthsize,
  1830. .encrypt = sa_aead_encrypt,
  1831. .decrypt = sa_aead_decrypt,
  1832. },
  1833. },
  1834. [SA_ALG_AUTHENC_SHA256_AES] = {
  1835. .type = CRYPTO_ALG_TYPE_AEAD,
  1836. .alg.aead = {
  1837. .base = {
  1838. .cra_name = "authenc(hmac(sha256),cbc(aes))",
  1839. .cra_driver_name =
  1840. "authenc(hmac(sha256),cbc(aes))-sa2ul",
  1841. .cra_blocksize = AES_BLOCK_SIZE,
  1842. .cra_flags = CRYPTO_ALG_TYPE_AEAD |
  1843. CRYPTO_ALG_KERN_DRIVER_ONLY |
  1844. CRYPTO_ALG_ASYNC |
  1845. CRYPTO_ALG_NEED_FALLBACK,
  1846. .cra_ctxsize = sizeof(struct sa_tfm_ctx),
  1847. .cra_module = THIS_MODULE,
  1848. .cra_alignmask = 0,
  1849. .cra_priority = 3000,
  1850. },
  1851. .ivsize = AES_BLOCK_SIZE,
  1852. .maxauthsize = SHA256_DIGEST_SIZE,
  1853. .init = sa_cra_init_aead_sha256,
  1854. .exit = sa_exit_tfm_aead,
  1855. .setkey = sa_aead_cbc_sha256_setkey,
  1856. .setauthsize = sa_aead_setauthsize,
  1857. .encrypt = sa_aead_encrypt,
  1858. .decrypt = sa_aead_decrypt,
  1859. },
  1860. },
  1861. };
  1862. /* Register the algorithms in crypto framework */
  1863. static void sa_register_algos(struct sa_crypto_data *dev_data)
  1864. {
  1865. const struct sa_match_data *match_data = dev_data->match_data;
  1866. struct device *dev = dev_data->dev;
  1867. char *alg_name;
  1868. u32 type;
  1869. int i, err;
  1870. for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
  1871. /* Skip unsupported algos */
  1872. if (!(match_data->supported_algos & BIT(i)))
  1873. continue;
  1874. type = sa_algs[i].type;
  1875. if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
  1876. alg_name = sa_algs[i].alg.skcipher.base.cra_name;
  1877. err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
  1878. } else if (type == CRYPTO_ALG_TYPE_AHASH) {
  1879. alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
  1880. err = crypto_register_ahash(&sa_algs[i].alg.ahash);
  1881. } else if (type == CRYPTO_ALG_TYPE_AEAD) {
  1882. alg_name = sa_algs[i].alg.aead.base.cra_name;
  1883. err = crypto_register_aead(&sa_algs[i].alg.aead);
  1884. } else {
  1885. dev_err(dev,
  1886. "un-supported crypto algorithm (%d)",
  1887. sa_algs[i].type);
  1888. continue;
  1889. }
  1890. if (err)
  1891. dev_err(dev, "Failed to register '%s'\n", alg_name);
  1892. else
  1893. sa_algs[i].registered = true;
  1894. }
  1895. }
  1896. /* Unregister the algorithms in crypto framework */
  1897. static void sa_unregister_algos(const struct device *dev)
  1898. {
  1899. u32 type;
  1900. int i;
  1901. for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
  1902. type = sa_algs[i].type;
  1903. if (!sa_algs[i].registered)
  1904. continue;
  1905. if (type == CRYPTO_ALG_TYPE_SKCIPHER)
  1906. crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
  1907. else if (type == CRYPTO_ALG_TYPE_AHASH)
  1908. crypto_unregister_ahash(&sa_algs[i].alg.ahash);
  1909. else if (type == CRYPTO_ALG_TYPE_AEAD)
  1910. crypto_unregister_aead(&sa_algs[i].alg.aead);
  1911. sa_algs[i].registered = false;
  1912. }
  1913. }
  1914. static int sa_init_mem(struct sa_crypto_data *dev_data)
  1915. {
  1916. struct device *dev = &dev_data->pdev->dev;
  1917. /* Setup dma pool for security context buffers */
  1918. dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
  1919. SA_CTX_MAX_SZ, 64, 0);
  1920. if (!dev_data->sc_pool) {
  1921. dev_err(dev, "Failed to create dma pool");
  1922. return -ENOMEM;
  1923. }
  1924. return 0;
  1925. }
  1926. static int sa_dma_init(struct sa_crypto_data *dd)
  1927. {
  1928. int ret;
  1929. struct dma_slave_config cfg;
  1930. dd->dma_rx1 = NULL;
  1931. dd->dma_tx = NULL;
  1932. dd->dma_rx2 = NULL;
  1933. ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
  1934. if (ret)
  1935. return ret;
  1936. dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
  1937. if (IS_ERR(dd->dma_rx1))
  1938. return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
  1939. "Unable to request rx1 DMA channel\n");
  1940. dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
  1941. if (IS_ERR(dd->dma_rx2)) {
  1942. ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
  1943. "Unable to request rx2 DMA channel\n");
  1944. goto err_dma_rx2;
  1945. }
  1946. dd->dma_tx = dma_request_chan(dd->dev, "tx");
  1947. if (IS_ERR(dd->dma_tx)) {
  1948. ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
  1949. "Unable to request tx DMA channel\n");
  1950. goto err_dma_tx;
  1951. }
  1952. memzero_explicit(&cfg, sizeof(cfg));
  1953. cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  1954. cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  1955. cfg.src_maxburst = 4;
  1956. cfg.dst_maxburst = 4;
  1957. ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
  1958. if (ret) {
  1959. dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
  1960. ret);
  1961. goto err_dma_config;
  1962. }
  1963. ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
  1964. if (ret) {
  1965. dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
  1966. ret);
  1967. goto err_dma_config;
  1968. }
  1969. ret = dmaengine_slave_config(dd->dma_tx, &cfg);
  1970. if (ret) {
  1971. dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
  1972. ret);
  1973. goto err_dma_config;
  1974. }
  1975. return 0;
  1976. err_dma_config:
  1977. dma_release_channel(dd->dma_tx);
  1978. err_dma_tx:
  1979. dma_release_channel(dd->dma_rx2);
  1980. err_dma_rx2:
  1981. dma_release_channel(dd->dma_rx1);
  1982. return ret;
  1983. }
  1984. static int sa_link_child(struct device *dev, void *data)
  1985. {
  1986. struct device *parent = data;
  1987. device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
  1988. return 0;
  1989. }
  1990. static struct sa_match_data am654_match_data = {
  1991. .priv = 1,
  1992. .priv_id = 1,
  1993. .supported_algos = BIT(SA_ALG_CBC_AES) |
  1994. BIT(SA_ALG_EBC_AES) |
  1995. BIT(SA_ALG_CBC_DES3) |
  1996. BIT(SA_ALG_ECB_DES3) |
  1997. BIT(SA_ALG_SHA1) |
  1998. BIT(SA_ALG_SHA256) |
  1999. BIT(SA_ALG_SHA512) |
  2000. BIT(SA_ALG_AUTHENC_SHA1_AES) |
  2001. BIT(SA_ALG_AUTHENC_SHA256_AES),
  2002. };
  2003. static struct sa_match_data am64_match_data = {
  2004. .priv = 0,
  2005. .priv_id = 0,
  2006. .supported_algos = BIT(SA_ALG_CBC_AES) |
  2007. BIT(SA_ALG_EBC_AES) |
  2008. BIT(SA_ALG_SHA256) |
  2009. BIT(SA_ALG_SHA512) |
  2010. BIT(SA_ALG_AUTHENC_SHA256_AES),
  2011. };
  2012. static const struct of_device_id of_match[] = {
  2013. { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, },
  2014. { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, },
  2015. { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, },
  2016. { .compatible = "ti,am62-sa3ul", .data = &am64_match_data, },
  2017. {},
  2018. };
  2019. MODULE_DEVICE_TABLE(of, of_match);
  2020. static int sa_ul_probe(struct platform_device *pdev)
  2021. {
  2022. struct device *dev = &pdev->dev;
  2023. struct device_node *node = dev->of_node;
  2024. static void __iomem *saul_base;
  2025. struct sa_crypto_data *dev_data;
  2026. u32 status, val;
  2027. int ret;
  2028. dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
  2029. if (!dev_data)
  2030. return -ENOMEM;
  2031. dev_data->match_data = of_device_get_match_data(dev);
  2032. if (!dev_data->match_data)
  2033. return -ENODEV;
  2034. saul_base = devm_platform_ioremap_resource(pdev, 0);
  2035. if (IS_ERR(saul_base))
  2036. return PTR_ERR(saul_base);
  2037. sa_k3_dev = dev;
  2038. dev_data->dev = dev;
  2039. dev_data->pdev = pdev;
  2040. dev_data->base = saul_base;
  2041. platform_set_drvdata(pdev, dev_data);
  2042. dev_set_drvdata(sa_k3_dev, dev_data);
  2043. pm_runtime_enable(dev);
  2044. ret = pm_runtime_resume_and_get(dev);
  2045. if (ret < 0) {
  2046. dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret);
  2047. pm_runtime_disable(dev);
  2048. return ret;
  2049. }
  2050. sa_init_mem(dev_data);
  2051. ret = sa_dma_init(dev_data);
  2052. if (ret)
  2053. goto destroy_dma_pool;
  2054. spin_lock_init(&dev_data->scid_lock);
  2055. val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
  2056. SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
  2057. SA_EEC_TRNG_EN;
  2058. status = readl_relaxed(saul_base + SA_ENGINE_STATUS);
  2059. /* Only enable engines if all are not already enabled */
  2060. if (val & ~status)
  2061. writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
  2062. sa_register_algos(dev_data);
  2063. ret = of_platform_populate(node, NULL, NULL, dev);
  2064. if (ret)
  2065. goto release_dma;
  2066. device_for_each_child(dev, dev, sa_link_child);
  2067. return 0;
  2068. release_dma:
  2069. sa_unregister_algos(dev);
  2070. dma_release_channel(dev_data->dma_rx2);
  2071. dma_release_channel(dev_data->dma_rx1);
  2072. dma_release_channel(dev_data->dma_tx);
  2073. destroy_dma_pool:
  2074. dma_pool_destroy(dev_data->sc_pool);
  2075. pm_runtime_put_sync(dev);
  2076. pm_runtime_disable(dev);
  2077. return ret;
  2078. }
  2079. static int sa_ul_remove(struct platform_device *pdev)
  2080. {
  2081. struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
  2082. of_platform_depopulate(&pdev->dev);
  2083. sa_unregister_algos(&pdev->dev);
  2084. dma_release_channel(dev_data->dma_rx2);
  2085. dma_release_channel(dev_data->dma_rx1);
  2086. dma_release_channel(dev_data->dma_tx);
  2087. dma_pool_destroy(dev_data->sc_pool);
  2088. platform_set_drvdata(pdev, NULL);
  2089. pm_runtime_put_sync(&pdev->dev);
  2090. pm_runtime_disable(&pdev->dev);
  2091. return 0;
  2092. }
  2093. static struct platform_driver sa_ul_driver = {
  2094. .probe = sa_ul_probe,
  2095. .remove = sa_ul_remove,
  2096. .driver = {
  2097. .name = "saul-crypto",
  2098. .of_match_table = of_match,
  2099. },
  2100. };
  2101. module_platform_driver(sa_ul_driver);
  2102. MODULE_LICENSE("GPL v2");