atmel-aes.c 67 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Cryptographic API.
  4. *
  5. * Support for ATMEL AES HW acceleration.
  6. *
  7. * Copyright (c) 2012 Eukréa Electromatique - ATMEL
  8. * Author: Nicolas Royer <[email protected]>
  9. *
  10. * Some ideas are from omap-aes.c driver.
  11. */
  12. #include <linux/kernel.h>
  13. #include <linux/module.h>
  14. #include <linux/slab.h>
  15. #include <linux/err.h>
  16. #include <linux/clk.h>
  17. #include <linux/io.h>
  18. #include <linux/hw_random.h>
  19. #include <linux/platform_device.h>
  20. #include <linux/device.h>
  21. #include <linux/dmaengine.h>
  22. #include <linux/init.h>
  23. #include <linux/errno.h>
  24. #include <linux/interrupt.h>
  25. #include <linux/irq.h>
  26. #include <linux/scatterlist.h>
  27. #include <linux/dma-mapping.h>
  28. #include <linux/of_device.h>
  29. #include <linux/delay.h>
  30. #include <linux/crypto.h>
  31. #include <crypto/scatterwalk.h>
  32. #include <crypto/algapi.h>
  33. #include <crypto/aes.h>
  34. #include <crypto/gcm.h>
  35. #include <crypto/xts.h>
  36. #include <crypto/internal/aead.h>
  37. #include <crypto/internal/skcipher.h>
  38. #include "atmel-aes-regs.h"
  39. #include "atmel-authenc.h"
  40. #define ATMEL_AES_PRIORITY 300
  41. #define ATMEL_AES_BUFFER_ORDER 2
  42. #define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER)
  43. #define CFB8_BLOCK_SIZE 1
  44. #define CFB16_BLOCK_SIZE 2
  45. #define CFB32_BLOCK_SIZE 4
  46. #define CFB64_BLOCK_SIZE 8
  47. #define SIZE_IN_WORDS(x) ((x) >> 2)
  48. /* AES flags */
  49. /* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */
  50. #define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC
  51. #define AES_FLAGS_GTAGEN AES_MR_GTAGEN
  52. #define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK)
  53. #define AES_FLAGS_ECB AES_MR_OPMOD_ECB
  54. #define AES_FLAGS_CBC AES_MR_OPMOD_CBC
  55. #define AES_FLAGS_OFB AES_MR_OPMOD_OFB
  56. #define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b)
  57. #define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b)
  58. #define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b)
  59. #define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b)
  60. #define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b)
  61. #define AES_FLAGS_CTR AES_MR_OPMOD_CTR
  62. #define AES_FLAGS_GCM AES_MR_OPMOD_GCM
  63. #define AES_FLAGS_XTS AES_MR_OPMOD_XTS
  64. #define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \
  65. AES_FLAGS_ENCRYPT | \
  66. AES_FLAGS_GTAGEN)
  67. #define AES_FLAGS_BUSY BIT(3)
  68. #define AES_FLAGS_DUMP_REG BIT(4)
  69. #define AES_FLAGS_OWN_SHA BIT(5)
  70. #define AES_FLAGS_PERSISTENT AES_FLAGS_BUSY
  71. #define ATMEL_AES_QUEUE_LENGTH 50
  72. #define ATMEL_AES_DMA_THRESHOLD 256
  73. struct atmel_aes_caps {
  74. bool has_dualbuff;
  75. bool has_cfb64;
  76. bool has_gcm;
  77. bool has_xts;
  78. bool has_authenc;
  79. u32 max_burst_size;
  80. };
  81. struct atmel_aes_dev;
  82. typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *);
  83. struct atmel_aes_base_ctx {
  84. struct atmel_aes_dev *dd;
  85. atmel_aes_fn_t start;
  86. int keylen;
  87. u32 key[AES_KEYSIZE_256 / sizeof(u32)];
  88. u16 block_size;
  89. bool is_aead;
  90. };
  91. struct atmel_aes_ctx {
  92. struct atmel_aes_base_ctx base;
  93. };
  94. struct atmel_aes_ctr_ctx {
  95. struct atmel_aes_base_ctx base;
  96. __be32 iv[AES_BLOCK_SIZE / sizeof(u32)];
  97. size_t offset;
  98. struct scatterlist src[2];
  99. struct scatterlist dst[2];
  100. u32 blocks;
  101. };
  102. struct atmel_aes_gcm_ctx {
  103. struct atmel_aes_base_ctx base;
  104. struct scatterlist src[2];
  105. struct scatterlist dst[2];
  106. __be32 j0[AES_BLOCK_SIZE / sizeof(u32)];
  107. u32 tag[AES_BLOCK_SIZE / sizeof(u32)];
  108. __be32 ghash[AES_BLOCK_SIZE / sizeof(u32)];
  109. size_t textlen;
  110. const __be32 *ghash_in;
  111. __be32 *ghash_out;
  112. atmel_aes_fn_t ghash_resume;
  113. };
  114. struct atmel_aes_xts_ctx {
  115. struct atmel_aes_base_ctx base;
  116. u32 key2[AES_KEYSIZE_256 / sizeof(u32)];
  117. struct crypto_skcipher *fallback_tfm;
  118. };
  119. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  120. struct atmel_aes_authenc_ctx {
  121. struct atmel_aes_base_ctx base;
  122. struct atmel_sha_authenc_ctx *auth;
  123. };
  124. #endif
  125. struct atmel_aes_reqctx {
  126. unsigned long mode;
  127. u8 lastc[AES_BLOCK_SIZE];
  128. struct skcipher_request fallback_req;
  129. };
  130. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  131. struct atmel_aes_authenc_reqctx {
  132. struct atmel_aes_reqctx base;
  133. struct scatterlist src[2];
  134. struct scatterlist dst[2];
  135. size_t textlen;
  136. u32 digest[SHA512_DIGEST_SIZE / sizeof(u32)];
  137. /* auth_req MUST be place last. */
  138. struct ahash_request auth_req;
  139. };
  140. #endif
  141. struct atmel_aes_dma {
  142. struct dma_chan *chan;
  143. struct scatterlist *sg;
  144. int nents;
  145. unsigned int remainder;
  146. unsigned int sg_len;
  147. };
  148. struct atmel_aes_dev {
  149. struct list_head list;
  150. unsigned long phys_base;
  151. void __iomem *io_base;
  152. struct crypto_async_request *areq;
  153. struct atmel_aes_base_ctx *ctx;
  154. bool is_async;
  155. atmel_aes_fn_t resume;
  156. atmel_aes_fn_t cpu_transfer_complete;
  157. struct device *dev;
  158. struct clk *iclk;
  159. int irq;
  160. unsigned long flags;
  161. spinlock_t lock;
  162. struct crypto_queue queue;
  163. struct tasklet_struct done_task;
  164. struct tasklet_struct queue_task;
  165. size_t total;
  166. size_t datalen;
  167. u32 *data;
  168. struct atmel_aes_dma src;
  169. struct atmel_aes_dma dst;
  170. size_t buflen;
  171. void *buf;
  172. struct scatterlist aligned_sg;
  173. struct scatterlist *real_dst;
  174. struct atmel_aes_caps caps;
  175. u32 hw_version;
  176. };
  177. struct atmel_aes_drv {
  178. struct list_head dev_list;
  179. spinlock_t lock;
  180. };
  181. static struct atmel_aes_drv atmel_aes = {
  182. .dev_list = LIST_HEAD_INIT(atmel_aes.dev_list),
  183. .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock),
  184. };
  185. #ifdef VERBOSE_DEBUG
  186. static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz)
  187. {
  188. switch (offset) {
  189. case AES_CR:
  190. return "CR";
  191. case AES_MR:
  192. return "MR";
  193. case AES_ISR:
  194. return "ISR";
  195. case AES_IMR:
  196. return "IMR";
  197. case AES_IER:
  198. return "IER";
  199. case AES_IDR:
  200. return "IDR";
  201. case AES_KEYWR(0):
  202. case AES_KEYWR(1):
  203. case AES_KEYWR(2):
  204. case AES_KEYWR(3):
  205. case AES_KEYWR(4):
  206. case AES_KEYWR(5):
  207. case AES_KEYWR(6):
  208. case AES_KEYWR(7):
  209. snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2);
  210. break;
  211. case AES_IDATAR(0):
  212. case AES_IDATAR(1):
  213. case AES_IDATAR(2):
  214. case AES_IDATAR(3):
  215. snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2);
  216. break;
  217. case AES_ODATAR(0):
  218. case AES_ODATAR(1):
  219. case AES_ODATAR(2):
  220. case AES_ODATAR(3):
  221. snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2);
  222. break;
  223. case AES_IVR(0):
  224. case AES_IVR(1):
  225. case AES_IVR(2):
  226. case AES_IVR(3):
  227. snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2);
  228. break;
  229. case AES_AADLENR:
  230. return "AADLENR";
  231. case AES_CLENR:
  232. return "CLENR";
  233. case AES_GHASHR(0):
  234. case AES_GHASHR(1):
  235. case AES_GHASHR(2):
  236. case AES_GHASHR(3):
  237. snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2);
  238. break;
  239. case AES_TAGR(0):
  240. case AES_TAGR(1):
  241. case AES_TAGR(2):
  242. case AES_TAGR(3):
  243. snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2);
  244. break;
  245. case AES_CTRR:
  246. return "CTRR";
  247. case AES_GCMHR(0):
  248. case AES_GCMHR(1):
  249. case AES_GCMHR(2):
  250. case AES_GCMHR(3):
  251. snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2);
  252. break;
  253. case AES_EMR:
  254. return "EMR";
  255. case AES_TWR(0):
  256. case AES_TWR(1):
  257. case AES_TWR(2):
  258. case AES_TWR(3):
  259. snprintf(tmp, sz, "TWR[%u]", (offset - AES_TWR(0)) >> 2);
  260. break;
  261. case AES_ALPHAR(0):
  262. case AES_ALPHAR(1):
  263. case AES_ALPHAR(2):
  264. case AES_ALPHAR(3):
  265. snprintf(tmp, sz, "ALPHAR[%u]", (offset - AES_ALPHAR(0)) >> 2);
  266. break;
  267. default:
  268. snprintf(tmp, sz, "0x%02x", offset);
  269. break;
  270. }
  271. return tmp;
  272. }
  273. #endif /* VERBOSE_DEBUG */
  274. /* Shared functions */
  275. static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset)
  276. {
  277. u32 value = readl_relaxed(dd->io_base + offset);
  278. #ifdef VERBOSE_DEBUG
  279. if (dd->flags & AES_FLAGS_DUMP_REG) {
  280. char tmp[16];
  281. dev_vdbg(dd->dev, "read 0x%08x from %s\n", value,
  282. atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
  283. }
  284. #endif /* VERBOSE_DEBUG */
  285. return value;
  286. }
  287. static inline void atmel_aes_write(struct atmel_aes_dev *dd,
  288. u32 offset, u32 value)
  289. {
  290. #ifdef VERBOSE_DEBUG
  291. if (dd->flags & AES_FLAGS_DUMP_REG) {
  292. char tmp[16];
  293. dev_vdbg(dd->dev, "write 0x%08x into %s\n", value,
  294. atmel_aes_reg_name(offset, tmp, sizeof(tmp)));
  295. }
  296. #endif /* VERBOSE_DEBUG */
  297. writel_relaxed(value, dd->io_base + offset);
  298. }
  299. static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset,
  300. u32 *value, int count)
  301. {
  302. for (; count--; value++, offset += 4)
  303. *value = atmel_aes_read(dd, offset);
  304. }
  305. static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset,
  306. const u32 *value, int count)
  307. {
  308. for (; count--; value++, offset += 4)
  309. atmel_aes_write(dd, offset, *value);
  310. }
  311. static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset,
  312. void *value)
  313. {
  314. atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));
  315. }
  316. static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset,
  317. const void *value)
  318. {
  319. atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE));
  320. }
  321. static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd,
  322. atmel_aes_fn_t resume)
  323. {
  324. u32 isr = atmel_aes_read(dd, AES_ISR);
  325. if (unlikely(isr & AES_INT_DATARDY))
  326. return resume(dd);
  327. dd->resume = resume;
  328. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  329. return -EINPROGRESS;
  330. }
  331. static inline size_t atmel_aes_padlen(size_t len, size_t block_size)
  332. {
  333. len &= block_size - 1;
  334. return len ? block_size - len : 0;
  335. }
  336. static struct atmel_aes_dev *atmel_aes_dev_alloc(struct atmel_aes_base_ctx *ctx)
  337. {
  338. struct atmel_aes_dev *aes_dd;
  339. spin_lock_bh(&atmel_aes.lock);
  340. /* One AES IP per SoC. */
  341. aes_dd = list_first_entry_or_null(&atmel_aes.dev_list,
  342. struct atmel_aes_dev, list);
  343. spin_unlock_bh(&atmel_aes.lock);
  344. return aes_dd;
  345. }
  346. static int atmel_aes_hw_init(struct atmel_aes_dev *dd)
  347. {
  348. int err;
  349. err = clk_enable(dd->iclk);
  350. if (err)
  351. return err;
  352. atmel_aes_write(dd, AES_CR, AES_CR_SWRST);
  353. atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET);
  354. return 0;
  355. }
  356. static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd)
  357. {
  358. return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff;
  359. }
  360. static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd)
  361. {
  362. int err;
  363. err = atmel_aes_hw_init(dd);
  364. if (err)
  365. return err;
  366. dd->hw_version = atmel_aes_get_version(dd);
  367. dev_info(dd->dev, "version: 0x%x\n", dd->hw_version);
  368. clk_disable(dd->iclk);
  369. return 0;
  370. }
  371. static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd,
  372. const struct atmel_aes_reqctx *rctx)
  373. {
  374. /* Clear all but persistent flags and set request flags. */
  375. dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode;
  376. }
  377. static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd)
  378. {
  379. return (dd->flags & AES_FLAGS_ENCRYPT);
  380. }
  381. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  382. static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err);
  383. #endif
  384. static void atmel_aes_set_iv_as_last_ciphertext_block(struct atmel_aes_dev *dd)
  385. {
  386. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  387. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  388. struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
  389. unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
  390. if (req->cryptlen < ivsize)
  391. return;
  392. if (rctx->mode & AES_FLAGS_ENCRYPT) {
  393. scatterwalk_map_and_copy(req->iv, req->dst,
  394. req->cryptlen - ivsize, ivsize, 0);
  395. } else {
  396. if (req->src == req->dst)
  397. memcpy(req->iv, rctx->lastc, ivsize);
  398. else
  399. scatterwalk_map_and_copy(req->iv, req->src,
  400. req->cryptlen - ivsize,
  401. ivsize, 0);
  402. }
  403. }
  404. static inline struct atmel_aes_ctr_ctx *
  405. atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx)
  406. {
  407. return container_of(ctx, struct atmel_aes_ctr_ctx, base);
  408. }
  409. static void atmel_aes_ctr_update_req_iv(struct atmel_aes_dev *dd)
  410. {
  411. struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
  412. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  413. struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
  414. unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
  415. int i;
  416. /*
  417. * The CTR transfer works in fragments of data of maximum 1 MByte
  418. * because of the 16 bit CTR counter embedded in the IP. When reaching
  419. * here, ctx->blocks contains the number of blocks of the last fragment
  420. * processed, there is no need to explicit cast it to u16.
  421. */
  422. for (i = 0; i < ctx->blocks; i++)
  423. crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE);
  424. memcpy(req->iv, ctx->iv, ivsize);
  425. }
  426. static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err)
  427. {
  428. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  429. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  430. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  431. if (dd->ctx->is_aead)
  432. atmel_aes_authenc_complete(dd, err);
  433. #endif
  434. clk_disable(dd->iclk);
  435. dd->flags &= ~AES_FLAGS_BUSY;
  436. if (!err && !dd->ctx->is_aead &&
  437. (rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_ECB) {
  438. if ((rctx->mode & AES_FLAGS_OPMODE_MASK) != AES_FLAGS_CTR)
  439. atmel_aes_set_iv_as_last_ciphertext_block(dd);
  440. else
  441. atmel_aes_ctr_update_req_iv(dd);
  442. }
  443. if (dd->is_async)
  444. dd->areq->complete(dd->areq, err);
  445. tasklet_schedule(&dd->queue_task);
  446. return err;
  447. }
  448. static void atmel_aes_write_ctrl_key(struct atmel_aes_dev *dd, bool use_dma,
  449. const __be32 *iv, const u32 *key, int keylen)
  450. {
  451. u32 valmr = 0;
  452. /* MR register must be set before IV registers */
  453. if (keylen == AES_KEYSIZE_128)
  454. valmr |= AES_MR_KEYSIZE_128;
  455. else if (keylen == AES_KEYSIZE_192)
  456. valmr |= AES_MR_KEYSIZE_192;
  457. else
  458. valmr |= AES_MR_KEYSIZE_256;
  459. valmr |= dd->flags & AES_FLAGS_MODE_MASK;
  460. if (use_dma) {
  461. valmr |= AES_MR_SMOD_IDATAR0;
  462. if (dd->caps.has_dualbuff)
  463. valmr |= AES_MR_DUALBUFF;
  464. } else {
  465. valmr |= AES_MR_SMOD_AUTO;
  466. }
  467. atmel_aes_write(dd, AES_MR, valmr);
  468. atmel_aes_write_n(dd, AES_KEYWR(0), key, SIZE_IN_WORDS(keylen));
  469. if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB)
  470. atmel_aes_write_block(dd, AES_IVR(0), iv);
  471. }
  472. static inline void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma,
  473. const __be32 *iv)
  474. {
  475. atmel_aes_write_ctrl_key(dd, use_dma, iv,
  476. dd->ctx->key, dd->ctx->keylen);
  477. }
  478. /* CPU transfer */
  479. static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd)
  480. {
  481. int err = 0;
  482. u32 isr;
  483. for (;;) {
  484. atmel_aes_read_block(dd, AES_ODATAR(0), dd->data);
  485. dd->data += 4;
  486. dd->datalen -= AES_BLOCK_SIZE;
  487. if (dd->datalen < AES_BLOCK_SIZE)
  488. break;
  489. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  490. isr = atmel_aes_read(dd, AES_ISR);
  491. if (!(isr & AES_INT_DATARDY)) {
  492. dd->resume = atmel_aes_cpu_transfer;
  493. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  494. return -EINPROGRESS;
  495. }
  496. }
  497. if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),
  498. dd->buf, dd->total))
  499. err = -EINVAL;
  500. if (err)
  501. return atmel_aes_complete(dd, err);
  502. return dd->cpu_transfer_complete(dd);
  503. }
  504. static int atmel_aes_cpu_start(struct atmel_aes_dev *dd,
  505. struct scatterlist *src,
  506. struct scatterlist *dst,
  507. size_t len,
  508. atmel_aes_fn_t resume)
  509. {
  510. size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE);
  511. if (unlikely(len == 0))
  512. return -EINVAL;
  513. sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);
  514. dd->total = len;
  515. dd->real_dst = dst;
  516. dd->cpu_transfer_complete = resume;
  517. dd->datalen = len + padlen;
  518. dd->data = (u32 *)dd->buf;
  519. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  520. return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer);
  521. }
  522. /* DMA transfer */
  523. static void atmel_aes_dma_callback(void *data);
  524. static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd,
  525. struct scatterlist *sg,
  526. size_t len,
  527. struct atmel_aes_dma *dma)
  528. {
  529. int nents;
  530. if (!IS_ALIGNED(len, dd->ctx->block_size))
  531. return false;
  532. for (nents = 0; sg; sg = sg_next(sg), ++nents) {
  533. if (!IS_ALIGNED(sg->offset, sizeof(u32)))
  534. return false;
  535. if (len <= sg->length) {
  536. if (!IS_ALIGNED(len, dd->ctx->block_size))
  537. return false;
  538. dma->nents = nents+1;
  539. dma->remainder = sg->length - len;
  540. sg->length = len;
  541. return true;
  542. }
  543. if (!IS_ALIGNED(sg->length, dd->ctx->block_size))
  544. return false;
  545. len -= sg->length;
  546. }
  547. return false;
  548. }
  549. static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma)
  550. {
  551. struct scatterlist *sg = dma->sg;
  552. int nents = dma->nents;
  553. if (!dma->remainder)
  554. return;
  555. while (--nents > 0 && sg)
  556. sg = sg_next(sg);
  557. if (!sg)
  558. return;
  559. sg->length += dma->remainder;
  560. }
  561. static int atmel_aes_map(struct atmel_aes_dev *dd,
  562. struct scatterlist *src,
  563. struct scatterlist *dst,
  564. size_t len)
  565. {
  566. bool src_aligned, dst_aligned;
  567. size_t padlen;
  568. dd->total = len;
  569. dd->src.sg = src;
  570. dd->dst.sg = dst;
  571. dd->real_dst = dst;
  572. src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src);
  573. if (src == dst)
  574. dst_aligned = src_aligned;
  575. else
  576. dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst);
  577. if (!src_aligned || !dst_aligned) {
  578. padlen = atmel_aes_padlen(len, dd->ctx->block_size);
  579. if (dd->buflen < len + padlen)
  580. return -ENOMEM;
  581. if (!src_aligned) {
  582. sg_copy_to_buffer(src, sg_nents(src), dd->buf, len);
  583. dd->src.sg = &dd->aligned_sg;
  584. dd->src.nents = 1;
  585. dd->src.remainder = 0;
  586. }
  587. if (!dst_aligned) {
  588. dd->dst.sg = &dd->aligned_sg;
  589. dd->dst.nents = 1;
  590. dd->dst.remainder = 0;
  591. }
  592. sg_init_table(&dd->aligned_sg, 1);
  593. sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen);
  594. }
  595. if (dd->src.sg == dd->dst.sg) {
  596. dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,
  597. DMA_BIDIRECTIONAL);
  598. dd->dst.sg_len = dd->src.sg_len;
  599. if (!dd->src.sg_len)
  600. return -EFAULT;
  601. } else {
  602. dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents,
  603. DMA_TO_DEVICE);
  604. if (!dd->src.sg_len)
  605. return -EFAULT;
  606. dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents,
  607. DMA_FROM_DEVICE);
  608. if (!dd->dst.sg_len) {
  609. dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
  610. DMA_TO_DEVICE);
  611. return -EFAULT;
  612. }
  613. }
  614. return 0;
  615. }
  616. static void atmel_aes_unmap(struct atmel_aes_dev *dd)
  617. {
  618. if (dd->src.sg == dd->dst.sg) {
  619. dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
  620. DMA_BIDIRECTIONAL);
  621. if (dd->src.sg != &dd->aligned_sg)
  622. atmel_aes_restore_sg(&dd->src);
  623. } else {
  624. dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents,
  625. DMA_FROM_DEVICE);
  626. if (dd->dst.sg != &dd->aligned_sg)
  627. atmel_aes_restore_sg(&dd->dst);
  628. dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents,
  629. DMA_TO_DEVICE);
  630. if (dd->src.sg != &dd->aligned_sg)
  631. atmel_aes_restore_sg(&dd->src);
  632. }
  633. if (dd->dst.sg == &dd->aligned_sg)
  634. sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst),
  635. dd->buf, dd->total);
  636. }
  637. static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
  638. enum dma_slave_buswidth addr_width,
  639. enum dma_transfer_direction dir,
  640. u32 maxburst)
  641. {
  642. struct dma_async_tx_descriptor *desc;
  643. struct dma_slave_config config;
  644. dma_async_tx_callback callback;
  645. struct atmel_aes_dma *dma;
  646. int err;
  647. memset(&config, 0, sizeof(config));
  648. config.src_addr_width = addr_width;
  649. config.dst_addr_width = addr_width;
  650. config.src_maxburst = maxburst;
  651. config.dst_maxburst = maxburst;
  652. switch (dir) {
  653. case DMA_MEM_TO_DEV:
  654. dma = &dd->src;
  655. callback = NULL;
  656. config.dst_addr = dd->phys_base + AES_IDATAR(0);
  657. break;
  658. case DMA_DEV_TO_MEM:
  659. dma = &dd->dst;
  660. callback = atmel_aes_dma_callback;
  661. config.src_addr = dd->phys_base + AES_ODATAR(0);
  662. break;
  663. default:
  664. return -EINVAL;
  665. }
  666. err = dmaengine_slave_config(dma->chan, &config);
  667. if (err)
  668. return err;
  669. desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir,
  670. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  671. if (!desc)
  672. return -ENOMEM;
  673. desc->callback = callback;
  674. desc->callback_param = dd;
  675. dmaengine_submit(desc);
  676. dma_async_issue_pending(dma->chan);
  677. return 0;
  678. }
  679. static int atmel_aes_dma_start(struct atmel_aes_dev *dd,
  680. struct scatterlist *src,
  681. struct scatterlist *dst,
  682. size_t len,
  683. atmel_aes_fn_t resume)
  684. {
  685. enum dma_slave_buswidth addr_width;
  686. u32 maxburst;
  687. int err;
  688. switch (dd->ctx->block_size) {
  689. case CFB8_BLOCK_SIZE:
  690. addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE;
  691. maxburst = 1;
  692. break;
  693. case CFB16_BLOCK_SIZE:
  694. addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES;
  695. maxburst = 1;
  696. break;
  697. case CFB32_BLOCK_SIZE:
  698. case CFB64_BLOCK_SIZE:
  699. addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  700. maxburst = 1;
  701. break;
  702. case AES_BLOCK_SIZE:
  703. addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  704. maxburst = dd->caps.max_burst_size;
  705. break;
  706. default:
  707. err = -EINVAL;
  708. goto exit;
  709. }
  710. err = atmel_aes_map(dd, src, dst, len);
  711. if (err)
  712. goto exit;
  713. dd->resume = resume;
  714. /* Set output DMA transfer first */
  715. err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM,
  716. maxburst);
  717. if (err)
  718. goto unmap;
  719. /* Then set input DMA transfer */
  720. err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV,
  721. maxburst);
  722. if (err)
  723. goto output_transfer_stop;
  724. return -EINPROGRESS;
  725. output_transfer_stop:
  726. dmaengine_terminate_sync(dd->dst.chan);
  727. unmap:
  728. atmel_aes_unmap(dd);
  729. exit:
  730. return atmel_aes_complete(dd, err);
  731. }
  732. static void atmel_aes_dma_callback(void *data)
  733. {
  734. struct atmel_aes_dev *dd = data;
  735. atmel_aes_unmap(dd);
  736. dd->is_async = true;
  737. (void)dd->resume(dd);
  738. }
  739. static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
  740. struct crypto_async_request *new_areq)
  741. {
  742. struct crypto_async_request *areq, *backlog;
  743. struct atmel_aes_base_ctx *ctx;
  744. unsigned long flags;
  745. bool start_async;
  746. int err, ret = 0;
  747. spin_lock_irqsave(&dd->lock, flags);
  748. if (new_areq)
  749. ret = crypto_enqueue_request(&dd->queue, new_areq);
  750. if (dd->flags & AES_FLAGS_BUSY) {
  751. spin_unlock_irqrestore(&dd->lock, flags);
  752. return ret;
  753. }
  754. backlog = crypto_get_backlog(&dd->queue);
  755. areq = crypto_dequeue_request(&dd->queue);
  756. if (areq)
  757. dd->flags |= AES_FLAGS_BUSY;
  758. spin_unlock_irqrestore(&dd->lock, flags);
  759. if (!areq)
  760. return ret;
  761. if (backlog)
  762. backlog->complete(backlog, -EINPROGRESS);
  763. ctx = crypto_tfm_ctx(areq->tfm);
  764. dd->areq = areq;
  765. dd->ctx = ctx;
  766. start_async = (areq != new_areq);
  767. dd->is_async = start_async;
  768. /* WARNING: ctx->start() MAY change dd->is_async. */
  769. err = ctx->start(dd);
  770. return (start_async) ? ret : err;
  771. }
  772. /* AES async block ciphers */
  773. static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd)
  774. {
  775. return atmel_aes_complete(dd, 0);
  776. }
  777. static int atmel_aes_start(struct atmel_aes_dev *dd)
  778. {
  779. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  780. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  781. bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD ||
  782. dd->ctx->block_size != AES_BLOCK_SIZE);
  783. int err;
  784. atmel_aes_set_mode(dd, rctx);
  785. err = atmel_aes_hw_init(dd);
  786. if (err)
  787. return atmel_aes_complete(dd, err);
  788. atmel_aes_write_ctrl(dd, use_dma, (void *)req->iv);
  789. if (use_dma)
  790. return atmel_aes_dma_start(dd, req->src, req->dst,
  791. req->cryptlen,
  792. atmel_aes_transfer_complete);
  793. return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,
  794. atmel_aes_transfer_complete);
  795. }
  796. static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd)
  797. {
  798. struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
  799. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  800. struct scatterlist *src, *dst;
  801. size_t datalen;
  802. u32 ctr;
  803. u16 start, end;
  804. bool use_dma, fragmented = false;
  805. /* Check for transfer completion. */
  806. ctx->offset += dd->total;
  807. if (ctx->offset >= req->cryptlen)
  808. return atmel_aes_transfer_complete(dd);
  809. /* Compute data length. */
  810. datalen = req->cryptlen - ctx->offset;
  811. ctx->blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE);
  812. ctr = be32_to_cpu(ctx->iv[3]);
  813. /* Check 16bit counter overflow. */
  814. start = ctr & 0xffff;
  815. end = start + ctx->blocks - 1;
  816. if (ctx->blocks >> 16 || end < start) {
  817. ctr |= 0xffff;
  818. datalen = AES_BLOCK_SIZE * (0x10000 - start);
  819. fragmented = true;
  820. }
  821. use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD);
  822. /* Jump to offset. */
  823. src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset);
  824. dst = ((req->src == req->dst) ? src :
  825. scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset));
  826. /* Configure hardware. */
  827. atmel_aes_write_ctrl(dd, use_dma, ctx->iv);
  828. if (unlikely(fragmented)) {
  829. /*
  830. * Increment the counter manually to cope with the hardware
  831. * counter overflow.
  832. */
  833. ctx->iv[3] = cpu_to_be32(ctr);
  834. crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE);
  835. }
  836. if (use_dma)
  837. return atmel_aes_dma_start(dd, src, dst, datalen,
  838. atmel_aes_ctr_transfer);
  839. return atmel_aes_cpu_start(dd, src, dst, datalen,
  840. atmel_aes_ctr_transfer);
  841. }
  842. static int atmel_aes_ctr_start(struct atmel_aes_dev *dd)
  843. {
  844. struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx);
  845. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  846. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  847. int err;
  848. atmel_aes_set_mode(dd, rctx);
  849. err = atmel_aes_hw_init(dd);
  850. if (err)
  851. return atmel_aes_complete(dd, err);
  852. memcpy(ctx->iv, req->iv, AES_BLOCK_SIZE);
  853. ctx->offset = 0;
  854. dd->total = 0;
  855. return atmel_aes_ctr_transfer(dd);
  856. }
  857. static int atmel_aes_xts_fallback(struct skcipher_request *req, bool enc)
  858. {
  859. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  860. struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(
  861. crypto_skcipher_reqtfm(req));
  862. skcipher_request_set_tfm(&rctx->fallback_req, ctx->fallback_tfm);
  863. skcipher_request_set_callback(&rctx->fallback_req, req->base.flags,
  864. req->base.complete, req->base.data);
  865. skcipher_request_set_crypt(&rctx->fallback_req, req->src, req->dst,
  866. req->cryptlen, req->iv);
  867. return enc ? crypto_skcipher_encrypt(&rctx->fallback_req) :
  868. crypto_skcipher_decrypt(&rctx->fallback_req);
  869. }
  870. static int atmel_aes_crypt(struct skcipher_request *req, unsigned long mode)
  871. {
  872. struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req);
  873. struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(skcipher);
  874. struct atmel_aes_reqctx *rctx;
  875. u32 opmode = mode & AES_FLAGS_OPMODE_MASK;
  876. if (opmode == AES_FLAGS_XTS) {
  877. if (req->cryptlen < XTS_BLOCK_SIZE)
  878. return -EINVAL;
  879. if (!IS_ALIGNED(req->cryptlen, XTS_BLOCK_SIZE))
  880. return atmel_aes_xts_fallback(req,
  881. mode & AES_FLAGS_ENCRYPT);
  882. }
  883. /*
  884. * ECB, CBC, CFB, OFB or CTR mode require the plaintext and ciphertext
  885. * to have a positve integer length.
  886. */
  887. if (!req->cryptlen && opmode != AES_FLAGS_XTS)
  888. return 0;
  889. if ((opmode == AES_FLAGS_ECB || opmode == AES_FLAGS_CBC) &&
  890. !IS_ALIGNED(req->cryptlen, crypto_skcipher_blocksize(skcipher)))
  891. return -EINVAL;
  892. switch (mode & AES_FLAGS_OPMODE_MASK) {
  893. case AES_FLAGS_CFB8:
  894. ctx->block_size = CFB8_BLOCK_SIZE;
  895. break;
  896. case AES_FLAGS_CFB16:
  897. ctx->block_size = CFB16_BLOCK_SIZE;
  898. break;
  899. case AES_FLAGS_CFB32:
  900. ctx->block_size = CFB32_BLOCK_SIZE;
  901. break;
  902. case AES_FLAGS_CFB64:
  903. ctx->block_size = CFB64_BLOCK_SIZE;
  904. break;
  905. default:
  906. ctx->block_size = AES_BLOCK_SIZE;
  907. break;
  908. }
  909. ctx->is_aead = false;
  910. rctx = skcipher_request_ctx(req);
  911. rctx->mode = mode;
  912. if (opmode != AES_FLAGS_ECB &&
  913. !(mode & AES_FLAGS_ENCRYPT) && req->src == req->dst) {
  914. unsigned int ivsize = crypto_skcipher_ivsize(skcipher);
  915. if (req->cryptlen >= ivsize)
  916. scatterwalk_map_and_copy(rctx->lastc, req->src,
  917. req->cryptlen - ivsize,
  918. ivsize, 0);
  919. }
  920. return atmel_aes_handle_queue(ctx->dd, &req->base);
  921. }
  922. static int atmel_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
  923. unsigned int keylen)
  924. {
  925. struct atmel_aes_base_ctx *ctx = crypto_skcipher_ctx(tfm);
  926. if (keylen != AES_KEYSIZE_128 &&
  927. keylen != AES_KEYSIZE_192 &&
  928. keylen != AES_KEYSIZE_256)
  929. return -EINVAL;
  930. memcpy(ctx->key, key, keylen);
  931. ctx->keylen = keylen;
  932. return 0;
  933. }
  934. static int atmel_aes_ecb_encrypt(struct skcipher_request *req)
  935. {
  936. return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
  937. }
  938. static int atmel_aes_ecb_decrypt(struct skcipher_request *req)
  939. {
  940. return atmel_aes_crypt(req, AES_FLAGS_ECB);
  941. }
  942. static int atmel_aes_cbc_encrypt(struct skcipher_request *req)
  943. {
  944. return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
  945. }
  946. static int atmel_aes_cbc_decrypt(struct skcipher_request *req)
  947. {
  948. return atmel_aes_crypt(req, AES_FLAGS_CBC);
  949. }
  950. static int atmel_aes_ofb_encrypt(struct skcipher_request *req)
  951. {
  952. return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT);
  953. }
  954. static int atmel_aes_ofb_decrypt(struct skcipher_request *req)
  955. {
  956. return atmel_aes_crypt(req, AES_FLAGS_OFB);
  957. }
  958. static int atmel_aes_cfb_encrypt(struct skcipher_request *req)
  959. {
  960. return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT);
  961. }
  962. static int atmel_aes_cfb_decrypt(struct skcipher_request *req)
  963. {
  964. return atmel_aes_crypt(req, AES_FLAGS_CFB128);
  965. }
  966. static int atmel_aes_cfb64_encrypt(struct skcipher_request *req)
  967. {
  968. return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT);
  969. }
  970. static int atmel_aes_cfb64_decrypt(struct skcipher_request *req)
  971. {
  972. return atmel_aes_crypt(req, AES_FLAGS_CFB64);
  973. }
  974. static int atmel_aes_cfb32_encrypt(struct skcipher_request *req)
  975. {
  976. return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT);
  977. }
  978. static int atmel_aes_cfb32_decrypt(struct skcipher_request *req)
  979. {
  980. return atmel_aes_crypt(req, AES_FLAGS_CFB32);
  981. }
  982. static int atmel_aes_cfb16_encrypt(struct skcipher_request *req)
  983. {
  984. return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT);
  985. }
  986. static int atmel_aes_cfb16_decrypt(struct skcipher_request *req)
  987. {
  988. return atmel_aes_crypt(req, AES_FLAGS_CFB16);
  989. }
  990. static int atmel_aes_cfb8_encrypt(struct skcipher_request *req)
  991. {
  992. return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT);
  993. }
  994. static int atmel_aes_cfb8_decrypt(struct skcipher_request *req)
  995. {
  996. return atmel_aes_crypt(req, AES_FLAGS_CFB8);
  997. }
  998. static int atmel_aes_ctr_encrypt(struct skcipher_request *req)
  999. {
  1000. return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT);
  1001. }
  1002. static int atmel_aes_ctr_decrypt(struct skcipher_request *req)
  1003. {
  1004. return atmel_aes_crypt(req, AES_FLAGS_CTR);
  1005. }
  1006. static int atmel_aes_init_tfm(struct crypto_skcipher *tfm)
  1007. {
  1008. struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  1009. struct atmel_aes_dev *dd;
  1010. dd = atmel_aes_dev_alloc(&ctx->base);
  1011. if (!dd)
  1012. return -ENODEV;
  1013. crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  1014. ctx->base.dd = dd;
  1015. ctx->base.start = atmel_aes_start;
  1016. return 0;
  1017. }
  1018. static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
  1019. {
  1020. struct atmel_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
  1021. struct atmel_aes_dev *dd;
  1022. dd = atmel_aes_dev_alloc(&ctx->base);
  1023. if (!dd)
  1024. return -ENODEV;
  1025. crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  1026. ctx->base.dd = dd;
  1027. ctx->base.start = atmel_aes_ctr_start;
  1028. return 0;
  1029. }
  1030. static struct skcipher_alg aes_algs[] = {
  1031. {
  1032. .base.cra_name = "ecb(aes)",
  1033. .base.cra_driver_name = "atmel-ecb-aes",
  1034. .base.cra_blocksize = AES_BLOCK_SIZE,
  1035. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1036. .init = atmel_aes_init_tfm,
  1037. .min_keysize = AES_MIN_KEY_SIZE,
  1038. .max_keysize = AES_MAX_KEY_SIZE,
  1039. .setkey = atmel_aes_setkey,
  1040. .encrypt = atmel_aes_ecb_encrypt,
  1041. .decrypt = atmel_aes_ecb_decrypt,
  1042. },
  1043. {
  1044. .base.cra_name = "cbc(aes)",
  1045. .base.cra_driver_name = "atmel-cbc-aes",
  1046. .base.cra_blocksize = AES_BLOCK_SIZE,
  1047. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1048. .init = atmel_aes_init_tfm,
  1049. .min_keysize = AES_MIN_KEY_SIZE,
  1050. .max_keysize = AES_MAX_KEY_SIZE,
  1051. .setkey = atmel_aes_setkey,
  1052. .encrypt = atmel_aes_cbc_encrypt,
  1053. .decrypt = atmel_aes_cbc_decrypt,
  1054. .ivsize = AES_BLOCK_SIZE,
  1055. },
  1056. {
  1057. .base.cra_name = "ofb(aes)",
  1058. .base.cra_driver_name = "atmel-ofb-aes",
  1059. .base.cra_blocksize = 1,
  1060. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1061. .init = atmel_aes_init_tfm,
  1062. .min_keysize = AES_MIN_KEY_SIZE,
  1063. .max_keysize = AES_MAX_KEY_SIZE,
  1064. .setkey = atmel_aes_setkey,
  1065. .encrypt = atmel_aes_ofb_encrypt,
  1066. .decrypt = atmel_aes_ofb_decrypt,
  1067. .ivsize = AES_BLOCK_SIZE,
  1068. },
  1069. {
  1070. .base.cra_name = "cfb(aes)",
  1071. .base.cra_driver_name = "atmel-cfb-aes",
  1072. .base.cra_blocksize = AES_BLOCK_SIZE,
  1073. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1074. .init = atmel_aes_init_tfm,
  1075. .min_keysize = AES_MIN_KEY_SIZE,
  1076. .max_keysize = AES_MAX_KEY_SIZE,
  1077. .setkey = atmel_aes_setkey,
  1078. .encrypt = atmel_aes_cfb_encrypt,
  1079. .decrypt = atmel_aes_cfb_decrypt,
  1080. .ivsize = AES_BLOCK_SIZE,
  1081. },
  1082. {
  1083. .base.cra_name = "cfb32(aes)",
  1084. .base.cra_driver_name = "atmel-cfb32-aes",
  1085. .base.cra_blocksize = CFB32_BLOCK_SIZE,
  1086. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1087. .init = atmel_aes_init_tfm,
  1088. .min_keysize = AES_MIN_KEY_SIZE,
  1089. .max_keysize = AES_MAX_KEY_SIZE,
  1090. .setkey = atmel_aes_setkey,
  1091. .encrypt = atmel_aes_cfb32_encrypt,
  1092. .decrypt = atmel_aes_cfb32_decrypt,
  1093. .ivsize = AES_BLOCK_SIZE,
  1094. },
  1095. {
  1096. .base.cra_name = "cfb16(aes)",
  1097. .base.cra_driver_name = "atmel-cfb16-aes",
  1098. .base.cra_blocksize = CFB16_BLOCK_SIZE,
  1099. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1100. .init = atmel_aes_init_tfm,
  1101. .min_keysize = AES_MIN_KEY_SIZE,
  1102. .max_keysize = AES_MAX_KEY_SIZE,
  1103. .setkey = atmel_aes_setkey,
  1104. .encrypt = atmel_aes_cfb16_encrypt,
  1105. .decrypt = atmel_aes_cfb16_decrypt,
  1106. .ivsize = AES_BLOCK_SIZE,
  1107. },
  1108. {
  1109. .base.cra_name = "cfb8(aes)",
  1110. .base.cra_driver_name = "atmel-cfb8-aes",
  1111. .base.cra_blocksize = CFB8_BLOCK_SIZE,
  1112. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1113. .init = atmel_aes_init_tfm,
  1114. .min_keysize = AES_MIN_KEY_SIZE,
  1115. .max_keysize = AES_MAX_KEY_SIZE,
  1116. .setkey = atmel_aes_setkey,
  1117. .encrypt = atmel_aes_cfb8_encrypt,
  1118. .decrypt = atmel_aes_cfb8_decrypt,
  1119. .ivsize = AES_BLOCK_SIZE,
  1120. },
  1121. {
  1122. .base.cra_name = "ctr(aes)",
  1123. .base.cra_driver_name = "atmel-ctr-aes",
  1124. .base.cra_blocksize = 1,
  1125. .base.cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx),
  1126. .init = atmel_aes_ctr_init_tfm,
  1127. .min_keysize = AES_MIN_KEY_SIZE,
  1128. .max_keysize = AES_MAX_KEY_SIZE,
  1129. .setkey = atmel_aes_setkey,
  1130. .encrypt = atmel_aes_ctr_encrypt,
  1131. .decrypt = atmel_aes_ctr_decrypt,
  1132. .ivsize = AES_BLOCK_SIZE,
  1133. },
  1134. };
  1135. static struct skcipher_alg aes_cfb64_alg = {
  1136. .base.cra_name = "cfb64(aes)",
  1137. .base.cra_driver_name = "atmel-cfb64-aes",
  1138. .base.cra_blocksize = CFB64_BLOCK_SIZE,
  1139. .base.cra_ctxsize = sizeof(struct atmel_aes_ctx),
  1140. .init = atmel_aes_init_tfm,
  1141. .min_keysize = AES_MIN_KEY_SIZE,
  1142. .max_keysize = AES_MAX_KEY_SIZE,
  1143. .setkey = atmel_aes_setkey,
  1144. .encrypt = atmel_aes_cfb64_encrypt,
  1145. .decrypt = atmel_aes_cfb64_decrypt,
  1146. .ivsize = AES_BLOCK_SIZE,
  1147. };
  1148. /* gcm aead functions */
  1149. static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
  1150. const u32 *data, size_t datalen,
  1151. const __be32 *ghash_in, __be32 *ghash_out,
  1152. atmel_aes_fn_t resume);
  1153. static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd);
  1154. static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd);
  1155. static int atmel_aes_gcm_start(struct atmel_aes_dev *dd);
  1156. static int atmel_aes_gcm_process(struct atmel_aes_dev *dd);
  1157. static int atmel_aes_gcm_length(struct atmel_aes_dev *dd);
  1158. static int atmel_aes_gcm_data(struct atmel_aes_dev *dd);
  1159. static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd);
  1160. static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd);
  1161. static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd);
  1162. static inline struct atmel_aes_gcm_ctx *
  1163. atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx)
  1164. {
  1165. return container_of(ctx, struct atmel_aes_gcm_ctx, base);
  1166. }
  1167. static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd,
  1168. const u32 *data, size_t datalen,
  1169. const __be32 *ghash_in, __be32 *ghash_out,
  1170. atmel_aes_fn_t resume)
  1171. {
  1172. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1173. dd->data = (u32 *)data;
  1174. dd->datalen = datalen;
  1175. ctx->ghash_in = ghash_in;
  1176. ctx->ghash_out = ghash_out;
  1177. ctx->ghash_resume = resume;
  1178. atmel_aes_write_ctrl(dd, false, NULL);
  1179. return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init);
  1180. }
  1181. static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd)
  1182. {
  1183. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1184. /* Set the data length. */
  1185. atmel_aes_write(dd, AES_AADLENR, dd->total);
  1186. atmel_aes_write(dd, AES_CLENR, 0);
  1187. /* If needed, overwrite the GCM Intermediate Hash Word Registers */
  1188. if (ctx->ghash_in)
  1189. atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in);
  1190. return atmel_aes_gcm_ghash_finalize(dd);
  1191. }
  1192. static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd)
  1193. {
  1194. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1195. u32 isr;
  1196. /* Write data into the Input Data Registers. */
  1197. while (dd->datalen > 0) {
  1198. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  1199. dd->data += 4;
  1200. dd->datalen -= AES_BLOCK_SIZE;
  1201. isr = atmel_aes_read(dd, AES_ISR);
  1202. if (!(isr & AES_INT_DATARDY)) {
  1203. dd->resume = atmel_aes_gcm_ghash_finalize;
  1204. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  1205. return -EINPROGRESS;
  1206. }
  1207. }
  1208. /* Read the computed hash from GHASHRx. */
  1209. atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out);
  1210. return ctx->ghash_resume(dd);
  1211. }
  1212. static int atmel_aes_gcm_start(struct atmel_aes_dev *dd)
  1213. {
  1214. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1215. struct aead_request *req = aead_request_cast(dd->areq);
  1216. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1217. struct atmel_aes_reqctx *rctx = aead_request_ctx(req);
  1218. size_t ivsize = crypto_aead_ivsize(tfm);
  1219. size_t datalen, padlen;
  1220. const void *iv = req->iv;
  1221. u8 *data = dd->buf;
  1222. int err;
  1223. atmel_aes_set_mode(dd, rctx);
  1224. err = atmel_aes_hw_init(dd);
  1225. if (err)
  1226. return atmel_aes_complete(dd, err);
  1227. if (likely(ivsize == GCM_AES_IV_SIZE)) {
  1228. memcpy(ctx->j0, iv, ivsize);
  1229. ctx->j0[3] = cpu_to_be32(1);
  1230. return atmel_aes_gcm_process(dd);
  1231. }
  1232. padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE);
  1233. datalen = ivsize + padlen + AES_BLOCK_SIZE;
  1234. if (datalen > dd->buflen)
  1235. return atmel_aes_complete(dd, -EINVAL);
  1236. memcpy(data, iv, ivsize);
  1237. memset(data + ivsize, 0, padlen + sizeof(u64));
  1238. ((__be64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8);
  1239. return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen,
  1240. NULL, ctx->j0, atmel_aes_gcm_process);
  1241. }
  1242. static int atmel_aes_gcm_process(struct atmel_aes_dev *dd)
  1243. {
  1244. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1245. struct aead_request *req = aead_request_cast(dd->areq);
  1246. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1247. bool enc = atmel_aes_is_encrypt(dd);
  1248. u32 authsize;
  1249. /* Compute text length. */
  1250. authsize = crypto_aead_authsize(tfm);
  1251. ctx->textlen = req->cryptlen - (enc ? 0 : authsize);
  1252. /*
  1253. * According to tcrypt test suite, the GCM Automatic Tag Generation
  1254. * fails when both the message and its associated data are empty.
  1255. */
  1256. if (likely(req->assoclen != 0 || ctx->textlen != 0))
  1257. dd->flags |= AES_FLAGS_GTAGEN;
  1258. atmel_aes_write_ctrl(dd, false, NULL);
  1259. return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length);
  1260. }
  1261. static int atmel_aes_gcm_length(struct atmel_aes_dev *dd)
  1262. {
  1263. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1264. struct aead_request *req = aead_request_cast(dd->areq);
  1265. __be32 j0_lsw, *j0 = ctx->j0;
  1266. size_t padlen;
  1267. /* Write incr32(J0) into IV. */
  1268. j0_lsw = j0[3];
  1269. be32_add_cpu(&j0[3], 1);
  1270. atmel_aes_write_block(dd, AES_IVR(0), j0);
  1271. j0[3] = j0_lsw;
  1272. /* Set aad and text lengths. */
  1273. atmel_aes_write(dd, AES_AADLENR, req->assoclen);
  1274. atmel_aes_write(dd, AES_CLENR, ctx->textlen);
  1275. /* Check whether AAD are present. */
  1276. if (unlikely(req->assoclen == 0)) {
  1277. dd->datalen = 0;
  1278. return atmel_aes_gcm_data(dd);
  1279. }
  1280. /* Copy assoc data and add padding. */
  1281. padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE);
  1282. if (unlikely(req->assoclen + padlen > dd->buflen))
  1283. return atmel_aes_complete(dd, -EINVAL);
  1284. sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen);
  1285. /* Write assoc data into the Input Data register. */
  1286. dd->data = (u32 *)dd->buf;
  1287. dd->datalen = req->assoclen + padlen;
  1288. return atmel_aes_gcm_data(dd);
  1289. }
  1290. static int atmel_aes_gcm_data(struct atmel_aes_dev *dd)
  1291. {
  1292. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1293. struct aead_request *req = aead_request_cast(dd->areq);
  1294. bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD);
  1295. struct scatterlist *src, *dst;
  1296. u32 isr, mr;
  1297. /* Write AAD first. */
  1298. while (dd->datalen > 0) {
  1299. atmel_aes_write_block(dd, AES_IDATAR(0), dd->data);
  1300. dd->data += 4;
  1301. dd->datalen -= AES_BLOCK_SIZE;
  1302. isr = atmel_aes_read(dd, AES_ISR);
  1303. if (!(isr & AES_INT_DATARDY)) {
  1304. dd->resume = atmel_aes_gcm_data;
  1305. atmel_aes_write(dd, AES_IER, AES_INT_DATARDY);
  1306. return -EINPROGRESS;
  1307. }
  1308. }
  1309. /* GMAC only. */
  1310. if (unlikely(ctx->textlen == 0))
  1311. return atmel_aes_gcm_tag_init(dd);
  1312. /* Prepare src and dst scatter lists to transfer cipher/plain texts */
  1313. src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen);
  1314. dst = ((req->src == req->dst) ? src :
  1315. scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen));
  1316. if (use_dma) {
  1317. /* Update the Mode Register for DMA transfers. */
  1318. mr = atmel_aes_read(dd, AES_MR);
  1319. mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF);
  1320. mr |= AES_MR_SMOD_IDATAR0;
  1321. if (dd->caps.has_dualbuff)
  1322. mr |= AES_MR_DUALBUFF;
  1323. atmel_aes_write(dd, AES_MR, mr);
  1324. return atmel_aes_dma_start(dd, src, dst, ctx->textlen,
  1325. atmel_aes_gcm_tag_init);
  1326. }
  1327. return atmel_aes_cpu_start(dd, src, dst, ctx->textlen,
  1328. atmel_aes_gcm_tag_init);
  1329. }
  1330. static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd)
  1331. {
  1332. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1333. struct aead_request *req = aead_request_cast(dd->areq);
  1334. __be64 *data = dd->buf;
  1335. if (likely(dd->flags & AES_FLAGS_GTAGEN)) {
  1336. if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) {
  1337. dd->resume = atmel_aes_gcm_tag_init;
  1338. atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY);
  1339. return -EINPROGRESS;
  1340. }
  1341. return atmel_aes_gcm_finalize(dd);
  1342. }
  1343. /* Read the GCM Intermediate Hash Word Registers. */
  1344. atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash);
  1345. data[0] = cpu_to_be64(req->assoclen * 8);
  1346. data[1] = cpu_to_be64(ctx->textlen * 8);
  1347. return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE,
  1348. ctx->ghash, ctx->ghash, atmel_aes_gcm_tag);
  1349. }
  1350. static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd)
  1351. {
  1352. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1353. unsigned long flags;
  1354. /*
  1355. * Change mode to CTR to complete the tag generation.
  1356. * Use J0 as Initialization Vector.
  1357. */
  1358. flags = dd->flags;
  1359. dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN);
  1360. dd->flags |= AES_FLAGS_CTR;
  1361. atmel_aes_write_ctrl(dd, false, ctx->j0);
  1362. dd->flags = flags;
  1363. atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash);
  1364. return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize);
  1365. }
  1366. static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd)
  1367. {
  1368. struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx);
  1369. struct aead_request *req = aead_request_cast(dd->areq);
  1370. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1371. bool enc = atmel_aes_is_encrypt(dd);
  1372. u32 offset, authsize, itag[4], *otag = ctx->tag;
  1373. int err;
  1374. /* Read the computed tag. */
  1375. if (likely(dd->flags & AES_FLAGS_GTAGEN))
  1376. atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag);
  1377. else
  1378. atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag);
  1379. offset = req->assoclen + ctx->textlen;
  1380. authsize = crypto_aead_authsize(tfm);
  1381. if (enc) {
  1382. scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1);
  1383. err = 0;
  1384. } else {
  1385. scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0);
  1386. err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0;
  1387. }
  1388. return atmel_aes_complete(dd, err);
  1389. }
  1390. static int atmel_aes_gcm_crypt(struct aead_request *req,
  1391. unsigned long mode)
  1392. {
  1393. struct atmel_aes_base_ctx *ctx;
  1394. struct atmel_aes_reqctx *rctx;
  1395. ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
  1396. ctx->block_size = AES_BLOCK_SIZE;
  1397. ctx->is_aead = true;
  1398. rctx = aead_request_ctx(req);
  1399. rctx->mode = AES_FLAGS_GCM | mode;
  1400. return atmel_aes_handle_queue(ctx->dd, &req->base);
  1401. }
  1402. static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
  1403. unsigned int keylen)
  1404. {
  1405. struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
  1406. if (keylen != AES_KEYSIZE_256 &&
  1407. keylen != AES_KEYSIZE_192 &&
  1408. keylen != AES_KEYSIZE_128)
  1409. return -EINVAL;
  1410. memcpy(ctx->key, key, keylen);
  1411. ctx->keylen = keylen;
  1412. return 0;
  1413. }
  1414. static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm,
  1415. unsigned int authsize)
  1416. {
  1417. return crypto_gcm_check_authsize(authsize);
  1418. }
  1419. static int atmel_aes_gcm_encrypt(struct aead_request *req)
  1420. {
  1421. return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT);
  1422. }
  1423. static int atmel_aes_gcm_decrypt(struct aead_request *req)
  1424. {
  1425. return atmel_aes_gcm_crypt(req, 0);
  1426. }
  1427. static int atmel_aes_gcm_init(struct crypto_aead *tfm)
  1428. {
  1429. struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm);
  1430. struct atmel_aes_dev *dd;
  1431. dd = atmel_aes_dev_alloc(&ctx->base);
  1432. if (!dd)
  1433. return -ENODEV;
  1434. crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
  1435. ctx->base.dd = dd;
  1436. ctx->base.start = atmel_aes_gcm_start;
  1437. return 0;
  1438. }
  1439. static struct aead_alg aes_gcm_alg = {
  1440. .setkey = atmel_aes_gcm_setkey,
  1441. .setauthsize = atmel_aes_gcm_setauthsize,
  1442. .encrypt = atmel_aes_gcm_encrypt,
  1443. .decrypt = atmel_aes_gcm_decrypt,
  1444. .init = atmel_aes_gcm_init,
  1445. .ivsize = GCM_AES_IV_SIZE,
  1446. .maxauthsize = AES_BLOCK_SIZE,
  1447. .base = {
  1448. .cra_name = "gcm(aes)",
  1449. .cra_driver_name = "atmel-gcm-aes",
  1450. .cra_blocksize = 1,
  1451. .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx),
  1452. },
  1453. };
  1454. /* xts functions */
  1455. static inline struct atmel_aes_xts_ctx *
  1456. atmel_aes_xts_ctx_cast(struct atmel_aes_base_ctx *ctx)
  1457. {
  1458. return container_of(ctx, struct atmel_aes_xts_ctx, base);
  1459. }
  1460. static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd);
  1461. static int atmel_aes_xts_start(struct atmel_aes_dev *dd)
  1462. {
  1463. struct atmel_aes_xts_ctx *ctx = atmel_aes_xts_ctx_cast(dd->ctx);
  1464. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  1465. struct atmel_aes_reqctx *rctx = skcipher_request_ctx(req);
  1466. unsigned long flags;
  1467. int err;
  1468. atmel_aes_set_mode(dd, rctx);
  1469. err = atmel_aes_hw_init(dd);
  1470. if (err)
  1471. return atmel_aes_complete(dd, err);
  1472. /* Compute the tweak value from req->iv with ecb(aes). */
  1473. flags = dd->flags;
  1474. dd->flags &= ~AES_FLAGS_MODE_MASK;
  1475. dd->flags |= (AES_FLAGS_ECB | AES_FLAGS_ENCRYPT);
  1476. atmel_aes_write_ctrl_key(dd, false, NULL,
  1477. ctx->key2, ctx->base.keylen);
  1478. dd->flags = flags;
  1479. atmel_aes_write_block(dd, AES_IDATAR(0), req->iv);
  1480. return atmel_aes_wait_for_data_ready(dd, atmel_aes_xts_process_data);
  1481. }
  1482. static int atmel_aes_xts_process_data(struct atmel_aes_dev *dd)
  1483. {
  1484. struct skcipher_request *req = skcipher_request_cast(dd->areq);
  1485. bool use_dma = (req->cryptlen >= ATMEL_AES_DMA_THRESHOLD);
  1486. u32 tweak[AES_BLOCK_SIZE / sizeof(u32)];
  1487. static const __le32 one[AES_BLOCK_SIZE / sizeof(u32)] = {cpu_to_le32(1), };
  1488. u8 *tweak_bytes = (u8 *)tweak;
  1489. int i;
  1490. /* Read the computed ciphered tweak value. */
  1491. atmel_aes_read_block(dd, AES_ODATAR(0), tweak);
  1492. /*
  1493. * Hardware quirk:
  1494. * the order of the ciphered tweak bytes need to be reversed before
  1495. * writing them into the ODATARx registers.
  1496. */
  1497. for (i = 0; i < AES_BLOCK_SIZE/2; ++i)
  1498. swap(tweak_bytes[i], tweak_bytes[AES_BLOCK_SIZE - 1 - i]);
  1499. /* Process the data. */
  1500. atmel_aes_write_ctrl(dd, use_dma, NULL);
  1501. atmel_aes_write_block(dd, AES_TWR(0), tweak);
  1502. atmel_aes_write_block(dd, AES_ALPHAR(0), one);
  1503. if (use_dma)
  1504. return atmel_aes_dma_start(dd, req->src, req->dst,
  1505. req->cryptlen,
  1506. atmel_aes_transfer_complete);
  1507. return atmel_aes_cpu_start(dd, req->src, req->dst, req->cryptlen,
  1508. atmel_aes_transfer_complete);
  1509. }
  1510. static int atmel_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key,
  1511. unsigned int keylen)
  1512. {
  1513. struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  1514. int err;
  1515. err = xts_check_key(crypto_skcipher_tfm(tfm), key, keylen);
  1516. if (err)
  1517. return err;
  1518. crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK);
  1519. crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags &
  1520. CRYPTO_TFM_REQ_MASK);
  1521. err = crypto_skcipher_setkey(ctx->fallback_tfm, key, keylen);
  1522. if (err)
  1523. return err;
  1524. memcpy(ctx->base.key, key, keylen/2);
  1525. memcpy(ctx->key2, key + keylen/2, keylen/2);
  1526. ctx->base.keylen = keylen/2;
  1527. return 0;
  1528. }
  1529. static int atmel_aes_xts_encrypt(struct skcipher_request *req)
  1530. {
  1531. return atmel_aes_crypt(req, AES_FLAGS_XTS | AES_FLAGS_ENCRYPT);
  1532. }
  1533. static int atmel_aes_xts_decrypt(struct skcipher_request *req)
  1534. {
  1535. return atmel_aes_crypt(req, AES_FLAGS_XTS);
  1536. }
  1537. static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm)
  1538. {
  1539. struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  1540. struct atmel_aes_dev *dd;
  1541. const char *tfm_name = crypto_tfm_alg_name(&tfm->base);
  1542. dd = atmel_aes_dev_alloc(&ctx->base);
  1543. if (!dd)
  1544. return -ENODEV;
  1545. ctx->fallback_tfm = crypto_alloc_skcipher(tfm_name, 0,
  1546. CRYPTO_ALG_NEED_FALLBACK);
  1547. if (IS_ERR(ctx->fallback_tfm))
  1548. return PTR_ERR(ctx->fallback_tfm);
  1549. crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) +
  1550. crypto_skcipher_reqsize(ctx->fallback_tfm));
  1551. ctx->base.dd = dd;
  1552. ctx->base.start = atmel_aes_xts_start;
  1553. return 0;
  1554. }
  1555. static void atmel_aes_xts_exit_tfm(struct crypto_skcipher *tfm)
  1556. {
  1557. struct atmel_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
  1558. crypto_free_skcipher(ctx->fallback_tfm);
  1559. }
  1560. static struct skcipher_alg aes_xts_alg = {
  1561. .base.cra_name = "xts(aes)",
  1562. .base.cra_driver_name = "atmel-xts-aes",
  1563. .base.cra_blocksize = AES_BLOCK_SIZE,
  1564. .base.cra_ctxsize = sizeof(struct atmel_aes_xts_ctx),
  1565. .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK,
  1566. .min_keysize = 2 * AES_MIN_KEY_SIZE,
  1567. .max_keysize = 2 * AES_MAX_KEY_SIZE,
  1568. .ivsize = AES_BLOCK_SIZE,
  1569. .setkey = atmel_aes_xts_setkey,
  1570. .encrypt = atmel_aes_xts_encrypt,
  1571. .decrypt = atmel_aes_xts_decrypt,
  1572. .init = atmel_aes_xts_init_tfm,
  1573. .exit = atmel_aes_xts_exit_tfm,
  1574. };
  1575. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1576. /* authenc aead functions */
  1577. static int atmel_aes_authenc_start(struct atmel_aes_dev *dd);
  1578. static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
  1579. bool is_async);
  1580. static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
  1581. bool is_async);
  1582. static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd);
  1583. static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
  1584. bool is_async);
  1585. static void atmel_aes_authenc_complete(struct atmel_aes_dev *dd, int err)
  1586. {
  1587. struct aead_request *req = aead_request_cast(dd->areq);
  1588. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1589. if (err && (dd->flags & AES_FLAGS_OWN_SHA))
  1590. atmel_sha_authenc_abort(&rctx->auth_req);
  1591. dd->flags &= ~AES_FLAGS_OWN_SHA;
  1592. }
  1593. static int atmel_aes_authenc_start(struct atmel_aes_dev *dd)
  1594. {
  1595. struct aead_request *req = aead_request_cast(dd->areq);
  1596. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1597. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1598. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1599. int err;
  1600. atmel_aes_set_mode(dd, &rctx->base);
  1601. err = atmel_aes_hw_init(dd);
  1602. if (err)
  1603. return atmel_aes_complete(dd, err);
  1604. return atmel_sha_authenc_schedule(&rctx->auth_req, ctx->auth,
  1605. atmel_aes_authenc_init, dd);
  1606. }
  1607. static int atmel_aes_authenc_init(struct atmel_aes_dev *dd, int err,
  1608. bool is_async)
  1609. {
  1610. struct aead_request *req = aead_request_cast(dd->areq);
  1611. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1612. if (is_async)
  1613. dd->is_async = true;
  1614. if (err)
  1615. return atmel_aes_complete(dd, err);
  1616. /* If here, we've got the ownership of the SHA device. */
  1617. dd->flags |= AES_FLAGS_OWN_SHA;
  1618. /* Configure the SHA device. */
  1619. return atmel_sha_authenc_init(&rctx->auth_req,
  1620. req->src, req->assoclen,
  1621. rctx->textlen,
  1622. atmel_aes_authenc_transfer, dd);
  1623. }
  1624. static int atmel_aes_authenc_transfer(struct atmel_aes_dev *dd, int err,
  1625. bool is_async)
  1626. {
  1627. struct aead_request *req = aead_request_cast(dd->areq);
  1628. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1629. bool enc = atmel_aes_is_encrypt(dd);
  1630. struct scatterlist *src, *dst;
  1631. __be32 iv[AES_BLOCK_SIZE / sizeof(u32)];
  1632. u32 emr;
  1633. if (is_async)
  1634. dd->is_async = true;
  1635. if (err)
  1636. return atmel_aes_complete(dd, err);
  1637. /* Prepare src and dst scatter-lists to transfer cipher/plain texts. */
  1638. src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
  1639. dst = src;
  1640. if (req->src != req->dst)
  1641. dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
  1642. /* Configure the AES device. */
  1643. memcpy(iv, req->iv, sizeof(iv));
  1644. /*
  1645. * Here we always set the 2nd parameter of atmel_aes_write_ctrl() to
  1646. * 'true' even if the data transfer is actually performed by the CPU (so
  1647. * not by the DMA) because we must force the AES_MR_SMOD bitfield to the
  1648. * value AES_MR_SMOD_IDATAR0. Indeed, both AES_MR_SMOD and SHA_MR_SMOD
  1649. * must be set to *_MR_SMOD_IDATAR0.
  1650. */
  1651. atmel_aes_write_ctrl(dd, true, iv);
  1652. emr = AES_EMR_PLIPEN;
  1653. if (!enc)
  1654. emr |= AES_EMR_PLIPD;
  1655. atmel_aes_write(dd, AES_EMR, emr);
  1656. /* Transfer data. */
  1657. return atmel_aes_dma_start(dd, src, dst, rctx->textlen,
  1658. atmel_aes_authenc_digest);
  1659. }
  1660. static int atmel_aes_authenc_digest(struct atmel_aes_dev *dd)
  1661. {
  1662. struct aead_request *req = aead_request_cast(dd->areq);
  1663. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1664. /* atmel_sha_authenc_final() releases the SHA device. */
  1665. dd->flags &= ~AES_FLAGS_OWN_SHA;
  1666. return atmel_sha_authenc_final(&rctx->auth_req,
  1667. rctx->digest, sizeof(rctx->digest),
  1668. atmel_aes_authenc_final, dd);
  1669. }
  1670. static int atmel_aes_authenc_final(struct atmel_aes_dev *dd, int err,
  1671. bool is_async)
  1672. {
  1673. struct aead_request *req = aead_request_cast(dd->areq);
  1674. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1675. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1676. bool enc = atmel_aes_is_encrypt(dd);
  1677. u32 idigest[SHA512_DIGEST_SIZE / sizeof(u32)], *odigest = rctx->digest;
  1678. u32 offs, authsize;
  1679. if (is_async)
  1680. dd->is_async = true;
  1681. if (err)
  1682. goto complete;
  1683. offs = req->assoclen + rctx->textlen;
  1684. authsize = crypto_aead_authsize(tfm);
  1685. if (enc) {
  1686. scatterwalk_map_and_copy(odigest, req->dst, offs, authsize, 1);
  1687. } else {
  1688. scatterwalk_map_and_copy(idigest, req->src, offs, authsize, 0);
  1689. if (crypto_memneq(idigest, odigest, authsize))
  1690. err = -EBADMSG;
  1691. }
  1692. complete:
  1693. return atmel_aes_complete(dd, err);
  1694. }
  1695. static int atmel_aes_authenc_setkey(struct crypto_aead *tfm, const u8 *key,
  1696. unsigned int keylen)
  1697. {
  1698. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1699. struct crypto_authenc_keys keys;
  1700. int err;
  1701. if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
  1702. goto badkey;
  1703. if (keys.enckeylen > sizeof(ctx->base.key))
  1704. goto badkey;
  1705. /* Save auth key. */
  1706. err = atmel_sha_authenc_setkey(ctx->auth,
  1707. keys.authkey, keys.authkeylen,
  1708. crypto_aead_get_flags(tfm));
  1709. if (err) {
  1710. memzero_explicit(&keys, sizeof(keys));
  1711. return err;
  1712. }
  1713. /* Save enc key. */
  1714. ctx->base.keylen = keys.enckeylen;
  1715. memcpy(ctx->base.key, keys.enckey, keys.enckeylen);
  1716. memzero_explicit(&keys, sizeof(keys));
  1717. return 0;
  1718. badkey:
  1719. memzero_explicit(&keys, sizeof(keys));
  1720. return -EINVAL;
  1721. }
  1722. static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
  1723. unsigned long auth_mode)
  1724. {
  1725. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1726. unsigned int auth_reqsize = atmel_sha_authenc_get_reqsize();
  1727. struct atmel_aes_dev *dd;
  1728. dd = atmel_aes_dev_alloc(&ctx->base);
  1729. if (!dd)
  1730. return -ENODEV;
  1731. ctx->auth = atmel_sha_authenc_spawn(auth_mode);
  1732. if (IS_ERR(ctx->auth))
  1733. return PTR_ERR(ctx->auth);
  1734. crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
  1735. auth_reqsize));
  1736. ctx->base.dd = dd;
  1737. ctx->base.start = atmel_aes_authenc_start;
  1738. return 0;
  1739. }
  1740. static int atmel_aes_authenc_hmac_sha1_init_tfm(struct crypto_aead *tfm)
  1741. {
  1742. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA1);
  1743. }
  1744. static int atmel_aes_authenc_hmac_sha224_init_tfm(struct crypto_aead *tfm)
  1745. {
  1746. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA224);
  1747. }
  1748. static int atmel_aes_authenc_hmac_sha256_init_tfm(struct crypto_aead *tfm)
  1749. {
  1750. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA256);
  1751. }
  1752. static int atmel_aes_authenc_hmac_sha384_init_tfm(struct crypto_aead *tfm)
  1753. {
  1754. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA384);
  1755. }
  1756. static int atmel_aes_authenc_hmac_sha512_init_tfm(struct crypto_aead *tfm)
  1757. {
  1758. return atmel_aes_authenc_init_tfm(tfm, SHA_FLAGS_HMAC_SHA512);
  1759. }
  1760. static void atmel_aes_authenc_exit_tfm(struct crypto_aead *tfm)
  1761. {
  1762. struct atmel_aes_authenc_ctx *ctx = crypto_aead_ctx(tfm);
  1763. atmel_sha_authenc_free(ctx->auth);
  1764. }
  1765. static int atmel_aes_authenc_crypt(struct aead_request *req,
  1766. unsigned long mode)
  1767. {
  1768. struct atmel_aes_authenc_reqctx *rctx = aead_request_ctx(req);
  1769. struct crypto_aead *tfm = crypto_aead_reqtfm(req);
  1770. struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm);
  1771. u32 authsize = crypto_aead_authsize(tfm);
  1772. bool enc = (mode & AES_FLAGS_ENCRYPT);
  1773. /* Compute text length. */
  1774. if (!enc && req->cryptlen < authsize)
  1775. return -EINVAL;
  1776. rctx->textlen = req->cryptlen - (enc ? 0 : authsize);
  1777. /*
  1778. * Currently, empty messages are not supported yet:
  1779. * the SHA auto-padding can be used only on non-empty messages.
  1780. * Hence a special case needs to be implemented for empty message.
  1781. */
  1782. if (!rctx->textlen && !req->assoclen)
  1783. return -EINVAL;
  1784. rctx->base.mode = mode;
  1785. ctx->block_size = AES_BLOCK_SIZE;
  1786. ctx->is_aead = true;
  1787. return atmel_aes_handle_queue(ctx->dd, &req->base);
  1788. }
  1789. static int atmel_aes_authenc_cbc_aes_encrypt(struct aead_request *req)
  1790. {
  1791. return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT);
  1792. }
  1793. static int atmel_aes_authenc_cbc_aes_decrypt(struct aead_request *req)
  1794. {
  1795. return atmel_aes_authenc_crypt(req, AES_FLAGS_CBC);
  1796. }
  1797. static struct aead_alg aes_authenc_algs[] = {
  1798. {
  1799. .setkey = atmel_aes_authenc_setkey,
  1800. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1801. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1802. .init = atmel_aes_authenc_hmac_sha1_init_tfm,
  1803. .exit = atmel_aes_authenc_exit_tfm,
  1804. .ivsize = AES_BLOCK_SIZE,
  1805. .maxauthsize = SHA1_DIGEST_SIZE,
  1806. .base = {
  1807. .cra_name = "authenc(hmac(sha1),cbc(aes))",
  1808. .cra_driver_name = "atmel-authenc-hmac-sha1-cbc-aes",
  1809. .cra_blocksize = AES_BLOCK_SIZE,
  1810. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1811. },
  1812. },
  1813. {
  1814. .setkey = atmel_aes_authenc_setkey,
  1815. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1816. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1817. .init = atmel_aes_authenc_hmac_sha224_init_tfm,
  1818. .exit = atmel_aes_authenc_exit_tfm,
  1819. .ivsize = AES_BLOCK_SIZE,
  1820. .maxauthsize = SHA224_DIGEST_SIZE,
  1821. .base = {
  1822. .cra_name = "authenc(hmac(sha224),cbc(aes))",
  1823. .cra_driver_name = "atmel-authenc-hmac-sha224-cbc-aes",
  1824. .cra_blocksize = AES_BLOCK_SIZE,
  1825. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1826. },
  1827. },
  1828. {
  1829. .setkey = atmel_aes_authenc_setkey,
  1830. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1831. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1832. .init = atmel_aes_authenc_hmac_sha256_init_tfm,
  1833. .exit = atmel_aes_authenc_exit_tfm,
  1834. .ivsize = AES_BLOCK_SIZE,
  1835. .maxauthsize = SHA256_DIGEST_SIZE,
  1836. .base = {
  1837. .cra_name = "authenc(hmac(sha256),cbc(aes))",
  1838. .cra_driver_name = "atmel-authenc-hmac-sha256-cbc-aes",
  1839. .cra_blocksize = AES_BLOCK_SIZE,
  1840. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1841. },
  1842. },
  1843. {
  1844. .setkey = atmel_aes_authenc_setkey,
  1845. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1846. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1847. .init = atmel_aes_authenc_hmac_sha384_init_tfm,
  1848. .exit = atmel_aes_authenc_exit_tfm,
  1849. .ivsize = AES_BLOCK_SIZE,
  1850. .maxauthsize = SHA384_DIGEST_SIZE,
  1851. .base = {
  1852. .cra_name = "authenc(hmac(sha384),cbc(aes))",
  1853. .cra_driver_name = "atmel-authenc-hmac-sha384-cbc-aes",
  1854. .cra_blocksize = AES_BLOCK_SIZE,
  1855. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1856. },
  1857. },
  1858. {
  1859. .setkey = atmel_aes_authenc_setkey,
  1860. .encrypt = atmel_aes_authenc_cbc_aes_encrypt,
  1861. .decrypt = atmel_aes_authenc_cbc_aes_decrypt,
  1862. .init = atmel_aes_authenc_hmac_sha512_init_tfm,
  1863. .exit = atmel_aes_authenc_exit_tfm,
  1864. .ivsize = AES_BLOCK_SIZE,
  1865. .maxauthsize = SHA512_DIGEST_SIZE,
  1866. .base = {
  1867. .cra_name = "authenc(hmac(sha512),cbc(aes))",
  1868. .cra_driver_name = "atmel-authenc-hmac-sha512-cbc-aes",
  1869. .cra_blocksize = AES_BLOCK_SIZE,
  1870. .cra_ctxsize = sizeof(struct atmel_aes_authenc_ctx),
  1871. },
  1872. },
  1873. };
  1874. #endif /* CONFIG_CRYPTO_DEV_ATMEL_AUTHENC */
  1875. /* Probe functions */
  1876. static int atmel_aes_buff_init(struct atmel_aes_dev *dd)
  1877. {
  1878. dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER);
  1879. dd->buflen = ATMEL_AES_BUFFER_SIZE;
  1880. dd->buflen &= ~(AES_BLOCK_SIZE - 1);
  1881. if (!dd->buf) {
  1882. dev_err(dd->dev, "unable to alloc pages.\n");
  1883. return -ENOMEM;
  1884. }
  1885. return 0;
  1886. }
  1887. static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd)
  1888. {
  1889. free_page((unsigned long)dd->buf);
  1890. }
  1891. static int atmel_aes_dma_init(struct atmel_aes_dev *dd)
  1892. {
  1893. int ret;
  1894. /* Try to grab 2 DMA channels */
  1895. dd->src.chan = dma_request_chan(dd->dev, "tx");
  1896. if (IS_ERR(dd->src.chan)) {
  1897. ret = PTR_ERR(dd->src.chan);
  1898. goto err_dma_in;
  1899. }
  1900. dd->dst.chan = dma_request_chan(dd->dev, "rx");
  1901. if (IS_ERR(dd->dst.chan)) {
  1902. ret = PTR_ERR(dd->dst.chan);
  1903. goto err_dma_out;
  1904. }
  1905. return 0;
  1906. err_dma_out:
  1907. dma_release_channel(dd->src.chan);
  1908. err_dma_in:
  1909. dev_err(dd->dev, "no DMA channel available\n");
  1910. return ret;
  1911. }
  1912. static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd)
  1913. {
  1914. dma_release_channel(dd->dst.chan);
  1915. dma_release_channel(dd->src.chan);
  1916. }
  1917. static void atmel_aes_queue_task(unsigned long data)
  1918. {
  1919. struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;
  1920. atmel_aes_handle_queue(dd, NULL);
  1921. }
  1922. static void atmel_aes_done_task(unsigned long data)
  1923. {
  1924. struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data;
  1925. dd->is_async = true;
  1926. (void)dd->resume(dd);
  1927. }
  1928. static irqreturn_t atmel_aes_irq(int irq, void *dev_id)
  1929. {
  1930. struct atmel_aes_dev *aes_dd = dev_id;
  1931. u32 reg;
  1932. reg = atmel_aes_read(aes_dd, AES_ISR);
  1933. if (reg & atmel_aes_read(aes_dd, AES_IMR)) {
  1934. atmel_aes_write(aes_dd, AES_IDR, reg);
  1935. if (AES_FLAGS_BUSY & aes_dd->flags)
  1936. tasklet_schedule(&aes_dd->done_task);
  1937. else
  1938. dev_warn(aes_dd->dev, "AES interrupt when no active requests.\n");
  1939. return IRQ_HANDLED;
  1940. }
  1941. return IRQ_NONE;
  1942. }
  1943. static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd)
  1944. {
  1945. int i;
  1946. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1947. if (dd->caps.has_authenc)
  1948. for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++)
  1949. crypto_unregister_aead(&aes_authenc_algs[i]);
  1950. #endif
  1951. if (dd->caps.has_xts)
  1952. crypto_unregister_skcipher(&aes_xts_alg);
  1953. if (dd->caps.has_gcm)
  1954. crypto_unregister_aead(&aes_gcm_alg);
  1955. if (dd->caps.has_cfb64)
  1956. crypto_unregister_skcipher(&aes_cfb64_alg);
  1957. for (i = 0; i < ARRAY_SIZE(aes_algs); i++)
  1958. crypto_unregister_skcipher(&aes_algs[i]);
  1959. }
  1960. static void atmel_aes_crypto_alg_init(struct crypto_alg *alg)
  1961. {
  1962. alg->cra_flags |= CRYPTO_ALG_ASYNC;
  1963. alg->cra_alignmask = 0xf;
  1964. alg->cra_priority = ATMEL_AES_PRIORITY;
  1965. alg->cra_module = THIS_MODULE;
  1966. }
  1967. static int atmel_aes_register_algs(struct atmel_aes_dev *dd)
  1968. {
  1969. int err, i, j;
  1970. for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
  1971. atmel_aes_crypto_alg_init(&aes_algs[i].base);
  1972. err = crypto_register_skcipher(&aes_algs[i]);
  1973. if (err)
  1974. goto err_aes_algs;
  1975. }
  1976. if (dd->caps.has_cfb64) {
  1977. atmel_aes_crypto_alg_init(&aes_cfb64_alg.base);
  1978. err = crypto_register_skcipher(&aes_cfb64_alg);
  1979. if (err)
  1980. goto err_aes_cfb64_alg;
  1981. }
  1982. if (dd->caps.has_gcm) {
  1983. atmel_aes_crypto_alg_init(&aes_gcm_alg.base);
  1984. err = crypto_register_aead(&aes_gcm_alg);
  1985. if (err)
  1986. goto err_aes_gcm_alg;
  1987. }
  1988. if (dd->caps.has_xts) {
  1989. atmel_aes_crypto_alg_init(&aes_xts_alg.base);
  1990. err = crypto_register_skcipher(&aes_xts_alg);
  1991. if (err)
  1992. goto err_aes_xts_alg;
  1993. }
  1994. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  1995. if (dd->caps.has_authenc) {
  1996. for (i = 0; i < ARRAY_SIZE(aes_authenc_algs); i++) {
  1997. atmel_aes_crypto_alg_init(&aes_authenc_algs[i].base);
  1998. err = crypto_register_aead(&aes_authenc_algs[i]);
  1999. if (err)
  2000. goto err_aes_authenc_alg;
  2001. }
  2002. }
  2003. #endif
  2004. return 0;
  2005. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  2006. /* i = ARRAY_SIZE(aes_authenc_algs); */
  2007. err_aes_authenc_alg:
  2008. for (j = 0; j < i; j++)
  2009. crypto_unregister_aead(&aes_authenc_algs[j]);
  2010. crypto_unregister_skcipher(&aes_xts_alg);
  2011. #endif
  2012. err_aes_xts_alg:
  2013. crypto_unregister_aead(&aes_gcm_alg);
  2014. err_aes_gcm_alg:
  2015. crypto_unregister_skcipher(&aes_cfb64_alg);
  2016. err_aes_cfb64_alg:
  2017. i = ARRAY_SIZE(aes_algs);
  2018. err_aes_algs:
  2019. for (j = 0; j < i; j++)
  2020. crypto_unregister_skcipher(&aes_algs[j]);
  2021. return err;
  2022. }
  2023. static void atmel_aes_get_cap(struct atmel_aes_dev *dd)
  2024. {
  2025. dd->caps.has_dualbuff = 0;
  2026. dd->caps.has_cfb64 = 0;
  2027. dd->caps.has_gcm = 0;
  2028. dd->caps.has_xts = 0;
  2029. dd->caps.has_authenc = 0;
  2030. dd->caps.max_burst_size = 1;
  2031. /* keep only major version number */
  2032. switch (dd->hw_version & 0xff0) {
  2033. case 0x700:
  2034. case 0x500:
  2035. dd->caps.has_dualbuff = 1;
  2036. dd->caps.has_cfb64 = 1;
  2037. dd->caps.has_gcm = 1;
  2038. dd->caps.has_xts = 1;
  2039. dd->caps.has_authenc = 1;
  2040. dd->caps.max_burst_size = 4;
  2041. break;
  2042. case 0x200:
  2043. dd->caps.has_dualbuff = 1;
  2044. dd->caps.has_cfb64 = 1;
  2045. dd->caps.has_gcm = 1;
  2046. dd->caps.max_burst_size = 4;
  2047. break;
  2048. case 0x130:
  2049. dd->caps.has_dualbuff = 1;
  2050. dd->caps.has_cfb64 = 1;
  2051. dd->caps.max_burst_size = 4;
  2052. break;
  2053. case 0x120:
  2054. break;
  2055. default:
  2056. dev_warn(dd->dev,
  2057. "Unmanaged aes version, set minimum capabilities\n");
  2058. break;
  2059. }
  2060. }
  2061. #if defined(CONFIG_OF)
  2062. static const struct of_device_id atmel_aes_dt_ids[] = {
  2063. { .compatible = "atmel,at91sam9g46-aes" },
  2064. { /* sentinel */ }
  2065. };
  2066. MODULE_DEVICE_TABLE(of, atmel_aes_dt_ids);
  2067. #endif
  2068. static int atmel_aes_probe(struct platform_device *pdev)
  2069. {
  2070. struct atmel_aes_dev *aes_dd;
  2071. struct device *dev = &pdev->dev;
  2072. struct resource *aes_res;
  2073. int err;
  2074. aes_dd = devm_kzalloc(&pdev->dev, sizeof(*aes_dd), GFP_KERNEL);
  2075. if (!aes_dd)
  2076. return -ENOMEM;
  2077. aes_dd->dev = dev;
  2078. platform_set_drvdata(pdev, aes_dd);
  2079. INIT_LIST_HEAD(&aes_dd->list);
  2080. spin_lock_init(&aes_dd->lock);
  2081. tasklet_init(&aes_dd->done_task, atmel_aes_done_task,
  2082. (unsigned long)aes_dd);
  2083. tasklet_init(&aes_dd->queue_task, atmel_aes_queue_task,
  2084. (unsigned long)aes_dd);
  2085. crypto_init_queue(&aes_dd->queue, ATMEL_AES_QUEUE_LENGTH);
  2086. /* Get the base address */
  2087. aes_res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  2088. if (!aes_res) {
  2089. dev_err(dev, "no MEM resource info\n");
  2090. err = -ENODEV;
  2091. goto err_tasklet_kill;
  2092. }
  2093. aes_dd->phys_base = aes_res->start;
  2094. /* Get the IRQ */
  2095. aes_dd->irq = platform_get_irq(pdev, 0);
  2096. if (aes_dd->irq < 0) {
  2097. err = aes_dd->irq;
  2098. goto err_tasklet_kill;
  2099. }
  2100. err = devm_request_irq(&pdev->dev, aes_dd->irq, atmel_aes_irq,
  2101. IRQF_SHARED, "atmel-aes", aes_dd);
  2102. if (err) {
  2103. dev_err(dev, "unable to request aes irq.\n");
  2104. goto err_tasklet_kill;
  2105. }
  2106. /* Initializing the clock */
  2107. aes_dd->iclk = devm_clk_get(&pdev->dev, "aes_clk");
  2108. if (IS_ERR(aes_dd->iclk)) {
  2109. dev_err(dev, "clock initialization failed.\n");
  2110. err = PTR_ERR(aes_dd->iclk);
  2111. goto err_tasklet_kill;
  2112. }
  2113. aes_dd->io_base = devm_ioremap_resource(&pdev->dev, aes_res);
  2114. if (IS_ERR(aes_dd->io_base)) {
  2115. dev_err(dev, "can't ioremap\n");
  2116. err = PTR_ERR(aes_dd->io_base);
  2117. goto err_tasklet_kill;
  2118. }
  2119. err = clk_prepare(aes_dd->iclk);
  2120. if (err)
  2121. goto err_tasklet_kill;
  2122. err = atmel_aes_hw_version_init(aes_dd);
  2123. if (err)
  2124. goto err_iclk_unprepare;
  2125. atmel_aes_get_cap(aes_dd);
  2126. #if IS_ENABLED(CONFIG_CRYPTO_DEV_ATMEL_AUTHENC)
  2127. if (aes_dd->caps.has_authenc && !atmel_sha_authenc_is_ready()) {
  2128. err = -EPROBE_DEFER;
  2129. goto err_iclk_unprepare;
  2130. }
  2131. #endif
  2132. err = atmel_aes_buff_init(aes_dd);
  2133. if (err)
  2134. goto err_iclk_unprepare;
  2135. err = atmel_aes_dma_init(aes_dd);
  2136. if (err)
  2137. goto err_buff_cleanup;
  2138. spin_lock(&atmel_aes.lock);
  2139. list_add_tail(&aes_dd->list, &atmel_aes.dev_list);
  2140. spin_unlock(&atmel_aes.lock);
  2141. err = atmel_aes_register_algs(aes_dd);
  2142. if (err)
  2143. goto err_algs;
  2144. dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n",
  2145. dma_chan_name(aes_dd->src.chan),
  2146. dma_chan_name(aes_dd->dst.chan));
  2147. return 0;
  2148. err_algs:
  2149. spin_lock(&atmel_aes.lock);
  2150. list_del(&aes_dd->list);
  2151. spin_unlock(&atmel_aes.lock);
  2152. atmel_aes_dma_cleanup(aes_dd);
  2153. err_buff_cleanup:
  2154. atmel_aes_buff_cleanup(aes_dd);
  2155. err_iclk_unprepare:
  2156. clk_unprepare(aes_dd->iclk);
  2157. err_tasklet_kill:
  2158. tasklet_kill(&aes_dd->done_task);
  2159. tasklet_kill(&aes_dd->queue_task);
  2160. return err;
  2161. }
  2162. static int atmel_aes_remove(struct platform_device *pdev)
  2163. {
  2164. struct atmel_aes_dev *aes_dd;
  2165. aes_dd = platform_get_drvdata(pdev);
  2166. spin_lock(&atmel_aes.lock);
  2167. list_del(&aes_dd->list);
  2168. spin_unlock(&atmel_aes.lock);
  2169. atmel_aes_unregister_algs(aes_dd);
  2170. tasklet_kill(&aes_dd->done_task);
  2171. tasklet_kill(&aes_dd->queue_task);
  2172. atmel_aes_dma_cleanup(aes_dd);
  2173. atmel_aes_buff_cleanup(aes_dd);
  2174. clk_unprepare(aes_dd->iclk);
  2175. return 0;
  2176. }
  2177. static struct platform_driver atmel_aes_driver = {
  2178. .probe = atmel_aes_probe,
  2179. .remove = atmel_aes_remove,
  2180. .driver = {
  2181. .name = "atmel_aes",
  2182. .of_match_table = of_match_ptr(atmel_aes_dt_ids),
  2183. },
  2184. };
  2185. module_platform_driver(atmel_aes_driver);
  2186. MODULE_DESCRIPTION("Atmel AES hw acceleration support.");
  2187. MODULE_LICENSE("GPL v2");
  2188. MODULE_AUTHOR("Nicolas Royer - Eukréa Electromatique");