omap-sham.c 53 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Cryptographic API.
  4. *
  5. * Support for OMAP SHA1/MD5 HW acceleration.
  6. *
  7. * Copyright (c) 2010 Nokia Corporation
  8. * Author: Dmitry Kasatkin <[email protected]>
  9. * Copyright (c) 2011 Texas Instruments Incorporated
  10. *
  11. * Some ideas are from old omap-sha1-md5.c driver.
  12. */
  13. #define pr_fmt(fmt) "%s: " fmt, __func__
  14. #include <linux/err.h>
  15. #include <linux/device.h>
  16. #include <linux/module.h>
  17. #include <linux/init.h>
  18. #include <linux/errno.h>
  19. #include <linux/interrupt.h>
  20. #include <linux/kernel.h>
  21. #include <linux/irq.h>
  22. #include <linux/io.h>
  23. #include <linux/platform_device.h>
  24. #include <linux/scatterlist.h>
  25. #include <linux/dma-mapping.h>
  26. #include <linux/dmaengine.h>
  27. #include <linux/pm_runtime.h>
  28. #include <linux/of.h>
  29. #include <linux/of_device.h>
  30. #include <linux/of_address.h>
  31. #include <linux/of_irq.h>
  32. #include <linux/delay.h>
  33. #include <linux/crypto.h>
  34. #include <crypto/scatterwalk.h>
  35. #include <crypto/algapi.h>
  36. #include <crypto/sha1.h>
  37. #include <crypto/sha2.h>
  38. #include <crypto/hash.h>
  39. #include <crypto/hmac.h>
  40. #include <crypto/internal/hash.h>
  41. #include <crypto/engine.h>
  42. #define MD5_DIGEST_SIZE 16
  43. #define SHA_REG_IDIGEST(dd, x) ((dd)->pdata->idigest_ofs + ((x)*0x04))
  44. #define SHA_REG_DIN(dd, x) ((dd)->pdata->din_ofs + ((x) * 0x04))
  45. #define SHA_REG_DIGCNT(dd) ((dd)->pdata->digcnt_ofs)
  46. #define SHA_REG_ODIGEST(dd, x) ((dd)->pdata->odigest_ofs + (x * 0x04))
  47. #define SHA_REG_CTRL 0x18
  48. #define SHA_REG_CTRL_LENGTH (0xFFFFFFFF << 5)
  49. #define SHA_REG_CTRL_CLOSE_HASH (1 << 4)
  50. #define SHA_REG_CTRL_ALGO_CONST (1 << 3)
  51. #define SHA_REG_CTRL_ALGO (1 << 2)
  52. #define SHA_REG_CTRL_INPUT_READY (1 << 1)
  53. #define SHA_REG_CTRL_OUTPUT_READY (1 << 0)
  54. #define SHA_REG_REV(dd) ((dd)->pdata->rev_ofs)
  55. #define SHA_REG_MASK(dd) ((dd)->pdata->mask_ofs)
  56. #define SHA_REG_MASK_DMA_EN (1 << 3)
  57. #define SHA_REG_MASK_IT_EN (1 << 2)
  58. #define SHA_REG_MASK_SOFTRESET (1 << 1)
  59. #define SHA_REG_AUTOIDLE (1 << 0)
  60. #define SHA_REG_SYSSTATUS(dd) ((dd)->pdata->sysstatus_ofs)
  61. #define SHA_REG_SYSSTATUS_RESETDONE (1 << 0)
  62. #define SHA_REG_MODE(dd) ((dd)->pdata->mode_ofs)
  63. #define SHA_REG_MODE_HMAC_OUTER_HASH (1 << 7)
  64. #define SHA_REG_MODE_HMAC_KEY_PROC (1 << 5)
  65. #define SHA_REG_MODE_CLOSE_HASH (1 << 4)
  66. #define SHA_REG_MODE_ALGO_CONSTANT (1 << 3)
  67. #define SHA_REG_MODE_ALGO_MASK (7 << 0)
  68. #define SHA_REG_MODE_ALGO_MD5_128 (0 << 1)
  69. #define SHA_REG_MODE_ALGO_SHA1_160 (1 << 1)
  70. #define SHA_REG_MODE_ALGO_SHA2_224 (2 << 1)
  71. #define SHA_REG_MODE_ALGO_SHA2_256 (3 << 1)
  72. #define SHA_REG_MODE_ALGO_SHA2_384 (1 << 0)
  73. #define SHA_REG_MODE_ALGO_SHA2_512 (3 << 0)
  74. #define SHA_REG_LENGTH(dd) ((dd)->pdata->length_ofs)
  75. #define SHA_REG_IRQSTATUS 0x118
  76. #define SHA_REG_IRQSTATUS_CTX_RDY (1 << 3)
  77. #define SHA_REG_IRQSTATUS_PARTHASH_RDY (1 << 2)
  78. #define SHA_REG_IRQSTATUS_INPUT_RDY (1 << 1)
  79. #define SHA_REG_IRQSTATUS_OUTPUT_RDY (1 << 0)
  80. #define SHA_REG_IRQENA 0x11C
  81. #define SHA_REG_IRQENA_CTX_RDY (1 << 3)
  82. #define SHA_REG_IRQENA_PARTHASH_RDY (1 << 2)
  83. #define SHA_REG_IRQENA_INPUT_RDY (1 << 1)
  84. #define SHA_REG_IRQENA_OUTPUT_RDY (1 << 0)
  85. #define DEFAULT_TIMEOUT_INTERVAL HZ
  86. #define DEFAULT_AUTOSUSPEND_DELAY 1000
  87. /* mostly device flags */
  88. #define FLAGS_FINAL 1
  89. #define FLAGS_DMA_ACTIVE 2
  90. #define FLAGS_OUTPUT_READY 3
  91. #define FLAGS_CPU 5
  92. #define FLAGS_DMA_READY 6
  93. #define FLAGS_AUTO_XOR 7
  94. #define FLAGS_BE32_SHA1 8
  95. #define FLAGS_SGS_COPIED 9
  96. #define FLAGS_SGS_ALLOCED 10
  97. #define FLAGS_HUGE 11
  98. /* context flags */
  99. #define FLAGS_FINUP 16
  100. #define FLAGS_MODE_SHIFT 18
  101. #define FLAGS_MODE_MASK (SHA_REG_MODE_ALGO_MASK << FLAGS_MODE_SHIFT)
  102. #define FLAGS_MODE_MD5 (SHA_REG_MODE_ALGO_MD5_128 << FLAGS_MODE_SHIFT)
  103. #define FLAGS_MODE_SHA1 (SHA_REG_MODE_ALGO_SHA1_160 << FLAGS_MODE_SHIFT)
  104. #define FLAGS_MODE_SHA224 (SHA_REG_MODE_ALGO_SHA2_224 << FLAGS_MODE_SHIFT)
  105. #define FLAGS_MODE_SHA256 (SHA_REG_MODE_ALGO_SHA2_256 << FLAGS_MODE_SHIFT)
  106. #define FLAGS_MODE_SHA384 (SHA_REG_MODE_ALGO_SHA2_384 << FLAGS_MODE_SHIFT)
  107. #define FLAGS_MODE_SHA512 (SHA_REG_MODE_ALGO_SHA2_512 << FLAGS_MODE_SHIFT)
  108. #define FLAGS_HMAC 21
  109. #define FLAGS_ERROR 22
  110. #define OP_UPDATE 1
  111. #define OP_FINAL 2
  112. #define OMAP_ALIGN_MASK (sizeof(u32)-1)
  113. #define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
  114. #define BUFLEN SHA512_BLOCK_SIZE
  115. #define OMAP_SHA_DMA_THRESHOLD 256
  116. #define OMAP_SHA_MAX_DMA_LEN (1024 * 2048)
  117. struct omap_sham_dev;
  118. struct omap_sham_reqctx {
  119. struct omap_sham_dev *dd;
  120. unsigned long flags;
  121. u8 op;
  122. u8 digest[SHA512_DIGEST_SIZE] OMAP_ALIGNED;
  123. size_t digcnt;
  124. size_t bufcnt;
  125. size_t buflen;
  126. /* walk state */
  127. struct scatterlist *sg;
  128. struct scatterlist sgl[2];
  129. int offset; /* offset in current sg */
  130. int sg_len;
  131. unsigned int total; /* total request */
  132. u8 buffer[] OMAP_ALIGNED;
  133. };
  134. struct omap_sham_hmac_ctx {
  135. struct crypto_shash *shash;
  136. u8 ipad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
  137. u8 opad[SHA512_BLOCK_SIZE] OMAP_ALIGNED;
  138. };
  139. struct omap_sham_ctx {
  140. struct crypto_engine_ctx enginectx;
  141. unsigned long flags;
  142. /* fallback stuff */
  143. struct crypto_shash *fallback;
  144. struct omap_sham_hmac_ctx base[];
  145. };
  146. #define OMAP_SHAM_QUEUE_LENGTH 10
  147. struct omap_sham_algs_info {
  148. struct ahash_alg *algs_list;
  149. unsigned int size;
  150. unsigned int registered;
  151. };
  152. struct omap_sham_pdata {
  153. struct omap_sham_algs_info *algs_info;
  154. unsigned int algs_info_size;
  155. unsigned long flags;
  156. int digest_size;
  157. void (*copy_hash)(struct ahash_request *req, int out);
  158. void (*write_ctrl)(struct omap_sham_dev *dd, size_t length,
  159. int final, int dma);
  160. void (*trigger)(struct omap_sham_dev *dd, size_t length);
  161. int (*poll_irq)(struct omap_sham_dev *dd);
  162. irqreturn_t (*intr_hdlr)(int irq, void *dev_id);
  163. u32 odigest_ofs;
  164. u32 idigest_ofs;
  165. u32 din_ofs;
  166. u32 digcnt_ofs;
  167. u32 rev_ofs;
  168. u32 mask_ofs;
  169. u32 sysstatus_ofs;
  170. u32 mode_ofs;
  171. u32 length_ofs;
  172. u32 major_mask;
  173. u32 major_shift;
  174. u32 minor_mask;
  175. u32 minor_shift;
  176. };
  177. struct omap_sham_dev {
  178. struct list_head list;
  179. unsigned long phys_base;
  180. struct device *dev;
  181. void __iomem *io_base;
  182. int irq;
  183. int err;
  184. struct dma_chan *dma_lch;
  185. struct tasklet_struct done_task;
  186. u8 polling_mode;
  187. u8 xmit_buf[BUFLEN] OMAP_ALIGNED;
  188. unsigned long flags;
  189. int fallback_sz;
  190. struct crypto_queue queue;
  191. struct ahash_request *req;
  192. struct crypto_engine *engine;
  193. const struct omap_sham_pdata *pdata;
  194. };
  195. struct omap_sham_drv {
  196. struct list_head dev_list;
  197. spinlock_t lock;
  198. unsigned long flags;
  199. };
  200. static struct omap_sham_drv sham = {
  201. .dev_list = LIST_HEAD_INIT(sham.dev_list),
  202. .lock = __SPIN_LOCK_UNLOCKED(sham.lock),
  203. };
  204. static int omap_sham_enqueue(struct ahash_request *req, unsigned int op);
  205. static void omap_sham_finish_req(struct ahash_request *req, int err);
  206. static inline u32 omap_sham_read(struct omap_sham_dev *dd, u32 offset)
  207. {
  208. return __raw_readl(dd->io_base + offset);
  209. }
  210. static inline void omap_sham_write(struct omap_sham_dev *dd,
  211. u32 offset, u32 value)
  212. {
  213. __raw_writel(value, dd->io_base + offset);
  214. }
  215. static inline void omap_sham_write_mask(struct omap_sham_dev *dd, u32 address,
  216. u32 value, u32 mask)
  217. {
  218. u32 val;
  219. val = omap_sham_read(dd, address);
  220. val &= ~mask;
  221. val |= value;
  222. omap_sham_write(dd, address, val);
  223. }
  224. static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
  225. {
  226. unsigned long timeout = jiffies + DEFAULT_TIMEOUT_INTERVAL;
  227. while (!(omap_sham_read(dd, offset) & bit)) {
  228. if (time_is_before_jiffies(timeout))
  229. return -ETIMEDOUT;
  230. }
  231. return 0;
  232. }
  233. static void omap_sham_copy_hash_omap2(struct ahash_request *req, int out)
  234. {
  235. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  236. struct omap_sham_dev *dd = ctx->dd;
  237. u32 *hash = (u32 *)ctx->digest;
  238. int i;
  239. for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
  240. if (out)
  241. hash[i] = omap_sham_read(dd, SHA_REG_IDIGEST(dd, i));
  242. else
  243. omap_sham_write(dd, SHA_REG_IDIGEST(dd, i), hash[i]);
  244. }
  245. }
  246. static void omap_sham_copy_hash_omap4(struct ahash_request *req, int out)
  247. {
  248. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  249. struct omap_sham_dev *dd = ctx->dd;
  250. int i;
  251. if (ctx->flags & BIT(FLAGS_HMAC)) {
  252. struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
  253. struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
  254. struct omap_sham_hmac_ctx *bctx = tctx->base;
  255. u32 *opad = (u32 *)bctx->opad;
  256. for (i = 0; i < dd->pdata->digest_size / sizeof(u32); i++) {
  257. if (out)
  258. opad[i] = omap_sham_read(dd,
  259. SHA_REG_ODIGEST(dd, i));
  260. else
  261. omap_sham_write(dd, SHA_REG_ODIGEST(dd, i),
  262. opad[i]);
  263. }
  264. }
  265. omap_sham_copy_hash_omap2(req, out);
  266. }
  267. static void omap_sham_copy_ready_hash(struct ahash_request *req)
  268. {
  269. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  270. u32 *in = (u32 *)ctx->digest;
  271. u32 *hash = (u32 *)req->result;
  272. int i, d, big_endian = 0;
  273. if (!hash)
  274. return;
  275. switch (ctx->flags & FLAGS_MODE_MASK) {
  276. case FLAGS_MODE_MD5:
  277. d = MD5_DIGEST_SIZE / sizeof(u32);
  278. break;
  279. case FLAGS_MODE_SHA1:
  280. /* OMAP2 SHA1 is big endian */
  281. if (test_bit(FLAGS_BE32_SHA1, &ctx->dd->flags))
  282. big_endian = 1;
  283. d = SHA1_DIGEST_SIZE / sizeof(u32);
  284. break;
  285. case FLAGS_MODE_SHA224:
  286. d = SHA224_DIGEST_SIZE / sizeof(u32);
  287. break;
  288. case FLAGS_MODE_SHA256:
  289. d = SHA256_DIGEST_SIZE / sizeof(u32);
  290. break;
  291. case FLAGS_MODE_SHA384:
  292. d = SHA384_DIGEST_SIZE / sizeof(u32);
  293. break;
  294. case FLAGS_MODE_SHA512:
  295. d = SHA512_DIGEST_SIZE / sizeof(u32);
  296. break;
  297. default:
  298. d = 0;
  299. }
  300. if (big_endian)
  301. for (i = 0; i < d; i++)
  302. hash[i] = be32_to_cpup((__be32 *)in + i);
  303. else
  304. for (i = 0; i < d; i++)
  305. hash[i] = le32_to_cpup((__le32 *)in + i);
  306. }
  307. static void omap_sham_write_ctrl_omap2(struct omap_sham_dev *dd, size_t length,
  308. int final, int dma)
  309. {
  310. struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
  311. u32 val = length << 5, mask;
  312. if (likely(ctx->digcnt))
  313. omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
  314. omap_sham_write_mask(dd, SHA_REG_MASK(dd),
  315. SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
  316. SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
  317. /*
  318. * Setting ALGO_CONST only for the first iteration
  319. * and CLOSE_HASH only for the last one.
  320. */
  321. if ((ctx->flags & FLAGS_MODE_MASK) == FLAGS_MODE_SHA1)
  322. val |= SHA_REG_CTRL_ALGO;
  323. if (!ctx->digcnt)
  324. val |= SHA_REG_CTRL_ALGO_CONST;
  325. if (final)
  326. val |= SHA_REG_CTRL_CLOSE_HASH;
  327. mask = SHA_REG_CTRL_ALGO_CONST | SHA_REG_CTRL_CLOSE_HASH |
  328. SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
  329. omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
  330. }
  331. static void omap_sham_trigger_omap2(struct omap_sham_dev *dd, size_t length)
  332. {
  333. }
  334. static int omap_sham_poll_irq_omap2(struct omap_sham_dev *dd)
  335. {
  336. return omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY);
  337. }
  338. static int get_block_size(struct omap_sham_reqctx *ctx)
  339. {
  340. int d;
  341. switch (ctx->flags & FLAGS_MODE_MASK) {
  342. case FLAGS_MODE_MD5:
  343. case FLAGS_MODE_SHA1:
  344. d = SHA1_BLOCK_SIZE;
  345. break;
  346. case FLAGS_MODE_SHA224:
  347. case FLAGS_MODE_SHA256:
  348. d = SHA256_BLOCK_SIZE;
  349. break;
  350. case FLAGS_MODE_SHA384:
  351. case FLAGS_MODE_SHA512:
  352. d = SHA512_BLOCK_SIZE;
  353. break;
  354. default:
  355. d = 0;
  356. }
  357. return d;
  358. }
  359. static void omap_sham_write_n(struct omap_sham_dev *dd, u32 offset,
  360. u32 *value, int count)
  361. {
  362. for (; count--; value++, offset += 4)
  363. omap_sham_write(dd, offset, *value);
  364. }
  365. static void omap_sham_write_ctrl_omap4(struct omap_sham_dev *dd, size_t length,
  366. int final, int dma)
  367. {
  368. struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
  369. u32 val, mask;
  370. if (likely(ctx->digcnt))
  371. omap_sham_write(dd, SHA_REG_DIGCNT(dd), ctx->digcnt);
  372. /*
  373. * Setting ALGO_CONST only for the first iteration and
  374. * CLOSE_HASH only for the last one. Note that flags mode bits
  375. * correspond to algorithm encoding in mode register.
  376. */
  377. val = (ctx->flags & FLAGS_MODE_MASK) >> (FLAGS_MODE_SHIFT);
  378. if (!ctx->digcnt) {
  379. struct crypto_ahash *tfm = crypto_ahash_reqtfm(dd->req);
  380. struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
  381. struct omap_sham_hmac_ctx *bctx = tctx->base;
  382. int bs, nr_dr;
  383. val |= SHA_REG_MODE_ALGO_CONSTANT;
  384. if (ctx->flags & BIT(FLAGS_HMAC)) {
  385. bs = get_block_size(ctx);
  386. nr_dr = bs / (2 * sizeof(u32));
  387. val |= SHA_REG_MODE_HMAC_KEY_PROC;
  388. omap_sham_write_n(dd, SHA_REG_ODIGEST(dd, 0),
  389. (u32 *)bctx->ipad, nr_dr);
  390. omap_sham_write_n(dd, SHA_REG_IDIGEST(dd, 0),
  391. (u32 *)bctx->ipad + nr_dr, nr_dr);
  392. ctx->digcnt += bs;
  393. }
  394. }
  395. if (final) {
  396. val |= SHA_REG_MODE_CLOSE_HASH;
  397. if (ctx->flags & BIT(FLAGS_HMAC))
  398. val |= SHA_REG_MODE_HMAC_OUTER_HASH;
  399. }
  400. mask = SHA_REG_MODE_ALGO_CONSTANT | SHA_REG_MODE_CLOSE_HASH |
  401. SHA_REG_MODE_ALGO_MASK | SHA_REG_MODE_HMAC_OUTER_HASH |
  402. SHA_REG_MODE_HMAC_KEY_PROC;
  403. dev_dbg(dd->dev, "ctrl: %08x, flags: %08lx\n", val, ctx->flags);
  404. omap_sham_write_mask(dd, SHA_REG_MODE(dd), val, mask);
  405. omap_sham_write(dd, SHA_REG_IRQENA, SHA_REG_IRQENA_OUTPUT_RDY);
  406. omap_sham_write_mask(dd, SHA_REG_MASK(dd),
  407. SHA_REG_MASK_IT_EN |
  408. (dma ? SHA_REG_MASK_DMA_EN : 0),
  409. SHA_REG_MASK_IT_EN | SHA_REG_MASK_DMA_EN);
  410. }
  411. static void omap_sham_trigger_omap4(struct omap_sham_dev *dd, size_t length)
  412. {
  413. omap_sham_write(dd, SHA_REG_LENGTH(dd), length);
  414. }
  415. static int omap_sham_poll_irq_omap4(struct omap_sham_dev *dd)
  416. {
  417. return omap_sham_wait(dd, SHA_REG_IRQSTATUS,
  418. SHA_REG_IRQSTATUS_INPUT_RDY);
  419. }
  420. static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, size_t length,
  421. int final)
  422. {
  423. struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
  424. int count, len32, bs32, offset = 0;
  425. const u32 *buffer;
  426. int mlen;
  427. struct sg_mapping_iter mi;
  428. dev_dbg(dd->dev, "xmit_cpu: digcnt: %zd, length: %zd, final: %d\n",
  429. ctx->digcnt, length, final);
  430. dd->pdata->write_ctrl(dd, length, final, 0);
  431. dd->pdata->trigger(dd, length);
  432. /* should be non-zero before next lines to disable clocks later */
  433. ctx->digcnt += length;
  434. ctx->total -= length;
  435. if (final)
  436. set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
  437. set_bit(FLAGS_CPU, &dd->flags);
  438. len32 = DIV_ROUND_UP(length, sizeof(u32));
  439. bs32 = get_block_size(ctx) / sizeof(u32);
  440. sg_miter_start(&mi, ctx->sg, ctx->sg_len,
  441. SG_MITER_FROM_SG | SG_MITER_ATOMIC);
  442. mlen = 0;
  443. while (len32) {
  444. if (dd->pdata->poll_irq(dd))
  445. return -ETIMEDOUT;
  446. for (count = 0; count < min(len32, bs32); count++, offset++) {
  447. if (!mlen) {
  448. sg_miter_next(&mi);
  449. mlen = mi.length;
  450. if (!mlen) {
  451. pr_err("sg miter failure.\n");
  452. return -EINVAL;
  453. }
  454. offset = 0;
  455. buffer = mi.addr;
  456. }
  457. omap_sham_write(dd, SHA_REG_DIN(dd, count),
  458. buffer[offset]);
  459. mlen -= 4;
  460. }
  461. len32 -= min(len32, bs32);
  462. }
  463. sg_miter_stop(&mi);
  464. return -EINPROGRESS;
  465. }
  466. static void omap_sham_dma_callback(void *param)
  467. {
  468. struct omap_sham_dev *dd = param;
  469. set_bit(FLAGS_DMA_READY, &dd->flags);
  470. tasklet_schedule(&dd->done_task);
  471. }
  472. static int omap_sham_xmit_dma(struct omap_sham_dev *dd, size_t length,
  473. int final)
  474. {
  475. struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
  476. struct dma_async_tx_descriptor *tx;
  477. struct dma_slave_config cfg;
  478. int ret;
  479. dev_dbg(dd->dev, "xmit_dma: digcnt: %zd, length: %zd, final: %d\n",
  480. ctx->digcnt, length, final);
  481. if (!dma_map_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE)) {
  482. dev_err(dd->dev, "dma_map_sg error\n");
  483. return -EINVAL;
  484. }
  485. memset(&cfg, 0, sizeof(cfg));
  486. cfg.dst_addr = dd->phys_base + SHA_REG_DIN(dd, 0);
  487. cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
  488. cfg.dst_maxburst = get_block_size(ctx) / DMA_SLAVE_BUSWIDTH_4_BYTES;
  489. ret = dmaengine_slave_config(dd->dma_lch, &cfg);
  490. if (ret) {
  491. pr_err("omap-sham: can't configure dmaengine slave: %d\n", ret);
  492. return ret;
  493. }
  494. tx = dmaengine_prep_slave_sg(dd->dma_lch, ctx->sg, ctx->sg_len,
  495. DMA_MEM_TO_DEV,
  496. DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
  497. if (!tx) {
  498. dev_err(dd->dev, "prep_slave_sg failed\n");
  499. return -EINVAL;
  500. }
  501. tx->callback = omap_sham_dma_callback;
  502. tx->callback_param = dd;
  503. dd->pdata->write_ctrl(dd, length, final, 1);
  504. ctx->digcnt += length;
  505. ctx->total -= length;
  506. if (final)
  507. set_bit(FLAGS_FINAL, &dd->flags); /* catch last interrupt */
  508. set_bit(FLAGS_DMA_ACTIVE, &dd->flags);
  509. dmaengine_submit(tx);
  510. dma_async_issue_pending(dd->dma_lch);
  511. dd->pdata->trigger(dd, length);
  512. return -EINPROGRESS;
  513. }
  514. static int omap_sham_copy_sg_lists(struct omap_sham_reqctx *ctx,
  515. struct scatterlist *sg, int bs, int new_len)
  516. {
  517. int n = sg_nents(sg);
  518. struct scatterlist *tmp;
  519. int offset = ctx->offset;
  520. ctx->total = new_len;
  521. if (ctx->bufcnt)
  522. n++;
  523. ctx->sg = kmalloc_array(n, sizeof(*sg), GFP_KERNEL);
  524. if (!ctx->sg)
  525. return -ENOMEM;
  526. sg_init_table(ctx->sg, n);
  527. tmp = ctx->sg;
  528. ctx->sg_len = 0;
  529. if (ctx->bufcnt) {
  530. sg_set_buf(tmp, ctx->dd->xmit_buf, ctx->bufcnt);
  531. tmp = sg_next(tmp);
  532. ctx->sg_len++;
  533. new_len -= ctx->bufcnt;
  534. }
  535. while (sg && new_len) {
  536. int len = sg->length - offset;
  537. if (len <= 0) {
  538. offset -= sg->length;
  539. sg = sg_next(sg);
  540. continue;
  541. }
  542. if (new_len < len)
  543. len = new_len;
  544. if (len > 0) {
  545. new_len -= len;
  546. sg_set_page(tmp, sg_page(sg), len, sg->offset + offset);
  547. offset = 0;
  548. ctx->offset = 0;
  549. ctx->sg_len++;
  550. if (new_len <= 0)
  551. break;
  552. tmp = sg_next(tmp);
  553. }
  554. sg = sg_next(sg);
  555. }
  556. if (tmp)
  557. sg_mark_end(tmp);
  558. set_bit(FLAGS_SGS_ALLOCED, &ctx->dd->flags);
  559. ctx->offset += new_len - ctx->bufcnt;
  560. ctx->bufcnt = 0;
  561. return 0;
  562. }
  563. static int omap_sham_copy_sgs(struct omap_sham_reqctx *ctx,
  564. struct scatterlist *sg, int bs,
  565. unsigned int new_len)
  566. {
  567. int pages;
  568. void *buf;
  569. pages = get_order(new_len);
  570. buf = (void *)__get_free_pages(GFP_ATOMIC, pages);
  571. if (!buf) {
  572. pr_err("Couldn't allocate pages for unaligned cases.\n");
  573. return -ENOMEM;
  574. }
  575. if (ctx->bufcnt)
  576. memcpy(buf, ctx->dd->xmit_buf, ctx->bufcnt);
  577. scatterwalk_map_and_copy(buf + ctx->bufcnt, sg, ctx->offset,
  578. min(new_len, ctx->total) - ctx->bufcnt, 0);
  579. sg_init_table(ctx->sgl, 1);
  580. sg_set_buf(ctx->sgl, buf, new_len);
  581. ctx->sg = ctx->sgl;
  582. set_bit(FLAGS_SGS_COPIED, &ctx->dd->flags);
  583. ctx->sg_len = 1;
  584. ctx->offset += new_len - ctx->bufcnt;
  585. ctx->bufcnt = 0;
  586. ctx->total = new_len;
  587. return 0;
  588. }
  589. static int omap_sham_align_sgs(struct scatterlist *sg,
  590. int nbytes, int bs, bool final,
  591. struct omap_sham_reqctx *rctx)
  592. {
  593. int n = 0;
  594. bool aligned = true;
  595. bool list_ok = true;
  596. struct scatterlist *sg_tmp = sg;
  597. int new_len;
  598. int offset = rctx->offset;
  599. int bufcnt = rctx->bufcnt;
  600. if (!sg || !sg->length || !nbytes) {
  601. if (bufcnt) {
  602. bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs;
  603. sg_init_table(rctx->sgl, 1);
  604. sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt);
  605. rctx->sg = rctx->sgl;
  606. rctx->sg_len = 1;
  607. }
  608. return 0;
  609. }
  610. new_len = nbytes;
  611. if (offset)
  612. list_ok = false;
  613. if (final)
  614. new_len = DIV_ROUND_UP(new_len, bs) * bs;
  615. else
  616. new_len = (new_len - 1) / bs * bs;
  617. if (!new_len)
  618. return 0;
  619. if (nbytes != new_len)
  620. list_ok = false;
  621. while (nbytes > 0 && sg_tmp) {
  622. n++;
  623. if (bufcnt) {
  624. if (!IS_ALIGNED(bufcnt, bs)) {
  625. aligned = false;
  626. break;
  627. }
  628. nbytes -= bufcnt;
  629. bufcnt = 0;
  630. if (!nbytes)
  631. list_ok = false;
  632. continue;
  633. }
  634. #ifdef CONFIG_ZONE_DMA
  635. if (page_zonenum(sg_page(sg_tmp)) != ZONE_DMA) {
  636. aligned = false;
  637. break;
  638. }
  639. #endif
  640. if (offset < sg_tmp->length) {
  641. if (!IS_ALIGNED(offset + sg_tmp->offset, 4)) {
  642. aligned = false;
  643. break;
  644. }
  645. if (!IS_ALIGNED(sg_tmp->length - offset, bs)) {
  646. aligned = false;
  647. break;
  648. }
  649. }
  650. if (offset) {
  651. offset -= sg_tmp->length;
  652. if (offset < 0) {
  653. nbytes += offset;
  654. offset = 0;
  655. }
  656. } else {
  657. nbytes -= sg_tmp->length;
  658. }
  659. sg_tmp = sg_next(sg_tmp);
  660. if (nbytes < 0) {
  661. list_ok = false;
  662. break;
  663. }
  664. }
  665. if (new_len > OMAP_SHA_MAX_DMA_LEN) {
  666. new_len = OMAP_SHA_MAX_DMA_LEN;
  667. aligned = false;
  668. }
  669. if (!aligned)
  670. return omap_sham_copy_sgs(rctx, sg, bs, new_len);
  671. else if (!list_ok)
  672. return omap_sham_copy_sg_lists(rctx, sg, bs, new_len);
  673. rctx->total = new_len;
  674. rctx->offset += new_len;
  675. rctx->sg_len = n;
  676. if (rctx->bufcnt) {
  677. sg_init_table(rctx->sgl, 2);
  678. sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, rctx->bufcnt);
  679. sg_chain(rctx->sgl, 2, sg);
  680. rctx->sg = rctx->sgl;
  681. } else {
  682. rctx->sg = sg;
  683. }
  684. return 0;
  685. }
  686. static int omap_sham_prepare_request(struct crypto_engine *engine, void *areq)
  687. {
  688. struct ahash_request *req = container_of(areq, struct ahash_request,
  689. base);
  690. struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
  691. int bs;
  692. int ret;
  693. unsigned int nbytes;
  694. bool final = rctx->flags & BIT(FLAGS_FINUP);
  695. bool update = rctx->op == OP_UPDATE;
  696. int hash_later;
  697. bs = get_block_size(rctx);
  698. nbytes = rctx->bufcnt;
  699. if (update)
  700. nbytes += req->nbytes - rctx->offset;
  701. dev_dbg(rctx->dd->dev,
  702. "%s: nbytes=%d, bs=%d, total=%d, offset=%d, bufcnt=%zd\n",
  703. __func__, nbytes, bs, rctx->total, rctx->offset,
  704. rctx->bufcnt);
  705. if (!nbytes)
  706. return 0;
  707. rctx->total = nbytes;
  708. if (update && req->nbytes && (!IS_ALIGNED(rctx->bufcnt, bs))) {
  709. int len = bs - rctx->bufcnt % bs;
  710. if (len > req->nbytes)
  711. len = req->nbytes;
  712. scatterwalk_map_and_copy(rctx->buffer + rctx->bufcnt, req->src,
  713. 0, len, 0);
  714. rctx->bufcnt += len;
  715. rctx->offset = len;
  716. }
  717. if (rctx->bufcnt)
  718. memcpy(rctx->dd->xmit_buf, rctx->buffer, rctx->bufcnt);
  719. ret = omap_sham_align_sgs(req->src, nbytes, bs, final, rctx);
  720. if (ret)
  721. return ret;
  722. hash_later = nbytes - rctx->total;
  723. if (hash_later < 0)
  724. hash_later = 0;
  725. if (hash_later && hash_later <= rctx->buflen) {
  726. scatterwalk_map_and_copy(rctx->buffer,
  727. req->src,
  728. req->nbytes - hash_later,
  729. hash_later, 0);
  730. rctx->bufcnt = hash_later;
  731. } else {
  732. rctx->bufcnt = 0;
  733. }
  734. if (hash_later > rctx->buflen)
  735. set_bit(FLAGS_HUGE, &rctx->dd->flags);
  736. rctx->total = min(nbytes, rctx->total);
  737. return 0;
  738. }
  739. static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
  740. {
  741. struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
  742. dma_unmap_sg(dd->dev, ctx->sg, ctx->sg_len, DMA_TO_DEVICE);
  743. clear_bit(FLAGS_DMA_ACTIVE, &dd->flags);
  744. return 0;
  745. }
  746. static struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx)
  747. {
  748. struct omap_sham_dev *dd;
  749. if (ctx->dd)
  750. return ctx->dd;
  751. spin_lock_bh(&sham.lock);
  752. dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list);
  753. list_move_tail(&dd->list, &sham.dev_list);
  754. ctx->dd = dd;
  755. spin_unlock_bh(&sham.lock);
  756. return dd;
  757. }
  758. static int omap_sham_init(struct ahash_request *req)
  759. {
  760. struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
  761. struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
  762. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  763. struct omap_sham_dev *dd;
  764. int bs = 0;
  765. ctx->dd = NULL;
  766. dd = omap_sham_find_dev(ctx);
  767. if (!dd)
  768. return -ENODEV;
  769. ctx->flags = 0;
  770. dev_dbg(dd->dev, "init: digest size: %d\n",
  771. crypto_ahash_digestsize(tfm));
  772. switch (crypto_ahash_digestsize(tfm)) {
  773. case MD5_DIGEST_SIZE:
  774. ctx->flags |= FLAGS_MODE_MD5;
  775. bs = SHA1_BLOCK_SIZE;
  776. break;
  777. case SHA1_DIGEST_SIZE:
  778. ctx->flags |= FLAGS_MODE_SHA1;
  779. bs = SHA1_BLOCK_SIZE;
  780. break;
  781. case SHA224_DIGEST_SIZE:
  782. ctx->flags |= FLAGS_MODE_SHA224;
  783. bs = SHA224_BLOCK_SIZE;
  784. break;
  785. case SHA256_DIGEST_SIZE:
  786. ctx->flags |= FLAGS_MODE_SHA256;
  787. bs = SHA256_BLOCK_SIZE;
  788. break;
  789. case SHA384_DIGEST_SIZE:
  790. ctx->flags |= FLAGS_MODE_SHA384;
  791. bs = SHA384_BLOCK_SIZE;
  792. break;
  793. case SHA512_DIGEST_SIZE:
  794. ctx->flags |= FLAGS_MODE_SHA512;
  795. bs = SHA512_BLOCK_SIZE;
  796. break;
  797. }
  798. ctx->bufcnt = 0;
  799. ctx->digcnt = 0;
  800. ctx->total = 0;
  801. ctx->offset = 0;
  802. ctx->buflen = BUFLEN;
  803. if (tctx->flags & BIT(FLAGS_HMAC)) {
  804. if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) {
  805. struct omap_sham_hmac_ctx *bctx = tctx->base;
  806. memcpy(ctx->buffer, bctx->ipad, bs);
  807. ctx->bufcnt = bs;
  808. }
  809. ctx->flags |= BIT(FLAGS_HMAC);
  810. }
  811. return 0;
  812. }
  813. static int omap_sham_update_req(struct omap_sham_dev *dd)
  814. {
  815. struct ahash_request *req = dd->req;
  816. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  817. int err;
  818. bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
  819. !(dd->flags & BIT(FLAGS_HUGE));
  820. dev_dbg(dd->dev, "update_req: total: %u, digcnt: %zd, final: %d",
  821. ctx->total, ctx->digcnt, final);
  822. if (ctx->total < get_block_size(ctx) ||
  823. ctx->total < dd->fallback_sz)
  824. ctx->flags |= BIT(FLAGS_CPU);
  825. if (ctx->flags & BIT(FLAGS_CPU))
  826. err = omap_sham_xmit_cpu(dd, ctx->total, final);
  827. else
  828. err = omap_sham_xmit_dma(dd, ctx->total, final);
  829. /* wait for dma completion before can take more data */
  830. dev_dbg(dd->dev, "update: err: %d, digcnt: %zd\n", err, ctx->digcnt);
  831. return err;
  832. }
  833. static int omap_sham_final_req(struct omap_sham_dev *dd)
  834. {
  835. struct ahash_request *req = dd->req;
  836. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  837. int err = 0, use_dma = 1;
  838. if (dd->flags & BIT(FLAGS_HUGE))
  839. return 0;
  840. if ((ctx->total <= get_block_size(ctx)) || dd->polling_mode)
  841. /*
  842. * faster to handle last block with cpu or
  843. * use cpu when dma is not present.
  844. */
  845. use_dma = 0;
  846. if (use_dma)
  847. err = omap_sham_xmit_dma(dd, ctx->total, 1);
  848. else
  849. err = omap_sham_xmit_cpu(dd, ctx->total, 1);
  850. ctx->bufcnt = 0;
  851. dev_dbg(dd->dev, "final_req: err: %d\n", err);
  852. return err;
  853. }
  854. static int omap_sham_hash_one_req(struct crypto_engine *engine, void *areq)
  855. {
  856. struct ahash_request *req = container_of(areq, struct ahash_request,
  857. base);
  858. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  859. struct omap_sham_dev *dd = ctx->dd;
  860. int err;
  861. bool final = (ctx->flags & BIT(FLAGS_FINUP)) &&
  862. !(dd->flags & BIT(FLAGS_HUGE));
  863. dev_dbg(dd->dev, "hash-one: op: %u, total: %u, digcnt: %zd, final: %d",
  864. ctx->op, ctx->total, ctx->digcnt, final);
  865. err = pm_runtime_resume_and_get(dd->dev);
  866. if (err < 0) {
  867. dev_err(dd->dev, "failed to get sync: %d\n", err);
  868. return err;
  869. }
  870. dd->err = 0;
  871. dd->req = req;
  872. if (ctx->digcnt)
  873. dd->pdata->copy_hash(req, 0);
  874. if (ctx->op == OP_UPDATE)
  875. err = omap_sham_update_req(dd);
  876. else if (ctx->op == OP_FINAL)
  877. err = omap_sham_final_req(dd);
  878. if (err != -EINPROGRESS)
  879. omap_sham_finish_req(req, err);
  880. return 0;
  881. }
  882. static int omap_sham_finish_hmac(struct ahash_request *req)
  883. {
  884. struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
  885. struct omap_sham_hmac_ctx *bctx = tctx->base;
  886. int bs = crypto_shash_blocksize(bctx->shash);
  887. int ds = crypto_shash_digestsize(bctx->shash);
  888. SHASH_DESC_ON_STACK(shash, bctx->shash);
  889. shash->tfm = bctx->shash;
  890. return crypto_shash_init(shash) ?:
  891. crypto_shash_update(shash, bctx->opad, bs) ?:
  892. crypto_shash_finup(shash, req->result, ds, req->result);
  893. }
  894. static int omap_sham_finish(struct ahash_request *req)
  895. {
  896. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  897. struct omap_sham_dev *dd = ctx->dd;
  898. int err = 0;
  899. if (ctx->digcnt) {
  900. omap_sham_copy_ready_hash(req);
  901. if ((ctx->flags & BIT(FLAGS_HMAC)) &&
  902. !test_bit(FLAGS_AUTO_XOR, &dd->flags))
  903. err = omap_sham_finish_hmac(req);
  904. }
  905. dev_dbg(dd->dev, "digcnt: %zd, bufcnt: %zd\n", ctx->digcnt, ctx->bufcnt);
  906. return err;
  907. }
  908. static void omap_sham_finish_req(struct ahash_request *req, int err)
  909. {
  910. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  911. struct omap_sham_dev *dd = ctx->dd;
  912. if (test_bit(FLAGS_SGS_COPIED, &dd->flags))
  913. free_pages((unsigned long)sg_virt(ctx->sg),
  914. get_order(ctx->sg->length));
  915. if (test_bit(FLAGS_SGS_ALLOCED, &dd->flags))
  916. kfree(ctx->sg);
  917. ctx->sg = NULL;
  918. dd->flags &= ~(BIT(FLAGS_SGS_ALLOCED) | BIT(FLAGS_SGS_COPIED) |
  919. BIT(FLAGS_CPU) | BIT(FLAGS_DMA_READY) |
  920. BIT(FLAGS_OUTPUT_READY));
  921. if (!err)
  922. dd->pdata->copy_hash(req, 1);
  923. if (dd->flags & BIT(FLAGS_HUGE)) {
  924. /* Re-enqueue the request */
  925. omap_sham_enqueue(req, ctx->op);
  926. return;
  927. }
  928. if (!err) {
  929. if (test_bit(FLAGS_FINAL, &dd->flags))
  930. err = omap_sham_finish(req);
  931. } else {
  932. ctx->flags |= BIT(FLAGS_ERROR);
  933. }
  934. /* atomic operation is not needed here */
  935. dd->flags &= ~(BIT(FLAGS_FINAL) | BIT(FLAGS_CPU) |
  936. BIT(FLAGS_DMA_READY) | BIT(FLAGS_OUTPUT_READY));
  937. pm_runtime_mark_last_busy(dd->dev);
  938. pm_runtime_put_autosuspend(dd->dev);
  939. ctx->offset = 0;
  940. crypto_finalize_hash_request(dd->engine, req, err);
  941. }
  942. static int omap_sham_handle_queue(struct omap_sham_dev *dd,
  943. struct ahash_request *req)
  944. {
  945. return crypto_transfer_hash_request_to_engine(dd->engine, req);
  946. }
  947. static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
  948. {
  949. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  950. struct omap_sham_dev *dd = ctx->dd;
  951. ctx->op = op;
  952. return omap_sham_handle_queue(dd, req);
  953. }
  954. static int omap_sham_update(struct ahash_request *req)
  955. {
  956. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  957. struct omap_sham_dev *dd = omap_sham_find_dev(ctx);
  958. if (!req->nbytes)
  959. return 0;
  960. if (ctx->bufcnt + req->nbytes <= ctx->buflen) {
  961. scatterwalk_map_and_copy(ctx->buffer + ctx->bufcnt, req->src,
  962. 0, req->nbytes, 0);
  963. ctx->bufcnt += req->nbytes;
  964. return 0;
  965. }
  966. if (dd->polling_mode)
  967. ctx->flags |= BIT(FLAGS_CPU);
  968. return omap_sham_enqueue(req, OP_UPDATE);
  969. }
  970. static int omap_sham_final_shash(struct ahash_request *req)
  971. {
  972. struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
  973. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  974. int offset = 0;
  975. /*
  976. * If we are running HMAC on limited hardware support, skip
  977. * the ipad in the beginning of the buffer if we are going for
  978. * software fallback algorithm.
  979. */
  980. if (test_bit(FLAGS_HMAC, &ctx->flags) &&
  981. !test_bit(FLAGS_AUTO_XOR, &ctx->dd->flags))
  982. offset = get_block_size(ctx);
  983. return crypto_shash_tfm_digest(tctx->fallback, ctx->buffer + offset,
  984. ctx->bufcnt - offset, req->result);
  985. }
  986. static int omap_sham_final(struct ahash_request *req)
  987. {
  988. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  989. ctx->flags |= BIT(FLAGS_FINUP);
  990. if (ctx->flags & BIT(FLAGS_ERROR))
  991. return 0; /* uncompleted hash is not needed */
  992. /*
  993. * OMAP HW accel works only with buffers >= 9.
  994. * HMAC is always >= 9 because ipad == block size.
  995. * If buffersize is less than fallback_sz, we use fallback
  996. * SW encoding, as using DMA + HW in this case doesn't provide
  997. * any benefit.
  998. */
  999. if (!ctx->digcnt && ctx->bufcnt < ctx->dd->fallback_sz)
  1000. return omap_sham_final_shash(req);
  1001. else if (ctx->bufcnt)
  1002. return omap_sham_enqueue(req, OP_FINAL);
  1003. /* copy ready hash (+ finalize hmac) */
  1004. return omap_sham_finish(req);
  1005. }
  1006. static int omap_sham_finup(struct ahash_request *req)
  1007. {
  1008. struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
  1009. int err1, err2;
  1010. ctx->flags |= BIT(FLAGS_FINUP);
  1011. err1 = omap_sham_update(req);
  1012. if (err1 == -EINPROGRESS || err1 == -EBUSY)
  1013. return err1;
  1014. /*
  1015. * final() has to be always called to cleanup resources
  1016. * even if udpate() failed, except EINPROGRESS
  1017. */
  1018. err2 = omap_sham_final(req);
  1019. return err1 ?: err2;
  1020. }
  1021. static int omap_sham_digest(struct ahash_request *req)
  1022. {
  1023. return omap_sham_init(req) ?: omap_sham_finup(req);
  1024. }
  1025. static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key,
  1026. unsigned int keylen)
  1027. {
  1028. struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm);
  1029. struct omap_sham_hmac_ctx *bctx = tctx->base;
  1030. int bs = crypto_shash_blocksize(bctx->shash);
  1031. int ds = crypto_shash_digestsize(bctx->shash);
  1032. int err, i;
  1033. err = crypto_shash_setkey(tctx->fallback, key, keylen);
  1034. if (err)
  1035. return err;
  1036. if (keylen > bs) {
  1037. err = crypto_shash_tfm_digest(bctx->shash, key, keylen,
  1038. bctx->ipad);
  1039. if (err)
  1040. return err;
  1041. keylen = ds;
  1042. } else {
  1043. memcpy(bctx->ipad, key, keylen);
  1044. }
  1045. memset(bctx->ipad + keylen, 0, bs - keylen);
  1046. if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) {
  1047. memcpy(bctx->opad, bctx->ipad, bs);
  1048. for (i = 0; i < bs; i++) {
  1049. bctx->ipad[i] ^= HMAC_IPAD_VALUE;
  1050. bctx->opad[i] ^= HMAC_OPAD_VALUE;
  1051. }
  1052. }
  1053. return err;
  1054. }
  1055. static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
  1056. {
  1057. struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
  1058. const char *alg_name = crypto_tfm_alg_name(tfm);
  1059. /* Allocate a fallback and abort if it failed. */
  1060. tctx->fallback = crypto_alloc_shash(alg_name, 0,
  1061. CRYPTO_ALG_NEED_FALLBACK);
  1062. if (IS_ERR(tctx->fallback)) {
  1063. pr_err("omap-sham: fallback driver '%s' "
  1064. "could not be loaded.\n", alg_name);
  1065. return PTR_ERR(tctx->fallback);
  1066. }
  1067. crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
  1068. sizeof(struct omap_sham_reqctx) + BUFLEN);
  1069. if (alg_base) {
  1070. struct omap_sham_hmac_ctx *bctx = tctx->base;
  1071. tctx->flags |= BIT(FLAGS_HMAC);
  1072. bctx->shash = crypto_alloc_shash(alg_base, 0,
  1073. CRYPTO_ALG_NEED_FALLBACK);
  1074. if (IS_ERR(bctx->shash)) {
  1075. pr_err("omap-sham: base driver '%s' "
  1076. "could not be loaded.\n", alg_base);
  1077. crypto_free_shash(tctx->fallback);
  1078. return PTR_ERR(bctx->shash);
  1079. }
  1080. }
  1081. tctx->enginectx.op.do_one_request = omap_sham_hash_one_req;
  1082. tctx->enginectx.op.prepare_request = omap_sham_prepare_request;
  1083. tctx->enginectx.op.unprepare_request = NULL;
  1084. return 0;
  1085. }
  1086. static int omap_sham_cra_init(struct crypto_tfm *tfm)
  1087. {
  1088. return omap_sham_cra_init_alg(tfm, NULL);
  1089. }
  1090. static int omap_sham_cra_sha1_init(struct crypto_tfm *tfm)
  1091. {
  1092. return omap_sham_cra_init_alg(tfm, "sha1");
  1093. }
  1094. static int omap_sham_cra_sha224_init(struct crypto_tfm *tfm)
  1095. {
  1096. return omap_sham_cra_init_alg(tfm, "sha224");
  1097. }
  1098. static int omap_sham_cra_sha256_init(struct crypto_tfm *tfm)
  1099. {
  1100. return omap_sham_cra_init_alg(tfm, "sha256");
  1101. }
  1102. static int omap_sham_cra_md5_init(struct crypto_tfm *tfm)
  1103. {
  1104. return omap_sham_cra_init_alg(tfm, "md5");
  1105. }
  1106. static int omap_sham_cra_sha384_init(struct crypto_tfm *tfm)
  1107. {
  1108. return omap_sham_cra_init_alg(tfm, "sha384");
  1109. }
  1110. static int omap_sham_cra_sha512_init(struct crypto_tfm *tfm)
  1111. {
  1112. return omap_sham_cra_init_alg(tfm, "sha512");
  1113. }
  1114. static void omap_sham_cra_exit(struct crypto_tfm *tfm)
  1115. {
  1116. struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
  1117. crypto_free_shash(tctx->fallback);
  1118. tctx->fallback = NULL;
  1119. if (tctx->flags & BIT(FLAGS_HMAC)) {
  1120. struct omap_sham_hmac_ctx *bctx = tctx->base;
  1121. crypto_free_shash(bctx->shash);
  1122. }
  1123. }
  1124. static int omap_sham_export(struct ahash_request *req, void *out)
  1125. {
  1126. struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
  1127. memcpy(out, rctx, sizeof(*rctx) + rctx->bufcnt);
  1128. return 0;
  1129. }
  1130. static int omap_sham_import(struct ahash_request *req, const void *in)
  1131. {
  1132. struct omap_sham_reqctx *rctx = ahash_request_ctx(req);
  1133. const struct omap_sham_reqctx *ctx_in = in;
  1134. memcpy(rctx, in, sizeof(*rctx) + ctx_in->bufcnt);
  1135. return 0;
  1136. }
  1137. static struct ahash_alg algs_sha1_md5[] = {
  1138. {
  1139. .init = omap_sham_init,
  1140. .update = omap_sham_update,
  1141. .final = omap_sham_final,
  1142. .finup = omap_sham_finup,
  1143. .digest = omap_sham_digest,
  1144. .halg.digestsize = SHA1_DIGEST_SIZE,
  1145. .halg.base = {
  1146. .cra_name = "sha1",
  1147. .cra_driver_name = "omap-sha1",
  1148. .cra_priority = 400,
  1149. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1150. CRYPTO_ALG_ASYNC |
  1151. CRYPTO_ALG_NEED_FALLBACK,
  1152. .cra_blocksize = SHA1_BLOCK_SIZE,
  1153. .cra_ctxsize = sizeof(struct omap_sham_ctx),
  1154. .cra_alignmask = OMAP_ALIGN_MASK,
  1155. .cra_module = THIS_MODULE,
  1156. .cra_init = omap_sham_cra_init,
  1157. .cra_exit = omap_sham_cra_exit,
  1158. }
  1159. },
  1160. {
  1161. .init = omap_sham_init,
  1162. .update = omap_sham_update,
  1163. .final = omap_sham_final,
  1164. .finup = omap_sham_finup,
  1165. .digest = omap_sham_digest,
  1166. .halg.digestsize = MD5_DIGEST_SIZE,
  1167. .halg.base = {
  1168. .cra_name = "md5",
  1169. .cra_driver_name = "omap-md5",
  1170. .cra_priority = 400,
  1171. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1172. CRYPTO_ALG_ASYNC |
  1173. CRYPTO_ALG_NEED_FALLBACK,
  1174. .cra_blocksize = SHA1_BLOCK_SIZE,
  1175. .cra_ctxsize = sizeof(struct omap_sham_ctx),
  1176. .cra_alignmask = OMAP_ALIGN_MASK,
  1177. .cra_module = THIS_MODULE,
  1178. .cra_init = omap_sham_cra_init,
  1179. .cra_exit = omap_sham_cra_exit,
  1180. }
  1181. },
  1182. {
  1183. .init = omap_sham_init,
  1184. .update = omap_sham_update,
  1185. .final = omap_sham_final,
  1186. .finup = omap_sham_finup,
  1187. .digest = omap_sham_digest,
  1188. .setkey = omap_sham_setkey,
  1189. .halg.digestsize = SHA1_DIGEST_SIZE,
  1190. .halg.base = {
  1191. .cra_name = "hmac(sha1)",
  1192. .cra_driver_name = "omap-hmac-sha1",
  1193. .cra_priority = 400,
  1194. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1195. CRYPTO_ALG_ASYNC |
  1196. CRYPTO_ALG_NEED_FALLBACK,
  1197. .cra_blocksize = SHA1_BLOCK_SIZE,
  1198. .cra_ctxsize = sizeof(struct omap_sham_ctx) +
  1199. sizeof(struct omap_sham_hmac_ctx),
  1200. .cra_alignmask = OMAP_ALIGN_MASK,
  1201. .cra_module = THIS_MODULE,
  1202. .cra_init = omap_sham_cra_sha1_init,
  1203. .cra_exit = omap_sham_cra_exit,
  1204. }
  1205. },
  1206. {
  1207. .init = omap_sham_init,
  1208. .update = omap_sham_update,
  1209. .final = omap_sham_final,
  1210. .finup = omap_sham_finup,
  1211. .digest = omap_sham_digest,
  1212. .setkey = omap_sham_setkey,
  1213. .halg.digestsize = MD5_DIGEST_SIZE,
  1214. .halg.base = {
  1215. .cra_name = "hmac(md5)",
  1216. .cra_driver_name = "omap-hmac-md5",
  1217. .cra_priority = 400,
  1218. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1219. CRYPTO_ALG_ASYNC |
  1220. CRYPTO_ALG_NEED_FALLBACK,
  1221. .cra_blocksize = SHA1_BLOCK_SIZE,
  1222. .cra_ctxsize = sizeof(struct omap_sham_ctx) +
  1223. sizeof(struct omap_sham_hmac_ctx),
  1224. .cra_alignmask = OMAP_ALIGN_MASK,
  1225. .cra_module = THIS_MODULE,
  1226. .cra_init = omap_sham_cra_md5_init,
  1227. .cra_exit = omap_sham_cra_exit,
  1228. }
  1229. }
  1230. };
  1231. /* OMAP4 has some algs in addition to what OMAP2 has */
  1232. static struct ahash_alg algs_sha224_sha256[] = {
  1233. {
  1234. .init = omap_sham_init,
  1235. .update = omap_sham_update,
  1236. .final = omap_sham_final,
  1237. .finup = omap_sham_finup,
  1238. .digest = omap_sham_digest,
  1239. .halg.digestsize = SHA224_DIGEST_SIZE,
  1240. .halg.base = {
  1241. .cra_name = "sha224",
  1242. .cra_driver_name = "omap-sha224",
  1243. .cra_priority = 400,
  1244. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1245. CRYPTO_ALG_ASYNC |
  1246. CRYPTO_ALG_NEED_FALLBACK,
  1247. .cra_blocksize = SHA224_BLOCK_SIZE,
  1248. .cra_ctxsize = sizeof(struct omap_sham_ctx),
  1249. .cra_alignmask = OMAP_ALIGN_MASK,
  1250. .cra_module = THIS_MODULE,
  1251. .cra_init = omap_sham_cra_init,
  1252. .cra_exit = omap_sham_cra_exit,
  1253. }
  1254. },
  1255. {
  1256. .init = omap_sham_init,
  1257. .update = omap_sham_update,
  1258. .final = omap_sham_final,
  1259. .finup = omap_sham_finup,
  1260. .digest = omap_sham_digest,
  1261. .halg.digestsize = SHA256_DIGEST_SIZE,
  1262. .halg.base = {
  1263. .cra_name = "sha256",
  1264. .cra_driver_name = "omap-sha256",
  1265. .cra_priority = 400,
  1266. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1267. CRYPTO_ALG_ASYNC |
  1268. CRYPTO_ALG_NEED_FALLBACK,
  1269. .cra_blocksize = SHA256_BLOCK_SIZE,
  1270. .cra_ctxsize = sizeof(struct omap_sham_ctx),
  1271. .cra_alignmask = OMAP_ALIGN_MASK,
  1272. .cra_module = THIS_MODULE,
  1273. .cra_init = omap_sham_cra_init,
  1274. .cra_exit = omap_sham_cra_exit,
  1275. }
  1276. },
  1277. {
  1278. .init = omap_sham_init,
  1279. .update = omap_sham_update,
  1280. .final = omap_sham_final,
  1281. .finup = omap_sham_finup,
  1282. .digest = omap_sham_digest,
  1283. .setkey = omap_sham_setkey,
  1284. .halg.digestsize = SHA224_DIGEST_SIZE,
  1285. .halg.base = {
  1286. .cra_name = "hmac(sha224)",
  1287. .cra_driver_name = "omap-hmac-sha224",
  1288. .cra_priority = 400,
  1289. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1290. CRYPTO_ALG_ASYNC |
  1291. CRYPTO_ALG_NEED_FALLBACK,
  1292. .cra_blocksize = SHA224_BLOCK_SIZE,
  1293. .cra_ctxsize = sizeof(struct omap_sham_ctx) +
  1294. sizeof(struct omap_sham_hmac_ctx),
  1295. .cra_alignmask = OMAP_ALIGN_MASK,
  1296. .cra_module = THIS_MODULE,
  1297. .cra_init = omap_sham_cra_sha224_init,
  1298. .cra_exit = omap_sham_cra_exit,
  1299. }
  1300. },
  1301. {
  1302. .init = omap_sham_init,
  1303. .update = omap_sham_update,
  1304. .final = omap_sham_final,
  1305. .finup = omap_sham_finup,
  1306. .digest = omap_sham_digest,
  1307. .setkey = omap_sham_setkey,
  1308. .halg.digestsize = SHA256_DIGEST_SIZE,
  1309. .halg.base = {
  1310. .cra_name = "hmac(sha256)",
  1311. .cra_driver_name = "omap-hmac-sha256",
  1312. .cra_priority = 400,
  1313. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1314. CRYPTO_ALG_ASYNC |
  1315. CRYPTO_ALG_NEED_FALLBACK,
  1316. .cra_blocksize = SHA256_BLOCK_SIZE,
  1317. .cra_ctxsize = sizeof(struct omap_sham_ctx) +
  1318. sizeof(struct omap_sham_hmac_ctx),
  1319. .cra_alignmask = OMAP_ALIGN_MASK,
  1320. .cra_module = THIS_MODULE,
  1321. .cra_init = omap_sham_cra_sha256_init,
  1322. .cra_exit = omap_sham_cra_exit,
  1323. }
  1324. },
  1325. };
  1326. static struct ahash_alg algs_sha384_sha512[] = {
  1327. {
  1328. .init = omap_sham_init,
  1329. .update = omap_sham_update,
  1330. .final = omap_sham_final,
  1331. .finup = omap_sham_finup,
  1332. .digest = omap_sham_digest,
  1333. .halg.digestsize = SHA384_DIGEST_SIZE,
  1334. .halg.base = {
  1335. .cra_name = "sha384",
  1336. .cra_driver_name = "omap-sha384",
  1337. .cra_priority = 400,
  1338. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1339. CRYPTO_ALG_ASYNC |
  1340. CRYPTO_ALG_NEED_FALLBACK,
  1341. .cra_blocksize = SHA384_BLOCK_SIZE,
  1342. .cra_ctxsize = sizeof(struct omap_sham_ctx),
  1343. .cra_alignmask = OMAP_ALIGN_MASK,
  1344. .cra_module = THIS_MODULE,
  1345. .cra_init = omap_sham_cra_init,
  1346. .cra_exit = omap_sham_cra_exit,
  1347. }
  1348. },
  1349. {
  1350. .init = omap_sham_init,
  1351. .update = omap_sham_update,
  1352. .final = omap_sham_final,
  1353. .finup = omap_sham_finup,
  1354. .digest = omap_sham_digest,
  1355. .halg.digestsize = SHA512_DIGEST_SIZE,
  1356. .halg.base = {
  1357. .cra_name = "sha512",
  1358. .cra_driver_name = "omap-sha512",
  1359. .cra_priority = 400,
  1360. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1361. CRYPTO_ALG_ASYNC |
  1362. CRYPTO_ALG_NEED_FALLBACK,
  1363. .cra_blocksize = SHA512_BLOCK_SIZE,
  1364. .cra_ctxsize = sizeof(struct omap_sham_ctx),
  1365. .cra_alignmask = OMAP_ALIGN_MASK,
  1366. .cra_module = THIS_MODULE,
  1367. .cra_init = omap_sham_cra_init,
  1368. .cra_exit = omap_sham_cra_exit,
  1369. }
  1370. },
  1371. {
  1372. .init = omap_sham_init,
  1373. .update = omap_sham_update,
  1374. .final = omap_sham_final,
  1375. .finup = omap_sham_finup,
  1376. .digest = omap_sham_digest,
  1377. .setkey = omap_sham_setkey,
  1378. .halg.digestsize = SHA384_DIGEST_SIZE,
  1379. .halg.base = {
  1380. .cra_name = "hmac(sha384)",
  1381. .cra_driver_name = "omap-hmac-sha384",
  1382. .cra_priority = 400,
  1383. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1384. CRYPTO_ALG_ASYNC |
  1385. CRYPTO_ALG_NEED_FALLBACK,
  1386. .cra_blocksize = SHA384_BLOCK_SIZE,
  1387. .cra_ctxsize = sizeof(struct omap_sham_ctx) +
  1388. sizeof(struct omap_sham_hmac_ctx),
  1389. .cra_alignmask = OMAP_ALIGN_MASK,
  1390. .cra_module = THIS_MODULE,
  1391. .cra_init = omap_sham_cra_sha384_init,
  1392. .cra_exit = omap_sham_cra_exit,
  1393. }
  1394. },
  1395. {
  1396. .init = omap_sham_init,
  1397. .update = omap_sham_update,
  1398. .final = omap_sham_final,
  1399. .finup = omap_sham_finup,
  1400. .digest = omap_sham_digest,
  1401. .setkey = omap_sham_setkey,
  1402. .halg.digestsize = SHA512_DIGEST_SIZE,
  1403. .halg.base = {
  1404. .cra_name = "hmac(sha512)",
  1405. .cra_driver_name = "omap-hmac-sha512",
  1406. .cra_priority = 400,
  1407. .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
  1408. CRYPTO_ALG_ASYNC |
  1409. CRYPTO_ALG_NEED_FALLBACK,
  1410. .cra_blocksize = SHA512_BLOCK_SIZE,
  1411. .cra_ctxsize = sizeof(struct omap_sham_ctx) +
  1412. sizeof(struct omap_sham_hmac_ctx),
  1413. .cra_alignmask = OMAP_ALIGN_MASK,
  1414. .cra_module = THIS_MODULE,
  1415. .cra_init = omap_sham_cra_sha512_init,
  1416. .cra_exit = omap_sham_cra_exit,
  1417. }
  1418. },
  1419. };
  1420. static void omap_sham_done_task(unsigned long data)
  1421. {
  1422. struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
  1423. int err = 0;
  1424. dev_dbg(dd->dev, "%s: flags=%lx\n", __func__, dd->flags);
  1425. if (test_bit(FLAGS_CPU, &dd->flags)) {
  1426. if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags))
  1427. goto finish;
  1428. } else if (test_bit(FLAGS_DMA_READY, &dd->flags)) {
  1429. if (test_bit(FLAGS_DMA_ACTIVE, &dd->flags)) {
  1430. omap_sham_update_dma_stop(dd);
  1431. if (dd->err) {
  1432. err = dd->err;
  1433. goto finish;
  1434. }
  1435. }
  1436. if (test_and_clear_bit(FLAGS_OUTPUT_READY, &dd->flags)) {
  1437. /* hash or semi-hash ready */
  1438. clear_bit(FLAGS_DMA_READY, &dd->flags);
  1439. goto finish;
  1440. }
  1441. }
  1442. return;
  1443. finish:
  1444. dev_dbg(dd->dev, "update done: err: %d\n", err);
  1445. /* finish curent request */
  1446. omap_sham_finish_req(dd->req, err);
  1447. }
  1448. static irqreturn_t omap_sham_irq_common(struct omap_sham_dev *dd)
  1449. {
  1450. set_bit(FLAGS_OUTPUT_READY, &dd->flags);
  1451. tasklet_schedule(&dd->done_task);
  1452. return IRQ_HANDLED;
  1453. }
  1454. static irqreturn_t omap_sham_irq_omap2(int irq, void *dev_id)
  1455. {
  1456. struct omap_sham_dev *dd = dev_id;
  1457. if (unlikely(test_bit(FLAGS_FINAL, &dd->flags)))
  1458. /* final -> allow device to go to power-saving mode */
  1459. omap_sham_write_mask(dd, SHA_REG_CTRL, 0, SHA_REG_CTRL_LENGTH);
  1460. omap_sham_write_mask(dd, SHA_REG_CTRL, SHA_REG_CTRL_OUTPUT_READY,
  1461. SHA_REG_CTRL_OUTPUT_READY);
  1462. omap_sham_read(dd, SHA_REG_CTRL);
  1463. return omap_sham_irq_common(dd);
  1464. }
  1465. static irqreturn_t omap_sham_irq_omap4(int irq, void *dev_id)
  1466. {
  1467. struct omap_sham_dev *dd = dev_id;
  1468. omap_sham_write_mask(dd, SHA_REG_MASK(dd), 0, SHA_REG_MASK_IT_EN);
  1469. return omap_sham_irq_common(dd);
  1470. }
  1471. static struct omap_sham_algs_info omap_sham_algs_info_omap2[] = {
  1472. {
  1473. .algs_list = algs_sha1_md5,
  1474. .size = ARRAY_SIZE(algs_sha1_md5),
  1475. },
  1476. };
  1477. static const struct omap_sham_pdata omap_sham_pdata_omap2 = {
  1478. .algs_info = omap_sham_algs_info_omap2,
  1479. .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap2),
  1480. .flags = BIT(FLAGS_BE32_SHA1),
  1481. .digest_size = SHA1_DIGEST_SIZE,
  1482. .copy_hash = omap_sham_copy_hash_omap2,
  1483. .write_ctrl = omap_sham_write_ctrl_omap2,
  1484. .trigger = omap_sham_trigger_omap2,
  1485. .poll_irq = omap_sham_poll_irq_omap2,
  1486. .intr_hdlr = omap_sham_irq_omap2,
  1487. .idigest_ofs = 0x00,
  1488. .din_ofs = 0x1c,
  1489. .digcnt_ofs = 0x14,
  1490. .rev_ofs = 0x5c,
  1491. .mask_ofs = 0x60,
  1492. .sysstatus_ofs = 0x64,
  1493. .major_mask = 0xf0,
  1494. .major_shift = 4,
  1495. .minor_mask = 0x0f,
  1496. .minor_shift = 0,
  1497. };
  1498. #ifdef CONFIG_OF
  1499. static struct omap_sham_algs_info omap_sham_algs_info_omap4[] = {
  1500. {
  1501. .algs_list = algs_sha1_md5,
  1502. .size = ARRAY_SIZE(algs_sha1_md5),
  1503. },
  1504. {
  1505. .algs_list = algs_sha224_sha256,
  1506. .size = ARRAY_SIZE(algs_sha224_sha256),
  1507. },
  1508. };
  1509. static const struct omap_sham_pdata omap_sham_pdata_omap4 = {
  1510. .algs_info = omap_sham_algs_info_omap4,
  1511. .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap4),
  1512. .flags = BIT(FLAGS_AUTO_XOR),
  1513. .digest_size = SHA256_DIGEST_SIZE,
  1514. .copy_hash = omap_sham_copy_hash_omap4,
  1515. .write_ctrl = omap_sham_write_ctrl_omap4,
  1516. .trigger = omap_sham_trigger_omap4,
  1517. .poll_irq = omap_sham_poll_irq_omap4,
  1518. .intr_hdlr = omap_sham_irq_omap4,
  1519. .idigest_ofs = 0x020,
  1520. .odigest_ofs = 0x0,
  1521. .din_ofs = 0x080,
  1522. .digcnt_ofs = 0x040,
  1523. .rev_ofs = 0x100,
  1524. .mask_ofs = 0x110,
  1525. .sysstatus_ofs = 0x114,
  1526. .mode_ofs = 0x44,
  1527. .length_ofs = 0x48,
  1528. .major_mask = 0x0700,
  1529. .major_shift = 8,
  1530. .minor_mask = 0x003f,
  1531. .minor_shift = 0,
  1532. };
  1533. static struct omap_sham_algs_info omap_sham_algs_info_omap5[] = {
  1534. {
  1535. .algs_list = algs_sha1_md5,
  1536. .size = ARRAY_SIZE(algs_sha1_md5),
  1537. },
  1538. {
  1539. .algs_list = algs_sha224_sha256,
  1540. .size = ARRAY_SIZE(algs_sha224_sha256),
  1541. },
  1542. {
  1543. .algs_list = algs_sha384_sha512,
  1544. .size = ARRAY_SIZE(algs_sha384_sha512),
  1545. },
  1546. };
  1547. static const struct omap_sham_pdata omap_sham_pdata_omap5 = {
  1548. .algs_info = omap_sham_algs_info_omap5,
  1549. .algs_info_size = ARRAY_SIZE(omap_sham_algs_info_omap5),
  1550. .flags = BIT(FLAGS_AUTO_XOR),
  1551. .digest_size = SHA512_DIGEST_SIZE,
  1552. .copy_hash = omap_sham_copy_hash_omap4,
  1553. .write_ctrl = omap_sham_write_ctrl_omap4,
  1554. .trigger = omap_sham_trigger_omap4,
  1555. .poll_irq = omap_sham_poll_irq_omap4,
  1556. .intr_hdlr = omap_sham_irq_omap4,
  1557. .idigest_ofs = 0x240,
  1558. .odigest_ofs = 0x200,
  1559. .din_ofs = 0x080,
  1560. .digcnt_ofs = 0x280,
  1561. .rev_ofs = 0x100,
  1562. .mask_ofs = 0x110,
  1563. .sysstatus_ofs = 0x114,
  1564. .mode_ofs = 0x284,
  1565. .length_ofs = 0x288,
  1566. .major_mask = 0x0700,
  1567. .major_shift = 8,
  1568. .minor_mask = 0x003f,
  1569. .minor_shift = 0,
  1570. };
  1571. static const struct of_device_id omap_sham_of_match[] = {
  1572. {
  1573. .compatible = "ti,omap2-sham",
  1574. .data = &omap_sham_pdata_omap2,
  1575. },
  1576. {
  1577. .compatible = "ti,omap3-sham",
  1578. .data = &omap_sham_pdata_omap2,
  1579. },
  1580. {
  1581. .compatible = "ti,omap4-sham",
  1582. .data = &omap_sham_pdata_omap4,
  1583. },
  1584. {
  1585. .compatible = "ti,omap5-sham",
  1586. .data = &omap_sham_pdata_omap5,
  1587. },
  1588. {},
  1589. };
  1590. MODULE_DEVICE_TABLE(of, omap_sham_of_match);
  1591. static int omap_sham_get_res_of(struct omap_sham_dev *dd,
  1592. struct device *dev, struct resource *res)
  1593. {
  1594. struct device_node *node = dev->of_node;
  1595. int err = 0;
  1596. dd->pdata = of_device_get_match_data(dev);
  1597. if (!dd->pdata) {
  1598. dev_err(dev, "no compatible OF match\n");
  1599. err = -EINVAL;
  1600. goto err;
  1601. }
  1602. err = of_address_to_resource(node, 0, res);
  1603. if (err < 0) {
  1604. dev_err(dev, "can't translate OF node address\n");
  1605. err = -EINVAL;
  1606. goto err;
  1607. }
  1608. dd->irq = irq_of_parse_and_map(node, 0);
  1609. if (!dd->irq) {
  1610. dev_err(dev, "can't translate OF irq value\n");
  1611. err = -EINVAL;
  1612. goto err;
  1613. }
  1614. err:
  1615. return err;
  1616. }
  1617. #else
  1618. static const struct of_device_id omap_sham_of_match[] = {
  1619. {},
  1620. };
  1621. static int omap_sham_get_res_of(struct omap_sham_dev *dd,
  1622. struct device *dev, struct resource *res)
  1623. {
  1624. return -EINVAL;
  1625. }
  1626. #endif
  1627. static int omap_sham_get_res_pdev(struct omap_sham_dev *dd,
  1628. struct platform_device *pdev, struct resource *res)
  1629. {
  1630. struct device *dev = &pdev->dev;
  1631. struct resource *r;
  1632. int err = 0;
  1633. /* Get the base address */
  1634. r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
  1635. if (!r) {
  1636. dev_err(dev, "no MEM resource info\n");
  1637. err = -ENODEV;
  1638. goto err;
  1639. }
  1640. memcpy(res, r, sizeof(*res));
  1641. /* Get the IRQ */
  1642. dd->irq = platform_get_irq(pdev, 0);
  1643. if (dd->irq < 0) {
  1644. err = dd->irq;
  1645. goto err;
  1646. }
  1647. /* Only OMAP2/3 can be non-DT */
  1648. dd->pdata = &omap_sham_pdata_omap2;
  1649. err:
  1650. return err;
  1651. }
  1652. static ssize_t fallback_show(struct device *dev, struct device_attribute *attr,
  1653. char *buf)
  1654. {
  1655. struct omap_sham_dev *dd = dev_get_drvdata(dev);
  1656. return sprintf(buf, "%d\n", dd->fallback_sz);
  1657. }
  1658. static ssize_t fallback_store(struct device *dev, struct device_attribute *attr,
  1659. const char *buf, size_t size)
  1660. {
  1661. struct omap_sham_dev *dd = dev_get_drvdata(dev);
  1662. ssize_t status;
  1663. long value;
  1664. status = kstrtol(buf, 0, &value);
  1665. if (status)
  1666. return status;
  1667. /* HW accelerator only works with buffers > 9 */
  1668. if (value < 9) {
  1669. dev_err(dev, "minimum fallback size 9\n");
  1670. return -EINVAL;
  1671. }
  1672. dd->fallback_sz = value;
  1673. return size;
  1674. }
  1675. static ssize_t queue_len_show(struct device *dev, struct device_attribute *attr,
  1676. char *buf)
  1677. {
  1678. struct omap_sham_dev *dd = dev_get_drvdata(dev);
  1679. return sprintf(buf, "%d\n", dd->queue.max_qlen);
  1680. }
  1681. static ssize_t queue_len_store(struct device *dev,
  1682. struct device_attribute *attr, const char *buf,
  1683. size_t size)
  1684. {
  1685. struct omap_sham_dev *dd = dev_get_drvdata(dev);
  1686. ssize_t status;
  1687. long value;
  1688. status = kstrtol(buf, 0, &value);
  1689. if (status)
  1690. return status;
  1691. if (value < 1)
  1692. return -EINVAL;
  1693. /*
  1694. * Changing the queue size in fly is safe, if size becomes smaller
  1695. * than current size, it will just not accept new entries until
  1696. * it has shrank enough.
  1697. */
  1698. dd->queue.max_qlen = value;
  1699. return size;
  1700. }
  1701. static DEVICE_ATTR_RW(queue_len);
  1702. static DEVICE_ATTR_RW(fallback);
  1703. static struct attribute *omap_sham_attrs[] = {
  1704. &dev_attr_queue_len.attr,
  1705. &dev_attr_fallback.attr,
  1706. NULL,
  1707. };
  1708. static const struct attribute_group omap_sham_attr_group = {
  1709. .attrs = omap_sham_attrs,
  1710. };
  1711. static int omap_sham_probe(struct platform_device *pdev)
  1712. {
  1713. struct omap_sham_dev *dd;
  1714. struct device *dev = &pdev->dev;
  1715. struct resource res;
  1716. dma_cap_mask_t mask;
  1717. int err, i, j;
  1718. u32 rev;
  1719. dd = devm_kzalloc(dev, sizeof(struct omap_sham_dev), GFP_KERNEL);
  1720. if (dd == NULL) {
  1721. dev_err(dev, "unable to alloc data struct.\n");
  1722. err = -ENOMEM;
  1723. goto data_err;
  1724. }
  1725. dd->dev = dev;
  1726. platform_set_drvdata(pdev, dd);
  1727. INIT_LIST_HEAD(&dd->list);
  1728. tasklet_init(&dd->done_task, omap_sham_done_task, (unsigned long)dd);
  1729. crypto_init_queue(&dd->queue, OMAP_SHAM_QUEUE_LENGTH);
  1730. err = (dev->of_node) ? omap_sham_get_res_of(dd, dev, &res) :
  1731. omap_sham_get_res_pdev(dd, pdev, &res);
  1732. if (err)
  1733. goto data_err;
  1734. dd->io_base = devm_ioremap_resource(dev, &res);
  1735. if (IS_ERR(dd->io_base)) {
  1736. err = PTR_ERR(dd->io_base);
  1737. goto data_err;
  1738. }
  1739. dd->phys_base = res.start;
  1740. err = devm_request_irq(dev, dd->irq, dd->pdata->intr_hdlr,
  1741. IRQF_TRIGGER_NONE, dev_name(dev), dd);
  1742. if (err) {
  1743. dev_err(dev, "unable to request irq %d, err = %d\n",
  1744. dd->irq, err);
  1745. goto data_err;
  1746. }
  1747. dma_cap_zero(mask);
  1748. dma_cap_set(DMA_SLAVE, mask);
  1749. dd->dma_lch = dma_request_chan(dev, "rx");
  1750. if (IS_ERR(dd->dma_lch)) {
  1751. err = PTR_ERR(dd->dma_lch);
  1752. if (err == -EPROBE_DEFER)
  1753. goto data_err;
  1754. dd->polling_mode = 1;
  1755. dev_dbg(dev, "using polling mode instead of dma\n");
  1756. }
  1757. dd->flags |= dd->pdata->flags;
  1758. sham.flags |= dd->pdata->flags;
  1759. pm_runtime_use_autosuspend(dev);
  1760. pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY);
  1761. dd->fallback_sz = OMAP_SHA_DMA_THRESHOLD;
  1762. pm_runtime_enable(dev);
  1763. err = pm_runtime_resume_and_get(dev);
  1764. if (err < 0) {
  1765. dev_err(dev, "failed to get sync: %d\n", err);
  1766. goto err_pm;
  1767. }
  1768. rev = omap_sham_read(dd, SHA_REG_REV(dd));
  1769. pm_runtime_put_sync(&pdev->dev);
  1770. dev_info(dev, "hw accel on OMAP rev %u.%u\n",
  1771. (rev & dd->pdata->major_mask) >> dd->pdata->major_shift,
  1772. (rev & dd->pdata->minor_mask) >> dd->pdata->minor_shift);
  1773. spin_lock_bh(&sham.lock);
  1774. list_add_tail(&dd->list, &sham.dev_list);
  1775. spin_unlock_bh(&sham.lock);
  1776. dd->engine = crypto_engine_alloc_init(dev, 1);
  1777. if (!dd->engine) {
  1778. err = -ENOMEM;
  1779. goto err_engine;
  1780. }
  1781. err = crypto_engine_start(dd->engine);
  1782. if (err)
  1783. goto err_engine_start;
  1784. for (i = 0; i < dd->pdata->algs_info_size; i++) {
  1785. if (dd->pdata->algs_info[i].registered)
  1786. break;
  1787. for (j = 0; j < dd->pdata->algs_info[i].size; j++) {
  1788. struct ahash_alg *alg;
  1789. alg = &dd->pdata->algs_info[i].algs_list[j];
  1790. alg->export = omap_sham_export;
  1791. alg->import = omap_sham_import;
  1792. alg->halg.statesize = sizeof(struct omap_sham_reqctx) +
  1793. BUFLEN;
  1794. err = crypto_register_ahash(alg);
  1795. if (err)
  1796. goto err_algs;
  1797. dd->pdata->algs_info[i].registered++;
  1798. }
  1799. }
  1800. err = sysfs_create_group(&dev->kobj, &omap_sham_attr_group);
  1801. if (err) {
  1802. dev_err(dev, "could not create sysfs device attrs\n");
  1803. goto err_algs;
  1804. }
  1805. return 0;
  1806. err_algs:
  1807. for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
  1808. for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)
  1809. crypto_unregister_ahash(
  1810. &dd->pdata->algs_info[i].algs_list[j]);
  1811. err_engine_start:
  1812. crypto_engine_exit(dd->engine);
  1813. err_engine:
  1814. spin_lock_bh(&sham.lock);
  1815. list_del(&dd->list);
  1816. spin_unlock_bh(&sham.lock);
  1817. err_pm:
  1818. pm_runtime_dont_use_autosuspend(dev);
  1819. pm_runtime_disable(dev);
  1820. if (!dd->polling_mode)
  1821. dma_release_channel(dd->dma_lch);
  1822. data_err:
  1823. dev_err(dev, "initialization failed.\n");
  1824. return err;
  1825. }
  1826. static int omap_sham_remove(struct platform_device *pdev)
  1827. {
  1828. struct omap_sham_dev *dd;
  1829. int i, j;
  1830. dd = platform_get_drvdata(pdev);
  1831. spin_lock_bh(&sham.lock);
  1832. list_del(&dd->list);
  1833. spin_unlock_bh(&sham.lock);
  1834. for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)
  1835. for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) {
  1836. crypto_unregister_ahash(
  1837. &dd->pdata->algs_info[i].algs_list[j]);
  1838. dd->pdata->algs_info[i].registered--;
  1839. }
  1840. tasklet_kill(&dd->done_task);
  1841. pm_runtime_dont_use_autosuspend(&pdev->dev);
  1842. pm_runtime_disable(&pdev->dev);
  1843. if (!dd->polling_mode)
  1844. dma_release_channel(dd->dma_lch);
  1845. sysfs_remove_group(&dd->dev->kobj, &omap_sham_attr_group);
  1846. return 0;
  1847. }
  1848. static struct platform_driver omap_sham_driver = {
  1849. .probe = omap_sham_probe,
  1850. .remove = omap_sham_remove,
  1851. .driver = {
  1852. .name = "omap-sham",
  1853. .of_match_table = omap_sham_of_match,
  1854. },
  1855. };
  1856. module_platform_driver(omap_sham_driver);
  1857. MODULE_DESCRIPTION("OMAP SHA1/MD5 hw acceleration support.");
  1858. MODULE_LICENSE("GPL v2");
  1859. MODULE_AUTHOR("Dmitry Kasatkin");
  1860. MODULE_ALIAS("platform:omap-sham");