pcrypt.c 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * pcrypt - Parallel crypto wrapper.
  4. *
  5. * Copyright (C) 2009 secunet Security Networks AG
  6. * Copyright (C) 2009 Steffen Klassert <[email protected]>
  7. */
  8. #include <crypto/algapi.h>
  9. #include <crypto/internal/aead.h>
  10. #include <linux/atomic.h>
  11. #include <linux/err.h>
  12. #include <linux/init.h>
  13. #include <linux/module.h>
  14. #include <linux/slab.h>
  15. #include <linux/kobject.h>
  16. #include <linux/cpu.h>
  17. #include <crypto/pcrypt.h>
  18. static struct padata_instance *pencrypt;
  19. static struct padata_instance *pdecrypt;
  20. static struct kset *pcrypt_kset;
  21. struct pcrypt_instance_ctx {
  22. struct crypto_aead_spawn spawn;
  23. struct padata_shell *psenc;
  24. struct padata_shell *psdec;
  25. atomic_t tfm_count;
  26. };
  27. struct pcrypt_aead_ctx {
  28. struct crypto_aead *child;
  29. unsigned int cb_cpu;
  30. };
  31. static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx(
  32. struct crypto_aead *tfm)
  33. {
  34. return aead_instance_ctx(aead_alg_instance(tfm));
  35. }
  36. static int pcrypt_aead_setkey(struct crypto_aead *parent,
  37. const u8 *key, unsigned int keylen)
  38. {
  39. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
  40. return crypto_aead_setkey(ctx->child, key, keylen);
  41. }
  42. static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
  43. unsigned int authsize)
  44. {
  45. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
  46. return crypto_aead_setauthsize(ctx->child, authsize);
  47. }
  48. static void pcrypt_aead_serial(struct padata_priv *padata)
  49. {
  50. struct pcrypt_request *preq = pcrypt_padata_request(padata);
  51. struct aead_request *req = pcrypt_request_ctx(preq);
  52. aead_request_complete(req->base.data, padata->info);
  53. }
  54. static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
  55. {
  56. struct aead_request *req = areq->data;
  57. struct pcrypt_request *preq = aead_request_ctx(req);
  58. struct padata_priv *padata = pcrypt_request_padata(preq);
  59. padata->info = err;
  60. padata_do_serial(padata);
  61. }
  62. static void pcrypt_aead_enc(struct padata_priv *padata)
  63. {
  64. struct pcrypt_request *preq = pcrypt_padata_request(padata);
  65. struct aead_request *req = pcrypt_request_ctx(preq);
  66. int ret;
  67. ret = crypto_aead_encrypt(req);
  68. if (ret == -EINPROGRESS)
  69. return;
  70. padata->info = ret;
  71. padata_do_serial(padata);
  72. }
  73. static int pcrypt_aead_encrypt(struct aead_request *req)
  74. {
  75. int err;
  76. struct pcrypt_request *preq = aead_request_ctx(req);
  77. struct aead_request *creq = pcrypt_request_ctx(preq);
  78. struct padata_priv *padata = pcrypt_request_padata(preq);
  79. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  80. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
  81. u32 flags = aead_request_flags(req);
  82. struct pcrypt_instance_ctx *ictx;
  83. ictx = pcrypt_tfm_ictx(aead);
  84. memset(padata, 0, sizeof(struct padata_priv));
  85. padata->parallel = pcrypt_aead_enc;
  86. padata->serial = pcrypt_aead_serial;
  87. aead_request_set_tfm(creq, ctx->child);
  88. aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
  89. pcrypt_aead_done, req);
  90. aead_request_set_crypt(creq, req->src, req->dst,
  91. req->cryptlen, req->iv);
  92. aead_request_set_ad(creq, req->assoclen);
  93. err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu);
  94. if (!err)
  95. return -EINPROGRESS;
  96. if (err == -EBUSY)
  97. return -EAGAIN;
  98. return err;
  99. }
  100. static void pcrypt_aead_dec(struct padata_priv *padata)
  101. {
  102. struct pcrypt_request *preq = pcrypt_padata_request(padata);
  103. struct aead_request *req = pcrypt_request_ctx(preq);
  104. int ret;
  105. ret = crypto_aead_decrypt(req);
  106. if (ret == -EINPROGRESS)
  107. return;
  108. padata->info = ret;
  109. padata_do_serial(padata);
  110. }
  111. static int pcrypt_aead_decrypt(struct aead_request *req)
  112. {
  113. int err;
  114. struct pcrypt_request *preq = aead_request_ctx(req);
  115. struct aead_request *creq = pcrypt_request_ctx(preq);
  116. struct padata_priv *padata = pcrypt_request_padata(preq);
  117. struct crypto_aead *aead = crypto_aead_reqtfm(req);
  118. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
  119. u32 flags = aead_request_flags(req);
  120. struct pcrypt_instance_ctx *ictx;
  121. ictx = pcrypt_tfm_ictx(aead);
  122. memset(padata, 0, sizeof(struct padata_priv));
  123. padata->parallel = pcrypt_aead_dec;
  124. padata->serial = pcrypt_aead_serial;
  125. aead_request_set_tfm(creq, ctx->child);
  126. aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
  127. pcrypt_aead_done, req);
  128. aead_request_set_crypt(creq, req->src, req->dst,
  129. req->cryptlen, req->iv);
  130. aead_request_set_ad(creq, req->assoclen);
  131. err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu);
  132. if (!err)
  133. return -EINPROGRESS;
  134. if (err == -EBUSY)
  135. return -EAGAIN;
  136. return err;
  137. }
  138. static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
  139. {
  140. int cpu, cpu_index;
  141. struct aead_instance *inst = aead_alg_instance(tfm);
  142. struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
  143. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
  144. struct crypto_aead *cipher;
  145. cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
  146. cpumask_weight(cpu_online_mask);
  147. ctx->cb_cpu = cpumask_first(cpu_online_mask);
  148. for (cpu = 0; cpu < cpu_index; cpu++)
  149. ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
  150. cipher = crypto_spawn_aead(&ictx->spawn);
  151. if (IS_ERR(cipher))
  152. return PTR_ERR(cipher);
  153. ctx->child = cipher;
  154. crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) +
  155. sizeof(struct aead_request) +
  156. crypto_aead_reqsize(cipher));
  157. return 0;
  158. }
  159. static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm)
  160. {
  161. struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
  162. crypto_free_aead(ctx->child);
  163. }
  164. static void pcrypt_free(struct aead_instance *inst)
  165. {
  166. struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst);
  167. crypto_drop_aead(&ctx->spawn);
  168. padata_free_shell(ctx->psdec);
  169. padata_free_shell(ctx->psenc);
  170. kfree(inst);
  171. }
  172. static int pcrypt_init_instance(struct crypto_instance *inst,
  173. struct crypto_alg *alg)
  174. {
  175. if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
  176. "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
  177. return -ENAMETOOLONG;
  178. memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
  179. inst->alg.cra_priority = alg->cra_priority + 100;
  180. inst->alg.cra_blocksize = alg->cra_blocksize;
  181. inst->alg.cra_alignmask = alg->cra_alignmask;
  182. return 0;
  183. }
  184. static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb,
  185. struct crypto_attr_type *algt)
  186. {
  187. struct pcrypt_instance_ctx *ctx;
  188. struct aead_instance *inst;
  189. struct aead_alg *alg;
  190. u32 mask = crypto_algt_inherited_mask(algt);
  191. int err;
  192. inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
  193. if (!inst)
  194. return -ENOMEM;
  195. err = -ENOMEM;
  196. ctx = aead_instance_ctx(inst);
  197. ctx->psenc = padata_alloc_shell(pencrypt);
  198. if (!ctx->psenc)
  199. goto err_free_inst;
  200. ctx->psdec = padata_alloc_shell(pdecrypt);
  201. if (!ctx->psdec)
  202. goto err_free_inst;
  203. err = crypto_grab_aead(&ctx->spawn, aead_crypto_instance(inst),
  204. crypto_attr_alg_name(tb[1]), 0, mask);
  205. if (err)
  206. goto err_free_inst;
  207. alg = crypto_spawn_aead_alg(&ctx->spawn);
  208. err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base);
  209. if (err)
  210. goto err_free_inst;
  211. inst->alg.base.cra_flags |= CRYPTO_ALG_ASYNC;
  212. inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
  213. inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
  214. inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
  215. inst->alg.init = pcrypt_aead_init_tfm;
  216. inst->alg.exit = pcrypt_aead_exit_tfm;
  217. inst->alg.setkey = pcrypt_aead_setkey;
  218. inst->alg.setauthsize = pcrypt_aead_setauthsize;
  219. inst->alg.encrypt = pcrypt_aead_encrypt;
  220. inst->alg.decrypt = pcrypt_aead_decrypt;
  221. inst->free = pcrypt_free;
  222. err = aead_register_instance(tmpl, inst);
  223. if (err) {
  224. err_free_inst:
  225. pcrypt_free(inst);
  226. }
  227. return err;
  228. }
  229. static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb)
  230. {
  231. struct crypto_attr_type *algt;
  232. algt = crypto_get_attr_type(tb);
  233. if (IS_ERR(algt))
  234. return PTR_ERR(algt);
  235. switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
  236. case CRYPTO_ALG_TYPE_AEAD:
  237. return pcrypt_create_aead(tmpl, tb, algt);
  238. }
  239. return -EINVAL;
  240. }
  241. static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
  242. {
  243. int ret;
  244. pinst->kobj.kset = pcrypt_kset;
  245. ret = kobject_add(&pinst->kobj, NULL, "%s", name);
  246. if (!ret)
  247. kobject_uevent(&pinst->kobj, KOBJ_ADD);
  248. return ret;
  249. }
  250. static int pcrypt_init_padata(struct padata_instance **pinst, const char *name)
  251. {
  252. int ret = -ENOMEM;
  253. *pinst = padata_alloc(name);
  254. if (!*pinst)
  255. return ret;
  256. ret = pcrypt_sysfs_add(*pinst, name);
  257. if (ret)
  258. padata_free(*pinst);
  259. return ret;
  260. }
  261. static struct crypto_template pcrypt_tmpl = {
  262. .name = "pcrypt",
  263. .create = pcrypt_create,
  264. .module = THIS_MODULE,
  265. };
  266. static int __init pcrypt_init(void)
  267. {
  268. int err = -ENOMEM;
  269. pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
  270. if (!pcrypt_kset)
  271. goto err;
  272. err = pcrypt_init_padata(&pencrypt, "pencrypt");
  273. if (err)
  274. goto err_unreg_kset;
  275. err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
  276. if (err)
  277. goto err_deinit_pencrypt;
  278. return crypto_register_template(&pcrypt_tmpl);
  279. err_deinit_pencrypt:
  280. padata_free(pencrypt);
  281. err_unreg_kset:
  282. kset_unregister(pcrypt_kset);
  283. err:
  284. return err;
  285. }
  286. static void __exit pcrypt_exit(void)
  287. {
  288. crypto_unregister_template(&pcrypt_tmpl);
  289. padata_free(pencrypt);
  290. padata_free(pdecrypt);
  291. kset_unregister(pcrypt_kset);
  292. }
  293. subsys_initcall(pcrypt_init);
  294. module_exit(pcrypt_exit);
  295. MODULE_LICENSE("GPL");
  296. MODULE_AUTHOR("Steffen Klassert <[email protected]>");
  297. MODULE_DESCRIPTION("Parallel crypto wrapper");
  298. MODULE_ALIAS_CRYPTO("pcrypt");