cc_request_mgr.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */
  3. #include <linux/kernel.h>
  4. #include <linux/nospec.h>
  5. #include "cc_driver.h"
  6. #include "cc_buffer_mgr.h"
  7. #include "cc_request_mgr.h"
  8. #include "cc_pm.h"
  9. #define CC_MAX_POLL_ITER 10
  10. /* The highest descriptor count in used */
  11. #define CC_MAX_DESC_SEQ_LEN 23
  12. struct cc_req_mgr_handle {
  13. /* Request manager resources */
  14. unsigned int hw_queue_size; /* HW capability */
  15. unsigned int min_free_hw_slots;
  16. unsigned int max_used_sw_slots;
  17. struct cc_crypto_req req_queue[MAX_REQUEST_QUEUE_SIZE];
  18. u32 req_queue_head;
  19. u32 req_queue_tail;
  20. u32 axi_completed;
  21. u32 q_free_slots;
  22. /* This lock protects access to HW register
  23. * that must be single request at a time
  24. */
  25. spinlock_t hw_lock;
  26. struct cc_hw_desc compl_desc;
  27. u8 *dummy_comp_buff;
  28. dma_addr_t dummy_comp_buff_dma;
  29. /* backlog queue */
  30. struct list_head backlog;
  31. unsigned int bl_len;
  32. spinlock_t bl_lock; /* protect backlog queue */
  33. #ifdef COMP_IN_WQ
  34. struct workqueue_struct *workq;
  35. struct delayed_work compwork;
  36. #else
  37. struct tasklet_struct comptask;
  38. #endif
  39. };
  40. struct cc_bl_item {
  41. struct cc_crypto_req creq;
  42. struct cc_hw_desc desc[CC_MAX_DESC_SEQ_LEN];
  43. unsigned int len;
  44. struct list_head list;
  45. bool notif;
  46. };
  47. static const u32 cc_cpp_int_masks[CC_CPP_NUM_ALGS][CC_CPP_NUM_SLOTS] = {
  48. { BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_0_INT_BIT_SHIFT),
  49. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_1_INT_BIT_SHIFT),
  50. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_2_INT_BIT_SHIFT),
  51. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_3_INT_BIT_SHIFT),
  52. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_4_INT_BIT_SHIFT),
  53. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_5_INT_BIT_SHIFT),
  54. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_6_INT_BIT_SHIFT),
  55. BIT(CC_HOST_IRR_REE_OP_ABORTED_AES_7_INT_BIT_SHIFT) },
  56. { BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_0_INT_BIT_SHIFT),
  57. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_1_INT_BIT_SHIFT),
  58. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_2_INT_BIT_SHIFT),
  59. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_3_INT_BIT_SHIFT),
  60. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_4_INT_BIT_SHIFT),
  61. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_5_INT_BIT_SHIFT),
  62. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_6_INT_BIT_SHIFT),
  63. BIT(CC_HOST_IRR_REE_OP_ABORTED_SM_7_INT_BIT_SHIFT) }
  64. };
  65. static void comp_handler(unsigned long devarg);
  66. #ifdef COMP_IN_WQ
  67. static void comp_work_handler(struct work_struct *work);
  68. #endif
  69. static inline u32 cc_cpp_int_mask(enum cc_cpp_alg alg, int slot)
  70. {
  71. alg = array_index_nospec(alg, CC_CPP_NUM_ALGS);
  72. slot = array_index_nospec(slot, CC_CPP_NUM_SLOTS);
  73. return cc_cpp_int_masks[alg][slot];
  74. }
  75. void cc_req_mgr_fini(struct cc_drvdata *drvdata)
  76. {
  77. struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
  78. struct device *dev = drvdata_to_dev(drvdata);
  79. if (!req_mgr_h)
  80. return; /* Not allocated */
  81. if (req_mgr_h->dummy_comp_buff_dma) {
  82. dma_free_coherent(dev, sizeof(u32), req_mgr_h->dummy_comp_buff,
  83. req_mgr_h->dummy_comp_buff_dma);
  84. }
  85. dev_dbg(dev, "max_used_hw_slots=%d\n", (req_mgr_h->hw_queue_size -
  86. req_mgr_h->min_free_hw_slots));
  87. dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
  88. #ifdef COMP_IN_WQ
  89. destroy_workqueue(req_mgr_h->workq);
  90. #else
  91. /* Kill tasklet */
  92. tasklet_kill(&req_mgr_h->comptask);
  93. #endif
  94. kfree_sensitive(req_mgr_h);
  95. drvdata->request_mgr_handle = NULL;
  96. }
  97. int cc_req_mgr_init(struct cc_drvdata *drvdata)
  98. {
  99. struct cc_req_mgr_handle *req_mgr_h;
  100. struct device *dev = drvdata_to_dev(drvdata);
  101. int rc = 0;
  102. req_mgr_h = kzalloc(sizeof(*req_mgr_h), GFP_KERNEL);
  103. if (!req_mgr_h) {
  104. rc = -ENOMEM;
  105. goto req_mgr_init_err;
  106. }
  107. drvdata->request_mgr_handle = req_mgr_h;
  108. spin_lock_init(&req_mgr_h->hw_lock);
  109. spin_lock_init(&req_mgr_h->bl_lock);
  110. INIT_LIST_HEAD(&req_mgr_h->backlog);
  111. #ifdef COMP_IN_WQ
  112. dev_dbg(dev, "Initializing completion workqueue\n");
  113. req_mgr_h->workq = create_singlethread_workqueue("ccree");
  114. if (!req_mgr_h->workq) {
  115. dev_err(dev, "Failed creating work queue\n");
  116. rc = -ENOMEM;
  117. goto req_mgr_init_err;
  118. }
  119. INIT_DELAYED_WORK(&req_mgr_h->compwork, comp_work_handler);
  120. #else
  121. dev_dbg(dev, "Initializing completion tasklet\n");
  122. tasklet_init(&req_mgr_h->comptask, comp_handler,
  123. (unsigned long)drvdata);
  124. #endif
  125. req_mgr_h->hw_queue_size = cc_ioread(drvdata,
  126. CC_REG(DSCRPTR_QUEUE_SRAM_SIZE));
  127. dev_dbg(dev, "hw_queue_size=0x%08X\n", req_mgr_h->hw_queue_size);
  128. if (req_mgr_h->hw_queue_size < MIN_HW_QUEUE_SIZE) {
  129. dev_err(dev, "Invalid HW queue size = %u (Min. required is %u)\n",
  130. req_mgr_h->hw_queue_size, MIN_HW_QUEUE_SIZE);
  131. rc = -ENOMEM;
  132. goto req_mgr_init_err;
  133. }
  134. req_mgr_h->min_free_hw_slots = req_mgr_h->hw_queue_size;
  135. req_mgr_h->max_used_sw_slots = 0;
  136. /* Allocate DMA word for "dummy" completion descriptor use */
  137. req_mgr_h->dummy_comp_buff =
  138. dma_alloc_coherent(dev, sizeof(u32),
  139. &req_mgr_h->dummy_comp_buff_dma,
  140. GFP_KERNEL);
  141. if (!req_mgr_h->dummy_comp_buff) {
  142. dev_err(dev, "Not enough memory to allocate DMA (%zu) dropped buffer\n",
  143. sizeof(u32));
  144. rc = -ENOMEM;
  145. goto req_mgr_init_err;
  146. }
  147. /* Init. "dummy" completion descriptor */
  148. hw_desc_init(&req_mgr_h->compl_desc);
  149. set_din_const(&req_mgr_h->compl_desc, 0, sizeof(u32));
  150. set_dout_dlli(&req_mgr_h->compl_desc, req_mgr_h->dummy_comp_buff_dma,
  151. sizeof(u32), NS_BIT, 1);
  152. set_flow_mode(&req_mgr_h->compl_desc, BYPASS);
  153. set_queue_last_ind(drvdata, &req_mgr_h->compl_desc);
  154. return 0;
  155. req_mgr_init_err:
  156. cc_req_mgr_fini(drvdata);
  157. return rc;
  158. }
  159. static void enqueue_seq(struct cc_drvdata *drvdata, struct cc_hw_desc seq[],
  160. unsigned int seq_len)
  161. {
  162. int i, w;
  163. void __iomem *reg = drvdata->cc_base + CC_REG(DSCRPTR_QUEUE_WORD0);
  164. struct device *dev = drvdata_to_dev(drvdata);
  165. /*
  166. * We do indeed write all 6 command words to the same
  167. * register. The HW supports this.
  168. */
  169. for (i = 0; i < seq_len; i++) {
  170. for (w = 0; w <= 5; w++)
  171. writel_relaxed(seq[i].word[w], reg);
  172. if (cc_dump_desc)
  173. dev_dbg(dev, "desc[%02d]: 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X 0x%08X\n",
  174. i, seq[i].word[0], seq[i].word[1],
  175. seq[i].word[2], seq[i].word[3],
  176. seq[i].word[4], seq[i].word[5]);
  177. }
  178. }
  179. /**
  180. * request_mgr_complete() - Completion will take place if and only if user
  181. * requested completion by cc_send_sync_request().
  182. *
  183. * @dev: Device pointer
  184. * @dx_compl_h: The completion event to signal
  185. * @dummy: unused error code
  186. */
  187. static void request_mgr_complete(struct device *dev, void *dx_compl_h,
  188. int dummy)
  189. {
  190. struct completion *this_compl = dx_compl_h;
  191. complete(this_compl);
  192. }
  193. static int cc_queues_status(struct cc_drvdata *drvdata,
  194. struct cc_req_mgr_handle *req_mgr_h,
  195. unsigned int total_seq_len)
  196. {
  197. unsigned long poll_queue;
  198. struct device *dev = drvdata_to_dev(drvdata);
  199. /* SW queue is checked only once as it will not
  200. * be changed during the poll because the spinlock_bh
  201. * is held by the thread
  202. */
  203. if (((req_mgr_h->req_queue_head + 1) & (MAX_REQUEST_QUEUE_SIZE - 1)) ==
  204. req_mgr_h->req_queue_tail) {
  205. dev_err(dev, "SW FIFO is full. req_queue_head=%d sw_fifo_len=%d\n",
  206. req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE);
  207. return -ENOSPC;
  208. }
  209. if (req_mgr_h->q_free_slots >= total_seq_len)
  210. return 0;
  211. /* Wait for space in HW queue. Poll constant num of iterations. */
  212. for (poll_queue = 0; poll_queue < CC_MAX_POLL_ITER ; poll_queue++) {
  213. req_mgr_h->q_free_slots =
  214. cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
  215. if (req_mgr_h->q_free_slots < req_mgr_h->min_free_hw_slots)
  216. req_mgr_h->min_free_hw_slots = req_mgr_h->q_free_slots;
  217. if (req_mgr_h->q_free_slots >= total_seq_len) {
  218. /* If there is enough place return */
  219. return 0;
  220. }
  221. dev_dbg(dev, "HW FIFO is full. q_free_slots=%d total_seq_len=%d\n",
  222. req_mgr_h->q_free_slots, total_seq_len);
  223. }
  224. /* No room in the HW queue try again later */
  225. dev_dbg(dev, "HW FIFO full, timeout. req_queue_head=%d sw_fifo_len=%d q_free_slots=%d total_seq_len=%d\n",
  226. req_mgr_h->req_queue_head, MAX_REQUEST_QUEUE_SIZE,
  227. req_mgr_h->q_free_slots, total_seq_len);
  228. return -ENOSPC;
  229. }
  230. /**
  231. * cc_do_send_request() - Enqueue caller request to crypto hardware.
  232. * Need to be called with HW lock held and PM running
  233. *
  234. * @drvdata: Associated device driver context
  235. * @cc_req: The request to enqueue
  236. * @desc: The crypto sequence
  237. * @len: The crypto sequence length
  238. * @add_comp: If "true": add an artificial dout DMA to mark completion
  239. *
  240. */
  241. static void cc_do_send_request(struct cc_drvdata *drvdata,
  242. struct cc_crypto_req *cc_req,
  243. struct cc_hw_desc *desc, unsigned int len,
  244. bool add_comp)
  245. {
  246. struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
  247. unsigned int used_sw_slots;
  248. unsigned int total_seq_len = len; /*initial sequence length*/
  249. struct device *dev = drvdata_to_dev(drvdata);
  250. used_sw_slots = ((req_mgr_h->req_queue_head -
  251. req_mgr_h->req_queue_tail) &
  252. (MAX_REQUEST_QUEUE_SIZE - 1));
  253. if (used_sw_slots > req_mgr_h->max_used_sw_slots)
  254. req_mgr_h->max_used_sw_slots = used_sw_slots;
  255. /* Enqueue request - must be locked with HW lock*/
  256. req_mgr_h->req_queue[req_mgr_h->req_queue_head] = *cc_req;
  257. req_mgr_h->req_queue_head = (req_mgr_h->req_queue_head + 1) &
  258. (MAX_REQUEST_QUEUE_SIZE - 1);
  259. dev_dbg(dev, "Enqueue request head=%u\n", req_mgr_h->req_queue_head);
  260. /*
  261. * We are about to push command to the HW via the command registers
  262. * that may reference host memory. We need to issue a memory barrier
  263. * to make sure there are no outstanding memory writes
  264. */
  265. wmb();
  266. /* STAT_PHASE_4: Push sequence */
  267. enqueue_seq(drvdata, desc, len);
  268. if (add_comp) {
  269. enqueue_seq(drvdata, &req_mgr_h->compl_desc, 1);
  270. total_seq_len++;
  271. }
  272. if (req_mgr_h->q_free_slots < total_seq_len) {
  273. /* This situation should never occur. Maybe indicating problem
  274. * with resuming power. Set the free slot count to 0 and hope
  275. * for the best.
  276. */
  277. dev_err(dev, "HW free slot count mismatch.");
  278. req_mgr_h->q_free_slots = 0;
  279. } else {
  280. /* Update the free slots in HW queue */
  281. req_mgr_h->q_free_slots -= total_seq_len;
  282. }
  283. }
  284. static void cc_enqueue_backlog(struct cc_drvdata *drvdata,
  285. struct cc_bl_item *bli)
  286. {
  287. struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
  288. struct device *dev = drvdata_to_dev(drvdata);
  289. spin_lock_bh(&mgr->bl_lock);
  290. list_add_tail(&bli->list, &mgr->backlog);
  291. ++mgr->bl_len;
  292. dev_dbg(dev, "+++bl len: %d\n", mgr->bl_len);
  293. spin_unlock_bh(&mgr->bl_lock);
  294. tasklet_schedule(&mgr->comptask);
  295. }
  296. static void cc_proc_backlog(struct cc_drvdata *drvdata)
  297. {
  298. struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
  299. struct cc_bl_item *bli;
  300. struct cc_crypto_req *creq;
  301. void *req;
  302. struct device *dev = drvdata_to_dev(drvdata);
  303. int rc;
  304. spin_lock(&mgr->bl_lock);
  305. while (mgr->bl_len) {
  306. bli = list_first_entry(&mgr->backlog, struct cc_bl_item, list);
  307. dev_dbg(dev, "---bl len: %d\n", mgr->bl_len);
  308. spin_unlock(&mgr->bl_lock);
  309. creq = &bli->creq;
  310. req = creq->user_arg;
  311. /*
  312. * Notify the request we're moving out of the backlog
  313. * but only if we haven't done so already.
  314. */
  315. if (!bli->notif) {
  316. creq->user_cb(dev, req, -EINPROGRESS);
  317. bli->notif = true;
  318. }
  319. spin_lock(&mgr->hw_lock);
  320. rc = cc_queues_status(drvdata, mgr, bli->len);
  321. if (rc) {
  322. /*
  323. * There is still no room in the FIFO for
  324. * this request. Bail out. We'll return here
  325. * on the next completion irq.
  326. */
  327. spin_unlock(&mgr->hw_lock);
  328. return;
  329. }
  330. cc_do_send_request(drvdata, &bli->creq, bli->desc, bli->len,
  331. false);
  332. spin_unlock(&mgr->hw_lock);
  333. /* Remove ourselves from the backlog list */
  334. spin_lock(&mgr->bl_lock);
  335. list_del(&bli->list);
  336. --mgr->bl_len;
  337. kfree(bli);
  338. }
  339. spin_unlock(&mgr->bl_lock);
  340. }
  341. int cc_send_request(struct cc_drvdata *drvdata, struct cc_crypto_req *cc_req,
  342. struct cc_hw_desc *desc, unsigned int len,
  343. struct crypto_async_request *req)
  344. {
  345. int rc;
  346. struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
  347. struct device *dev = drvdata_to_dev(drvdata);
  348. bool backlog_ok = req->flags & CRYPTO_TFM_REQ_MAY_BACKLOG;
  349. gfp_t flags = cc_gfp_flags(req);
  350. struct cc_bl_item *bli;
  351. rc = cc_pm_get(dev);
  352. if (rc) {
  353. dev_err(dev, "cc_pm_get returned %x\n", rc);
  354. return rc;
  355. }
  356. spin_lock_bh(&mgr->hw_lock);
  357. rc = cc_queues_status(drvdata, mgr, len);
  358. #ifdef CC_DEBUG_FORCE_BACKLOG
  359. if (backlog_ok)
  360. rc = -ENOSPC;
  361. #endif /* CC_DEBUG_FORCE_BACKLOG */
  362. if (rc == -ENOSPC && backlog_ok) {
  363. spin_unlock_bh(&mgr->hw_lock);
  364. bli = kmalloc(sizeof(*bli), flags);
  365. if (!bli) {
  366. cc_pm_put_suspend(dev);
  367. return -ENOMEM;
  368. }
  369. memcpy(&bli->creq, cc_req, sizeof(*cc_req));
  370. memcpy(&bli->desc, desc, len * sizeof(*desc));
  371. bli->len = len;
  372. bli->notif = false;
  373. cc_enqueue_backlog(drvdata, bli);
  374. return -EBUSY;
  375. }
  376. if (!rc) {
  377. cc_do_send_request(drvdata, cc_req, desc, len, false);
  378. rc = -EINPROGRESS;
  379. }
  380. spin_unlock_bh(&mgr->hw_lock);
  381. return rc;
  382. }
  383. int cc_send_sync_request(struct cc_drvdata *drvdata,
  384. struct cc_crypto_req *cc_req, struct cc_hw_desc *desc,
  385. unsigned int len)
  386. {
  387. int rc;
  388. struct device *dev = drvdata_to_dev(drvdata);
  389. struct cc_req_mgr_handle *mgr = drvdata->request_mgr_handle;
  390. init_completion(&cc_req->seq_compl);
  391. cc_req->user_cb = request_mgr_complete;
  392. cc_req->user_arg = &cc_req->seq_compl;
  393. rc = cc_pm_get(dev);
  394. if (rc) {
  395. dev_err(dev, "cc_pm_get returned %x\n", rc);
  396. return rc;
  397. }
  398. while (true) {
  399. spin_lock_bh(&mgr->hw_lock);
  400. rc = cc_queues_status(drvdata, mgr, len + 1);
  401. if (!rc)
  402. break;
  403. spin_unlock_bh(&mgr->hw_lock);
  404. wait_for_completion_interruptible(&drvdata->hw_queue_avail);
  405. reinit_completion(&drvdata->hw_queue_avail);
  406. }
  407. cc_do_send_request(drvdata, cc_req, desc, len, true);
  408. spin_unlock_bh(&mgr->hw_lock);
  409. wait_for_completion(&cc_req->seq_compl);
  410. return 0;
  411. }
  412. /**
  413. * send_request_init() - Enqueue caller request to crypto hardware during init
  414. * process.
  415. * Assume this function is not called in the middle of a flow,
  416. * since we set QUEUE_LAST_IND flag in the last descriptor.
  417. *
  418. * @drvdata: Associated device driver context
  419. * @desc: The crypto sequence
  420. * @len: The crypto sequence length
  421. *
  422. * Return:
  423. * Returns "0" upon success
  424. */
  425. int send_request_init(struct cc_drvdata *drvdata, struct cc_hw_desc *desc,
  426. unsigned int len)
  427. {
  428. struct cc_req_mgr_handle *req_mgr_h = drvdata->request_mgr_handle;
  429. unsigned int total_seq_len = len; /*initial sequence length*/
  430. int rc = 0;
  431. /* Wait for space in HW and SW FIFO. Poll for as much as FIFO_TIMEOUT.
  432. */
  433. rc = cc_queues_status(drvdata, req_mgr_h, total_seq_len);
  434. if (rc)
  435. return rc;
  436. set_queue_last_ind(drvdata, &desc[(len - 1)]);
  437. /*
  438. * We are about to push command to the HW via the command registers
  439. * that may reference host memory. We need to issue a memory barrier
  440. * to make sure there are no outstanding memory writes
  441. */
  442. wmb();
  443. enqueue_seq(drvdata, desc, len);
  444. /* Update the free slots in HW queue */
  445. req_mgr_h->q_free_slots =
  446. cc_ioread(drvdata, CC_REG(DSCRPTR_QUEUE_CONTENT));
  447. return 0;
  448. }
  449. void complete_request(struct cc_drvdata *drvdata)
  450. {
  451. struct cc_req_mgr_handle *request_mgr_handle =
  452. drvdata->request_mgr_handle;
  453. complete(&drvdata->hw_queue_avail);
  454. #ifdef COMP_IN_WQ
  455. queue_delayed_work(request_mgr_handle->workq,
  456. &request_mgr_handle->compwork, 0);
  457. #else
  458. tasklet_schedule(&request_mgr_handle->comptask);
  459. #endif
  460. }
  461. #ifdef COMP_IN_WQ
  462. static void comp_work_handler(struct work_struct *work)
  463. {
  464. struct cc_drvdata *drvdata =
  465. container_of(work, struct cc_drvdata, compwork.work);
  466. comp_handler((unsigned long)drvdata);
  467. }
  468. #endif
  469. static void proc_completions(struct cc_drvdata *drvdata)
  470. {
  471. struct cc_crypto_req *cc_req;
  472. struct device *dev = drvdata_to_dev(drvdata);
  473. struct cc_req_mgr_handle *request_mgr_handle =
  474. drvdata->request_mgr_handle;
  475. unsigned int *tail = &request_mgr_handle->req_queue_tail;
  476. unsigned int *head = &request_mgr_handle->req_queue_head;
  477. int rc;
  478. u32 mask;
  479. while (request_mgr_handle->axi_completed) {
  480. request_mgr_handle->axi_completed--;
  481. /* Dequeue request */
  482. if (*head == *tail) {
  483. /* We are supposed to handle a completion but our
  484. * queue is empty. This is not normal. Return and
  485. * hope for the best.
  486. */
  487. dev_err(dev, "Request queue is empty head == tail %u\n",
  488. *head);
  489. break;
  490. }
  491. cc_req = &request_mgr_handle->req_queue[*tail];
  492. if (cc_req->cpp.is_cpp) {
  493. dev_dbg(dev, "CPP request completion slot: %d alg:%d\n",
  494. cc_req->cpp.slot, cc_req->cpp.alg);
  495. mask = cc_cpp_int_mask(cc_req->cpp.alg,
  496. cc_req->cpp.slot);
  497. rc = (drvdata->irq & mask ? -EPERM : 0);
  498. dev_dbg(dev, "Got mask: %x irq: %x rc: %d\n", mask,
  499. drvdata->irq, rc);
  500. } else {
  501. dev_dbg(dev, "None CPP request completion\n");
  502. rc = 0;
  503. }
  504. if (cc_req->user_cb)
  505. cc_req->user_cb(dev, cc_req->user_arg, rc);
  506. *tail = (*tail + 1) & (MAX_REQUEST_QUEUE_SIZE - 1);
  507. dev_dbg(dev, "Dequeue request tail=%u\n", *tail);
  508. dev_dbg(dev, "Request completed. axi_completed=%d\n",
  509. request_mgr_handle->axi_completed);
  510. cc_pm_put_suspend(dev);
  511. }
  512. }
  513. static inline u32 cc_axi_comp_count(struct cc_drvdata *drvdata)
  514. {
  515. return FIELD_GET(AXIM_MON_COMP_VALUE,
  516. cc_ioread(drvdata, drvdata->axim_mon_offset));
  517. }
  518. /* Deferred service handler, run as interrupt-fired tasklet */
  519. static void comp_handler(unsigned long devarg)
  520. {
  521. struct cc_drvdata *drvdata = (struct cc_drvdata *)devarg;
  522. struct cc_req_mgr_handle *request_mgr_handle =
  523. drvdata->request_mgr_handle;
  524. struct device *dev = drvdata_to_dev(drvdata);
  525. u32 irq;
  526. dev_dbg(dev, "Completion handler called!\n");
  527. irq = (drvdata->irq & drvdata->comp_mask);
  528. /* To avoid the interrupt from firing as we unmask it,
  529. * we clear it now
  530. */
  531. cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
  532. /* Avoid race with above clear: Test completion counter once more */
  533. request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
  534. dev_dbg(dev, "AXI completion after updated: %d\n",
  535. request_mgr_handle->axi_completed);
  536. while (request_mgr_handle->axi_completed) {
  537. do {
  538. drvdata->irq |= cc_ioread(drvdata, CC_REG(HOST_IRR));
  539. irq = (drvdata->irq & drvdata->comp_mask);
  540. proc_completions(drvdata);
  541. /* At this point (after proc_completions()),
  542. * request_mgr_handle->axi_completed is 0.
  543. */
  544. request_mgr_handle->axi_completed +=
  545. cc_axi_comp_count(drvdata);
  546. } while (request_mgr_handle->axi_completed > 0);
  547. cc_iowrite(drvdata, CC_REG(HOST_ICR), irq);
  548. request_mgr_handle->axi_completed += cc_axi_comp_count(drvdata);
  549. }
  550. /* after verifying that there is nothing to do,
  551. * unmask AXI completion interrupt
  552. */
  553. cc_iowrite(drvdata, CC_REG(HOST_IMR),
  554. cc_ioread(drvdata, CC_REG(HOST_IMR)) & ~drvdata->comp_mask);
  555. cc_proc_backlog(drvdata);
  556. dev_dbg(dev, "Comp. handler done.\n");
  557. }