qcedev: concurrency support in crypto driver

Support to have multiple qcedev requests at a time in
the driver.

Change-Id: I2ba8f22e2b659db04db348dfa3b06b70bf234d0b
Signed-off-by: Gaurav Kashyap <[email protected]>

crypto-qti/qce50.c

@@ -1129,10 +1129,10 @@ static int _ce_setup_cipher(struct qce_device *pce_dev, struct qce_req *creq,
 	if (creq->is_copy_op) {
 		pce->data = 0;
 	} else {
-	if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT))
-		pce->data = (creq->cryptlen + creq->authsize);
-	else
-		pce->data = creq->cryptlen;
+		if ((creq->mode == QCE_MODE_CCM) && (creq->dir == QCE_DECRYPT))
+			pce->data = (creq->cryptlen + creq->authsize);
+		else
+			pce->data = creq->cryptlen;
 	}
 
 	/* write encr seg start */
@@ -1143,7 +1143,7 @@ static int _ce_setup_cipher(struct qce_device *pce_dev, struct qce_req *creq,
 	pce = cmdlistinfo->seg_size;
 	pce->data = totallen_in;
 
-	if (is_offload_op(creq->offload_op)) {
+	if (!is_des_cipher) {
 		/* pattern info */
 		pce = cmdlistinfo->pattern_info;
 		pce->data = creq->pattern_info;
@@ -1155,9 +1155,7 @@ static int _ce_setup_cipher(struct qce_device *pce_dev, struct qce_req *creq,
 
 		/* IV counter size */
 		qce_set_iv_ctr_mask(pce_dev, creq);
-	}
 
-	if (!is_des_cipher) {
 		pce = cmdlistinfo->encr_mask_3;
 		pce->data = pce_dev->reg.encr_cntr_mask_3;
 		pce = cmdlistinfo->encr_mask_2;
@@ -2364,8 +2362,11 @@ int qce_manage_timeout(void *handle, int req_info)
 	if (_qce_unlock_other_pipes(pce_dev, req_info))
 		pr_err("%s: fail unlock other pipes\n", __func__);
 
+	if (!atomic_read(&preq_info->in_use)) {
+		pr_err("request information %d already done\n", req_info);
+		return -ENXIO;
+	}
 	qce_free_req_info(pce_dev, req_info, true);
-	qce_callback(areq, NULL, NULL, 0);
 	return 0;
 }
 EXPORT_SYMBOL(qce_manage_timeout);
@@ -2433,6 +2434,10 @@ static int _aead_complete(struct qce_device *pce_dev, int req_info)
 		result_status = -ENXIO;
 	}
 
+	if (!atomic_read(&preq_info->in_use)) {
+		pr_err("request information %d already done\n", req_info);
+		return -ENXIO;
+	}
 	if (preq_info->mode == QCE_MODE_CCM) {
 		/*
 		 * Not from result dump, instead, use the status we just
@@ -2506,6 +2511,11 @@ static int _sha_complete(struct qce_device *pce_dev, int req_info)
 			pce_sps_data->consumer_status);
 		result_status = -ENXIO;
 	}
+
+	if (!atomic_read(&preq_info->in_use)) {
+		pr_err("request information %d already done\n", req_info);
+		return -ENXIO;
+	}
 	qce_free_req_info(pce_dev, req_info, true);
 	qce_callback(areq, digest, (char *)bytecount32, result_status);
 	return 0;
@@ -2655,6 +2665,11 @@ static int _ablk_cipher_complete(struct qce_device *pce_dev, int req_info)
 				(char *)(pce_sps_data->result->encr_cntr_iv),
 				sizeof(iv));
 		}
+
+		if (!atomic_read(&preq_info->in_use)) {
+			pr_err("request information %d already done\n", req_info);
+			return -ENXIO;
+		}
 		qce_free_req_info(pce_dev, req_info, true);
 		qce_callback(areq, NULL, iv, result_status);
 	}
@@ -3468,6 +3483,10 @@ static void _sps_producer_callback(struct sps_event_notify *notify)
 	}
 
 	preq_info = &pce_dev->ce_request_info[req_info];
+	if (!atomic_read(&preq_info->in_use)) {
+		pr_err("request information %d already done\n", req_info);
+		return;
+	}
 	op = pce_dev->ce_request_info[req_info].offload_op;
 
 	pce_sps_data = &preq_info->ce_sps;

crypto-qti/qcedev.c

@@ -44,6 +44,14 @@
  */
 #define MAX_CRYPTO_WAIT_TIME 25
 
+#define MAX_REQUEST_TIME 5000
+
+enum qcedev_req_status {
+	QCEDEV_REQ_CURRENT = 0,
+	QCEDEV_REQ_WAITING = 1,
+	QCEDEV_REQ_SUBMITTED = 2,
+};
+
 static uint8_t  _std_init_vector_sha1_uint8[] =   {
 	0x67, 0x45, 0x23, 0x01, 0xEF, 0xCD, 0xAB, 0x89,
 	0x98, 0xBA, 0xDC, 0xFE, 0x10, 0x32, 0x54, 0x76,
@@ -311,42 +319,24 @@ static void req_done(unsigned long data)
 	struct qcedev_async_req *areq;
 	unsigned long flags = 0;
 	struct qcedev_async_req *new_req = NULL;
-	int ret = 0;
-	int current_req_info = 0;
 
 	spin_lock_irqsave(&podev->lock, flags);
 	areq = podev->active_command;
 	podev->active_command = NULL;
 
-again:
+	if (areq && !areq->timed_out)
+		complete(&areq->complete);
+
+	/* Look through queued requests and wake up the corresponding thread */
 	if (!list_empty(&podev->ready_commands)) {
 		new_req = container_of(podev->ready_commands.next,
 						struct qcedev_async_req, list);
 		list_del(&new_req->list);
-		podev->active_command = new_req;
-		new_req->err = 0;
-		if (new_req->op_type == QCEDEV_CRYPTO_OPER_CIPHER)
-			ret = start_cipher_req(podev, &current_req_info);
-		else if (new_req->op_type == QCEDEV_CRYPTO_OPER_OFFLOAD_CIPHER)
-			ret = start_offload_cipher_req(podev, &current_req_info);
-		else
-			ret = start_sha_req(podev, &current_req_info);
+		new_req->state = QCEDEV_REQ_CURRENT;
+		wake_up_interruptible(&new_req->wait_q);
 	}
 
 	spin_unlock_irqrestore(&podev->lock, flags);
-
-	if (areq)
-		complete(&areq->complete);
-
-	if (new_req && ret) {
-		complete(&new_req->complete);
-		spin_lock_irqsave(&podev->lock, flags);
-		podev->active_command = NULL;
-		areq = NULL;
-		ret = 0;
-		new_req = NULL;
-		goto again;
-	}
 }
 
 void qcedev_sha_req_cb(void *cookie, unsigned char *digest,
@@ -408,6 +398,7 @@ static int start_cipher_req(struct qcedev_control *podev,
 	struct qce_req creq;
 	int ret = 0;
 
+	memset(&creq, 0, sizeof(creq));
 	/* start the command on the podev->active_command */
 	qcedev_areq = podev->active_command;
 	qcedev_areq->cipher_req.cookie = qcedev_areq->handle;
@@ -461,6 +452,7 @@ static int start_cipher_req(struct qcedev_control *podev,
 
 	creq.iv = &qcedev_areq->cipher_op_req.iv[0];
 	creq.ivsize = qcedev_areq->cipher_op_req.ivlen;
+	creq.iv_ctr_size = 0;
 
 	creq.enckey =  &qcedev_areq->cipher_op_req.enckey[0];
 	creq.encklen = qcedev_areq->cipher_op_req.encklen;
@@ -495,7 +487,7 @@ static int start_cipher_req(struct qcedev_control *podev,
 	creq.qce_cb = qcedev_cipher_req_cb;
 	creq.areq = (void *)&qcedev_areq->cipher_req;
 	creq.flags = 0;
-	creq.offload_op = 0;
+	creq.offload_op = QCE_OFFLOAD_NONE;
 	ret = qce_ablk_cipher_req(podev->qce, &creq);
 	*current_req_info = creq.current_req_info;
 unsupported:
@@ -750,29 +742,73 @@ static int submit_req(struct qcedev_async_req *qcedev_areq,
 	int current_req_info = 0;
 	int wait = MAX_CRYPTO_WAIT_TIME;
 	bool print_sts = false;
+	struct qcedev_async_req *new_req = NULL;
 
 	qcedev_areq->err = 0;
 	podev = handle->cntl;
+	init_waitqueue_head(&qcedev_areq->wait_q);
+
 
 	spin_lock_irqsave(&podev->lock, flags);
 
-	if (podev->active_command == NULL) {
-		podev->active_command = qcedev_areq;
-		if (qcedev_areq->op_type == QCEDEV_CRYPTO_OPER_CIPHER)
-			ret = start_cipher_req(podev, &current_req_info);
-		else if (qcedev_areq->op_type == QCEDEV_CRYPTO_OPER_OFFLOAD_CIPHER)
-			ret = start_offload_cipher_req(podev, &current_req_info);
-		else
-			ret = start_sha_req(podev, &current_req_info);
-	} else {
-		list_add_tail(&qcedev_areq->list, &podev->ready_commands);
-	}
+	/*
+	 * Service only one crypto request at a time.
+	 * Any other new requests are queued in ready_commands and woken up
+	 * only when the active command has finished successfully or when the
+	 * request times out or when the command failed when setting up.
+	 */
+	do {
+		if (podev->active_command == NULL) {
+			podev->active_command = qcedev_areq;
+			qcedev_areq->state = QCEDEV_REQ_SUBMITTED;
+			switch (qcedev_areq->op_type) {
+			case QCEDEV_CRYPTO_OPER_CIPHER:
+				ret = start_cipher_req(podev,
+						&current_req_info);
+				break;
+			case QCEDEV_CRYPTO_OPER_OFFLOAD_CIPHER:
+				ret = start_offload_cipher_req(podev,
+						&current_req_info);
+				break;
+			default:
+
+				ret = start_sha_req(podev,
+						&current_req_info);
+				break;
+			}
+		} else {
+			list_add_tail(&qcedev_areq->list,
+					&podev->ready_commands);
+			qcedev_areq->state = QCEDEV_REQ_WAITING;
+			if (wait_event_interruptible_lock_irq_timeout(
+				qcedev_areq->wait_q,
+				(qcedev_areq->state == QCEDEV_REQ_CURRENT),
+				podev->lock,
+				msecs_to_jiffies(MAX_REQUEST_TIME)) == 0) {
+				pr_err("%s: request timed out\n", __func__);
+				return qcedev_areq->err;
+			}
+		}
+	} while (qcedev_areq->state != QCEDEV_REQ_SUBMITTED);
 
-	if (ret != 0)
+	if (ret != 0) {
 		podev->active_command = NULL;
+		/*
+		 * Look through queued requests and wake up the corresponding
+		 * thread.
+		 */
+		if (!list_empty(&podev->ready_commands)) {
+			new_req = container_of(podev->ready_commands.next,
+						struct qcedev_async_req, list);
+			list_del(&new_req->list);
+			new_req->state = QCEDEV_REQ_CURRENT;
+			wake_up_interruptible(&new_req->wait_q);
+		}
+	}
 
 	spin_unlock_irqrestore(&podev->lock, flags);
 
+	qcedev_areq->timed_out = false;
 	if (ret == 0)
 		wait = wait_for_completion_timeout(&qcedev_areq->complete,
 				msecs_to_jiffies(MAX_CRYPTO_WAIT_TIME));
@@ -788,7 +824,14 @@ static int submit_req(struct qcedev_async_req *qcedev_areq,
 					current_req_info);
 		print_sts = true;
 		qcedev_check_crypto_status(qcedev_areq, podev->qce, print_sts);
-		qce_manage_timeout(podev->qce, current_req_info);
+		qcedev_areq->timed_out = true;
+		ret = qce_manage_timeout(podev->qce, current_req_info);
+		if (ret) {
+			pr_err("%s: error during manage timeout", __func__);
+			qcedev_areq->err = -EIO;
+			return qcedev_areq->err;
+		}
+		tasklet_schedule(&podev->done_tasklet);
 		if (qcedev_areq->offload_cipher_op_req.err !=
 						QCEDEV_OFFLOAD_NO_ERROR)
 			return 0;

crypto-qti/qcedevi.h

@@ -66,6 +66,9 @@ struct qcedev_async_req {
 	};
 	struct qcedev_handle			*handle;
 	int					err;
+	wait_queue_head_t			wait_q;
+	uint16_t				state;
+	bool					timed_out;
 };
 
 /**********************************************************************