qcedev: better error handling for crypto requests

Handle crypto errors and timeouts better leaving room
for lesser sync issues (especially during timeouts).
Also, support the crypto status changes in the v5.8
CE engine.

Tests: oemcrypto tests on pineapple.

Change-Id: I8d381c9a2b80853260bc779afbea58ae600bccaf
Signed-off-by: Gaurav Kashyap <[email protected]>

crypto-qti/qce.h

@@ -198,6 +198,13 @@ struct qce_pm_table {
 
 extern struct qce_pm_table qce_pm_table;
 
+struct qce_error {
+    bool no_error;
+    bool timer_error;
+    bool key_paused;
+    bool generic_error;
+};
+
 void *qce_open(struct platform_device *pdev, int *rc);
 int qce_close(void *handle);
 int qce_aead_req(void *handle, struct qce_req *req);
@@ -209,8 +216,6 @@ int qce_disable_clk(void *handle);
 void qce_get_driver_stats(void *handle);
 void qce_clear_driver_stats(void *handle);
 void qce_dump_req(void *handle);
-void qce_get_crypto_status(void *handle, unsigned int *s1, unsigned int *s2,
-			   unsigned int *s3, unsigned int *s4,
-			   unsigned int *s5, unsigned int *s6);
+void qce_get_crypto_status(void *handle, struct qce_error *error);
 int qce_manage_timeout(void *handle, int req_info);
 #endif /* __CRYPTO_MSM_QCE_H */

crypto-qti/qce50.c

@@ -84,7 +84,26 @@ static LIST_HEAD(qce50_bam_list);
 #define TOTAL_IOVEC_SPACE_PER_PIPE (QCE_MAX_NUM_DSCR * sizeof(struct sps_iovec))
 
 #define AES_CTR_IV_CTR_SIZE	64
-#define STATUS1_ERR_INTR_MASK	0x10
+
+#define QCE_STATUS1_NO_ERROR	0x2000006
+
+// Crypto Engines 5.7 and below
+// Key timer expiry for pipes 1-15 (Status3)
+#define CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS3	0x0000FF00
+// Key timer expiry for pipes 16-19 (Status6)
+#define CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS6	0x00000300
+// Key pause for pipes 1-15 (Status3)
+#define CRYPTO5_LEGACY_KEY_PAUSE_STATUS3		0xFF000000
+// Key pause for pipes 16-19 (Status6)
+#define CRYPTO5_LEGACY_KEY_PAUSE_STATUS6		0x3000000
+
+// Crypto Engines 5.8 and above
+// Key timer expiry for all pipes (Status3)
+#define CRYPTO58_TIMER_EXPIRED		0x00000010
+// Key pause for all pipes (Status3)
+#define CRYPTO58_KEY_PAUSE			0x00001000
+// Key index for Status3 (Timer and Key Pause)
+#define KEY_INDEX_SHIFT				16
 
 enum qce_owner {
 	QCE_OWNER_NONE   = 0,
@@ -200,36 +219,73 @@ static uint32_t qce_get_config_be(struct qce_device *pce_dev,
 		pipe_pair << CRYPTO_PIPE_SET_SELECT);
 }
 
-static void dump_status_regs(unsigned int s1, unsigned int s2,unsigned int s3,
-			unsigned int s4, unsigned int s5,unsigned int s6)
+static void dump_status_regs(unsigned int *status)
 {
-	pr_info("%s: CRYPTO_STATUS_REG = 0x%x\n", __func__, s1);
-	pr_info("%s: CRYPTO_STATUS2_REG = 0x%x\n", __func__, s2);
-	pr_info("%s: CRYPTO_STATUS3_REG = 0x%x\n", __func__, s3);
-	pr_info("%s: CRYPTO_STATUS4_REG = 0x%x\n", __func__, s4);
-	pr_info("%s: CRYPTO_STATUS5_REG = 0x%x\n", __func__, s5);
-	pr_info("%s: CRYPTO_STATUS6_REG = 0x%x\n", __func__, s6);
+	pr_info("%s: CRYPTO_STATUS_REG = 0x%x\n", __func__, status[0]);
+	pr_info("%s: CRYPTO_STATUS2_REG = 0x%x\n", __func__, status[1]);
+	pr_info("%s: CRYPTO_STATUS3_REG = 0x%x\n", __func__, status[2]);
+	pr_info("%s: CRYPTO_STATUS4_REG = 0x%x\n", __func__, status[3]);
+	pr_info("%s: CRYPTO_STATUS5_REG = 0x%x\n", __func__, status[4]);
+	pr_info("%s: CRYPTO_STATUS6_REG = 0x%x\n", __func__, status[5]);
 }
 
-void qce_get_crypto_status(void *handle, unsigned int *s1, unsigned int *s2,
-			   unsigned int *s3, unsigned int *s4,
-			   unsigned int *s5, unsigned int *s6)
+void qce_get_crypto_status(void *handle, struct qce_error *error)
 {
 	struct qce_device *pce_dev = (struct qce_device *) handle;
+	unsigned int status[6] = {0};
 
-	*s1 = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG);
-	*s2 = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS2_REG);
-	*s3 = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS3_REG);
-	*s4 = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS4_REG);
-	*s5 = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS5_REG);
-	*s6 = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS6_REG);
+	status[0] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS_REG);
+	status[1] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS2_REG);
+	status[2] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS3_REG);
+	status[3] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS4_REG);
+	status[4] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS5_REG);
+	status[5] = readl_relaxed(pce_dev->iobase + CRYPTO_STATUS6_REG);
 
 #ifdef QCE_DEBUG
-	dump_status_regs(*s1, *s2, *s3, *s4, *s5, *s6);
-#else
-	if (*s1 & STATUS1_ERR_INTR_MASK)
-		dump_status_regs(*s1, *s2, *s3, *s4, *s5, *s6);
+	dump_status_regs(status);
 #endif
+
+	if (status[0] != QCE_STATUS1_NO_ERROR) {
+		if (pce_dev->ce_bam_info.minor_version >= 8) {
+			if (status[2] & CRYPTO58_TIMER_EXPIRED) {
+				error->timer_error = true;
+				pr_err("%s: timer expired, index = 0x%x\n",
+					__func__, (status[2] >> KEY_INDEX_SHIFT));
+			} else if (status[2] & CRYPTO58_KEY_PAUSE) {
+				error->key_paused = true;
+				pr_err("%s: key paused, index = 0x%x\n",
+					__func__, (status[2] >> KEY_INDEX_SHIFT));
+			} else {
+				pr_err("%s: generic error, refer all status\n",
+					__func__);
+				error->generic_error = true;
+			}
+		} else {
+			if ((status[2] & CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS3) ||
+				(status[5] & CRYPTO5_LEGACY_TIMER_EXPIRED_STATUS6)) {
+				error->timer_error = true;
+				pr_err("%s: timer expired, refer status 3 and 6\n",
+					__func__);
+			}
+			else if ((status[2] & CRYPTO5_LEGACY_KEY_PAUSE_STATUS3) ||
+					(status[5] & CRYPTO5_LEGACY_KEY_PAUSE_STATUS6)) {
+				error->key_paused = true;
+				pr_err("%s: key paused, reder status 3 and 6\n",
+					__func__);
+			} else {
+				pr_err("%s: generic error, refer all status\n",
+					__func__);
+				error->generic_error = true;
+			}
+		}
+		dump_status_regs(status);
+		return;
+	}
+
+	error->no_error = true;
+	pr_err("%s: No crypto error, status1 = 0x%x\n",
+		   __func__, status[0]);
+
 	return;
 }
 EXPORT_SYMBOL(qce_get_crypto_status);
@@ -413,6 +469,7 @@ static int _probe_ce_engine(struct qce_device *pce_dev)
 		pce_dev->no_ccm_mac_status_get_around = false;
 
 	pce_dev->ce_bam_info.minor_version = min_rev;
+	pce_dev->ce_bam_info.major_version = maj_rev;
 
 	pce_dev->engines_avail = readl_relaxed(pce_dev->iobase +
 					CRYPTO_ENGINES_AVAIL);

crypto-qti/qce50.h

@@ -200,6 +200,7 @@ struct ce_bam_info {
 	unsigned long			bam_handle;
 	int				ce_burst_size;
 	uint32_t			minor_version;
+	uint32_t			major_version;
 	struct qce_sps_ep_conn_data	producer[QCE_OFFLOAD_OPER_LAST];
 	struct qce_sps_ep_conn_data	consumer[QCE_OFFLOAD_OPER_LAST];
 };

crypto-qti/qcedev.c

@@ -25,6 +25,7 @@
 #include "linux/platform_data/qcom_crypto_device.h"
 #include "linux/qcedev.h"
 #include <linux/interconnect.h>
+#include <linux/delay.h>
 
 #include <crypto/hash.h>
 #include "qcedevi.h"
@@ -50,6 +51,7 @@ enum qcedev_req_status {
 	QCEDEV_REQ_CURRENT = 0,
 	QCEDEV_REQ_WAITING = 1,
 	QCEDEV_REQ_SUBMITTED = 2,
+	QCEDEV_REQ_DONE = 3,
 };
 
 static uint8_t  _std_init_vector_sha1_uint8[] =   {
@@ -69,17 +71,6 @@ static uint8_t _std_init_vector_sha256_uint8[] = {
 #define QCEDEV_CTX_USE_HW_KEY		0x00000001
 #define QCEDEV_CTX_USE_PIPE_KEY		0x00000002
 
-// Key timer expiry for pipes 1-15 (Status3)
-#define PIPE_KEY_TIMER_EXPIRED_STATUS3_MASK	0x000000FF
-// Key timer expiry for pipes 16-19 (Status6)
-#define PIPE_KEY_TIMER_EXPIRED_STATUS6_MASK	0x00000003
-// Key pause for pipes 1-15 (Status3)
-#define PIPE_KEY_PAUSE_STATUS3_MASK	0xFF0000
-// Key pause for pipes 16-19 (Status6)
-#define PIPE_KEY_PAUSE_STATUS6_MASK	0x30000
-
-#define QCEDEV_STATUS1_ERR_INTR_MASK	0x10
-
 static DEFINE_MUTEX(send_cmd_lock);
 static DEFINE_MUTEX(qcedev_sent_bw_req);
 static DEFINE_MUTEX(hash_access_lock);
@@ -324,8 +315,10 @@ static void req_done(unsigned long data)
 	areq = podev->active_command;
 	podev->active_command = NULL;
 
-	if (areq && !areq->timed_out)
+	if (areq && !areq->timed_out) {
 		complete(&areq->complete);
+		areq->state = QCEDEV_REQ_DONE;
+	}
 
 	/* Look through queued requests and wake up the corresponding thread */
 	if (!list_empty(&podev->ready_commands)) {
@@ -686,52 +679,29 @@ static int start_sha_req(struct qcedev_control *podev,
 };
 
 static void qcedev_check_crypto_status(
-			struct qcedev_async_req *qcedev_areq, void *handle,
-			bool print_err)
+			struct qcedev_async_req *qcedev_areq, void *handle)
 {
-	unsigned int s1, s2, s3, s4, s5, s6;
+	struct qce_error error = {0};
 
 	qcedev_areq->offload_cipher_op_req.err = QCEDEV_OFFLOAD_NO_ERROR;
-	qce_get_crypto_status(handle, &s1, &s2, &s3, &s4, &s5, &s6);
+	qce_get_crypto_status(handle, &error);
 
-	if (print_err) {
-		pr_err("%s: sts = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", __func__,
-			s1, s2, s3, s4, s5, s6);
-	}
-
-	// Check for key timer expiry
-	if ((s6 & PIPE_KEY_TIMER_EXPIRED_STATUS6_MASK) ||
-		(s3 & PIPE_KEY_TIMER_EXPIRED_STATUS3_MASK)) {
-		pr_info("%s: crypto timer expired\n", __func__);
-		pr_info("%s: sts = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", __func__,
-			s1, s2, s3, s4, s5, s6);
+	if (error.timer_error) {
 		qcedev_areq->offload_cipher_op_req.err =
-					QCEDEV_OFFLOAD_KEY_TIMER_EXPIRED_ERROR;
-		return;
-	}
-
-	// Check for key pause
-	if ((s6 & PIPE_KEY_PAUSE_STATUS6_MASK) ||
-		(s3 & PIPE_KEY_PAUSE_STATUS3_MASK)) {
-		pr_info("%s: crypto key paused\n", __func__);
-		pr_info("%s: sts = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", __func__,
-			s1, s2, s3, s4, s5, s6);
+			QCEDEV_OFFLOAD_KEY_TIMER_EXPIRED_ERROR;
+	} else if (error.key_paused) {
 		qcedev_areq->offload_cipher_op_req.err =
-					QCEDEV_OFFLOAD_KEY_PAUSE_ERROR;
-		return;
-	}
-
-	// Check for generic error
-	if (s1 & QCEDEV_STATUS1_ERR_INTR_MASK) {
-		pr_err("%s: generic crypto error\n", __func__);
-		pr_info("%s: sts = 0x%x 0x%x 0x%x 0x%x 0x%x 0x%x\n", __func__,
-			s1, s2, s3, s4, s5, s6);
+			QCEDEV_OFFLOAD_KEY_PAUSE_ERROR;
+	} else if (error.generic_error) {
 		qcedev_areq->offload_cipher_op_req.err =
-					QCEDEV_OFFLOAD_GENERIC_ERROR;
-		return;
+			QCEDEV_OFFLOAD_GENERIC_ERROR;
 	}
+
+	return;
 }
 
+#define MAX_RETRIES	333
+
 static int submit_req(struct qcedev_async_req *qcedev_areq,
 					struct qcedev_handle *handle)
 {
@@ -741,14 +711,13 @@ static int submit_req(struct qcedev_async_req *qcedev_areq,
 	struct qcedev_stat *pstat;
 	int current_req_info = 0;
 	int wait = MAX_CRYPTO_WAIT_TIME;
-	bool print_sts = false;
 	struct qcedev_async_req *new_req = NULL;
+	int retries = 0;
 
 	qcedev_areq->err = 0;
 	podev = handle->cntl;
 	init_waitqueue_head(&qcedev_areq->wait_q);
 
-
 	spin_lock_irqsave(&podev->lock, flags);
 
 	/*
@@ -823,17 +792,25 @@ static int submit_req(struct qcedev_async_req *qcedev_areq,
 	 */
 		pr_err("%s: wait timed out, req info = %d\n", __func__,
 					current_req_info);
-		print_sts = true;
+		qcedev_check_crypto_status(qcedev_areq, podev->qce);
+		if (qcedev_areq->offload_cipher_op_req.err ==
+			QCEDEV_OFFLOAD_NO_ERROR) {
+			pr_err("%s: no error, wait for request to be done", __func__);
+			while (qcedev_areq->state != QCEDEV_REQ_DONE &&
+				retries < MAX_RETRIES) {
+				usleep_range(3000, 5000);
+				retries++;
+				pr_err("%s: waiting for req state to be done, retries = %d",
+					__func__, retries);
+			}
+			return 0;
+		}
 		spin_lock_irqsave(&podev->lock, flags);
-		qcedev_check_crypto_status(qcedev_areq, podev->qce, print_sts);
 		qcedev_areq->timed_out = true;
 		ret = qce_manage_timeout(podev->qce, current_req_info);
-		if (ret) {
+		if (ret)
 			pr_err("%s: error during manage timeout", __func__);
-			qcedev_areq->err = -EIO;
-			spin_unlock_irqrestore(&podev->lock, flags);
-			return qcedev_areq->err;
-		}
+
 		spin_unlock_irqrestore(&podev->lock, flags);
 		tasklet_schedule(&podev->done_tasklet);
 		if (qcedev_areq->offload_cipher_op_req.err !=