mm-drivers: hw_fence: add fence error support for clients without rxq

Add HW Fence Driver support to notify waiting clients of fence error in
HLOS. This is a requirement by clients that do not have Rx Queue. Such
clients can register a fence error callback function with data that will
be passed back with callback.

The fence error callback function is called by HW Fence Driver when:
1. Client registers for a fence already signaled with error.
2. Error is signaled for a fence that the client registered to wait on.

Change-Id: I2892333838001bed1152118b947cfe12b1a8dd04
Signed-off-by: Grace An <[email protected]>

hw_fence/include/hw_fence_drv_priv.h

@@ -131,9 +131,12 @@ struct msm_hw_fence_queue {
 
 /**
  * enum payload_type - Enum with the queue payload types.
+ * HW_FENCE_PAYLOAD_TYPE_1: client queue payload
+ * HW_FENCE_PAYLOAD_TYPE_2: ctrl queue payload for fence error; client_data stores client_id
  */
 enum payload_type {
-	HW_FENCE_PAYLOAD_TYPE_1 = 1
+	HW_FENCE_PAYLOAD_TYPE_1 = 1,
+	HW_FENCE_PAYLOAD_TYPE_2
 };
 
 /**
@@ -144,6 +147,10 @@ enum payload_type {
  * @mem_descriptor: hfi header memory descriptor
  * @queues: queues descriptor
  * @queues_num: number of client queues
+ * @fence_error_cb: function called for waiting clients that need HLOS notification of fence error
+ * @fence_error_cb_userdata: opaque pointer registered with fence error callback and passed to
+ *                           client during invocation of callback function
+ * @error_cb_lock: lock to synchronize access to fence error cb and fence error cb data
  * @ipc_signal_id: id of the signal to be triggered for this client
  * @ipc_client_vid: virtual id of the ipc client for this hw fence driver client
  * @ipc_client_pid: physical id of the ipc client for this hw fence driver client
@@ -158,6 +165,9 @@ struct msm_hw_fence_client {
 	struct msm_hw_fence_mem_addr mem_descriptor;
 	struct msm_hw_fence_queue queues[HW_FENCE_CLIENT_QUEUES];
 	int queues_num;
+	msm_hw_fence_error_cb_t fence_error_cb;
+	void *fence_error_cb_userdata;
+	struct mutex error_cb_lock;
 	int ipc_signal_id;
 	int ipc_client_vid;
 	int ipc_client_pid;
@@ -508,6 +518,8 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data);
 int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	struct msm_hw_fence_queue_payload *payload, int queue_type);
+int hw_fence_read_queue_helper(struct msm_hw_fence_queue *queue,
+	struct msm_hw_fence_queue_payload *payload);
 int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 	struct dma_fence *fence, struct msm_hw_fence_client *hw_fence_client, u64 context,
 	u64 seqno, u64 *hash, u64 client_data);

hw_fence/include/hw_fence_drv_utils.h

@@ -122,6 +122,21 @@ int hw_fence_utils_cleanup_fence(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, struct msm_hw_fence *hw_fence, u64 hash,
 	u32 reset_flags);
 
+/**
+ * hw_fence_utils_fence_error_cb() - Invokes fence error callback registered by specified client
+ *
+ * @hw_fence_client: client, for which fence error callback must be invoked
+ * @ctxt_id: context id of the hw-fence
+ * @seqno: sequence number of the hw-fence
+ * @hash: hash of the hw-fence
+ * @flags: flags of the hw-fence
+ * @error: error of the hw-fence
+ *
+ * Returns zero if success, otherwise returns negative error code
+ */
+int hw_fence_utils_fence_error_cb(struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id,
+	u64 seqno, u64 hash, u64 flags, u32 error);
+
 /**
  * hw_fence_utils_get_client_id_priv() - Gets the index into clients struct within hw fence driver
  *                                       from the client_id used externally

hw_fence/src/hw_fence_drv_priv.c

@@ -187,15 +187,7 @@ char *_get_queue_type(int queue_type)
 int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 		 struct msm_hw_fence_queue_payload *payload, int queue_type)
 {
-	struct msm_hw_fence_hfi_queue_header *hfi_header;
 	struct msm_hw_fence_queue *queue;
-	u32 read_idx;
-	u32 write_idx;
-	u32 to_read_idx;
-	u32 *read_ptr;
-	u32 payload_size_u32;
-	u32 q_size_u32;
-	struct msm_hw_fence_queue_payload *read_ptr_payload;
 
 	if (queue_type >= HW_FENCE_CLIENT_QUEUES || !hw_fence_client || !payload) {
 		HWFNC_ERR("Invalid queue type:%s hw_fence_client:0x%pK payload:0x%pK\n", queue_type,
@@ -204,6 +196,20 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	}
 
 	queue = &hw_fence_client->queues[queue_type];
+	HWFNC_DBG_Q("read client:%lu queue:0x%pK\n", hw_fence_client->client_id, queue);
+
+	return hw_fence_read_queue_helper(queue, payload);
+}
+
+int hw_fence_read_queue_helper(struct msm_hw_fence_queue *queue,
+		 struct msm_hw_fence_queue_payload *payload)
+{
+	struct msm_hw_fence_hfi_queue_header *hfi_header;
+	u32 read_idx, write_idx, to_read_idx;
+	u32 *read_ptr;
+	u32 payload_size_u32, q_size_u32;
+	struct msm_hw_fence_queue_payload *read_ptr_payload;
+
 	hfi_header = queue->va_header;
 
 	q_size_u32 = (queue->q_size_bytes / sizeof(u32));
@@ -230,13 +236,12 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 			read_idx, write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
 	}
 
-	HWFNC_DBG_Q("read client:%d rd_ptr:0x%pK wr_ptr:0x%pK rd_idx:%d wr_idx:%d queue:0x%pK\n",
-		hw_fence_client->client_id, &hfi_header->read_index, &hfi_header->write_index,
-		read_idx, write_idx, queue);
+	HWFNC_DBG_Q("read rd_ptr:0x%pK wr_ptr:0x%pK rd_idx:%d wr_idx:%d queue:0x%pK\n",
+		&hfi_header->read_index, &hfi_header->write_index, read_idx, write_idx, queue);
 
 	if (read_idx == write_idx) {
 		HWFNC_DBG_Q("Nothing to read!\n");
-		return 0;
+		return -EINVAL;
 	}
 
 	/* Move the pointer where we need to read and cast it */
@@ -264,12 +269,7 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	}
 
 	/* Read the Client Queue */
-	payload->ctxt_id = readq_relaxed(&read_ptr_payload->ctxt_id);
-	payload->seqno = readq_relaxed(&read_ptr_payload->seqno);
-	payload->hash = readq_relaxed(&read_ptr_payload->hash);
-	payload->flags = readq_relaxed(&read_ptr_payload->flags);
-	payload->client_data = readq_relaxed(&read_ptr_payload->client_data);
-	payload->error = readl_relaxed(&read_ptr_payload->error);
+	*payload = *read_ptr_payload;
 
 	/* update the read index */
 	writel_relaxed(to_read_idx, &hfi_header->read_index);
@@ -1204,15 +1204,22 @@ static void _fence_ctl_signal(struct hw_fence_driver_data *drv_data,
 
 	HWFNC_DBG_H("We must signal the client now! hfence hash:%llu\n", hash);
 
-	/* Write to Rx queue */
-	if (hw_fence_client->update_rxq)
-		hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id,
-			hw_fence->seq_id, hash, flags, client_data, error, HW_FENCE_RX_QUEUE - 1);
-
-	/* Signal the hw fence now */
-	if (hw_fence_client->send_ipc)
-		hw_fence_ipcc_trigger_signal(drv_data, tx_client_id, rx_client_id,
-			hw_fence_client->ipc_signal_id);
+	/* Call fence error callback */
+	if (error && hw_fence_client->fence_error_cb) {
+		hw_fence_utils_fence_error_cb(hw_fence_client, hw_fence->ctx_id, hw_fence->seq_id,
+			hash, flags, error);
+	} else {
+		/* Write to Rx queue */
+		if (hw_fence_client->update_rxq)
+			hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id,
+				hw_fence->seq_id, hash, flags, client_data, error,
+				HW_FENCE_RX_QUEUE - 1);
+
+		/* Signal the hw fence now */
+		if (hw_fence_client->send_ipc)
+			hw_fence_ipcc_trigger_signal(drv_data, tx_client_id, rx_client_id,
+				hw_fence_client->ipc_signal_id);
+	}
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 	if (hw_fence_client->client_id >= HW_FENCE_CLIENT_ID_VAL0
@@ -1355,6 +1362,7 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 
 			/* child fence is already signaled */
 			GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 1); /* lock */
+			join_fence->error |= hw_fence_child->error;
 			if (--join_fence->pending_child_cnt == 0)
 				signal_join_fence = true;
 
@@ -1400,8 +1408,8 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 	if (signal_join_fence) {
 
 		/* signal the join hw fence */
-		_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0, 0,
-			client_data);
+		_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0,
+			client_data, join_fence->error);
 		set_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &array->base.flags);
 
 		/*

hw_fence/src/hw_fence_drv_utils.c

@@ -53,11 +53,34 @@
 #define HW_FENCE_CLIENT_TYPE_MAX_VPU 32
 #define HW_FENCE_CLIENT_TYPE_MAX_IFE 32
 
-/*
- * Each bit in this mask represents each of the loopback clients supported in
- * the enum hw_fence_client_id
+/**
+ * HW_FENCE_CTRL_QUEUE_DOORBELL:
+ * Bit set in doorbell flags mask if hw fence driver should read ctrl rx queue
+ */
+#define HW_FENCE_CTRL_QUEUE_DOORBELL 0
+
+/**
+ * HW_FENCE_DOORBELL_FLAGS_ID_LAST:
+ * Last doorbell flags id for which HW Fence Driver can receive doorbell
+ */
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+#define HW_FENCE_DOORBELL_FLAGS_ID_LAST HW_FENCE_CLIENT_ID_VAL6
+#else
+#define HW_FENCE_DOORBELL_FLAGS_ID_LAST HW_FENCE_CTRL_QUEUE_DOORBELL
+#endif /* CONFIG_DEBUG_FS */
+
+/**
+ * HW_FENCE_DOORBELL_MASK:
+ * Each bit in this mask represents possible doorbell flag ids for which hw fence driver can receive
+ */
+#define HW_FENCE_DOORBELL_MASK \
+	GENMASK(HW_FENCE_DOORBELL_FLAGS_ID_LAST, HW_FENCE_CTRL_QUEUE_DOORBELL)
+
+/**
+ * HW_FENCE_MAX_ITER_READ:
+ * Maximum number of iterations when reading queue
  */
-#define HW_FENCE_LOOPBACK_CLIENTS_MASK 0x7fff
+#define HW_FENCE_MAX_ITER_READ 100
 
 /**
  * HW_FENCE_MAX_EVENTS:
@@ -179,12 +202,110 @@ void global_atomic_store(struct hw_fence_driver_data *drv_data, uint64_t *lock,
 	}
 }
 
-static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int client_id)
+int hw_fence_utils_fence_error_cb(struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id,
+	u64 seqno, u64 hash, u64 flags, u32 error)
+{
+	struct msm_hw_fence_cb_data cb_data;
+	struct dma_fence fence;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(hw_fence_client)) {
+		HWFNC_ERR("Invalid client:0x%pK\n", hw_fence_client);
+		return -EINVAL;
+	}
+
+	mutex_lock(&hw_fence_client->error_cb_lock);
+	if (!error || !hw_fence_client->fence_error_cb) {
+		HWFNC_ERR("Invalid error:%d fence_error_cb:0x%pK\n", error,
+			hw_fence_client->fence_error_cb);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	/* initialize cb_data info */
+	fence.context = ctxt_id;
+	fence.seqno = seqno;
+	fence.flags = flags;
+	fence.error = error;
+	cb_data.fence = &fence;
+	cb_data.data = hw_fence_client->fence_error_cb_userdata;
+
+	HWFNC_DBG_L("invoking cb for client:%d ctx:%llu seq:%llu flags:%llu e:%u data:0x%pK\n",
+		hw_fence_client->client_id, ctxt_id, seqno, flags, error,
+		hw_fence_client->fence_error_cb_userdata);
+
+	hw_fence_client->fence_error_cb(hash, error, &cb_data);
+
+exit:
+	mutex_unlock(&hw_fence_client->error_cb_lock);
+
+	return ret;
+}
+
+static int _process_fence_error_client_loopback(struct hw_fence_driver_data *drv_data,
+	int db_flag_id)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+	struct msm_hw_fence_queue_payload payload;
+	int i, cb_ret, ret = 0, read = 1;
+	u32 client_id;
+
+	for (i = 0; read && i < HW_FENCE_MAX_ITER_READ; i++) {
+		read = hw_fence_read_queue_helper(&drv_data->ctrl_queues[HW_FENCE_RX_QUEUE - 1],
+			&payload);
+		if (read < 0) {
+			HWFNC_DBG_Q("unable to read ctrl rxq for db_flag_id:%d\n", db_flag_id);
+			return read;
+		}
+		if (payload.type != HW_FENCE_PAYLOAD_TYPE_2) {
+			HWFNC_ERR("unsupported payload type in ctrl rxq received:%u expected:%u\n",
+				payload.type, HW_FENCE_PAYLOAD_TYPE_2);
+			ret = -EINVAL;
+			continue;
+		}
+		if (payload.client_data < HW_FENCE_CLIENT_ID_CTX0 ||
+				payload.client_data >= drv_data->clients_num) {
+			HWFNC_ERR("read invalid client_id:%llu from ctrl rxq min:%u max:%u\n",
+				payload.client_data, HW_FENCE_CLIENT_ID_CTX0,
+				drv_data->clients_num);
+			ret = -EINVAL;
+			continue;
+		}
+
+		client_id = payload.client_data;
+		HWFNC_DBG_Q("ctrl rxq rd: it:%d h:%llu ctx:%llu seq:%llu f:%llu e:%u client:%u\n",
+			i, payload.hash, payload.ctxt_id, payload.seqno, payload.flags,
+			payload.error, client_id);
+
+		hw_fence_client = drv_data->clients[client_id];
+		if (!hw_fence_client) {
+			HWFNC_ERR("processing fence error cb for unregistered client_id:%u\n",
+				client_id);
+			ret = -EINVAL;
+			continue;
+		}
+
+		cb_ret = hw_fence_utils_fence_error_cb(hw_fence_client, payload.ctxt_id,
+			payload.seqno, payload.hash, payload.flags, payload.error);
+		if (cb_ret) {
+			HWFNC_ERR("fence_error_cb failed for client:%u ctx:%llu seq:%llu err:%u\n",
+				client_id, payload.ctxt_id, payload.seqno, payload.error);
+			ret = cb_ret;
+		}
+	}
+
+	return ret;
+}
+
+static int _process_doorbell_id(struct hw_fence_driver_data *drv_data, int db_flag_id)
 {
 	int ret;
 
-	HWFNC_DBG_H("Processing doorbell client_id:%d\n", client_id);
-	switch (client_id) {
+	HWFNC_DBG_H("Processing doorbell mask id:%d\n", db_flag_id);
+	switch (db_flag_id) {
+	case HW_FENCE_CTRL_QUEUE_DOORBELL:
+		ret = _process_fence_error_client_loopback(drv_data, db_flag_id);
+		break;
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 	case HW_FENCE_CLIENT_ID_VAL0:
 	case HW_FENCE_CLIENT_ID_VAL1:
@@ -193,11 +314,11 @@ static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int c
 	case HW_FENCE_CLIENT_ID_VAL4:
 	case HW_FENCE_CLIENT_ID_VAL5:
 	case HW_FENCE_CLIENT_ID_VAL6:
-		ret = process_validation_client_loopback(drv_data, client_id);
+		ret = process_validation_client_loopback(drv_data, db_flag_id);
 		break;
 #endif /* CONFIG_DEBUG_FS */
 	default:
-		HWFNC_ERR("unknown client:%d\n", client_id);
+		HWFNC_ERR("unknown mask id:%d\n", db_flag_id);
 		ret = -EINVAL;
 	}
 
@@ -206,22 +327,21 @@ static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int c
 
 void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data, u64 db_flags)
 {
-	int client_id = HW_FENCE_CLIENT_ID_CTL0;
+	int db_flag_id = HW_FENCE_CTRL_QUEUE_DOORBELL;
 	u64 mask;
 
-	for (; client_id <= HW_FENCE_CLIENT_ID_VAL6; client_id++) {
-		mask = 1 << client_id;
+	for (; db_flag_id <= HW_FENCE_DOORBELL_FLAGS_ID_LAST; db_flag_id++) {
+		mask = 1 << db_flag_id;
 		if (mask & db_flags) {
-			HWFNC_DBG_H("client_id:%d signaled! flags:0x%llx\n", client_id, db_flags);
+			HWFNC_DBG_H("db_flag:%d signaled! flags:0x%llx\n", db_flag_id, db_flags);
 
-			/* process client */
-			if (_process_doorbell_client(drv_data, client_id))
-				HWFNC_ERR("Failed to process client:%d\n", client_id);
+			if (_process_doorbell_id(drv_data, db_flag_id))
+				HWFNC_ERR("Failed to process db_flag_id:%d\n", db_flag_id);
 
-			/* clear mask for this client and if nothing else pending finish */
+			/* clear mask for this flag id if nothing else pending finish */
 			db_flags = db_flags & ~(mask);
-			HWFNC_DBG_H("client_id:%d cleared flags:0x%llx mask:0x%llx ~mask:0x%llx\n",
-				client_id, db_flags, mask, ~(mask));
+			HWFNC_DBG_H("db_flag_id:%d cleared flags:0x%llx mask:0x%llx ~mask:0x%llx\n",
+				db_flag_id, db_flags, mask, ~(mask));
 			if (!db_flags)
 				break;
 		}
@@ -232,7 +352,7 @@ void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data,
 static void _hw_fence_cb(int irq, void *data)
 {
 	struct hw_fence_driver_data *drv_data = (struct hw_fence_driver_data *)data;
-	gh_dbl_flags_t clear_flags = HW_FENCE_LOOPBACK_CLIENTS_MASK;
+	gh_dbl_flags_t clear_flags = HW_FENCE_DOORBELL_MASK;
 	int ret;
 
 	if (!drv_data)

hw_fence/src/msm_hw_fence.c

@@ -118,6 +118,8 @@ void *msm_hw_fence_register(enum hw_fence_client_id client_id_ext,
 	if (ret)
 		goto error;
 
+	mutex_init(&hw_fence_client->error_cb_lock);
+
 	HWFNC_DBG_INIT("Initialized ptr:0x%p client_id:%d q_num:%d ipc signal:%d vid:%d pid:%d\n",
 		hw_fence_client, hw_fence_client->client_id, hw_fence_client->queues_num,
 		hw_fence_client->ipc_signal_id, hw_fence_client->ipc_client_vid,
@@ -476,6 +478,73 @@ int msm_hw_fence_trigger_signal(void *client_handle,
 }
 EXPORT_SYMBOL(msm_hw_fence_trigger_signal);
 
+int msm_hw_fence_register_error_cb(void *client_handle, msm_hw_fence_error_cb_t cb, void *data)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
+		HWFNC_ERR("hw fence driver not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle) || IS_ERR_OR_NULL(cb) || IS_ERR_OR_NULL(data)) {
+		HWFNC_ERR("Invalid params client:0x%pK cb_func:0x%pK data:0x%pK\n", client_handle,
+			cb, data);
+		return -EINVAL;
+	}
+
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+	if (hw_fence_client->fence_error_cb) {
+		HWFNC_ERR("client_id:%d client_id_ext:%d already registered cb_func:%pK data:%pK\n",
+			hw_fence_client->client_id, hw_fence_client->client_id_ext,
+			hw_fence_client->fence_error_cb, hw_fence_client->fence_error_cb_userdata);
+		return -EINVAL;
+	}
+
+	hw_fence_client->fence_error_cb_userdata = data;
+	hw_fence_client->fence_error_cb = cb;
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_register_error_cb);
+
+int msm_hw_fence_deregister_error_cb(void *client_handle)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
+		HWFNC_ERR("hw fence driver not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle)) {
+		HWFNC_ERR("Invalid client: 0x%pK\n", client_handle);
+		return -EINVAL;
+	}
+
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+	if (!mutex_trylock(&hw_fence_client->error_cb_lock)) {
+		HWFNC_ERR("client_id:%d is modifying or using fence_error_cb:0x%pK data:0x%pK\n",
+			hw_fence_client->client_id, hw_fence_client->fence_error_cb,
+			hw_fence_client->fence_error_cb_userdata);
+		return -EAGAIN;
+	}
+
+	if (!hw_fence_client->fence_error_cb) {
+		HWFNC_ERR("client_id:%d client_id_ext:%d did not register cb:%pK data:%pK\n",
+			hw_fence_client->client_id, hw_fence_client->client_id_ext,
+			hw_fence_client->fence_error_cb, hw_fence_client->fence_error_cb_userdata);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	hw_fence_client->fence_error_cb = NULL;
+	hw_fence_client->fence_error_cb_userdata = NULL;
+
+exit:
+	mutex_unlock(&hw_fence_client->error_cb_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_deregister_error_cb);
+
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 int msm_hw_fence_dump_debug_data(void *client_handle, u32 dump_flags, u32 dump_clients_mask)
 {