Merge 07a41b0f006dffbfb2a13bbb40bdb19535e8e7ee on remote branch

Change-Id: I664bd0ccc671cd0637d878716a2410ed0480b223

hw_fence/include/hw_fence_drv_debug.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 /*
- * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
  */
 
 #ifndef __HW_FENCE_DRV_DEBUG
@@ -60,6 +60,9 @@ extern u32 msm_hw_fence_debug_level;
 #define HWFNC_DBG_LOCK(fmt, ...) \
 	dprintk(HW_FENCE_LOCK, "[hwfence:%s:%d][dbglock]"fmt, __func__, __LINE__, ##__VA_ARGS__)
 
+#define HWFNC_DBG_DUMP(prio, fmt, ...) \
+	dprintk(prio, "[hwfence:%s:%d][dbgd]"fmt, __func__, __LINE__, ##__VA_ARGS__)
+
 #define HWFNC_WARN(fmt, ...) \
 	pr_warn("[hwfence:%s:%d][warn][%pS] "fmt, __func__, __LINE__, \
 	__builtin_return_address(0), ##__VA_ARGS__)
@@ -70,6 +73,13 @@ int hw_fence_debug_debugfs_register(struct hw_fence_driver_data *drv_data);
 
 int process_validation_client_loopback(struct hw_fence_driver_data *drv_data, int client_id);
 
+void hw_fence_debug_dump_queues(enum hw_fence_drv_prio prio,
+	struct msm_hw_fence_client *hw_fence_client);
+void hw_fence_debug_dump_fence(enum hw_fence_drv_prio prio, struct msm_hw_fence *hw_fence, u64 hash,
+	u32 count);
+void hw_fence_debug_dump_table(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data);
+void hw_fence_debug_dump_events(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data);
+
 extern const struct file_operations hw_sync_debugfs_fops;
 
 struct hw_fence_out_clients_map {

hw_fence/include/hw_fence_drv_priv.h

@@ -131,9 +131,12 @@ struct msm_hw_fence_queue {
 
 /**
  * enum payload_type - Enum with the queue payload types.
+ * HW_FENCE_PAYLOAD_TYPE_1: client queue payload
+ * HW_FENCE_PAYLOAD_TYPE_2: ctrl queue payload for fence error; client_data stores client_id
  */
 enum payload_type {
-	HW_FENCE_PAYLOAD_TYPE_1 = 1
+	HW_FENCE_PAYLOAD_TYPE_1 = 1,
+	HW_FENCE_PAYLOAD_TYPE_2
 };
 
 /**
@@ -144,6 +147,10 @@ enum payload_type {
  * @mem_descriptor: hfi header memory descriptor
  * @queues: queues descriptor
  * @queues_num: number of client queues
+ * @fence_error_cb: function called for waiting clients that need HLOS notification of fence error
+ * @fence_error_cb_userdata: opaque pointer registered with fence error callback and passed to
+ *                           client during invocation of callback function
+ * @error_cb_lock: lock to synchronize access to fence error cb and fence error cb data
  * @ipc_signal_id: id of the signal to be triggered for this client
  * @ipc_client_vid: virtual id of the ipc client for this hw fence driver client
  * @ipc_client_pid: physical id of the ipc client for this hw fence driver client
@@ -158,6 +165,9 @@ struct msm_hw_fence_client {
 	struct msm_hw_fence_mem_addr mem_descriptor;
 	struct msm_hw_fence_queue queues[HW_FENCE_CLIENT_QUEUES];
 	int queues_num;
+	msm_hw_fence_error_cb_t fence_error_cb;
+	void *fence_error_cb_userdata;
+	struct mutex error_cb_lock;
 	int ipc_signal_id;
 	int ipc_client_vid;
 	int ipc_client_pid;
@@ -505,9 +515,13 @@ int hw_fence_process_fence(struct hw_fence_driver_data *drv_data,
 int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id, u64 seqno, u64 hash,
 	u64 flags, u64 client_data, u32 error, int queue_type);
+int hw_fence_update_existing_txq_payload(struct hw_fence_driver_data *drv_data,
+	struct msm_hw_fence_client *hw_fence_client, u64 hash, u32 error);
 inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data);
 int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	struct msm_hw_fence_queue_payload *payload, int queue_type);
+int hw_fence_read_queue_helper(struct msm_hw_fence_queue *queue,
+	struct msm_hw_fence_queue_payload *payload);
 int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 	struct dma_fence *fence, struct msm_hw_fence_client *hw_fence_client, u64 context,
 	u64 seqno, u64 *hash, u64 client_data);

hw_fence/include/hw_fence_drv_utils.h

@@ -122,6 +122,21 @@ int hw_fence_utils_cleanup_fence(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, struct msm_hw_fence *hw_fence, u64 hash,
 	u32 reset_flags);
 
+/**
+ * hw_fence_utils_fence_error_cb() - Invokes fence error callback registered by specified client
+ *
+ * @hw_fence_client: client, for which fence error callback must be invoked
+ * @ctxt_id: context id of the hw-fence
+ * @seqno: sequence number of the hw-fence
+ * @hash: hash of the hw-fence
+ * @flags: flags of the hw-fence
+ * @error: error of the hw-fence
+ *
+ * Returns zero if success, otherwise returns negative error code
+ */
+int hw_fence_utils_fence_error_cb(struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id,
+	u64 seqno, u64 hash, u64 flags, u32 error);
+
 /**
  * hw_fence_utils_get_client_id_priv() - Gets the index into clients struct within hw fence driver
  *                                       from the client_id used externally

hw_fence/src/hw_fence_drv_debug.c

@@ -14,6 +14,14 @@
 
 #define HW_FENCE_DEBUG_MAX_LOOPS 200
 
+#define HFENCE_TBL_MSG \
+	"[%d]hfence[%d] v:%d err:%lu ctx:%llu seq:%llu wait:0x%llx alloc:%d f:0x%llx child_cnt:%d" \
+	"%s ct:%llu tt:%llu wt:%llu\n"
+
+/* each hwfence parent includes one "32-bit" element + "," separator */
+#define HW_FENCE_MAX_PARENTS_SUBLIST_DUMP (MSM_HW_FENCE_MAX_JOIN_PARENTS * 9)
+#define HW_FENCE_MAX_PARENTS_DUMP (sizeof("parent_list[] ") + HW_FENCE_MAX_PARENTS_SUBLIST_DUMP)
+
 /* event dump data includes one "32-bit" element + "|" separator */
 #define HW_FENCE_MAX_DATA_PER_EVENT_DUMP (HW_FENCE_EVENT_MAX_DATA * 9)
 
@@ -473,29 +481,82 @@ static ssize_t hw_fence_dbg_create_wr(struct file *file,
 	return count;
 }
 
-#define HFENCE_TBL_MSG \
-	"[%d]hfence[%d] v:%d err:%d ctx:%d seqno:%d wait:0x%llx alloc:%d f:0x%lx tt:%llu wt:%llu\n"
+static void _dump_fence_helper(enum hw_fence_drv_prio prio, struct msm_hw_fence *hw_fence,
+	char *parents_dump, u64 hash, u32 count)
+{
+	char sublist[HW_FENCE_MAX_PARENTS_SUBLIST_DUMP];
+	u32 parents_cnt;
+	int i, len = 0;
+
+	if (!hw_fence || !parents_dump) {
+		HWFNC_ERR("invalid params hw_fence:0x%pK parents_dump:0x%pK\n", hw_fence,
+			parents_dump);
+		return;
+	}
+
+	memset(parents_dump, 0, sizeof(char) * HW_FENCE_MAX_PARENTS_DUMP);
+	if (hw_fence->parents_cnt) {
+		if (hw_fence->parents_cnt > MSM_HW_FENCE_MAX_JOIN_PARENTS) {
+			HWFNC_ERR("hfence[%d] has invalid parents_cnt:%d greater than max:%d\n",
+				hash, hw_fence->parents_cnt, MSM_HW_FENCE_MAX_JOIN_PARENTS);
+			parents_cnt = MSM_HW_FENCE_MAX_JOIN_PARENTS;
+		} else {
+			parents_cnt = hw_fence->parents_cnt;
+		}
+
+		memset(sublist, 0, sizeof(sublist));
+		for (i = 0; i < parents_cnt; i++)
+			len += scnprintf(sublist + len, HW_FENCE_MAX_PARENTS_SUBLIST_DUMP - len,
+				"%lu,", hw_fence->parent_list[i]);
+		scnprintf(parents_dump, HW_FENCE_MAX_PARENTS_DUMP, " p:[%s]", sublist);
+	}
+
+	HWFNC_DBG_DUMP(prio, HFENCE_TBL_MSG,
+		count, hash, hw_fence->valid, hw_fence->error, hw_fence->ctx_id, hw_fence->seq_id,
+		hw_fence->wait_client_mask, hw_fence->fence_allocator, hw_fence->flags,
+		hw_fence->pending_child_cnt, parents_dump, hw_fence->fence_create_time,
+		hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
+}
+
+void hw_fence_debug_dump_fence(enum hw_fence_drv_prio prio, struct msm_hw_fence *hw_fence, u64 hash,
+	u32 count)
+{
+	char parents_dump[HW_FENCE_MAX_PARENTS_DUMP];
+
+	return _dump_fence_helper(prio, hw_fence, parents_dump, hash, count);
+}
 
 static inline int _dump_fence(struct msm_hw_fence *hw_fence, char *buf, int len, int max_size,
 		u32 index, u32 cnt)
 {
 	int ret;
+	char parents_dump[HW_FENCE_MAX_PARENTS_DUMP];
 
-	ret = scnprintf(buf + len, max_size - len, HFENCE_TBL_MSG,
-		cnt, index, hw_fence->valid, hw_fence->error,
-		hw_fence->ctx_id, hw_fence->seq_id,
-		hw_fence->wait_client_mask, hw_fence->fence_allocator,
-		hw_fence->flags, hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
+	_dump_fence_helper(HW_FENCE_INFO, hw_fence, parents_dump, index, cnt);
 
-	HWFNC_DBG_L(HFENCE_TBL_MSG,
-		cnt, index, hw_fence->valid, hw_fence->error,
-		hw_fence->ctx_id, hw_fence->seq_id,
-		hw_fence->wait_client_mask, hw_fence->fence_allocator,
-		hw_fence->flags, hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
+	ret = scnprintf(buf + len, max_size - len, HFENCE_TBL_MSG,
+		cnt, index, hw_fence->valid, hw_fence->error, hw_fence->ctx_id, hw_fence->seq_id,
+		hw_fence->wait_client_mask, hw_fence->fence_allocator, hw_fence->flags,
+		hw_fence->pending_child_cnt, parents_dump, hw_fence->fence_create_time,
+		hw_fence->fence_trigger_time, hw_fence->fence_wait_time);
 
 	return ret;
 }
 
+void hw_fence_debug_dump_table(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data)
+{
+	u32 i, cnt = 0;
+	struct msm_hw_fence *hw_fence;
+
+	for (i = 0; i < drv_data->hw_fences_tbl_cnt; i++) {
+		hw_fence = &drv_data->hw_fences_tbl[i];
+		if (!hw_fence->valid)
+			continue;
+		hw_fence_debug_dump_fence(prio, hw_fence, i, cnt);
+		cnt++;
+	}
+}
+
 static int dump_single_entry(struct hw_fence_driver_data *drv_data, char *buf, u32 *index,
 	int max_size)
 {
@@ -545,17 +606,40 @@ static int dump_full_table(struct hw_fence_driver_data *drv_data, char *buf, u32
 	return len;
 }
 
-static inline int _dump_event(struct msm_hw_fence_event *event, char *buf, int len, int max_size,
-	u32 index)
+static void _find_earliest_event(struct hw_fence_driver_data *drv_data, u32 *start_index,
+	u64 *start_time)
+{
+	u32 i;
+
+	if (!start_index || !start_time) {
+		HWFNC_ERR("invalid params start_index:0x%pK start_time:0x%pK\n", start_index,
+			start_time);
+		return;
+	}
+
+	mb(); /* make sure data is ready before read */
+	for (i = 0; i < drv_data->total_events; i++) {
+		u64 time = drv_data->events[i].time;
+
+		if (time && (!*start_time || time < *start_time)) {
+			*start_time = time;
+			*start_index = i;
+		}
+	}
+}
+
+static void _dump_event(enum hw_fence_drv_prio prio, struct msm_hw_fence_event *event,
+	char *data, u32 index)
 {
-	char data[HW_FENCE_MAX_DATA_PER_EVENT_DUMP];
 	u32 data_cnt;
-	int i, tmp_len = 0, ret = 0;
+	int i, len = 0;
 
-	if (!event->time)
-		return 0;
+	if (!event || !data) {
+		HWFNC_ERR("invalid params event:0x%pK data:0x%pK\n", event, data);
+		return;
+	}
 
-	memset(&data, 0, sizeof(data));
+	memset(data, 0, sizeof(char) * HW_FENCE_MAX_DATA_PER_EVENT_DUMP);
 	if (event->data_cnt > HW_FENCE_EVENT_MAX_DATA) {
 		HWFNC_ERR("event[%d] has invalid data_cnt:%lu greater than max_data_cnt:%lu\n",
 			index, event->data_cnt, HW_FENCE_EVENT_MAX_DATA);
@@ -565,15 +649,29 @@ static inline int _dump_event(struct msm_hw_fence_event *event, char *buf, int l
 	}
 
 	for (i = 0; i < data_cnt; i++)
-		tmp_len += scnprintf(data + tmp_len, HW_FENCE_MAX_DATA_PER_EVENT_DUMP - tmp_len,
+		len += scnprintf(data + len, HW_FENCE_MAX_DATA_PER_EVENT_DUMP - len,
 			"%lx|", event->data[i]);
 
-	ret = scnprintf(buf + len, max_size - len, HFENCE_EVT_MSG, index, event->cpu, event->time,
-		event->data_cnt, data);
+	HWFNC_DBG_DUMP(prio, HFENCE_EVT_MSG, index, event->cpu, event->time, event->data_cnt, data);
+}
 
-	HWFNC_DBG_INFO(HFENCE_EVT_MSG, index, event->cpu, event->time, event->data_cnt, data);
+void hw_fence_debug_dump_events(enum hw_fence_drv_prio prio, struct hw_fence_driver_data *drv_data)
+{
+	char data[HW_FENCE_MAX_DATA_PER_EVENT_DUMP];
+	u32 start_index;
+	u64 start_time;
+	int i;
 
-	return ret;
+	if (!drv_data->events) {
+		HWFNC_ERR("events not supported\n");
+		return;
+	}
+
+	_find_earliest_event(drv_data, &start_index, &start_time);
+	for (i = start_index; i < drv_data->total_events && drv_data->events[i].time; i++)
+		_dump_event(prio, &drv_data->events[i], data, i);
+	for (i = 0; i < start_index; i++)
+		_dump_event(prio, &drv_data->events[i], data, i);
 }
 
 /**
@@ -626,15 +724,7 @@ static ssize_t hw_fence_dbg_dump_events_rd(struct file *file, char __user *user_
 
 	/* find index of earliest event */
 	if (!start_time) {
-		mb(); /* make sure data is ready before read */
-		for (index = 0; index < drv_data->total_events; index++) {
-			u64 time = drv_data->events[index].time;
-
-			if (time && (!start_time || time < start_time)) {
-				start_time = time;
-				start_index = index;
-			}
-		}
+		_find_earliest_event(drv_data, &start_index, &start_time);
 		index = start_index;
 		HWFNC_DBG_H("events:0x%pK start_index:%d start_time:%llu total_events:%d\n",
 			drv_data->events, start_index, start_time, drv_data->total_events);
@@ -642,7 +732,15 @@ static ssize_t hw_fence_dbg_dump_events_rd(struct file *file, char __user *user_
 
 	HWFNC_DBG_H("++ dump_events index:%d qtime:%llu\n", index, hw_fence_get_qtime(drv_data));
 	while ((!wraparound || index < start_index) && len < (max_size - entry_size)) {
-		len += _dump_event(&drv_data->events[index], buf, len, max_size, index);
+		char data[HW_FENCE_MAX_DATA_PER_EVENT_DUMP];
+
+		if (drv_data->events[index].time) {
+			_dump_event(HW_FENCE_INFO, &drv_data->events[index], data, index);
+			len += scnprintf(buf + len, max_size - len, HFENCE_EVT_MSG, index,
+				drv_data->events[index].cpu, drv_data->events[index].time,
+				drv_data->events[index].data_cnt, data);
+		}
+
 		index++;
 		if (index >= drv_data->total_events) {
 			index = 0;
@@ -668,6 +766,63 @@ exit:
 	return len;
 }
 
+static void _dump_queue(enum hw_fence_drv_prio prio, struct msm_hw_fence_client *hw_fence_client,
+	int queue_type)
+{
+	struct msm_hw_fence_queue *queue;
+	struct msm_hw_fence_hfi_queue_header *hfi_header;
+	struct msm_hw_fence_queue_payload *payload;
+	u64 timestamp;
+	u32 *read_ptr, queue_entries;
+	int i;
+
+	queue = &hw_fence_client->queues[queue_type - 1];
+
+	if ((queue_type > hw_fence_client->queues_num) || !queue || !queue->va_header
+			|| !queue->va_queue) {
+		HWFNC_ERR("Cannot dump client:%d q_type:%s q_ptr:0x%pK q_header:0x%pK q_va:0x%pK\n",
+			hw_fence_client->client_id,
+			(queue_type == HW_FENCE_TX_QUEUE) ? "TX QUEUE" : "RX QUEUE",
+			queue, queue ? queue->va_header : NULL, queue ? queue->va_queue : NULL);
+		return;
+	}
+	hfi_header = (struct msm_hw_fence_hfi_queue_header *)queue->va_header;
+
+	mb(); /* make sure data is ready before read */
+	HWFNC_DBG_DUMP(prio, "%s va:0x%pK rd_idx:%lu wr_idx:%lu tx_wm:%lu q_size_bytes:%lu\n",
+		(queue_type == HW_FENCE_TX_QUEUE) ? "TX QUEUE" : "RX QUEUE", queue->va_queue,
+		hfi_header->read_index, hfi_header->write_index, hfi_header->tx_wm,
+		queue->q_size_bytes);
+	queue_entries = queue->q_size_bytes / HW_FENCE_CLIENT_QUEUE_PAYLOAD;
+
+	for (i = 0; i < queue_entries; i++) {
+		read_ptr = ((u32 *)queue->va_queue +
+			(i * (sizeof(struct msm_hw_fence_queue_payload) / sizeof(u32))));
+		payload = (struct msm_hw_fence_queue_payload *)read_ptr;
+		timestamp = (u64)payload->timestamp_lo | ((u64)payload->timestamp_hi << 32);
+
+		HWFNC_DBG_DUMP(prio,
+			"%s[%d]: hash:%d ctx:%llu seqno:%llu f:%llu d:%llu err:%u time:%llu\n",
+			(queue_type == HW_FENCE_TX_QUEUE) ? "tx" : "rx", i, payload->hash,
+			payload->ctxt_id, payload->seqno, payload->flags, payload->client_data,
+			payload->error, timestamp);
+	}
+}
+
+void hw_fence_debug_dump_queues(enum hw_fence_drv_prio prio,
+	struct msm_hw_fence_client *hw_fence_client)
+{
+	if (!hw_fence_client) {
+		HWFNC_ERR("Invalid params client:0x%pK\n", hw_fence_client);
+		return;
+	}
+
+	HWFNC_DBG_DUMP(prio, "Queues for client %d\n", hw_fence_client->client_id);
+	if (hw_fence_client->queues_num == HW_FENCE_CLIENT_QUEUES)
+		_dump_queue(prio, hw_fence_client, HW_FENCE_RX_QUEUE);
+	_dump_queue(prio, hw_fence_client, HW_FENCE_TX_QUEUE);
+}
+
 /**
  * hw_fence_dbg_dump_queues_wr() - debugfs wr to dump the hw-fences queues.
  * @file: file handler.
@@ -682,12 +837,7 @@ static ssize_t hw_fence_dbg_dump_queues_wr(struct file *file, const char __user
 	size_t count, loff_t *ppos)
 {
 	struct hw_fence_driver_data *drv_data;
-	struct msm_hw_fence_queue *rx_queue;
-	struct msm_hw_fence_queue *tx_queue;
-	u64 hash, ctx_id, seqno, timestamp, flags, client_data;
-	u32 *read_ptr, error;
-	int client_id, i;
-	struct msm_hw_fence_queue_payload *read_ptr_payload;
+	int client_id;
 
 	if (!file || !file->private_data) {
 		HWFNC_ERR("unexpected data %d\n", file);
@@ -699,53 +849,11 @@ static ssize_t hw_fence_dbg_dump_queues_wr(struct file *file, const char __user
 	if (client_id < 0)
 		return -EINVAL;
 
-	if (!drv_data->clients[client_id] ||
-		IS_ERR_OR_NULL(&drv_data->clients[client_id]->queues[HW_FENCE_RX_QUEUE - 1]) ||
-		IS_ERR_OR_NULL(&drv_data->clients[client_id]->queues[HW_FENCE_TX_QUEUE - 1])) {
+	if (!drv_data->clients[client_id]) {
 		HWFNC_ERR("client %d not initialized\n", client_id);
 		return -EINVAL;
 	}
-
-	HWFNC_DBG_L("Queues for client %d\n", client_id);
-
-	rx_queue = &drv_data->clients[client_id]->queues[HW_FENCE_RX_QUEUE - 1];
-	tx_queue = &drv_data->clients[client_id]->queues[HW_FENCE_TX_QUEUE - 1];
-
-	HWFNC_DBG_L("-------RX QUEUE------\n");
-	for (i = 0; i < drv_data->hw_fence_queue_entries; i++) {
-		read_ptr = ((u32 *)rx_queue->va_queue +
-			(i * (sizeof(struct msm_hw_fence_queue_payload) / sizeof(u32))));
-		read_ptr_payload = (struct msm_hw_fence_queue_payload *)read_ptr;
-
-		ctx_id = readq_relaxed(&read_ptr_payload->ctxt_id);
-		seqno = readq_relaxed(&read_ptr_payload->seqno);
-		hash = readq_relaxed(&read_ptr_payload->hash);
-		flags = readq_relaxed(&read_ptr_payload->flags);
-		client_data = readq_relaxed(&read_ptr_payload->client_data);
-		error = readl_relaxed(&read_ptr_payload->error);
-		timestamp = (u64)readl_relaxed(&read_ptr_payload->timestamp_lo) |
-			((u64)readl_relaxed(&read_ptr_payload->timestamp_hi) << 32);
-
-		HWFNC_DBG_L("rx[%d]: hash:%d ctx:%llu seqno:%llu f:%llu d:%llu err:%u time:%llu\n",
-			i, hash, ctx_id, seqno, flags, client_data, error, timestamp);
-	}
-
-	HWFNC_DBG_L("-------TX QUEUE------\n");
-	for (i = 0; i < drv_data->hw_fence_queue_entries; i++) {
-		read_ptr = ((u32 *)tx_queue->va_queue +
-			(i * (sizeof(struct msm_hw_fence_queue_payload) / sizeof(u32))));
-		read_ptr_payload = (struct msm_hw_fence_queue_payload *)read_ptr;
-
-		ctx_id = readq_relaxed(&read_ptr_payload->ctxt_id);
-		seqno = readq_relaxed(&read_ptr_payload->seqno);
-		hash = readq_relaxed(&read_ptr_payload->hash);
-		flags = readq_relaxed(&read_ptr_payload->flags);
-		error = readl_relaxed(&read_ptr_payload->error);
-		timestamp = (u64)readl_relaxed(&read_ptr_payload->timestamp_lo) |
-			((u64)readl_relaxed(&read_ptr_payload->timestamp_hi) << 32);
-		HWFNC_DBG_L("tx[%d]: hash:%d ctx:%llu seqno:%llu f:%llu err:%u time:%llu\n",
-			i, hash, ctx_id, seqno, flags, error, timestamp);
-	}
+	hw_fence_debug_dump_queues(HW_FENCE_PRINTK, drv_data->clients[client_id]);
 
 	return count;
 }

hw_fence/src/hw_fence_drv_priv.c

@@ -17,6 +17,15 @@
 
 #define IS_HW_FENCE_TX_QUEUE(queue_type) ((queue_type) == HW_FENCE_TX_QUEUE - 1)
 
+#define REQUIRES_IDX_TRANSLATION(queue) \
+	((queue)->rd_wr_idx_factor && ((queue)->rd_wr_idx_start || (queue)->rd_wr_idx_factor > 1))
+
+#define IDX_TRANSLATE_CUSTOM_TO_DEFAULT(queue, idx) \
+	(((idx) - (queue)->rd_wr_idx_start) * (queue)->rd_wr_idx_factor)
+
+#define IDX_TRANSLATE_DEFAULT_TO_CUSTOM(queue, idx) \
+	(((idx) / (queue)->rd_wr_idx_factor) + (queue)->rd_wr_idx_start)
+
 inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data)
 {
 #ifdef HWFENCE_USE_SLEEP_TIMER
@@ -184,18 +193,21 @@ char *_get_queue_type(int queue_type)
 	return (queue_type == (HW_FENCE_RX_QUEUE - 1)) ? "RXQ" : "TXQ";
 }
 
+static void _translate_queue_indexes_custom_to_default(struct msm_hw_fence_queue *queue,
+	u32 *read_idx, u32 *write_idx)
+{
+	if (REQUIRES_IDX_TRANSLATION(queue)) {
+		*read_idx = IDX_TRANSLATE_CUSTOM_TO_DEFAULT(queue, *read_idx);
+		*write_idx = IDX_TRANSLATE_CUSTOM_TO_DEFAULT(queue, *write_idx);
+		HWFNC_DBG_Q("rd_idx_u32:%lu wr_idx_u32:%lu rd_wr_idx start:%lu factor:%lu\n",
+			*read_idx, *write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
+	}
+}
+
 int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 		 struct msm_hw_fence_queue_payload *payload, int queue_type)
 {
-	struct msm_hw_fence_hfi_queue_header *hfi_header;
 	struct msm_hw_fence_queue *queue;
-	u32 read_idx;
-	u32 write_idx;
-	u32 to_read_idx;
-	u32 *read_ptr;
-	u32 payload_size_u32;
-	u32 q_size_u32;
-	struct msm_hw_fence_queue_payload *read_ptr_payload;
 
 	if (queue_type >= HW_FENCE_CLIENT_QUEUES || !hw_fence_client || !payload) {
 		HWFNC_ERR("Invalid queue type:%s hw_fence_client:0x%pK payload:0x%pK\n", queue_type,
@@ -204,6 +216,20 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	}
 
 	queue = &hw_fence_client->queues[queue_type];
+	HWFNC_DBG_Q("read client:%lu queue:0x%pK\n", hw_fence_client->client_id, queue);
+
+	return hw_fence_read_queue_helper(queue, payload);
+}
+
+int hw_fence_read_queue_helper(struct msm_hw_fence_queue *queue,
+		 struct msm_hw_fence_queue_payload *payload)
+{
+	struct msm_hw_fence_hfi_queue_header *hfi_header;
+	u32 read_idx, write_idx, to_read_idx;
+	u32 *read_ptr;
+	u32 payload_size_u32, q_size_u32;
+	struct msm_hw_fence_queue_payload *read_ptr_payload;
+
 	hfi_header = queue->va_header;
 
 	q_size_u32 = (queue->q_size_bytes / sizeof(u32));
@@ -223,20 +249,14 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	write_idx = readl_relaxed(&hfi_header->write_index);
 
 	/* translate read and write indexes from custom indexing to dwords with no offset */
-	if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) {
-		read_idx = (read_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
-		write_idx = (write_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
-		HWFNC_DBG_Q("rd_idx_u32:%lu wr_idx_u32:%lu rd_wr_idx start:%lu factor:%lu\n",
-			read_idx, write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
-	}
+	_translate_queue_indexes_custom_to_default(queue, &read_idx, &write_idx);
 
-	HWFNC_DBG_Q("read client:%d rd_ptr:0x%pK wr_ptr:0x%pK rd_idx:%d wr_idx:%d queue:0x%pK\n",
-		hw_fence_client->client_id, &hfi_header->read_index, &hfi_header->write_index,
-		read_idx, write_idx, queue);
+	HWFNC_DBG_Q("read rd_ptr:0x%pK wr_ptr:0x%pK rd_idx:%d wr_idx:%d queue:0x%pK\n",
+		&hfi_header->read_index, &hfi_header->write_index, read_idx, write_idx, queue);
 
 	if (read_idx == write_idx) {
 		HWFNC_DBG_Q("Nothing to read!\n");
-		return 0;
+		return -EINVAL;
 	}
 
 	/* Move the pointer where we need to read and cast it */
@@ -257,19 +277,14 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 		to_read_idx = 0;
 
 	/* translate to_read_idx to custom indexing with offset */
-	if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) {
-		to_read_idx = (to_read_idx / queue->rd_wr_idx_factor) + queue->rd_wr_idx_start;
+	if (REQUIRES_IDX_TRANSLATION(queue)) {
+		to_read_idx = IDX_TRANSLATE_DEFAULT_TO_CUSTOM(queue, to_read_idx);
 		HWFNC_DBG_Q("translated to_read_idx:%lu rd_wr_idx start:%lu factor:%lu\n",
 			to_read_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
 	}
 
 	/* Read the Client Queue */
-	payload->ctxt_id = readq_relaxed(&read_ptr_payload->ctxt_id);
-	payload->seqno = readq_relaxed(&read_ptr_payload->seqno);
-	payload->hash = readq_relaxed(&read_ptr_payload->hash);
-	payload->flags = readq_relaxed(&read_ptr_payload->flags);
-	payload->client_data = readq_relaxed(&read_ptr_payload->client_data);
-	payload->error = readl_relaxed(&read_ptr_payload->error);
+	*payload = *read_ptr_payload;
 
 	/* update the read index */
 	writel_relaxed(to_read_idx, &hfi_header->read_index);
@@ -281,6 +296,34 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	return to_read_idx == write_idx ? 0 : 1;
 }
 
+static int _get_update_queue_params(struct msm_hw_fence_queue *queue,
+	struct msm_hw_fence_hfi_queue_header **hfi_header, u32 *q_size_u32, u32 *payload_size,
+	u32 *payload_size_u32, u32 **wr_ptr)
+{
+	if (!queue) {
+		HWFNC_ERR("invalid queue\n");
+		return -EINVAL;
+	}
+
+	*hfi_header = queue->va_header;
+	if (!*hfi_header) {
+		HWFNC_ERR("Invalid queue hfi_header\n");
+		return -EINVAL;
+	}
+
+	*q_size_u32 = (queue->q_size_bytes / sizeof(u32));
+	*payload_size = sizeof(struct msm_hw_fence_queue_payload);
+	*payload_size_u32 = (*payload_size / sizeof(u32));
+
+	/* if skipping update wr_index, then use hfi_header->tx_wm instead */
+	if (queue->skip_wr_idx)
+		*wr_ptr = &((*hfi_header)->tx_wm);
+	else
+		*wr_ptr = &((*hfi_header)->write_index);
+
+	return 0;
+}
+
 /*
  * This function writes to the queue of the client. The 'queue_type' determines
  * if this function is writing to the rx or tx queue
@@ -312,23 +355,13 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 	}
 
 	queue = &hw_fence_client->queues[queue_type];
-	hfi_header = queue->va_header;
-
-	q_size_u32 = (queue->q_size_bytes / sizeof(u32));
-	payload_size = sizeof(struct msm_hw_fence_queue_payload);
-	payload_size_u32 = (payload_size / sizeof(u32));
-
-	if (!hfi_header) {
-		HWFNC_ERR("Invalid queue\n");
+	if (_get_update_queue_params(queue, &hfi_header, &q_size_u32, &payload_size,
+			&payload_size_u32, &wr_ptr)) {
+		HWFNC_ERR("Invalid client:%d q_type:%d queue\n", hw_fence_client->client_id,
+			queue_type);
 		return -EINVAL;
 	}
 
-	/* if skipping update wr_index, then use hfi_header->tx_wm instead */
-	if (queue->skip_wr_idx)
-		wr_ptr = &hfi_header->tx_wm;
-	else
-		wr_ptr = &hfi_header->write_index;
-
 	/*
 	 * We need to lock the client if there is an Rx Queue update, since that
 	 * is the only time when HW Fence driver can have a race condition updating
@@ -361,12 +394,7 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 		read_idx, write_idx, queue, queue_type, queue->skip_wr_idx ? "true" : "false");
 
 	/* translate read and write indexes from custom indexing to dwords with no offset */
-	if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) {
-		read_idx = (read_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
-		write_idx = (write_idx - queue->rd_wr_idx_start) * queue->rd_wr_idx_factor;
-		HWFNC_DBG_Q("rd_idx_u32:%lu wr_idx_u32:%lu rd_wr_idx start:%lu factor:%lu\n",
-			read_idx, write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
-	}
+	_translate_queue_indexes_custom_to_default(queue, &read_idx, &write_idx);
 
 	/* Check queue to make sure message will fit */
 	q_free_u32 = read_idx <= write_idx ? (q_size_u32 - (write_idx - read_idx)) :
@@ -402,8 +430,8 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 		to_write_idx = 0;
 
 	/* translate to_write_idx to custom indexing with offset */
-	if (queue->rd_wr_idx_start || queue->rd_wr_idx_factor != 1) {
-		to_write_idx = (to_write_idx / queue->rd_wr_idx_factor) + queue->rd_wr_idx_start;
+	if (REQUIRES_IDX_TRANSLATION(queue)) {
+		to_write_idx = IDX_TRANSLATE_DEFAULT_TO_CUSTOM(queue, to_write_idx);
 		HWFNC_DBG_Q("translated to_write_idx:%lu rd_wr_idx start:%lu factor:%lu\n",
 			to_write_idx, queue->rd_wr_idx_start, queue->rd_wr_idx_factor);
 	}
@@ -438,6 +466,90 @@ exit:
 	return ret;
 }
 
+int hw_fence_update_existing_txq_payload(struct hw_fence_driver_data *drv_data,
+	struct msm_hw_fence_client *hw_fence_client, u64 hash, u32 error)
+{
+	u32 q_size_u32, payload_size, payload_size_u32, read_idx, write_idx, second_idx, *wr_ptr;
+	struct msm_hw_fence_queue_payload tmp, *first_payload, *second_payload;
+	struct msm_hw_fence_hfi_queue_header *hfi_header;
+	struct msm_hw_fence_queue *queue;
+	int ret = 0;
+
+	queue = &hw_fence_client->queues[HW_FENCE_TX_QUEUE - 1];
+	if (_get_update_queue_params(queue, &hfi_header, &q_size_u32, &payload_size,
+			&payload_size_u32, &wr_ptr)) {
+		HWFNC_ERR("Invalid client:%d tx queue\n", hw_fence_client->client_id);
+		return -EINVAL;
+	}
+
+	/* Make sure data is ready before read */
+	mb();
+
+	/* Get read and write index */
+	read_idx = hfi_header->read_index;
+	write_idx = *wr_ptr;
+
+	/* translate read and write indexes from custom indexing to dwords with no offset */
+	_translate_queue_indexes_custom_to_default(queue, &read_idx, &write_idx);
+
+	if (read_idx == write_idx) {
+		HWFNC_DBG_Q("Empty queue, no entry matches with hash:%llu\n", hash);
+		return -EINVAL;
+	}
+
+	first_payload = (struct msm_hw_fence_queue_payload *)((u32 *)queue->va_queue + read_idx);
+	HWFNC_DBG_Q("client:%d txq: va=0x%pK pa=0x%pK idx:%d ptr_payload:0x%pK\n",
+		hw_fence_client->client_id, queue->va_queue, queue->pa_queue, read_idx,
+		first_payload);
+
+	if (first_payload->hash == hash) {
+		/* Swap not needed, update first payload in client queue with fence error */
+		first_payload->error = error;
+	} else {
+		/* Check whether second entry matches hash */
+		second_idx = read_idx + payload_size_u32;
+
+		/* wrap-around case */
+		if (second_idx >= q_size_u32)
+			second_idx = 0;
+
+		if (second_idx == write_idx) {
+			HWFNC_ERR("Failed to find matching entry with hash:%llu\n", hash);
+			return -EINVAL;
+		}
+
+		second_payload = (struct msm_hw_fence_queue_payload *)
+			((u32 *)queue->va_queue + second_idx);
+		HWFNC_DBG_Q("client:%d txq: va=0x%pK pa=0x%pK idx:%d ptr_payload:0x%pK\n",
+			hw_fence_client->client_id, queue->va_queue, queue->pa_queue, second_idx,
+			second_payload);
+
+		if (second_payload->hash != hash) {
+			HWFNC_ERR("hash:%llu not found in first two queue payloads:%u, %u\n", hash,
+				read_idx, second_idx);
+			return -EINVAL;
+		}
+
+		/* swap first and second payload, updating error field in new first payload */
+		tmp = *first_payload;
+		*first_payload = *second_payload;
+		first_payload->error = error;
+		*second_payload = tmp;
+
+		HWFNC_DBG_L("client_id:%d txq move from idx:%u to idx:%u hash:%llu c:%llu s:%llu\n",
+			hw_fence_client->client_id, read_idx, second_idx, hash, tmp.ctxt_id,
+			tmp.seqno);
+	}
+
+	/* update memory for the messages */
+	wmb();
+
+	HWFNC_DBG_L("client_id:%d update tx queue index:%u hash:%llu error:%u\n",
+		hw_fence_client->client_id, read_idx, hash, error);
+
+	return ret;
+}
+
 static int init_global_locks(struct hw_fence_driver_data *drv_data)
 {
 	struct msm_hw_fence_mem_addr *mem_descriptor;
@@ -1204,15 +1316,22 @@ static void _fence_ctl_signal(struct hw_fence_driver_data *drv_data,
 
 	HWFNC_DBG_H("We must signal the client now! hfence hash:%llu\n", hash);
 
-	/* Write to Rx queue */
-	if (hw_fence_client->update_rxq)
-		hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id,
-			hw_fence->seq_id, hash, flags, client_data, error, HW_FENCE_RX_QUEUE - 1);
-
-	/* Signal the hw fence now */
-	if (hw_fence_client->send_ipc)
-		hw_fence_ipcc_trigger_signal(drv_data, tx_client_id, rx_client_id,
-			hw_fence_client->ipc_signal_id);
+	/* Call fence error callback */
+	if (error && hw_fence_client->fence_error_cb) {
+		hw_fence_utils_fence_error_cb(hw_fence_client, hw_fence->ctx_id, hw_fence->seq_id,
+			hash, flags, error);
+	} else {
+		/* Write to Rx queue */
+		if (hw_fence_client->update_rxq)
+			hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id,
+				hw_fence->seq_id, hash, flags, client_data, error,
+				HW_FENCE_RX_QUEUE - 1);
+
+		/* Signal the hw fence now */
+		if (hw_fence_client->send_ipc)
+			hw_fence_ipcc_trigger_signal(drv_data, tx_client_id, rx_client_id,
+				hw_fence_client->ipc_signal_id);
+	}
 
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 	if (hw_fence_client->client_id >= HW_FENCE_CLIENT_ID_VAL0
@@ -1355,6 +1474,7 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 
 			/* child fence is already signaled */
 			GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 1); /* lock */
+			join_fence->error |= hw_fence_child->error;
 			if (--join_fence->pending_child_cnt == 0)
 				signal_join_fence = true;
 
@@ -1400,8 +1520,8 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 	if (signal_join_fence) {
 
 		/* signal the join hw fence */
-		_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0, 0,
-			client_data);
+		_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0,
+			client_data, join_fence->error);
 		set_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &array->base.flags);
 
 		/*

hw_fence/src/hw_fence_drv_utils.c

@@ -53,11 +53,34 @@
 #define HW_FENCE_CLIENT_TYPE_MAX_VPU 32
 #define HW_FENCE_CLIENT_TYPE_MAX_IFE 32
 
-/*
- * Each bit in this mask represents each of the loopback clients supported in
- * the enum hw_fence_client_id
+/**
+ * HW_FENCE_CTRL_QUEUE_DOORBELL:
+ * Bit set in doorbell flags mask if hw fence driver should read ctrl rx queue
+ */
+#define HW_FENCE_CTRL_QUEUE_DOORBELL 0
+
+/**
+ * HW_FENCE_DOORBELL_FLAGS_ID_LAST:
+ * Last doorbell flags id for which HW Fence Driver can receive doorbell
+ */
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+#define HW_FENCE_DOORBELL_FLAGS_ID_LAST HW_FENCE_CLIENT_ID_VAL6
+#else
+#define HW_FENCE_DOORBELL_FLAGS_ID_LAST HW_FENCE_CTRL_QUEUE_DOORBELL
+#endif /* CONFIG_DEBUG_FS */
+
+/**
+ * HW_FENCE_DOORBELL_MASK:
+ * Each bit in this mask represents possible doorbell flag ids for which hw fence driver can receive
+ */
+#define HW_FENCE_DOORBELL_MASK \
+	GENMASK(HW_FENCE_DOORBELL_FLAGS_ID_LAST, HW_FENCE_CTRL_QUEUE_DOORBELL)
+
+/**
+ * HW_FENCE_MAX_ITER_READ:
+ * Maximum number of iterations when reading queue
  */
-#define HW_FENCE_LOOPBACK_CLIENTS_MASK 0x7fff
+#define HW_FENCE_MAX_ITER_READ 100
 
 /**
  * HW_FENCE_MAX_EVENTS:
@@ -179,12 +202,110 @@ void global_atomic_store(struct hw_fence_driver_data *drv_data, uint64_t *lock,
 	}
 }
 
-static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int client_id)
+int hw_fence_utils_fence_error_cb(struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id,
+	u64 seqno, u64 hash, u64 flags, u32 error)
+{
+	struct msm_hw_fence_cb_data cb_data;
+	struct dma_fence fence;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(hw_fence_client)) {
+		HWFNC_ERR("Invalid client:0x%pK\n", hw_fence_client);
+		return -EINVAL;
+	}
+
+	mutex_lock(&hw_fence_client->error_cb_lock);
+	if (!error || !hw_fence_client->fence_error_cb) {
+		HWFNC_ERR("Invalid error:%d fence_error_cb:0x%pK\n", error,
+			hw_fence_client->fence_error_cb);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	/* initialize cb_data info */
+	fence.context = ctxt_id;
+	fence.seqno = seqno;
+	fence.flags = flags;
+	fence.error = error;
+	cb_data.fence = &fence;
+	cb_data.data = hw_fence_client->fence_error_cb_userdata;
+
+	HWFNC_DBG_L("invoking cb for client:%d ctx:%llu seq:%llu flags:%llu e:%u data:0x%pK\n",
+		hw_fence_client->client_id, ctxt_id, seqno, flags, error,
+		hw_fence_client->fence_error_cb_userdata);
+
+	hw_fence_client->fence_error_cb(hash, error, &cb_data);
+
+exit:
+	mutex_unlock(&hw_fence_client->error_cb_lock);
+
+	return ret;
+}
+
+static int _process_fence_error_client_loopback(struct hw_fence_driver_data *drv_data,
+	int db_flag_id)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+	struct msm_hw_fence_queue_payload payload;
+	int i, cb_ret, ret = 0, read = 1;
+	u32 client_id;
+
+	for (i = 0; read && i < HW_FENCE_MAX_ITER_READ; i++) {
+		read = hw_fence_read_queue_helper(&drv_data->ctrl_queues[HW_FENCE_RX_QUEUE - 1],
+			&payload);
+		if (read < 0) {
+			HWFNC_DBG_Q("unable to read ctrl rxq for db_flag_id:%d\n", db_flag_id);
+			return read;
+		}
+		if (payload.type != HW_FENCE_PAYLOAD_TYPE_2) {
+			HWFNC_ERR("unsupported payload type in ctrl rxq received:%u expected:%u\n",
+				payload.type, HW_FENCE_PAYLOAD_TYPE_2);
+			ret = -EINVAL;
+			continue;
+		}
+		if (payload.client_data < HW_FENCE_CLIENT_ID_CTX0 ||
+				payload.client_data >= drv_data->clients_num) {
+			HWFNC_ERR("read invalid client_id:%llu from ctrl rxq min:%u max:%u\n",
+				payload.client_data, HW_FENCE_CLIENT_ID_CTX0,
+				drv_data->clients_num);
+			ret = -EINVAL;
+			continue;
+		}
+
+		client_id = payload.client_data;
+		HWFNC_DBG_Q("ctrl rxq rd: it:%d h:%llu ctx:%llu seq:%llu f:%llu e:%u client:%u\n",
+			i, payload.hash, payload.ctxt_id, payload.seqno, payload.flags,
+			payload.error, client_id);
+
+		hw_fence_client = drv_data->clients[client_id];
+		if (!hw_fence_client) {
+			HWFNC_ERR("processing fence error cb for unregistered client_id:%u\n",
+				client_id);
+			ret = -EINVAL;
+			continue;
+		}
+
+		cb_ret = hw_fence_utils_fence_error_cb(hw_fence_client, payload.ctxt_id,
+			payload.seqno, payload.hash, payload.flags, payload.error);
+		if (cb_ret) {
+			HWFNC_ERR("fence_error_cb failed for client:%u ctx:%llu seq:%llu err:%u\n",
+				client_id, payload.ctxt_id, payload.seqno, payload.error);
+			ret = cb_ret;
+		}
+	}
+
+	return ret;
+}
+
+static int _process_doorbell_id(struct hw_fence_driver_data *drv_data, int db_flag_id)
 {
 	int ret;
 
-	HWFNC_DBG_H("Processing doorbell client_id:%d\n", client_id);
-	switch (client_id) {
+	HWFNC_DBG_H("Processing doorbell mask id:%d\n", db_flag_id);
+	switch (db_flag_id) {
+	case HW_FENCE_CTRL_QUEUE_DOORBELL:
+		ret = _process_fence_error_client_loopback(drv_data, db_flag_id);
+		break;
 #if IS_ENABLED(CONFIG_DEBUG_FS)
 	case HW_FENCE_CLIENT_ID_VAL0:
 	case HW_FENCE_CLIENT_ID_VAL1:
@@ -193,11 +314,11 @@ static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int c
 	case HW_FENCE_CLIENT_ID_VAL4:
 	case HW_FENCE_CLIENT_ID_VAL5:
 	case HW_FENCE_CLIENT_ID_VAL6:
-		ret = process_validation_client_loopback(drv_data, client_id);
+		ret = process_validation_client_loopback(drv_data, db_flag_id);
 		break;
 #endif /* CONFIG_DEBUG_FS */
 	default:
-		HWFNC_ERR("unknown client:%d\n", client_id);
+		HWFNC_ERR("unknown mask id:%d\n", db_flag_id);
 		ret = -EINVAL;
 	}
 
@@ -206,22 +327,21 @@ static int _process_doorbell_client(struct hw_fence_driver_data *drv_data, int c
 
 void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data, u64 db_flags)
 {
-	int client_id = HW_FENCE_CLIENT_ID_CTL0;
+	int db_flag_id = HW_FENCE_CTRL_QUEUE_DOORBELL;
 	u64 mask;
 
-	for (; client_id <= HW_FENCE_CLIENT_ID_VAL6; client_id++) {
-		mask = 1 << client_id;
+	for (; db_flag_id <= HW_FENCE_DOORBELL_FLAGS_ID_LAST; db_flag_id++) {
+		mask = 1 << db_flag_id;
 		if (mask & db_flags) {
-			HWFNC_DBG_H("client_id:%d signaled! flags:0x%llx\n", client_id, db_flags);
+			HWFNC_DBG_H("db_flag:%d signaled! flags:0x%llx\n", db_flag_id, db_flags);
 
-			/* process client */
-			if (_process_doorbell_client(drv_data, client_id))
-				HWFNC_ERR("Failed to process client:%d\n", client_id);
+			if (_process_doorbell_id(drv_data, db_flag_id))
+				HWFNC_ERR("Failed to process db_flag_id:%d\n", db_flag_id);
 
-			/* clear mask for this client and if nothing else pending finish */
+			/* clear mask for this flag id if nothing else pending finish */
 			db_flags = db_flags & ~(mask);
-			HWFNC_DBG_H("client_id:%d cleared flags:0x%llx mask:0x%llx ~mask:0x%llx\n",
-				client_id, db_flags, mask, ~(mask));
+			HWFNC_DBG_H("db_flag_id:%d cleared flags:0x%llx mask:0x%llx ~mask:0x%llx\n",
+				db_flag_id, db_flags, mask, ~(mask));
 			if (!db_flags)
 				break;
 		}
@@ -232,7 +352,7 @@ void hw_fence_utils_process_doorbell_mask(struct hw_fence_driver_data *drv_data,
 static void _hw_fence_cb(int irq, void *data)
 {
 	struct hw_fence_driver_data *drv_data = (struct hw_fence_driver_data *)data;
-	gh_dbl_flags_t clear_flags = HW_FENCE_LOOPBACK_CLIENTS_MASK;
+	gh_dbl_flags_t clear_flags = HW_FENCE_DOORBELL_MASK;
 	int ret;
 
 	if (!drv_data)

hw_fence/src/msm_hw_fence.c

@@ -118,6 +118,8 @@ void *msm_hw_fence_register(enum hw_fence_client_id client_id_ext,
 	if (ret)
 		goto error;
 
+	mutex_init(&hw_fence_client->error_cb_lock);
+
 	HWFNC_DBG_INIT("Initialized ptr:0x%p client_id:%d q_num:%d ipc signal:%d vid:%d pid:%d\n",
 		hw_fence_client, hw_fence_client->client_id, hw_fence_client->queues_num,
 		hw_fence_client->ipc_signal_id, hw_fence_client->ipc_client_vid,
@@ -451,6 +453,35 @@ int msm_hw_fence_update_txq(void *client_handle, u64 handle, u64 flags, u32 erro
 }
 EXPORT_SYMBOL(msm_hw_fence_update_txq);
 
+
+int msm_hw_fence_update_txq_error(void *client_handle, u64 handle, u32 error, u32 update_flags)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready ||
+			!hw_fence_drv_data->vm_ready) {
+		HWFNC_ERR("hw fence driver or vm not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle) ||
+			(handle >= hw_fence_drv_data->hw_fences_tbl_cnt) || !error) {
+		HWFNC_ERR("Invalid client_handle:0x%pK or fence handle:%d max:%d or error:%d\n",
+			client_handle, handle, hw_fence_drv_data->hw_fences_tbl_cnt, error);
+		return -EINVAL;
+	} else if (update_flags != MSM_HW_FENCE_UPDATE_ERROR_WITH_MOVE) {
+		HWFNC_ERR("invalid flags:0x%x expected:0x%x no support of in-place error update\n",
+			update_flags, MSM_HW_FENCE_UPDATE_ERROR_WITH_MOVE);
+		return -EINVAL;
+	}
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+
+	/* Write to Tx queue */
+	hw_fence_update_existing_txq_payload(hw_fence_drv_data, hw_fence_client,
+		handle, error);
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_update_txq_error);
+
 /* tx client has to be the physical, rx client virtual id*/
 int msm_hw_fence_trigger_signal(void *client_handle,
 	u32 tx_client_pid, u32 rx_client_vid,
@@ -476,6 +507,142 @@ int msm_hw_fence_trigger_signal(void *client_handle,
 }
 EXPORT_SYMBOL(msm_hw_fence_trigger_signal);
 
+int msm_hw_fence_register_error_cb(void *client_handle, msm_hw_fence_error_cb_t cb, void *data)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
+		HWFNC_ERR("hw fence driver not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle) || IS_ERR_OR_NULL(cb) || IS_ERR_OR_NULL(data)) {
+		HWFNC_ERR("Invalid params client:0x%pK cb_func:0x%pK data:0x%pK\n", client_handle,
+			cb, data);
+		return -EINVAL;
+	}
+
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+	if (hw_fence_client->fence_error_cb) {
+		HWFNC_ERR("client_id:%d client_id_ext:%d already registered cb_func:%pK data:%pK\n",
+			hw_fence_client->client_id, hw_fence_client->client_id_ext,
+			hw_fence_client->fence_error_cb, hw_fence_client->fence_error_cb_userdata);
+		return -EINVAL;
+	}
+
+	hw_fence_client->fence_error_cb_userdata = data;
+	hw_fence_client->fence_error_cb = cb;
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_register_error_cb);
+
+int msm_hw_fence_deregister_error_cb(void *client_handle)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+	int ret = 0;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
+		HWFNC_ERR("hw fence driver not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle)) {
+		HWFNC_ERR("Invalid client: 0x%pK\n", client_handle);
+		return -EINVAL;
+	}
+
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+	if (!mutex_trylock(&hw_fence_client->error_cb_lock)) {
+		HWFNC_ERR("client_id:%d is modifying or using fence_error_cb:0x%pK data:0x%pK\n",
+			hw_fence_client->client_id, hw_fence_client->fence_error_cb,
+			hw_fence_client->fence_error_cb_userdata);
+		return -EAGAIN;
+	}
+
+	if (!hw_fence_client->fence_error_cb) {
+		HWFNC_ERR("client_id:%d client_id_ext:%d did not register cb:%pK data:%pK\n",
+			hw_fence_client->client_id, hw_fence_client->client_id_ext,
+			hw_fence_client->fence_error_cb, hw_fence_client->fence_error_cb_userdata);
+		ret = -EINVAL;
+		goto exit;
+	}
+
+	hw_fence_client->fence_error_cb = NULL;
+	hw_fence_client->fence_error_cb_userdata = NULL;
+
+exit:
+	mutex_unlock(&hw_fence_client->error_cb_lock);
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_deregister_error_cb);
+
+#if IS_ENABLED(CONFIG_DEBUG_FS)
+int msm_hw_fence_dump_debug_data(void *client_handle, u32 dump_flags, u32 dump_clients_mask)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+	int client_id;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
+		HWFNC_ERR("hw fence driver not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle)) {
+		HWFNC_ERR("Invalid client handle:%d\n", IS_ERR_OR_NULL(client_handle));
+		return -EINVAL;
+	}
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+
+	if (dump_flags & MSM_HW_FENCE_DBG_DUMP_QUEUES) {
+		hw_fence_debug_dump_queues(HW_FENCE_PRINTK, hw_fence_client);
+
+		if (dump_clients_mask)
+			for (client_id = 0; client_id < HW_FENCE_CLIENT_MAX; client_id++)
+				if ((dump_clients_mask & (1 << client_id)) &&
+						hw_fence_drv_data->clients[client_id])
+					hw_fence_debug_dump_queues(HW_FENCE_PRINTK,
+						hw_fence_drv_data->clients[client_id]);
+	}
+
+	if (dump_flags & MSM_HW_FENCE_DBG_DUMP_TABLE)
+		hw_fence_debug_dump_table(HW_FENCE_PRINTK, hw_fence_drv_data);
+
+	if (dump_flags & MSM_HW_FENCE_DBG_DUMP_EVENTS)
+		hw_fence_debug_dump_events(HW_FENCE_PRINTK, hw_fence_drv_data);
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_dump_debug_data);
+
+int msm_hw_fence_dump_fence(void *client_handle, struct dma_fence *fence)
+{
+	struct msm_hw_fence_client *hw_fence_client;
+	struct msm_hw_fence *hw_fence;
+	u64 hash;
+
+	if (IS_ERR_OR_NULL(hw_fence_drv_data) || !hw_fence_drv_data->resources_ready) {
+		HWFNC_ERR("hw fence driver not ready\n");
+		return -EAGAIN;
+	} else if (IS_ERR_OR_NULL(client_handle)) {
+		HWFNC_ERR("Invalid client handle:%d\n", IS_ERR_OR_NULL(client_handle));
+		return -EINVAL;
+	} else if (!test_bit(MSM_HW_FENCE_FLAG_ENABLED_BIT, &fence->flags)) {
+		HWFNC_ERR("DMA Fence is not a HW Fence ctx:%llu seqno:%llu flags:0x%llx\n",
+			fence->context, fence->seqno, fence->flags);
+		return -EINVAL;
+	}
+	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+
+	hw_fence = msm_hw_fence_find(hw_fence_drv_data, hw_fence_client, fence->context,
+		fence->seqno, &hash);
+	if (!hw_fence) {
+		HWFNC_ERR("failed to find hw-fence client_id:%d fence:0x%pK ctx:%llu seqno:%llu\n",
+			hw_fence_client->client_id, fence, fence->context, fence->seqno);
+		return -EINVAL;
+	}
+	hw_fence_debug_dump_fence(HW_FENCE_PRINTK, hw_fence, hash, 0);
+
+	return 0;
+}
+EXPORT_SYMBOL(msm_hw_fence_dump_fence);
+#endif /* CONFIG_DEBUG_FS */
+
 /* Function used for simulation purposes only. */
 int msm_hw_fence_driver_doorbell_sim(u64 db_mask)
 {