mm-drivers: hw_fence: add support for 64-bit client_data

Add support of the option to pass a 64-bit client_data value to
the hw fence driver when a client registers as a waiting client
for a hardware fence. Then during fence signaling, this client_data
is returned to the client via the RxQ.

If no client_data is passed to the driver for the hw fence, then a
default value of zero is registered as the client_data.

Change-Id: I34cf3e50413639d53cbfa8251c98b9ff1d3cbf4a
Signed-off-by: Grace An <[email protected]>

hw_fence/include/hw_fence_drv_priv.h

@@ -121,6 +121,27 @@ enum hw_fence_loopback_id {
 
 #define HW_FENCE_MAX_DPU_LOOPBACK_CLIENTS (HW_FENCE_LOOPBACK_DPU_CTL_5 + 1)
 
+/**
+ * enum hw_fence_client_data_id - Enum with the clients having client_data, an optional
+ *                                parameter passed from the waiting client and returned
+ *                                to it upon fence signaling
+ * @HW_FENCE_CLIENT_DATA_ID_CTX0: GFX Client.
+ * @HW_FENCE_CLIENT_DATA_ID_IPE: IPE Client.
+ * @HW_FENCE_CLIENT_DATA_ID_VPU: VPU Client.
+ * @HW_FENCE_CLIENT_DATA_ID_VAL0: Debug validation client 0.
+ * @HW_FENCE_CLIENT_DATA_ID_VAL1: Debug validation client 1.
+ * @HW_FENCE_MAX_CLIENTS_WITH_DATA: Max number of clients with data, also indicates an
+ *                                  invalid hw_fence_client_data_id
+ */
+enum hw_fence_client_data_id {
+	HW_FENCE_CLIENT_DATA_ID_CTX0,
+	HW_FENCE_CLIENT_DATA_ID_IPE,
+	HW_FENCE_CLIENT_DATA_ID_VPU,
+	HW_FENCE_CLIENT_DATA_ID_VAL0,
+	HW_FENCE_CLIENT_DATA_ID_VAL1,
+	HW_FENCE_MAX_CLIENTS_WITH_DATA,
+};
+
 /**
  * struct msm_hw_fence_queue - Structure holding the data of the hw fence queues.
  * @va_queue: pointer to the virtual address of the queue elements
@@ -387,6 +408,8 @@ struct msm_hw_fence_queue_payload {
  * @fence_trigger_time: debug info with the trigger time timestamp
  * @fence_wait_time: debug info with the register-for-wait timestamp
  * @debug_refcount: refcount used for debugging
+ * @client_data: array of data optionally passed from and returned to clients waiting on the fence
+ *               during fence signaling
  */
 struct msm_hw_fence {
 	u32 valid;
@@ -405,6 +428,7 @@ struct msm_hw_fence {
 	u64 fence_trigger_time;
 	u64 fence_wait_time;
 	u64 debug_refcount;
+	u64 client_data[HW_FENCE_MAX_CLIENTS_WITH_DATA];
 };
 
 int hw_fence_init(struct hw_fence_driver_data *drv_data);
@@ -426,20 +450,22 @@ int hw_fence_destroy_with_hash(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, u64 hash);
 int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client,
-	struct dma_fence_array *array, u64 *hash_join_fence);
+	struct dma_fence_array *array, u64 *hash_join_fence, u64 client_data);
 int hw_fence_process_fence(struct hw_fence_driver_data *drv_data,
-	struct msm_hw_fence_client *hw_fence_client, struct dma_fence *fence, u64 *hash);
+	struct msm_hw_fence_client *hw_fence_client, struct dma_fence *fence, u64 *hash,
+	u64 client_data);
 int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id, u64 seqno, u64 hash,
-	u64 flags, u32 error, int queue_type);
+	u64 flags, u64 client_data, u32 error, int queue_type);
 inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data);
 int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	struct msm_hw_fence_queue_payload *payload, int queue_type);
 int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 	struct dma_fence *fence, struct msm_hw_fence_client *hw_fence_client, u64 context,
-	u64 seqno, u64 *hash);
+	u64 seqno, u64 *hash, u64 client_data);
 struct msm_hw_fence *msm_hw_fence_find(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client,
 	u64 context, u64 seqno, u64 *hash);
+enum hw_fence_client_data_id hw_fence_get_client_data_id(enum hw_fence_client_id client_id);
 
 #endif /* __HW_FENCE_DRV_INTERNAL_H */

hw_fence/src/hw_fence_drv_debug.c

@@ -338,14 +338,14 @@ static ssize_t hw_fence_dbg_tx_and_signal_clients_wr(struct file *file,
 
 		/* Write to Tx queue */
 		hw_fence_update_queue(drv_data, hw_fence_client, context, seqno, hash,
-			0, 0, HW_FENCE_TX_QUEUE - 1); // no flags and no error
+			0, 0, 0, HW_FENCE_TX_QUEUE - 1); /* no flags and no error */
 
 		/**********************************************/
 		/***** DST CLIENT - REGISTER WAIT CLIENT ******/
 		/**********************************************/
 		/* use same context and seqno that src client used to create fence */
 		ret = hw_fence_register_wait_client(drv_data, NULL, hw_fence_client_dst, context,
-			seqno, &hash);
+			seqno, &hash, 0);
 		if (ret) {
 			HWFNC_ERR("failed to register for wait\n");
 			return -EINVAL;
@@ -558,7 +558,7 @@ static ssize_t hw_fence_dbg_dump_queues_wr(struct file *file, const char __user
 	struct hw_fence_driver_data *drv_data;
 	struct msm_hw_fence_queue *rx_queue;
 	struct msm_hw_fence_queue *tx_queue;
-	u64 hash, ctx_id, seqno, timestamp, flags;
+	u64 hash, ctx_id, seqno, timestamp, flags, client_data;
 	u32 *read_ptr, error;
 	int client_id, i;
 	struct msm_hw_fence_queue_payload *read_ptr_payload;
@@ -595,12 +595,13 @@ static ssize_t hw_fence_dbg_dump_queues_wr(struct file *file, const char __user
 		seqno = readq_relaxed(&read_ptr_payload->seqno);
 		hash = readq_relaxed(&read_ptr_payload->hash);
 		flags = readq_relaxed(&read_ptr_payload->flags);
+		client_data = readq_relaxed(&read_ptr_payload->client_data);
 		error = readl_relaxed(&read_ptr_payload->error);
 		timestamp = (u64)readl_relaxed(&read_ptr_payload->timestamp_lo) |
 			((u64)readl_relaxed(&read_ptr_payload->timestamp_hi) << 32);
 
-		HWFNC_DBG_L("rx[%d]: hash:%d ctx:%llu seqno:%llu f:%llu err:%u time:%llu\n",
-			i, hash, ctx_id, seqno, flags, error, timestamp);
+		HWFNC_DBG_L("rx[%d]: hash:%d ctx:%llu seqno:%llu f:%llu d:%llu err:%u time:%llu\n",
+			i, hash, ctx_id, seqno, flags, client_data, error, timestamp);
 	}
 
 	HWFNC_DBG_L("-------TX QUEUE------\n");
@@ -855,7 +856,7 @@ static ssize_t hw_fence_dbg_create_join_fence(struct file *file,
 
 		/* Write to Tx queue */
 		hw_fence_update_queue(drv_data, hw_fence_client, client_info_src->dma_context,
-			hw_fence_dbg_seqno + i, hash, 0, 0,
+			hw_fence_dbg_seqno + i, hash, 0, 0, 0,
 			HW_FENCE_TX_QUEUE - 1);
 	}
 

hw_fence/src/hw_fence_drv_priv.c

@@ -208,6 +208,7 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
 	payload->seqno = readq_relaxed(&read_ptr_payload->seqno);
 	payload->hash = readq_relaxed(&read_ptr_payload->hash);
 	payload->flags = readq_relaxed(&read_ptr_payload->flags);
+	payload->client_data = readq_relaxed(&read_ptr_payload->client_data);
 	payload->error = readl_relaxed(&read_ptr_payload->error);
 
 	/* update the read index */
@@ -226,7 +227,7 @@ int hw_fence_read_queue(struct msm_hw_fence_client *hw_fence_client,
  */
 int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, u64 ctxt_id, u64 seqno, u64 hash,
-	u64 flags, u32 error, int queue_type)
+	u64 flags, u64 client_data, u32 error, int queue_type)
 {
 	struct msm_hw_fence_hfi_queue_header *hfi_header;
 	struct msm_hw_fence_queue *queue;
@@ -332,6 +333,7 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 	writeq_relaxed(seqno, &write_ptr_payload->seqno);
 	writeq_relaxed(hash, &write_ptr_payload->hash);
 	writeq_relaxed(flags, &write_ptr_payload->flags);
+	writeq_relaxed(client_data, &write_ptr_payload->client_data);
 	writel_relaxed(error, &write_ptr_payload->error);
 	timestamp = hw_fence_get_qtime(drv_data);
 	writel_relaxed(timestamp, &write_ptr_payload->timestamp_lo);
@@ -726,6 +728,8 @@ static void _cleanup_hw_fence(struct msm_hw_fence *hw_fence)
 
 	for (i = 0; i < MSM_HW_FENCE_MAX_JOIN_PARENTS; i++)
 		hw_fence->parent_list[i] = HW_FENCE_INVALID_PARENT_FENCE;
+
+	memset(hw_fence->client_data, 0, sizeof(hw_fence->client_data));
 }
 
 /* This function must be called with the hw fence lock */
@@ -1076,7 +1080,7 @@ struct msm_hw_fence *msm_hw_fence_find(struct hw_fence_driver_data *drv_data,
 
 static void _fence_ctl_signal(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, struct msm_hw_fence *hw_fence, u64 hash,
-	u64 flags, u32 error)
+	u64 flags, u64 client_data, u32 error)
 {
 	u32 tx_client_id = drv_data->ipcc_client_pid; /* phys id for tx client */
 	u32 rx_client_id = hw_fence_client->ipc_client_vid; /* virt id for rx client */
@@ -1086,7 +1090,7 @@ static void _fence_ctl_signal(struct hw_fence_driver_data *drv_data,
 	/* Write to Rx queue */
 	if (hw_fence_client->update_rxq)
 		hw_fence_update_queue(drv_data, hw_fence_client, hw_fence->ctx_id,
-			hw_fence->seq_id, hash, flags, error, HW_FENCE_RX_QUEUE - 1);
+			hw_fence->seq_id, hash, flags, client_data, error, HW_FENCE_RX_QUEUE - 1);
 
 	/* Signal the hw fence now */
 	if (hw_fence_client->send_ipc)
@@ -1151,7 +1155,7 @@ static void _cleanup_join_and_child_fences(struct hw_fence_driver_data *drv_data
 
 int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client, struct dma_fence_array *array,
-	u64 *hash_join_fence)
+	u64 *hash_join_fence, u64 client_data)
 {
 	struct msm_hw_fence *join_fence;
 	struct msm_hw_fence *hw_fence_child;
@@ -1159,6 +1163,16 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 	bool signal_join_fence = false;
 	u64 hash;
 	int i, ret = 0;
+	enum hw_fence_client_data_id data_id;
+
+	if (client_data) {
+		data_id = hw_fence_get_client_data_id(hw_fence_client->client_id);
+		if (data_id >= HW_FENCE_MAX_CLIENTS_WITH_DATA) {
+			HWFNC_ERR("Populating non-zero client_data:%llu with invalid client:%d\n",
+				client_data, hw_fence_client->client_id);
+			return -EINVAL;
+		}
+	}
 
 	/*
 	 * Create join fence from the join-fences table,
@@ -1248,11 +1262,15 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence_child->lock, 0); /* unlock */
 	}
 
+	if (client_data)
+		join_fence->client_data[data_id] = client_data;
+
 	/* all fences were signaled, signal client now */
 	if (signal_join_fence) {
 
 		/* signal the join hw fence */
-		_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0, 0);
+		_fence_ctl_signal(drv_data, hw_fence_client, join_fence, *hash_join_fence, 0, 0,
+			client_data);
 		set_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &array->base.flags);
 
 		/*
@@ -1275,9 +1293,19 @@ error_array:
 
 int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 		struct dma_fence *fence, struct msm_hw_fence_client *hw_fence_client, u64 context,
-		u64 seqno, u64 *hash)
+		u64 seqno, u64 *hash, u64 client_data)
 {
 	struct msm_hw_fence *hw_fence;
+	enum hw_fence_client_data_id data_id;
+
+	if (client_data) {
+		data_id = hw_fence_get_client_data_id(hw_fence_client->client_id);
+		if (data_id >= HW_FENCE_MAX_CLIENTS_WITH_DATA) {
+			HWFNC_ERR("Populating non-zero client_data:%llu with invalid client:%d\n",
+				client_data, hw_fence_client->client_id);
+			return -EINVAL;
+		}
+	}
 
 	/* find the hw fence within the table */
 	hw_fence = msm_hw_fence_find(drv_data, hw_fence_client, context, seqno, hash);
@@ -1292,6 +1320,8 @@ int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 	hw_fence->wait_client_mask |= BIT(hw_fence_client->client_id);
 	hw_fence->fence_wait_time = hw_fence_get_qtime(drv_data);
 	hw_fence->debug_refcount++;
+	if (client_data)
+		hw_fence->client_data[data_id] = client_data;
 
 	/* update memory for the table update */
 	wmb();
@@ -1302,7 +1332,7 @@ int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 	if (hw_fence->flags & MSM_HW_FENCE_FLAG_SIGNAL) {
 		if (fence != NULL)
 			set_bit(MSM_HW_FENCE_FLAG_SIGNALED_BIT, &fence->flags);
-		_fence_ctl_signal(drv_data, hw_fence_client, hw_fence, *hash, 0, 0);
+		_fence_ctl_signal(drv_data, hw_fence_client, hw_fence, *hash, 0, client_data, 0);
 	}
 
 	return 0;
@@ -1310,7 +1340,7 @@ int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 
 int hw_fence_process_fence(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence_client *hw_fence_client,
-	struct dma_fence *fence, u64 *hash)
+	struct dma_fence *fence, u64 *hash, u64 client_data)
 {
 	int ret = 0;
 
@@ -1325,7 +1355,7 @@ int hw_fence_process_fence(struct hw_fence_driver_data *drv_data,
 	}
 
 	ret = hw_fence_register_wait_client(drv_data, fence, hw_fence_client, fence->context,
-		fence->seqno, hash);
+		fence->seqno, hash, client_data);
 	if (ret)
 		HWFNC_ERR("Error registering for wait client:%d\n", hw_fence_client->client_id);
 
@@ -1336,16 +1366,22 @@ static void _signal_all_wait_clients(struct hw_fence_driver_data *drv_data,
 	struct msm_hw_fence *hw_fence, u64 hash, int error)
 {
 	enum hw_fence_client_id wait_client_id;
+	enum hw_fence_client_data_id data_id;
 	struct msm_hw_fence_client *hw_fence_wait_client;
+	u64 client_data = 0;
 
 	/* signal with an error all the waiting clients for this fence */
 	for (wait_client_id = 0; wait_client_id < HW_FENCE_CLIENT_MAX; wait_client_id++) {
 		if (hw_fence->wait_client_mask & BIT(wait_client_id)) {
 			hw_fence_wait_client = drv_data->clients[wait_client_id];
+			data_id = hw_fence_get_client_data_id(wait_client_id);
+
+			if (data_id < HW_FENCE_MAX_CLIENTS_WITH_DATA)
+				client_data = hw_fence->client_data[data_id];
 
 			if (hw_fence_wait_client)
 				_fence_ctl_signal(drv_data, hw_fence_wait_client, hw_fence,
-					hash, 0, error);
+					hash, 0, client_data, error);
 		}
 	}
 }
@@ -1389,3 +1425,25 @@ int hw_fence_utils_cleanup_fence(struct hw_fence_driver_data *drv_data,
 skip_destroy:
 	return ret;
 }
+
+enum hw_fence_client_data_id hw_fence_get_client_data_id(enum hw_fence_client_id client_id)
+{
+	enum hw_fence_client_data_id data_id;
+
+	switch (client_id) {
+	case HW_FENCE_CLIENT_ID_CTX0:
+		data_id = HW_FENCE_CLIENT_DATA_ID_CTX0;
+		break;
+	case HW_FENCE_CLIENT_ID_VAL0:
+		data_id = HW_FENCE_CLIENT_DATA_ID_VAL0;
+		break;
+	case HW_FENCE_CLIENT_ID_VAL1:
+		data_id = HW_FENCE_CLIENT_DATA_ID_VAL1;
+		break;
+	default:
+		data_id = HW_FENCE_MAX_CLIENTS_WITH_DATA;
+		break;
+	}
+
+	return data_id;
+}

hw_fence/src/msm_hw_fence.c

@@ -277,6 +277,7 @@ int msm_hw_fence_wait_update_v2(void *client_handle,
 	struct msm_hw_fence_client *hw_fence_client;
 	struct dma_fence_array *array;
 	int i, ret = 0;
+	enum hw_fence_client_data_id data_id;
 
 	if (IS_ERR_OR_NULL(client_handle) || !fence_list || !*fence_list) {
 		HWFNC_ERR("Invalid data\n");
@@ -289,19 +290,28 @@ int msm_hw_fence_wait_update_v2(void *client_handle,
 	}
 
 	hw_fence_client = (struct msm_hw_fence_client *)client_handle;
+	data_id = hw_fence_get_client_data_id(hw_fence_client->client_id);
+	if (client_data_list && data_id >= HW_FENCE_MAX_CLIENTS_WITH_DATA) {
+		HWFNC_ERR("Populating non-NULL client_data_list with unsupported client id:%d\n",
+			hw_fence_client->client_id);
+		return -EINVAL;
+	}
 
 	HWFNC_DBG_H("+\n");
 
 	/* Process all the list of fences */
 	for (i = 0; i < num_fences; i++) {
 		struct dma_fence *fence = fence_list[i];
-		u64 hash;
+		u64 hash, client_data = 0;
+
+		if (client_data_list)
+			client_data = client_data_list[i];
 
 		/* Process a Fence-Array */
 		array = to_dma_fence_array(fence);
 		if (array) {
 			ret = hw_fence_process_fence_array(hw_fence_drv_data, hw_fence_client,
-				array, &hash);
+				array, &hash, client_data);
 			if (ret) {
 				HWFNC_ERR("Failed to process FenceArray\n");
 				return ret;
@@ -309,7 +319,7 @@ int msm_hw_fence_wait_update_v2(void *client_handle,
 		} else {
 			/* Process individual Fence */
 			ret = hw_fence_process_fence(hw_fence_drv_data, hw_fence_client, fence,
-				&hash);
+				&hash, client_data);
 			if (ret) {
 				HWFNC_ERR("Failed to process Fence\n");
 				return ret;
@@ -394,7 +404,7 @@ int msm_hw_fence_update_txq(void *client_handle, u64 handle, u64 flags, u32 erro
 	hw_fence_update_queue(hw_fence_drv_data, hw_fence_client,
 		hw_fence_drv_data->hw_fences_tbl[handle].ctx_id,
 		hw_fence_drv_data->hw_fences_tbl[handle].seq_id, handle,
-		flags, error, HW_FENCE_TX_QUEUE - 1);
+		flags, 0, error, HW_FENCE_TX_QUEUE - 1);
 
 	return 0;
 }