mm-drivers: hw_fence: add inter-vm try lock

Add support for inter-vm try-lock between hlos and vm.

Change-Id: Iab9087acf82a4a746e9d43a736724ce2e7196237
Signed-off-by: Ingrid Gallardo <[email protected]>

hw_fence/include/hw_fence_drv_debug.h

@@ -18,6 +18,7 @@ enum hw_fence_drv_prio {
 	HW_FENCE_QUEUE = 0x000010,	/* Queue logs */
 	HW_FENCE_LUT = 0x000020,	/* Look-up and algorithm logs */
 	HW_FENCE_IRQ = 0x000040,	/* Interrupt-related messages */
+	HW_FENCE_LOCK = 0x000080,	/* Lock-related messages */
 	HW_FENCE_PRINTK = 0x010000,
 };
 
@@ -56,6 +57,9 @@ extern u32 msm_hw_fence_debug_level;
 #define HWFNC_DBG_IRQ(fmt, ...) \
 	dprintk(HW_FENCE_IRQ, "[hwfence:%s:%d][dbgirq]"fmt, __func__, __LINE__, ##__VA_ARGS__)
 
+#define HWFNC_DBG_LOCK(fmt, ...) \
+	dprintk(HW_FENCE_LOCK, "[hwfence:%s:%d][dbglock]"fmt, __func__, __LINE__, ##__VA_ARGS__)
+
 #define HWFNC_WARN(fmt, ...) \
 	pr_warn("[hwfence:%s:%d][warn][%pS] "fmt, __func__, __LINE__, \
 	__builtin_return_address(0), ##__VA_ARGS__)

hw_fence/include/hw_fence_drv_utils.h

@@ -22,10 +22,11 @@ enum hw_fence_mem_reserve {
 
 /**
  * global_atomic_store() - Inter-processor lock
+ * @drv_data: hw fence driver data
  * @lock: memory to lock
  * @val: if true, api locks the memory, if false it unlocks the memory
  */
-void global_atomic_store(uint64_t *lock, bool val);
+void global_atomic_store(struct hw_fence_driver_data *drv_data, uint64_t *lock, bool val);
 
 /**
  * hw_fence_utils_init_virq() - Initialilze doorbell (i.e. vIRQ) for SVM to HLOS signaling

hw_fence/src/hw_fence_drv_priv.c

@@ -13,7 +13,7 @@
 #include "hw_fence_drv_debug.h"
 
 /* Global atomic lock */
-#define GLOBAL_ATOMIC_STORE(lock, val) global_atomic_store(lock, val)
+#define GLOBAL_ATOMIC_STORE(drv_data, lock, val) global_atomic_store(drv_data, lock, val)
 
 inline u64 hw_fence_get_qtime(struct hw_fence_driver_data *drv_data)
 {
@@ -277,7 +277,7 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 		HWFNC_DBG_Q("Locking client id:%d: idx:%d\n", hw_fence_client->client_id, lock_idx);
 
 		/* lock the client rx queue to update */
-		GLOBAL_ATOMIC_STORE(&drv_data->client_lock_tbl[lock_idx], 1); /* lock */
+		GLOBAL_ATOMIC_STORE(drv_data, &drv_data->client_lock_tbl[lock_idx], 1); /* lock */
 	}
 
 	/* Make sure data is ready before read */
@@ -348,7 +348,7 @@ int hw_fence_update_queue(struct hw_fence_driver_data *drv_data,
 
 exit:
 	if (lock_client)
-		GLOBAL_ATOMIC_STORE(&drv_data->client_lock_tbl[lock_idx], 0); /* unlock */
+		GLOBAL_ATOMIC_STORE(drv_data, &drv_data->client_lock_tbl[lock_idx], 0); /* unlock */
 
 	return ret;
 }
@@ -882,7 +882,7 @@ struct msm_hw_fence *_hw_fence_lookup_and_process(struct hw_fence_driver_data *d
 			break;
 		}
 
-		GLOBAL_ATOMIC_STORE(&hw_fence->lock, 1);
+		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 1);
 
 		/* compare to either find a free fence or find an allocated fence */
 		if (compare_fnc(hw_fence, context, seqno)) {
@@ -907,7 +907,7 @@ struct msm_hw_fence *_hw_fence_lookup_and_process(struct hw_fence_driver_data *d
 				/* ctx & seqno must be unique creating a hw-fence */
 				HWFNC_ERR("cannot create hw fence with same ctx:%llu seqno:%llu\n",
 					context, seqno);
-				GLOBAL_ATOMIC_STORE(&hw_fence->lock, 0);
+				GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 0);
 				break;
 			}
 			/* compare can fail if we have a collision, we will linearly resolve it */
@@ -915,7 +915,7 @@ struct msm_hw_fence *_hw_fence_lookup_and_process(struct hw_fence_driver_data *d
 				context, seqno);
 		}
 
-		GLOBAL_ATOMIC_STORE(&hw_fence->lock, 0);
+		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 0);
 
 		/* Increment step for the next loop */
 		step++;
@@ -1090,7 +1090,7 @@ static void _cleanup_join_and_child_fences(struct hw_fence_driver_data *drv_data
 		}
 
 		/* lock the child while we clean it up from the parent join-fence */
-		GLOBAL_ATOMIC_STORE(&hw_fence_child->lock, 1); /* lock */
+		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence_child->lock, 1); /* lock */
 		for (j = hw_fence_child->parents_cnt; j > 0; j--) {
 
 			if (j > MSM_HW_FENCE_MAX_JOIN_PARENTS) {
@@ -1110,7 +1110,7 @@ static void _cleanup_join_and_child_fences(struct hw_fence_driver_data *drv_data
 				wmb();
 			}
 		}
-		GLOBAL_ATOMIC_STORE(&hw_fence_child->lock, 0); /* unlock */
+		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence_child->lock, 0); /* unlock */
 	}
 
 	/* destroy join fence */
@@ -1141,9 +1141,9 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 	}
 
 	/* update this as waiting client of the join-fence */
-	GLOBAL_ATOMIC_STORE(&join_fence->lock, 1); /* lock */
+	GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 1); /* lock */
 	join_fence->wait_client_mask |= BIT(hw_fence_client->client_id);
-	GLOBAL_ATOMIC_STORE(&join_fence->lock, 0); /* unlock */
+	GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 0); /* unlock */
 
 	/* Iterate through fences of the array */
 	for (i = 0; i < array->num_fences; i++) {
@@ -1173,18 +1173,18 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 			goto error_array;
 		}
 
-		GLOBAL_ATOMIC_STORE(&hw_fence_child->lock, 1); /* lock */
+		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence_child->lock, 1); /* lock */
 		if (hw_fence_child->flags & MSM_HW_FENCE_FLAG_SIGNAL) {
 
 			/* child fence is already signaled */
-			GLOBAL_ATOMIC_STORE(&join_fence->lock, 1); /* lock */
+			GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 1); /* lock */
 			if (--join_fence->pending_child_cnt == 0)
 				signal_join_fence = true;
 
 			/* update memory for the table update */
 			wmb();
 
-			GLOBAL_ATOMIC_STORE(&join_fence->lock, 0); /* unlock */
+			GLOBAL_ATOMIC_STORE(drv_data, &join_fence->lock, 0); /* unlock */
 		} else {
 
 			/* child fence is not signaled */
@@ -1201,7 +1201,8 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 				/* update memory for the table update */
 				wmb();
 
-				GLOBAL_ATOMIC_STORE(&hw_fence_child->lock, 0); /* unlock */
+				/* unlock */
+				GLOBAL_ATOMIC_STORE(drv_data, &hw_fence_child->lock, 0);
 				ret = -EINVAL;
 				goto error_array;
 			}
@@ -1212,7 +1213,7 @@ int hw_fence_process_fence_array(struct hw_fence_driver_data *drv_data,
 			/* update memory for the table update */
 			wmb();
 		}
-		GLOBAL_ATOMIC_STORE(&hw_fence_child->lock, 0); /* unlock */
+		GLOBAL_ATOMIC_STORE(drv_data, &hw_fence_child->lock, 0); /* unlock */
 	}
 
 	/* all fences were signaled, signal client now */
@@ -1254,7 +1255,7 @@ int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 		return -EINVAL;
 	}
 
-	GLOBAL_ATOMIC_STORE(&hw_fence->lock, 1); /* lock */
+	GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 1); /* lock */
 
 	/* register client in the hw fence */
 	hw_fence->wait_client_mask |= BIT(hw_fence_client->client_id);
@@ -1264,6 +1265,8 @@ int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 	/* update memory for the table update */
 	wmb();
 
+	GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 0); /* unlock */
+
 	/* if hw fence already signaled, signal the client */
 	if (hw_fence->flags & MSM_HW_FENCE_FLAG_SIGNAL) {
 		if (fence != NULL)
@@ -1271,8 +1274,6 @@ int hw_fence_register_wait_client(struct hw_fence_driver_data *drv_data,
 		_fence_ctl_signal(drv_data, hw_fence_client, hw_fence, hash, 0, 0);
 	}
 
-	GLOBAL_ATOMIC_STORE(&hw_fence->lock, 0); /* unlock */
-
 	return 0;
 }
 
@@ -1325,7 +1326,7 @@ int hw_fence_utils_cleanup_fence(struct hw_fence_driver_data *drv_data,
 	int ret = 0;
 	int error = (reset_flags & MSM_HW_FENCE_RESET_WITHOUT_ERROR) ? 0 : MSM_HW_FENCE_ERROR_RESET;
 
-	GLOBAL_ATOMIC_STORE(&hw_fence->lock, 1); /* lock */
+	GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 1); /* lock */
 	if (hw_fence->wait_client_mask & BIT(hw_fence_client->client_id)) {
 		HWFNC_DBG_H("clearing client:%d wait bit for fence: ctx:%d seqno:%d\n",
 			hw_fence_client->client_id, hw_fence->ctx_id,
@@ -1335,7 +1336,7 @@ int hw_fence_utils_cleanup_fence(struct hw_fence_driver_data *drv_data,
 		/* update memory for the table update */
 		wmb();
 	}
-	GLOBAL_ATOMIC_STORE(&hw_fence->lock, 0); /* unlock */
+	GLOBAL_ATOMIC_STORE(drv_data, &hw_fence->lock, 0); /* unlock */
 
 	if (hw_fence->fence_allocator == hw_fence_client->client_id) {
 

hw_fence/src/hw_fence_drv_utils.c

@@ -17,40 +17,64 @@
 
 static void _lock(uint64_t *wait)
 {
-	/* WFE Wait */
 #if defined(__aarch64__)
-	__asm__("SEVL\n\t"
+	__asm__(
+		// Sequence to wait for lock to be free (i.e. zero)
 		"PRFM PSTL1KEEP, [%x[i_lock]]\n\t"
 		"1:\n\t"
-		"WFE\n\t"
 		"LDAXR W5, [%x[i_lock]]\n\t"
 		"CBNZ W5, 1b\n\t"
-		"STXR W5, W0, [%x[i_lock]]\n\t"
-		"CBNZ W5, 1b\n"
+		// Sequence to set PVM BIT0
+		"LDR W7, =0x1\n\t"              // Load BIT0 (0x1) into W7
+		"STXR W5, W7, [%x[i_lock]]\n\t" // Atomic Store exclusive BIT0 (lock = 0x1)
+		"CBNZ W5, 1b\n\t"               // If cannot set it, goto 1
 		:
 		: [i_lock] "r" (wait)
 		: "memory");
 #endif
 }
 
-static void _unlock(uint64_t *lock)
+static void _unlock(struct hw_fence_driver_data *drv_data, uint64_t *lock)
 {
-	/* Signal Client */
+	uint64_t lock_val;
+
 #if defined(__aarch64__)
-	__asm__("STLR WZR, [%x[i_out]]\n\t"
-		"SEV\n"
+	__asm__(
+		// Sequence to clear PVM BIT0
+		"2:\n\t"
+		"LDAXR W5, [%x[i_out]]\n\t"             // Atomic Fetch Lock
+		"AND W6, W5, #0xFFFFFFFFFFFFFFFE\n\t"   // AND to clear BIT0 (lock &= ~0x1))
+		"STXR W5, W6, [%x[i_out]]\n\t"          // Store exclusive result
+		"CBNZ W5, 2b\n\t"                       // If cannot store exclusive, goto 2
 		:
 		: [i_out] "r" (lock)
 		: "memory");
 #endif
+	mb(); /* Make sure the memory is updated */
+
+	lock_val = *lock; /* Read the lock value */
+	HWFNC_DBG_LOCK("unlock: lock_val after:0x%llx\n", lock_val);
+	if (lock_val & 0x2) { /* check if SVM BIT1 is set*/
+		/*
+		 * SVM is in WFI state, since SVM acquire bit is set
+		 * Trigger IRQ to Wake-Up SVM Client
+		 */
+		HWFNC_DBG_LOCK("triggering ipc to unblock SVM lock_val:%d\n", lock_val);
+		hw_fence_ipcc_trigger_signal(drv_data,
+			drv_data->ipcc_client_pid,
+			drv_data->ipcc_client_vid, 30); /* Trigger APPS Signal 30 */
+	}
 }
 
-void global_atomic_store(uint64_t *lock, bool val)
+void global_atomic_store(struct hw_fence_driver_data *drv_data, uint64_t *lock, bool val)
 {
-	if (val)
+	if (val) {
+		preempt_disable();
 		_lock(lock);
-	else
-		_unlock(lock);
+	} else {
+		_unlock(drv_data, lock);
+		preempt_enable();
+	}
 }
 
 /*