drm/amdkfd: fix zero reading of VMID and PASID for Hawaii

Upon VM Fault, the VMID and PASID written by HW are zeros in
Hawaii. Instead of reading from ih_ring_entry, read directly
from the registers. This workaround fix the soft hang issues
caused by mishandled VM Fault in Hawaii.

Signed-off-by: Lan Xiao <Lan.Xiao@amd.com>
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
Acked-by: Christian König <christian.koenig@amd.com>
Signed-off-by: Oded Gabbay <oded.gabbay@gmail.com>

drivers/gpu/drm/amd/amdkfd/cik_event_interrupt.c

@@ -25,12 +25,39 @@
 #include "cik_int.h"
 
 static bool cik_event_interrupt_isr(struct kfd_dev *dev,
-					const uint32_t *ih_ring_entry)
+					const uint32_t *ih_ring_entry,
+					uint32_t *patched_ihre,
+					bool *patched_flag)
 {
 	const struct cik_ih_ring_entry *ihre =
 			(const struct cik_ih_ring_entry *)ih_ring_entry;
+	const struct kfd2kgd_calls *f2g = dev->kfd2kgd;
 	unsigned int vmid, pasid;
 
+	/* This workaround is due to HW/FW limitation on Hawaii that
+	 * VMID and PASID are not written into ih_ring_entry
+	 */
+	if ((ihre->source_id == CIK_INTSRC_GFX_PAGE_INV_FAULT ||
+		ihre->source_id == CIK_INTSRC_GFX_MEM_PROT_FAULT) &&
+		dev->device_info->asic_family == CHIP_HAWAII) {
+		struct cik_ih_ring_entry *tmp_ihre =
+			(struct cik_ih_ring_entry *)patched_ihre;
+
+		*patched_flag = true;
+		*tmp_ihre = *ihre;
+
+		vmid = f2g->read_vmid_from_vmfault_reg(dev->kgd);
+		pasid = f2g->get_atc_vmid_pasid_mapping_pasid(dev->kgd, vmid);
+
+		tmp_ihre->ring_id &= 0x000000ff;
+		tmp_ihre->ring_id |= vmid << 8;
+		tmp_ihre->ring_id |= pasid << 16;
+
+		return (pasid != 0) &&
+			vmid >= dev->vm_info.first_vmid_kfd &&
+			vmid <= dev->vm_info.last_vmid_kfd;
+	}
+
 	/* Only handle interrupts from KFD VMIDs */
 	vmid  = (ihre->ring_id & 0x0000ff00) >> 8;
 	if (vmid < dev->vm_info.first_vmid_kfd ||

drivers/gpu/drm/amd/amdkfd/kfd_device.c

@@ -577,14 +577,24 @@ dqm_start_error:
 /* This is called directly from KGD at ISR. */
 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry)
 {
+	uint32_t patched_ihre[KFD_MAX_RING_ENTRY_SIZE];
+	bool is_patched = false;
+
 	if (!kfd->init_complete)
 		return;
 
+	if (kfd->device_info->ih_ring_entry_size > sizeof(patched_ihre)) {
+		dev_err_once(kfd_device, "Ring entry too small\n");
+		return;
+	}
+
 	spin_lock(&kfd->interrupt_lock);
 
 	if (kfd->interrupts_active
-	    && interrupt_is_wanted(kfd, ih_ring_entry)
-	    && enqueue_ih_ring_entry(kfd, ih_ring_entry))
+	    && interrupt_is_wanted(kfd, ih_ring_entry,
+				   patched_ihre, &is_patched)
+	    && enqueue_ih_ring_entry(kfd,
+				     is_patched ? patched_ihre : ih_ring_entry))
 		queue_work(kfd->ih_wq, &kfd->interrupt_work);
 
 	spin_unlock(&kfd->interrupt_lock);

drivers/gpu/drm/amd/amdkfd/kfd_int_process_v9.c

@@ -26,7 +26,9 @@
 
 
 static bool event_interrupt_isr_v9(struct kfd_dev *dev,
-					const uint32_t *ih_ring_entry)
+					const uint32_t *ih_ring_entry,
+					uint32_t *patched_ihre,
+					bool *patched_flag)
 {
 	uint16_t source_id, client_id, pasid, vmid;
 	const uint32_t *data = ih_ring_entry;

drivers/gpu/drm/amd/amdkfd/kfd_interrupt.c

@@ -151,13 +151,15 @@ static void interrupt_wq(struct work_struct *work)
 								ih_ring_entry);
 }
 
-bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry)
+bool interrupt_is_wanted(struct kfd_dev *dev,
+			const uint32_t *ih_ring_entry,
+			uint32_t *patched_ihre, bool *flag)
 {
 	/* integer and bitwise OR so there is no boolean short-circuiting */
 	unsigned int wanted = 0;
 
 	wanted |= dev->device_info->event_interrupt_class->interrupt_isr(dev,
-								ih_ring_entry);
+					 ih_ring_entry, patched_ihre, flag);
 
 	return wanted != 0;
 }

drivers/gpu/drm/amd/amdkfd/kfd_priv.h

@@ -180,9 +180,10 @@ enum cache_policy {
 
 struct kfd_event_interrupt_class {
 	bool (*interrupt_isr)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+			const uint32_t *ih_ring_entry, uint32_t *patched_ihre,
+			bool *patched_flag);
 	void (*interrupt_wq)(struct kfd_dev *dev,
-				const uint32_t *ih_ring_entry);
+			const uint32_t *ih_ring_entry);
 };
 
 struct kfd_device_info {
@@ -806,7 +807,9 @@ int kfd_interrupt_init(struct kfd_dev *dev);
 void kfd_interrupt_exit(struct kfd_dev *dev);
 void kgd2kfd_interrupt(struct kfd_dev *kfd, const void *ih_ring_entry);
 bool enqueue_ih_ring_entry(struct kfd_dev *kfd,	const void *ih_ring_entry);
-bool interrupt_is_wanted(struct kfd_dev *dev, const uint32_t *ih_ring_entry);
+bool interrupt_is_wanted(struct kfd_dev *dev,
+				const uint32_t *ih_ring_entry,
+				uint32_t *patched_ihre, bool *flag);
 
 /* Power Management */
 void kgd2kfd_suspend(struct kfd_dev *kfd);