disp: msm: sde: add support for hw fence error handling

Register callback function to hw fence driver and implement the
callback funtion.

As part of fence error handling, address out of ordering of HW
fences, SW override for release fence signal and handle BW voting
in both cmd and video mode.

Change-Id: I22902762b4cc09a5f5a20cf0dd01fc336a0f0cb4
Signed-off-by: GG Hou <[email protected]>

msm/sde/sde_core_perf.c

@@ -1054,6 +1054,15 @@ void sde_core_perf_crtc_update(struct drm_crtc *crtc,
 	old = &sde_crtc->cur_perf;
 	new = &sde_crtc->new_perf;
 
+	/* avoid the voting in fence error case when there is decrease in BW vote */
+	if (!params_changed && !stop_req && sde_crtc->handle_fence_error_bw_update) {
+		new = &sde_crtc->cur_perf;
+		SDE_EVT32(kms->dev, params_changed, stop_req,
+			sde_crtc->handle_fence_error_bw_update);
+
+		sde_crtc->handle_fence_error_bw_update = false;
+	}
+
 	if (_sde_core_perf_crtc_is_power_on(crtc) && !stop_req) {
 		_sde_core_perf_crtc_update_check(crtc, params_changed,
 				&update_bus, &update_clk);

msm/sde/sde_crtc.c

@@ -4769,6 +4769,7 @@ void sde_crtc_commit_kickoff(struct drm_crtc *crtc,
 	idle_pc_state = sde_crtc_get_property(cstate, CRTC_PROP_IDLE_PC_STATE);
 
 	sde_crtc->kickoff_in_progress = true;
+	sde_crtc->handle_fence_error_bw_update = false;
 	list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
 		if (encoder->crtc != crtc)
 			continue;

msm/sde/sde_crtc.h

@@ -375,6 +375,7 @@ enum sde_crtc_hw_fence_flags {
  *                          sde_crtc_hw_fence_flags for available fields.
  * @hwfence_out_fences_skip: number of frames to skip before create a new hw-fence, this can be
  *                   used to slow-down creation of output hw-fences for debugging purposes.
+ * @hanle_fence_error_bw_update: bool to indicate if it is fence error and need to avoid bw vote.
  */
 struct sde_crtc {
 	struct drm_crtc base;
@@ -489,6 +490,7 @@ struct sde_crtc {
 
 	DECLARE_BITMAP(hwfence_features_mask, HW_FENCE_FEATURES_MAX);
 	u32 hwfence_out_fences_skip;
+	bool handle_fence_error_bw_update;
 };
 
 enum sde_crtc_dirty_flags {

msm/sde/sde_encoder.c

@@ -2021,6 +2021,76 @@ static void sde_encoder_misr_configure(struct drm_encoder *drm_enc,
 	sde_enc->misr_reconfigure = false;
 }
 
+static int sde_encoder_hw_fence_signal(struct sde_encoder_phys *phys_enc)
+{
+	struct sde_hw_ctl *hw_ctl;
+	struct sde_hw_fence_data *hwfence_data;
+	int pending_kickoff_cnt = -1;
+	int rc = 0;
+
+	if (!phys_enc || !phys_enc->parent || !phys_enc->hw_ctl) {
+		SDE_DEBUG("invalid parameters\n");
+		SDE_EVT32(SDE_EVTLOG_ERROR);
+		return -EINVAL;
+	}
+
+	hw_ctl = phys_enc->hw_ctl;
+	hwfence_data = &hw_ctl->hwfence_data;
+
+	pending_kickoff_cnt = atomic_read(&phys_enc->pending_kickoff_cnt);
+
+	/* out of order hw fence error signal is needed for video panel. */
+	if (sde_encoder_check_curr_mode(phys_enc->parent, MSM_DISPLAY_VIDEO_MODE)) {
+		/* out of order hw fence error signal */
+		msm_hw_fence_update_txq_error(hwfence_data->hw_fence_handle,
+			phys_enc->sde_hw_fence_handle, 1, MSM_HW_FENCE_UPDATE_ERROR_WITH_MOVE);
+	/* wait for frame done to avoid out of order signalling for cmd mode. */
+	} else if (pending_kickoff_cnt) {
+		SDE_EVT32(DRMID(phys_enc->parent), SDE_EVTLOG_FUNC_CASE1);
+		rc = sde_encoder_wait_for_event(phys_enc->parent, MSM_ENC_TX_COMPLETE);
+		if (rc && rc != -EWOULDBLOCK) {
+			SDE_DEBUG("wait for frame done failed %d\n", rc);
+			SDE_EVT32(DRMID(phys_enc->parent), rc, pending_kickoff_cnt,
+				SDE_EVTLOG_ERROR);
+		}
+	}
+
+	/* HW o/p fence override register */
+	if (hw_ctl->ops.trigger_output_fence_override) {
+		hw_ctl->ops.trigger_output_fence_override(hw_ctl);
+		SDE_DEBUG("trigger_output_fence_override executed.\n");
+		SDE_EVT32(DRMID(phys_enc->parent), SDE_EVTLOG_FUNC_CASE2);
+	}
+
+	return rc;
+}
+
+int sde_encoder_hw_fence_error_handle(struct drm_encoder *drm_enc)
+{
+	struct sde_encoder_virt *sde_enc;
+	struct sde_encoder_phys *phys_enc;
+	int rc = 0;
+
+	sde_enc = to_sde_encoder_virt(drm_enc);
+	if (!sde_enc || !sde_enc->phys_encs[0] ||
+		!sde_enc->phys_encs[0]->sde_hw_fence_error_status)
+		return 0;
+
+	SDE_EVT32(DRMID(drm_enc), SDE_EVTLOG_FUNC_ENTRY);
+
+	phys_enc = sde_enc->phys_encs[0];
+
+	rc = sde_encoder_hw_fence_signal(phys_enc);
+	if (rc) {
+		SDE_DEBUG("sde_encoder_hw_fence_signal error, rc = %d.\n", rc);
+		SDE_EVT32(DRMID(drm_enc), rc, SDE_EVTLOG_ERROR);
+	}
+
+	phys_enc->sde_hw_fence_error_status = false;
+	SDE_EVT32(DRMID(drm_enc), SDE_EVTLOG_FUNC_EXIT);
+	return rc;
+}
+
 static void sde_encoder_input_event_handler(struct input_handle *handle,
 	unsigned int type, unsigned int code, int value)
 {
@@ -4550,6 +4620,70 @@ void sde_encoder_early_wakeup(struct drm_encoder *drm_enc)
 	SDE_ATRACE_END("queue_early_wakeup_work");
 }
 
+void sde_encoder_handle_hw_fence_error(int ctl_idx, struct sde_kms *sde_kms, u32 handle, int error)
+{
+	struct drm_encoder *drm_enc;
+	struct sde_encoder_virt *sde_enc;
+	struct sde_encoder_phys *cur_master;
+	struct sde_crtc *sde_crtc;
+	struct sde_crtc_state *sde_crtc_state;
+	bool encoder_detected = false;
+	bool handle_fence_error;
+
+	SDE_EVT32(ctl_idx, handle, error, SDE_EVTLOG_FUNC_ENTRY);
+
+	if (!sde_kms || !sde_kms->dev) {
+		SDE_ERROR("Invalid sde_kms or sde_kms->dev\n");
+		return;
+	}
+
+	drm_for_each_encoder(drm_enc, sde_kms->dev) {
+		sde_enc = to_sde_encoder_virt(drm_enc);
+
+		if (sde_enc && sde_enc->phys_encs[0] && sde_enc->phys_encs[0]->hw_ctl &&
+			sde_enc->phys_encs[0]->hw_ctl->idx == ctl_idx) {
+			encoder_detected = true;
+			cur_master = sde_enc->phys_encs[0];
+			SDE_EVT32(ctl_idx, SDE_EVTLOG_FUNC_CASE1);
+			break;
+		}
+	}
+
+	if (!encoder_detected) {
+		SDE_DEBUG("failed to get the sde_encoder_phys.\n");
+		SDE_EVT32(ctl_idx, SDE_EVTLOG_FUNC_CASE2, SDE_EVTLOG_ERROR);
+		return;
+	}
+
+	if (!cur_master->parent || !cur_master->parent->crtc || !cur_master->parent->crtc->state) {
+		SDE_DEBUG("unexpected null pointer in cur_master.\n");
+		SDE_EVT32(ctl_idx, SDE_EVTLOG_FUNC_CASE3, SDE_EVTLOG_ERROR);
+		return;
+	}
+
+	sde_crtc = to_sde_crtc(cur_master->parent->crtc);
+	sde_crtc_state = to_sde_crtc_state(cur_master->parent->crtc->state);
+	handle_fence_error = sde_crtc_get_property(sde_crtc_state, CRTC_PROP_HANDLE_FENCE_ERROR);
+	if (!handle_fence_error) {
+		SDE_DEBUG("userspace not enabled handle fence error in kernel.\n");
+		SDE_EVT32(ctl_idx, SDE_EVTLOG_FUNC_CASE4);
+		return;
+	}
+
+	cur_master->sde_hw_fence_handle = handle;
+
+	if (error) {
+		sde_crtc->handle_fence_error_bw_update = true;
+		cur_master->sde_hw_fence_error_status = true;
+		cur_master->sde_hw_fence_error_value = error;
+	}
+
+	atomic_add_unless(&cur_master->pending_retire_fence_cnt, -1, 0);
+	wake_up_all(&cur_master->pending_kickoff_wq);
+
+	SDE_EVT32(ctl_idx, error, SDE_EVTLOG_FUNC_EXIT);
+}
+
 int sde_encoder_poll_line_counts(struct drm_encoder *drm_enc)
 {
 	static const uint64_t timeout_us = 50000;

msm/sde/sde_encoder.h

@@ -315,6 +315,22 @@ void sde_encoder_get_hw_resources(struct drm_encoder *encoder,
  */
 void sde_encoder_early_wakeup(struct drm_encoder *drm_enc);
 
+/**
+ * sde_encoder_handle_hw_fence_error - hw fence error handing in sde encoder
+ * @ctl_idx:	control path index
+ * @sde_kms:	Pointer to sde_kms
+ * @handle:	hash of fence signaled with error
+ * @error:	error signaled for fence from hw fence callback
+ */
+void sde_encoder_handle_hw_fence_error(int ctl_idx, struct sde_kms *sde_kms, u32 handle, int error);
+
+/**
+ * sde_encoder_hw_fence_error_handle - fence error handing while hw fence error
+ * @drm_enc: Pointer to drm encoder structure
+ * return: 0 on success; error code otherwise
+ */
+int sde_encoder_hw_fence_error_handle(struct drm_encoder *drm_enc);
+
 /**
  * sde_encoder_register_vblank_callback - provide callback to encoder that
  *	will be called on the next vblank.

msm/sde/sde_encoder_phys.h

@@ -336,6 +336,12 @@ struct sde_encoder_irq {
  * @vfp_cached:			cached vertical front porch to be used for
  *				programming ROT and MDP fetch start
  * @pf_time_in_us:		Programmable fetch time in micro-seconds
+ * @sde_hw_fence_error_status:	Hw fence error handing flag controled by userspace
+ *				that if handing fence error in driver
+ * @sde_hw_fence_error_value:	hw fence error value from cb function
+ * @sde_hw_fence_handle:	Hw fence driver client handle, this handle was returned
+ *				during the call 'msm_hw_fence_register' to register the
+ *				client
  * @frame_trigger_mode:		frame trigger mode indication for command
  *				mode display
  * @recovered:			flag set to true when recovered from pp timeout
@@ -388,6 +394,9 @@ struct sde_encoder_phys {
 	bool in_clone_mode;
 	int vfp_cached;
 	u32 pf_time_in_us;
+	bool sde_hw_fence_error_status;
+	int sde_hw_fence_error_value;
+	u64 sde_hw_fence_handle;
 	enum frame_trigger_mode_type frame_trigger_mode;
 	bool recovered;
 	bool autorefresh_disable_trans;
@@ -399,6 +408,13 @@ static inline int sde_encoder_phys_inc_pending(struct sde_encoder_phys *phys)
 	return atomic_inc_return(&phys->pending_kickoff_cnt);
 }
 
+/**
+ * sde_encoder_hw_fence_signal - hw fence related fence error handing
+ * @phys_enc: Pointer to physical encoder structure
+ * return: 0 on success; error code otherwise
+ */
+static inline int sde_encoder_hw_fence_signal(struct sde_encoder_phys *phys_enc);
+
 /**
  * struct sde_encoder_phys_vid - sub-class of sde_encoder_phys to handle video
  *	mode specific operations

msm/sde/sde_fence.c

@@ -12,6 +12,7 @@
 #include "msm_drv.h"
 #include "sde_kms.h"
 #include "sde_fence.h"
+#include "sde_encoder.h"
 
 #define TIMELINE_VAL_LENGTH		128
 #define SPEC_FENCE_FLAG_FENCE_ARRAY	0x10
@@ -102,12 +103,39 @@ struct sde_hw_fence_data hw_fence_data_dpu_client[SDE_HW_FENCE_CLIENT_MAX] = {
 		0, 8, 25, 0, 0}
 };
 
-int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, bool use_dpu_ipcc, struct msm_mmu *mmu)
+void msm_hw_fence_error_cb(u32 handle, int error, void *cb_data)
+{
+	struct msm_hw_fence_cb_data *msm_hw_fence_cb_data;
+	struct sde_hw_fence_error_cb_data *sde_hw_fence_error_data;
+
+	SDE_EVT32(handle, error, SDE_EVTLOG_FUNC_ENTRY);
+
+	msm_hw_fence_cb_data = (struct msm_hw_fence_cb_data *)cb_data;
+	if (!msm_hw_fence_cb_data) {
+		SDE_ERROR("msm hw fence cb data is NULL.\n");
+		SDE_EVT32(SDE_EVTLOG_FUNC_CASE1, SDE_EVTLOG_ERROR);
+		return;
+	}
+
+	sde_hw_fence_error_data = (struct sde_hw_fence_error_cb_data *)(msm_hw_fence_cb_data->data);
+	if (!sde_hw_fence_error_data) {
+		SDE_ERROR("sde hw fence cb data is NULL.\n");
+		SDE_EVT32(SDE_EVTLOG_FUNC_CASE2, SDE_EVTLOG_ERROR);
+		return;
+	}
+
+	sde_encoder_handle_hw_fence_error(sde_hw_fence_error_data->ctl_idx,
+		sde_hw_fence_error_data->sde_kms, handle, error);
+}
+
+int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, struct sde_kms *sde_kms, bool use_dpu_ipcc,
+		struct msm_mmu *mmu)
 {
 	struct msm_hw_fence_hfi_queue_header *hfi_queue_header_va, *hfi_queue_header_pa;
 	struct msm_hw_fence_hfi_queue_table_header *hfi_table_header;
 	struct sde_hw_fence_data *sde_hw_fence_data;
 	struct sde_hw_fence_data *hwfence_data;
+	struct sde_hw_fence_error_cb_data *sde_hw_fence_error_cb_data;
 	phys_addr_t queue_pa;
 	void *queue_va;
 	u32 qhdr0_offset, ctl_hfi_iova;
@@ -149,6 +177,17 @@ int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, bool use_dpu_ipcc, struct msm_m
 		return -EINVAL;
 	}
 
+	sde_hw_fence_error_cb_data = &(hwfence_data->sde_hw_fence_error_cb_data);
+	sde_hw_fence_error_cb_data->ctl_idx = hw_ctl->idx;
+	sde_hw_fence_error_cb_data->sde_kms = sde_kms;
+
+	ret = msm_hw_fence_register_error_cb(hwfence_data->hw_fence_handle,
+		msm_hw_fence_error_cb, (void *)sde_hw_fence_error_cb_data);
+	if (ret) {
+		SDE_EVT32(hw_ctl->idx, SDE_EVTLOG_ERROR);
+		SDE_DEBUG("hw fence cb register failed. ret = %d\n", ret);
+	}
+
 	/* one-to-one memory map of ctl-path client queues */
 	ctl_hfi_iova = HW_FENCE_HFI_MMAP_DPU_BA +
 		PAGE_ALIGN(hwfence_data->mem_descriptor.size * ctl_id);

msm/sde/sde_fence.h

@@ -50,6 +50,16 @@ struct sde_fence_context {
 	char name[SDE_FENCE_NAME_SIZE];
 };
 
+/**
+ * struct sde_hw_fence_error_cb_data - struct passed back in fence error callback
+ * @ctl_idx: control path index
+ * @sde_kms: handle to sde_kms
+ */
+struct sde_hw_fence_error_cb_data {
+	int ctl_idx;
+	struct sde_kms *sde_kms;
+};
+
 /**
  * enum sde_fence_event - sde fence event as hint fence operation
  * @SDE_FENCE_SIGNAL: Signal the fence cleanly with current timeline
@@ -78,6 +88,7 @@ enum sde_fence_event {
  * @ipcc_this_client: ipcc dpu client id (For Waipio: APPS, For Kailua: DPU HW)
  * @dma_context: per client dma context used to create join fences
  * @hw_fence_array_seqno: per-client seq number counter for join fences
+ * @sde_hw_fence_error_cb_data: data needed for hw fence cb function.
  */
 struct sde_hw_fence_data {
 	int client_id;
@@ -94,6 +105,7 @@ struct sde_hw_fence_data {
 	u32 ipcc_this_client;
 	u64 dma_context;
 	u32 hw_fence_array_seqno;
+	struct sde_hw_fence_error_cb_data sde_hw_fence_error_cb_data;
 };
 
 #if IS_ENABLED(CONFIG_SYNC_FILE)
@@ -156,12 +168,14 @@ struct sde_fence_context *sde_fence_init(const char *name,
  * sde_fence_hw_fence_init - initialize hw-fence clients
  *
  * @hw_ctl: hw ctl client to init.
+ * @sde_kms: used for hw fence error cb register.
  * @use_ipcc: boolean to indicate if hw should use dpu ipcc signals.
  * @mmu: mmu to map memory for queues
  *
  * Returns: Zero on success, otherwise returns an error code.
  */
-int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, bool use_dpu_ipcc, struct msm_mmu *mmu);
+int sde_hw_fence_init(struct sde_hw_ctl *hw_ctl, struct sde_kms *sde_kms, bool use_dpu_ipcc,
+	struct msm_mmu *mmu);
 
 /**
  * sde_fence_hw_fence_deinit - deinitialize hw-fence clients

msm/sde/sde_kms.c

@@ -1721,6 +1721,8 @@ static void sde_kms_wait_for_commit_done(struct msm_kms *kms,
 			break;
 		}
 
+		sde_encoder_hw_fence_error_handle(encoder);
+
 		sde_crtc_complete_flip(crtc, NULL);
 
 		if (cwb_disabling)

msm/sde/sde_rm.c

@@ -727,7 +727,7 @@ static int _init_hw_fences(struct sde_rm *rm, bool use_ipcc, struct sde_kms *sde
 
 		if (sde_kms->aspace[MSM_SMMU_DOMAIN_UNSECURE] &&
 				sde_kms->aspace[MSM_SMMU_DOMAIN_UNSECURE]->mmu) {
-			if (sde_hw_fence_init(ctl, use_ipcc,
+			if (sde_hw_fence_init(ctl, sde_kms, use_ipcc,
 					sde_kms->aspace[MSM_SMMU_DOMAIN_UNSECURE]->mmu)) {
 				SDE_DEBUG("failed to init hw_fence idx:%d\n", ctl->idx);
 				ret = -EINVAL;