disp: msm: sde: avoid wb done wait for cwb in wait_for_commit

Existing cwb implementation waits for WB done interrupt in
wait_for_commit_done API call. This serializes the cwb commit
and causes frame trigger delay on primary display. MDSS hw allows
to trigger the cwb frame when previous frame is in-progress. This
change updates driver to allow parallel frame trigger for cwb
enabled display. It releases frame N cwb output buffer in frame
N+1 wait_for_commit done call.

Change-Id: Id4f2a0cc78a3f24a1b5ce96dc907780246768dbf
Signed-off-by: Raviteja Tamatam <[email protected]>

msm/sde/sde_encoder_phys.h

@@ -388,13 +388,14 @@ struct sde_encoder_phys_cmd {
  * @hw_wb:		Hardware interface to the wb registers
  * @wbdone_timeout:	Timeout value for writeback done in msec
  * @bypass_irqreg:	Bypass irq register/unregister if non-zero
- * @wbdone_complete:	for wbdone irq synchronization
  * @wb_cfg:		Writeback hardware configuration
  * @cdp_cfg:		Writeback CDP configuration
  * @wb_roi:		Writeback region-of-interest
  * @wb_fmt:		Writeback pixel format
  * @wb_fb:		Pointer to current writeback framebuffer
  * @wb_aspace:		Pointer to current writeback address space
+ * @cwb_old_fb:		Pointer to old writeback framebuffer
+ * @cwb_old_aspace:	Pointer to old writeback address space
  * @frame_count:	Counter of completed writeback operations
  * @kickoff_count:	Counter of issued writeback operations
  * @aspace:		address space identifier for non-secure/secure domain
@@ -410,13 +411,14 @@ struct sde_encoder_phys_wb {
 	struct sde_hw_wb *hw_wb;
 	u32 wbdone_timeout;
 	u32 bypass_irqreg;
-	struct completion wbdone_complete;
 	struct sde_hw_wb_cfg wb_cfg;
 	struct sde_hw_wb_cdp_cfg cdp_cfg;
 	struct sde_rect wb_roi;
 	const struct sde_format *wb_fmt;
 	struct drm_framebuffer *wb_fb;
 	struct msm_gem_address_space *wb_aspace;
+	struct drm_framebuffer *cwb_old_fb;
+	struct msm_gem_address_space *cwb_old_aspace;
 	u32 frame_count;
 	u32 kickoff_count;
 	struct msm_gem_address_space *aspace[SDE_IOMMU_DOMAIN_MAX];

msm/sde/sde_encoder_phys_wb.c

@@ -88,20 +88,6 @@ static void sde_encoder_phys_wb_set_ot_limit(
 	sde_vbif_set_ot_limit(phys_enc->sde_kms, &ot_params);
 }
 
-/**
- * sde_encoder_phys_wb_set_traffic_shaper - set traffic shaper for writeback
- * @phys_enc:	Pointer to physical encoder
- */
-static void sde_encoder_phys_wb_set_traffic_shaper(
-		struct sde_encoder_phys *phys_enc)
-{
-	struct sde_encoder_phys_wb *wb_enc = to_sde_encoder_phys_wb(phys_enc);
-	struct sde_hw_wb_cfg *wb_cfg = &wb_enc->wb_cfg;
-
-	/* traffic shaper is only enabled for rotator */
-	wb_cfg->ts_cfg.en = false;
-}
-
 /**
  * sde_encoder_phys_wb_set_qos_remap - set QoS remapper for writeback
  * @phys_enc:	Pointer to physical encoder
@@ -1025,8 +1011,6 @@ static void sde_encoder_phys_wb_setup(
 
 	sde_encoder_phys_wb_set_ot_limit(phys_enc);
 
-	sde_encoder_phys_wb_set_traffic_shaper(phys_enc);
-
 	sde_encoder_phys_wb_set_qos_remap(phys_enc);
 
 	sde_encoder_phys_wb_set_qos(phys_enc);
@@ -1047,31 +1031,33 @@ static void _sde_encoder_phys_wb_frame_done_helper(void *arg, bool frame_error)
 	struct sde_hw_wb *hw_wb = wb_enc->hw_wb;
 	u32 event = frame_error ? SDE_ENCODER_FRAME_EVENT_ERROR : 0;
 
-	event |= SDE_ENCODER_FRAME_EVENT_DONE |
-		SDE_ENCODER_FRAME_EVENT_SIGNAL_RETIRE_FENCE;
-
 	SDE_DEBUG("[wb:%d,%u]\n", hw_wb->idx - WB_0, wb_enc->frame_count);
 
 	/* don't notify upper layer for internal commit */
 	if (phys_enc->enable_state == SDE_ENC_DISABLING)
 		goto complete;
 
-	if (!phys_enc->in_clone_mode)
-		event |= SDE_ENCODER_FRAME_EVENT_SIGNAL_RELEASE_FENCE;
+	if (phys_enc->parent_ops.handle_frame_done &&
+	    atomic_add_unless(&phys_enc->pending_retire_fence_cnt, -1, 0)) {
+		event |= SDE_ENCODER_FRAME_EVENT_DONE |
+			SDE_ENCODER_FRAME_EVENT_SIGNAL_RETIRE_FENCE;
+
+		if (!phys_enc->in_clone_mode)
+			event |= SDE_ENCODER_FRAME_EVENT_SIGNAL_RELEASE_FENCE;
 
-	atomic_add_unless(&phys_enc->pending_retire_fence_cnt, -1, 0);
-	if (phys_enc->parent_ops.handle_frame_done)
 		phys_enc->parent_ops.handle_frame_done(phys_enc->parent,
 				phys_enc, event);
+	}
 
 	if (phys_enc->parent_ops.handle_vblank_virt)
 		phys_enc->parent_ops.handle_vblank_virt(phys_enc->parent,
 				phys_enc);
 
-	SDE_EVT32_IRQ(DRMID(phys_enc->parent), hw_wb->idx - WB_0, event);
+	SDE_EVT32_IRQ(DRMID(phys_enc->parent), hw_wb->idx - WB_0, event,
+		frame_error);
 
 complete:
-	complete_all(&wb_enc->wbdone_complete);
+	wake_up_all(&phys_enc->pending_kickoff_wq);
 }
 
 /**
@@ -1201,20 +1187,37 @@ static void sde_encoder_phys_wb_mode_set(
 	}
 }
 
-/**
- * sde_encoder_phys_wb_wait_for_commit_done - wait until request is committed
- * @phys_enc:	Pointer to physical encoder
- */
-static int sde_encoder_phys_wb_wait_for_commit_done(
-		struct sde_encoder_phys *phys_enc)
+static int sde_encoder_phys_wb_frame_timeout(struct sde_encoder_phys *phys_enc)
+{
+	u32 event = 0;
+
+	while (atomic_add_unless(&phys_enc->pending_retire_fence_cnt, -1, 0) &&
+			phys_enc->parent_ops.handle_frame_done) {
+
+		event = SDE_ENCODER_FRAME_EVENT_SIGNAL_RETIRE_FENCE
+			| SDE_ENCODER_FRAME_EVENT_ERROR;
+
+		if (!phys_enc->in_clone_mode)
+			event |= SDE_ENCODER_FRAME_EVENT_SIGNAL_RELEASE_FENCE;
+
+		phys_enc->parent_ops.handle_frame_done(
+				phys_enc->parent, phys_enc, event);
+
+		SDE_EVT32(DRMID(phys_enc->parent), event,
+			atomic_read(&phys_enc->pending_retire_fence_cnt));
+	}
+
+	return event;
+}
+
+static int _sde_encoder_phys_wb_wait_for_commit_done(
+		struct sde_encoder_phys *phys_enc, bool is_disable)
 {
-	unsigned long ret;
 	struct sde_encoder_phys_wb *wb_enc = to_sde_encoder_phys_wb(phys_enc);
-	u32 irq_status, event = 0;
+	u32 event = 0;
 	u64 wb_time = 0;
 	int rc = 0;
-	int irq_idx = phys_enc->irq[INTR_IDX_WB_DONE].irq_idx;
-	u32 timeout = max_t(u32, wb_enc->wbdone_timeout, KICKOFF_TIMEOUT_MS);
+	struct sde_encoder_wait_info wait_info;
 
 	/* Return EWOULDBLOCK since we know the wait isn't necessary */
 	if (phys_enc->enable_state == SDE_ENC_DISABLED) {
@@ -1223,7 +1226,12 @@ static int sde_encoder_phys_wb_wait_for_commit_done(
 	}
 
 	SDE_EVT32(DRMID(phys_enc->parent), WBID(wb_enc), wb_enc->frame_count,
-			!!wb_enc->wb_fb);
+		wb_enc->kickoff_count, !!wb_enc->wb_fb, is_disable,
+		phys_enc->in_clone_mode);
+
+	if (!is_disable && phys_enc->in_clone_mode &&
+	    (atomic_read(&phys_enc->pending_retire_fence_cnt) <= 1))
+		goto skip_wait;
 
 	/* signal completion if commit with no framebuffer */
 	if (!wb_enc->wb_fb) {
@@ -1231,59 +1239,43 @@ static int sde_encoder_phys_wb_wait_for_commit_done(
 		_sde_encoder_phys_wb_frame_done_helper(wb_enc, false);
 	}
 
-	ret = wait_for_completion_timeout(&wb_enc->wbdone_complete,
-			msecs_to_jiffies(timeout));
-
-	if (!ret) {
+	wait_info.wq = &phys_enc->pending_kickoff_wq;
+	wait_info.atomic_cnt = &phys_enc->pending_retire_fence_cnt;
+	wait_info.timeout_ms = max_t(u32, wb_enc->wbdone_timeout,
+		KICKOFF_TIMEOUT_MS);
+	rc = sde_encoder_helper_wait_for_irq(phys_enc, INTR_IDX_WB_DONE,
+		&wait_info);
+	if (rc == -ETIMEDOUT) {
 		SDE_EVT32(DRMID(phys_enc->parent), WBID(wb_enc),
-				wb_enc->frame_count);
-		irq_status = sde_core_irq_read(phys_enc->sde_kms,
-				irq_idx, true);
-		if (irq_status) {
-			SDE_DEBUG("wb:%d done but irq not triggered\n",
-					WBID(wb_enc));
-			_sde_encoder_phys_wb_frame_done_helper(wb_enc, false);
-		} else {
-			SDE_ERROR("wb:%d kickoff timed out\n",
-					WBID(wb_enc));
-			atomic_add_unless(
-				&phys_enc->pending_retire_fence_cnt, -1, 0);
-
-			event = SDE_ENCODER_FRAME_EVENT_SIGNAL_RELEASE_FENCE
-				| SDE_ENCODER_FRAME_EVENT_SIGNAL_RETIRE_FENCE
-				| SDE_ENCODER_FRAME_EVENT_ERROR;
-			if (phys_enc->parent_ops.handle_frame_done)
-				phys_enc->parent_ops.handle_frame_done(
-					phys_enc->parent, phys_enc, event);
-			rc = -ETIMEDOUT;
-		}
-	}
+			wb_enc->frame_count, SDE_EVTLOG_ERROR);
+		SDE_ERROR("wb:%d kickoff timed out\n", WBID(wb_enc));
 
-	if (!rc)
-		wb_enc->end_time = ktime_get();
+		event = sde_encoder_phys_wb_frame_timeout(phys_enc);
+	}
 
-	/* once operation is done, disable traffic shaper */
-	if (wb_enc->wb_cfg.ts_cfg.en && wb_enc->hw_wb &&
-			wb_enc->hw_wb->ops.setup_trafficshaper) {
-		wb_enc->wb_cfg.ts_cfg.en = false;
-		wb_enc->hw_wb->ops.setup_trafficshaper(
-				wb_enc->hw_wb, &wb_enc->wb_cfg);
+	/* cleanup writeback framebuffer */
+	if (wb_enc->wb_fb && wb_enc->wb_aspace) {
+		msm_framebuffer_cleanup(wb_enc->wb_fb, wb_enc->wb_aspace);
+		wb_enc->wb_fb = NULL;
+		wb_enc->wb_aspace = NULL;
 	}
 
+skip_wait:
 	/* remove vote for iommu/clk/bus */
 	wb_enc->frame_count++;
 
 	if (!rc) {
+		wb_enc->end_time = ktime_get();
 		wb_time = (u64)ktime_to_us(wb_enc->end_time) -
 				(u64)ktime_to_us(wb_enc->start_time);
 		SDE_DEBUG("wb:%d took %llu us\n", WBID(wb_enc), wb_time);
 	}
 
-	/* cleanup writeback framebuffer */
-	if (wb_enc->wb_fb && wb_enc->wb_aspace) {
-		msm_framebuffer_cleanup(wb_enc->wb_fb, wb_enc->wb_aspace);
-		wb_enc->wb_fb = NULL;
-		wb_enc->wb_aspace = NULL;
+	/* cleanup previous buffer if pending */
+	if (wb_enc->cwb_old_fb && wb_enc->cwb_old_aspace) {
+		msm_framebuffer_cleanup(wb_enc->cwb_old_fb, wb_enc->cwb_old_aspace);
+		wb_enc->cwb_old_fb = NULL;
+		wb_enc->cwb_old_aspace = NULL;
 	}
 
 	SDE_EVT32(DRMID(phys_enc->parent), WBID(wb_enc), wb_enc->frame_count,
@@ -1292,6 +1284,16 @@ static int sde_encoder_phys_wb_wait_for_commit_done(
 	return rc;
 }
 
+/**
+ * sde_encoder_phys_wb_wait_for_commit_done - wait until request is committed
+ * @phys_enc:	Pointer to physical encoder
+ */
+static int sde_encoder_phys_wb_wait_for_commit_done(
+		struct sde_encoder_phys *phys_enc)
+{
+	return _sde_encoder_phys_wb_wait_for_commit_done(phys_enc, false);
+}
+
 /**
  * sde_encoder_phys_wb_prepare_for_kickoff - pre-kickoff processing
  * @phys_enc:	Pointer to physical encoder
@@ -1307,7 +1309,10 @@ static int sde_encoder_phys_wb_prepare_for_kickoff(
 	SDE_DEBUG("[wb:%d,%u]\n", wb_enc->hw_wb->idx - WB_0,
 			wb_enc->kickoff_count);
 
-	reinit_completion(&wb_enc->wbdone_complete);
+	if (phys_enc->in_clone_mode) {
+		wb_enc->cwb_old_fb = wb_enc->wb_fb;
+		wb_enc->cwb_old_aspace = wb_enc->wb_aspace;
+	}
 
 	wb_enc->kickoff_count++;
 
@@ -1321,7 +1326,9 @@ static int sde_encoder_phys_wb_prepare_for_kickoff(
 	/* vote for iommu/clk/bus */
 	wb_enc->start_time = ktime_get();
 
-	SDE_EVT32(DRMID(phys_enc->parent), WBID(wb_enc), wb_enc->kickoff_count);
+	SDE_EVT32(DRMID(phys_enc->parent), WBID(wb_enc),
+		wb_enc->kickoff_count, wb_enc->frame_count,
+		phys_enc->in_clone_mode);
 	return 0;
 }
 
@@ -1538,12 +1545,10 @@ static void sde_encoder_phys_wb_disable(struct sde_encoder_phys *phys_enc)
 		return;
 	}
 
-	if (wb_enc->frame_count != wb_enc->kickoff_count) {
-		SDE_DEBUG("[wait_for_done: wb:%d, frame:%u, kickoff:%u]\n",
-				hw_wb->idx - WB_0, wb_enc->frame_count,
-				wb_enc->kickoff_count);
-		sde_encoder_phys_wb_wait_for_commit_done(phys_enc);
-	}
+	SDE_DEBUG("[wait_for_done: wb:%d, frame:%u, kickoff:%u]\n",
+			hw_wb->idx - WB_0, wb_enc->frame_count,
+			wb_enc->kickoff_count);
+	_sde_encoder_phys_wb_wait_for_commit_done(phys_enc, true);
 
 	if (!phys_enc->hw_ctl || !phys_enc->parent ||
 			!phys_enc->sde_kms || !wb_enc->fb_disable) {
@@ -1583,9 +1588,24 @@ static void sde_encoder_phys_wb_disable(struct sde_encoder_phys *phys_enc)
 		phys_enc->hw_ctl->ops.trigger_flush(phys_enc->hw_ctl);
 
 	sde_encoder_helper_trigger_start(phys_enc);
-	sde_encoder_phys_wb_wait_for_commit_done(phys_enc);
+	_sde_encoder_phys_wb_wait_for_commit_done(phys_enc, true);
 	sde_encoder_phys_wb_irq_ctrl(phys_enc, false);
+
 exit:
+	/*
+	 * frame count and kickoff count are only used for debug purpose. Frame
+	 * count can be more than kickoff count at the end of disable call due
+	 * to extra frame_done wait. It does not cause any issue because
+	 * frame_done wait is based on retire_fence count. Leaving these
+	 * counters for debugging purpose.
+	 */
+	if (wb_enc->frame_count != wb_enc->kickoff_count) {
+		SDE_EVT32(DRMID(phys_enc->parent), WBID(wb_enc),
+			wb_enc->kickoff_count, wb_enc->frame_count,
+			phys_enc->in_clone_mode);
+		wb_enc->frame_count = wb_enc->kickoff_count;
+	}
+
 	phys_enc->enable_state = SDE_ENC_DISABLED;
 	wb_enc->crtc = NULL;
 }
@@ -1734,7 +1754,6 @@ struct sde_encoder_phys *sde_encoder_phys_wb_init(
 		goto fail_alloc;
 	}
 	wb_enc->wbdone_timeout = KICKOFF_TIMEOUT_MS;
-	init_completion(&wb_enc->wbdone_complete);
 
 	phys_enc = &wb_enc->base;
 
@@ -1797,6 +1816,7 @@ struct sde_encoder_phys *sde_encoder_phys_wb_init(
 	phys_enc->vblank_ctl_lock = p->vblank_ctl_lock;
 	atomic_set(&phys_enc->pending_retire_fence_cnt, 0);
 	atomic_set(&phys_enc->wbirq_refcount, 0);
+	init_waitqueue_head(&phys_enc->pending_kickoff_wq);
 
 	irq = &phys_enc->irq[INTR_IDX_WB_DONE];
 	INIT_LIST_HEAD(&irq->cb.list);

msm/sde/sde_hw_wb.h

@@ -16,7 +16,6 @@ struct sde_hw_wb;
 struct sde_hw_wb_cfg {
 	struct sde_hw_fmt_layout dest;
 	enum sde_intf_mode intf_mode;
-	struct traffic_shaper_cfg ts_cfg;
 	struct sde_rect roi;
 	bool is_secure;
 };