msm: camera: isp: Extend internal recovery scheme

This change extends internal scheme for certain specific
scenarios. If the mode switch is applied in between exposures
it could potentially lead to out_of_sync/hang. On the switching
frame if there is no sufficient common blanking among different
exposures, CSID HW flags it, internal recovery is triggered for this.
Also it's possible that due to packet delays, IFE & sensor go out of sync,
we can try internal recovery in this case as well on receiving
out of sync error consecutively for 3 frames.

CRs-Fixed: 3254775
Change-Id: I56b40bb9f73959e66174af382025c897c18ffed4
Signed-off-by: Karthik Anantha Ram <[email protected]>

drivers/cam_isp/cam_isp_context.c

@@ -7481,8 +7481,8 @@ static int __cam_isp_ctx_reset_and_recover(
 	__cam_isp_ctx_notify_v4l2_error_event(CAM_REQ_MGR_WARN_TYPE_KMD_RECOVERY,
 		0, req->request_id, ctx);
 
-	CAM_DBG(CAM_ISP, "Internal Start HW success ctx %u on link: 0x%x",
-		ctx->ctx_id, ctx->link_hdl);
+	CAM_INFO(CAM_ISP, "Internal Start HW success ctx %u on link: 0x%x for req: %llu",
+		ctx->ctx_id, ctx->link_hdl, req->request_id);
 
 end:
 	return rc;

drivers/cam_isp/isp_hw_mgr/cam_ife_hw_mgr.c

@@ -42,10 +42,20 @@
 #define CAM_ISP_GENERIC_BLOB_TYPE_MAX               \
 	(CAM_ISP_GENERIC_BLOB_TYPE_CSID_QCFA_CONFIG + 1)
 
-#define MAX_RETRY_ATTEMPTS 1
+#define MAX_INTERNAL_RECOVERY_ATTEMPTS    1
 
-#define CAM_ISP_CSID_ERROR_CAN_RECOVERY             \
-	CAM_ISP_HW_ERROR_RECOVERY_OVERFLOW
+#define CAM_ISP_NON_RECOVERABLE_CSID_ERRORS          \
+	(CAM_ISP_HW_ERROR_CSID_LANE_FIFO_OVERFLOW    |   \
+	 CAM_ISP_HW_ERROR_CSID_PKT_HDR_CORRUPTED     |   \
+	 CAM_ISP_HW_ERROR_CSID_MISSING_PKT_HDR_DATA  |   \
+	 CAM_ISP_HW_ERROR_CSID_FATAL                 |   \
+	 CAM_ISP_HW_ERROR_CSID_UNBOUNDED_FRAME       |   \
+	 CAM_ISP_HW_ERROR_CSID_MISSING_EOT           |   \
+	 CAM_ISP_HW_ERROR_CSID_PKT_PAYLOAD_CORRUPTED)
+
+#define CAM_ISP_RECOVERABLE_CSID_ERRORS              \
+	(CAM_ISP_HW_ERROR_CSID_SENSOR_SWITCH_ERROR   |   \
+	 CAM_ISP_HW_ERROR_CSID_SENSOR_FRAME_DROP)
 
 static uint32_t blob_type_hw_cmd_map[CAM_ISP_GENERIC_BLOB_TYPE_MAX] = {
 	CAM_ISP_HW_CMD_GET_HFR_UPDATE,
@@ -12892,24 +12902,25 @@ end:
 	return 0;
 }
 
-static int cam_ife_hw_mgr_handle_csid_frame_drop(
+static int cam_ife_hw_mgr_handle_csid_secondary_err_evts(
+	uint32_t                              err_type,
 	struct cam_isp_hw_event_info         *event_info,
 	struct cam_ife_hw_mgr_ctx            *ctx)
 {
-	int rc = 0;
+	int rc = -EINVAL;
+	struct cam_isp_hw_secondary_event_data sec_evt_data = {0};
 	cam_hw_event_cb_func ife_hw_irq_cb = ctx->common.event_cb;
 
 	/*
 	 * Support frame drop as secondary event
 	 */
-	if (event_info->is_secondary_evt) {
-		struct cam_isp_hw_secondary_event_data sec_evt_data;
+	if (err_type & CAM_ISP_HW_ERROR_CSID_SENSOR_FRAME_DROP) {
+		sec_evt_data.evt_type = CAM_ISP_HW_SEC_EVENT_OUT_OF_SYNC_FRAME_DROP;
 
 		CAM_DBG(CAM_ISP,
-			"Received CSID[%u] sensor sync frame drop res: %d as secondary evt",
-			event_info->hw_idx, event_info->res_id);
+			"Received CSID[%u] sensor sync frame drop res: %d as secondary evt on ctx: %u",
+			event_info->hw_idx, event_info->res_id, ctx->ctx_index);
 
-		sec_evt_data.evt_type = CAM_ISP_HW_SEC_EVENT_OUT_OF_SYNC_FRAME_DROP;
 		rc = ife_hw_irq_cb(ctx->common.cb_priv,
 			CAM_ISP_HW_SECONDARY_EVENT, (void *)&sec_evt_data);
 	}
@@ -12922,6 +12933,7 @@ static int cam_ife_hw_mgr_handle_csid_error(
 	struct cam_isp_hw_event_info   *event_info)
 {
 	int                                      rc = -EINVAL;
+	bool                                     recoverable = true;
 	uint32_t                                 err_type;
 	struct cam_isp_hw_error_event_info      *err_evt_info;
 	struct cam_isp_hw_error_event_data       error_event_data = {0};
@@ -12940,72 +12952,68 @@ static int cam_ife_hw_mgr_handle_csid_error(
 	CAM_DBG(CAM_ISP, "Entry CSID[%u] error %d", event_info->hw_idx, err_type);
 
 	spin_lock(&g_ife_hw_mgr.ctx_lock);
-	if (err_type & CAM_ISP_HW_ERROR_CSID_SENSOR_FRAME_DROP)
-		cam_ife_hw_mgr_handle_csid_frame_drop(event_info, ctx);
 
+	/* Secondary event handling */
+	if (event_info->is_secondary_evt) {
+		rc = cam_ife_hw_mgr_handle_csid_secondary_err_evts(err_type, event_info, ctx);
+		if (rc)
+			CAM_ERR(CAM_ISP,
+				"Failed to handle CSID[%u] sec event for res: %d err: 0x%x on ctx: %u",
+				event_info->hw_idx, event_info->res_id, err_type, ctx->ctx_index);
+		spin_unlock(&g_ife_hw_mgr.ctx_lock);
+		return rc;
+	}
+
+	/* Default error types */
 	recovery_data.error_type = CAM_ISP_HW_ERROR_OVERFLOW;
+	error_event_data.error_type = CAM_ISP_HW_ERROR_CSID_FATAL;
 
-	if ((err_type & (CAM_ISP_HW_ERROR_CSID_LANE_FIFO_OVERFLOW |
-		CAM_ISP_HW_ERROR_CSID_PKT_HDR_CORRUPTED |
-		CAM_ISP_HW_ERROR_CSID_MISSING_PKT_HDR_DATA |
-		CAM_ISP_HW_ERROR_CSID_SENSOR_SWITCH_ERROR |
-		CAM_ISP_HW_ERROR_CSID_FATAL |
-		CAM_ISP_HW_ERROR_CSID_UNBOUNDED_FRAME |
-		CAM_ISP_HW_ERROR_CSID_MISSING_EOT |
-		CAM_ISP_HW_ERROR_CSID_PKT_PAYLOAD_CORRUPTED)) &&
-		g_ife_hw_mgr.debug_cfg.enable_csid_recovery) {
+	/* Notify IFE/SFE devices, determine bus overflow */
+	if (err_type & (CAM_ISP_HW_ERROR_CSID_OUTPUT_FIFO_OVERFLOW |
+		CAM_ISP_HW_ERROR_RECOVERY_OVERFLOW |
+		CAM_ISP_HW_ERROR_CSID_FRAME_SIZE))
+		cam_ife_hw_mgr_check_and_notify_overflow(event_info,
+			ctx, &is_bus_overflow);
 
+	if (err_type & CAM_ISP_NON_RECOVERABLE_CSID_ERRORS) {
 		error_event_data.error_type |= err_type;
 		recovery_data.error_type = err_type;
-		rc = cam_ife_hw_mgr_find_affected_ctx(&error_event_data,
-			event_info->hw_idx, &recovery_data);
-		goto end;
+		recoverable = false;
 	}
 
-	if (err_type & (CAM_ISP_HW_ERROR_CSID_OUTPUT_FIFO_OVERFLOW |
-		CAM_ISP_HW_ERROR_RECOVERY_OVERFLOW |
-		CAM_ISP_HW_ERROR_CSID_FRAME_SIZE)) {
+	if (recoverable && (is_bus_overflow ||
+		(err_type & CAM_ISP_RECOVERABLE_CSID_ERRORS))) {
+		if (ctx->try_recovery_cnt < MAX_INTERNAL_RECOVERY_ATTEMPTS) {
+			error_event_data.try_internal_recovery = true;
 
-		cam_ife_hw_mgr_check_and_notify_overflow(event_info,
-			ctx, &is_bus_overflow);
+			if (!atomic_read(&ctx->overflow_pending))
+				ctx->try_recovery_cnt++;
 
-		/*
-		 * When CSID overflow IRQ comes, we need read bus overflow
-		 * status, to check if it's a bus overflow issue,
-		 * only do recovery in bus overflow cases.
-		 */
-		if ((err_type & CAM_ISP_CSID_ERROR_CAN_RECOVERY) &&
-			is_bus_overflow) {
-			if (ctx->try_recovery_cnt < MAX_RETRY_ATTEMPTS) {
-				error_event_data.try_internal_recovery = true;
-				if (!atomic_read(&ctx->overflow_pending))
-					ctx->try_recovery_cnt++;
-				if (!ctx->recovery_req_id)
-					ctx->recovery_req_id = ctx->applied_req_id;
-			}
-			CAM_DBG(CAM_ISP, "CSID[%u] Try recovery count %u on req %llu",
-				event_info->hw_idx,
-				ctx->try_recovery_cnt,
-				ctx->recovery_req_id);
+			if (!ctx->recovery_req_id)
+				ctx->recovery_req_id = ctx->applied_req_id;
 		}
 
+		CAM_DBG(CAM_ISP,
+			"CSID[%u] error: %u current_recovery_cnt: %u  recovery_req: %llu on ctx: %u",
+			event_info->hw_idx, err_type, ctx->try_recovery_cnt,
+			ctx->recovery_req_id, ctx->ctx_index);
+
 		error_event_data.error_type |= err_type;
 		recovery_data.error_type = err_type;
-		rc = cam_ife_hw_mgr_find_affected_ctx(&error_event_data,
-			event_info->hw_idx, &recovery_data);
 	}
 
-end:
-
+	rc = cam_ife_hw_mgr_find_affected_ctx(&error_event_data,
+			event_info->hw_idx, &recovery_data);
 	if (rc || !recovery_data.no_of_context)
-		goto skip_recovery;
+		goto end;
 
 	if (!error_event_data.try_internal_recovery)
 		cam_ife_hw_mgr_do_error_recovery(&recovery_data);
-	CAM_DBG(CAM_ISP, "Exit CSID[%u] error %d", event_info->hw_idx,
-		err_type);
 
-skip_recovery:
+	CAM_DBG(CAM_ISP, "Exit CSID[%u] error %d",
+		event_info->hw_idx, err_type);
+
+end:
 	spin_unlock(&g_ife_hw_mgr.ctx_lock);
 	return 0;
 }

drivers/cam_isp/isp_hw_mgr/isp_hw/ife_csid_hw/cam_ife_csid_hw_ver2.c

@@ -52,6 +52,9 @@
 /* Max CSI Rx irq error count threshold value */
 #define CAM_IFE_CSID_MAX_IRQ_ERROR_COUNT               100
 
+/* Max sensor switch out of sync threshold */
+#define CAM_IFE_CSID_MAX_OUT_OF_SYNC_ERR_COUNT         3
+
 #define CAM_CSID_IRQ_CTRL_NAME_LEN                     10
 
 static void cam_ife_csid_ver2_print_debug_reg_status(
@@ -1742,6 +1745,7 @@ static int cam_ife_csid_ver2_ipp_bottom_half(
 	uint32_t                                      err_type = 0;
 	void    __iomem                              *base;
 	int                                           rc = 0;
+	bool                                          out_of_sync_fatal = false;
 
 	if (!handler_priv || !evt_payload_priv) {
 		CAM_ERR(CAM_ISP, "Invalid params. evt_payload_priv: %s, handler_priv: %s",
@@ -1801,6 +1805,14 @@ static int cam_ife_csid_ver2_ipp_bottom_half(
 	if (irq_status_ipp & path_reg->epoch0_irq_mask)
 		csid_hw->event_cb(csid_hw->token, CAM_ISP_HW_EVENT_EPOCH, (void *)&evt_info);
 
+	if (irq_status_ipp & IFE_CSID_VER2_PATH_SENSOR_SWITCH_OUT_OF_SYNC_FRAME_DROP) {
+		atomic_inc(&path_cfg->switch_out_of_sync_cnt);
+		/* If threshold is seen, notify error */
+		if (atomic_read(&path_cfg->switch_out_of_sync_cnt) >=
+			CAM_IFE_CSID_MAX_OUT_OF_SYNC_ERR_COUNT)
+			out_of_sync_fatal = true;
+	}
+
 	err_mask = path_reg->fatal_err_mask | path_reg->non_fatal_err_mask;
 	spin_lock(&csid_hw->lock_state);
 	if (csid_hw->hw_info->hw_state != CAM_HW_STATE_POWER_UP) {
@@ -1810,16 +1822,17 @@ static int cam_ife_csid_ver2_ipp_bottom_half(
 	}
 
 	err_type = cam_ife_csid_ver2_parse_path_irq_status(
-		csid_hw, res,
-		CAM_IFE_CSID_IRQ_REG_IPP,
+		csid_hw, res, CAM_IFE_CSID_IRQ_REG_IPP,
 		err_mask, irq_status_ipp, payload);
 
-	if (err_type)
+	if (err_type || out_of_sync_fatal) {
+		if (out_of_sync_fatal)
+			err_type = CAM_ISP_HW_ERROR_CSID_SENSOR_FRAME_DROP;
+
 		cam_ife_csid_ver2_handle_event_err(csid_hw,
-			irq_status_ipp,
-			err_type,
-			false,
-			res);
+			irq_status_ipp, err_type, false, res);
+	}
+
 unlock:
 	spin_unlock(&csid_hw->lock_state);
 end:
@@ -2006,12 +2019,30 @@ static int cam_ife_csid_ver2_rdi_bottom_half(
 	evt_info.reg_val = irq_status_rdi;
 	evt_info.hw_type = CAM_ISP_HW_TYPE_CSID;
 
-	/* Check for specific secondary events */
-	if (path_cfg->sec_evt_config.en_secondary_evt &&
-		((irq_status_rdi & IFE_CSID_VER2_PATH_SENSOR_SWITCH_OUT_OF_SYNC_FRAME_DROP) &&
-		(path_cfg->sec_evt_config.evt_type & CAM_IFE_CSID_EVT_SENSOR_SYNC_FRAME_DROP)))
-		cam_ife_csid_ver2_handle_event_err(csid_hw, irq_status_rdi,
-			CAM_ISP_HW_ERROR_CSID_SENSOR_FRAME_DROP, true, res);
+	if (irq_status_rdi & IFE_CSID_VER2_PATH_SENSOR_SWITCH_OUT_OF_SYNC_FRAME_DROP) {
+		bool is_secondary = true;
+		bool do_notify = false;
+
+		/* Only notify if secondary event is subscribed for */
+		if ((path_cfg->sec_evt_config.en_secondary_evt) &&
+			(path_cfg->sec_evt_config.evt_type &
+			CAM_IFE_CSID_EVT_SENSOR_SYNC_FRAME_DROP))
+			do_notify = true;
+
+		/* Validate error threshold for primary RDI (master) */
+		if (res->is_rdi_primary_res) {
+			atomic_inc(&path_cfg->switch_out_of_sync_cnt);
+			if (atomic_read(&path_cfg->switch_out_of_sync_cnt) >=
+				CAM_IFE_CSID_MAX_OUT_OF_SYNC_ERR_COUNT) {
+				do_notify = true;
+				is_secondary = false;
+			}
+		}
+
+		if (do_notify)
+			cam_ife_csid_ver2_handle_event_err(csid_hw, irq_status_rdi,
+				CAM_ISP_HW_ERROR_CSID_SENSOR_FRAME_DROP, is_secondary, res);
+	}
 
 	if (irq_status_rdi & rdi_reg->eof_irq_mask)
 		csid_hw->event_cb(csid_hw->token, CAM_ISP_HW_EVENT_EOF, (void *)&evt_info);
@@ -2319,6 +2350,7 @@ static int cam_ife_csid_ver2_disable_path(
 	path_cfg->skip_discard_frame_cfg = false;
 	path_cfg->num_frames_discard = 0;
 	path_cfg->sof_cnt = 0;
+	atomic_set(&path_cfg->switch_out_of_sync_cnt, 0);
 	return rc;
 }
 

drivers/cam_isp/isp_hw_mgr/isp_hw/ife_csid_hw/cam_ife_csid_hw_ver2.h

@@ -162,6 +162,7 @@ struct cam_ife_csid_ver2_camif_data {
  * @sof_cnt:                SOF counter
  * @num_frames_discard:     number of frames to discard
  * @epoch_cfg:              Epoch configured value
+ * @switch_out_of_sync_cnt: Sensor out of sync error cnt
  * @sync_mode   :           Sync mode--> master/slave/none
  * @vfr_en   :              flag to indicate if variable frame rate is enabled
  * @frame_id_dec_en:        flag to indicate if frame id decoding is enabled
@@ -205,6 +206,7 @@ struct cam_ife_csid_ver2_path_cfg {
 	uint32_t                             sof_cnt;
 	uint32_t                             num_frames_discard;
 	uint32_t                             epoch_cfg;
+	atomic_t                             switch_out_of_sync_cnt;
 	enum cam_isp_hw_sync_mode            sync_mode;
 	bool                                 vfr_en;
 	bool                                 frame_id_dec_en;