msm: camera: cdm: Add CDM hang detect and debug registers dump support

Add support to dump the cdm core debug registers and CDM hang detect
support for better debugging purpose in case of cdm timeout at config
ife. Add a debugfs for CDM command buffer dump for cdm hang events.
Fix the possible NULL derefernce while dumping the cdm registers.
Turn on debugfs using following command in adb shell:
echo 1 >> /sys/kernel/debug/camera_isp_ctx/enable_cdm_cmd_buffer_dump.

CRs-Fixed: 2748715, 2782720, 2770565
Change-Id: Ibb9aa1d232d742ca1b6e64c16e9718bfc0bc8624
Signed-off-by: Jigar Agrawal <[email protected]>

drivers/cam_cdm/cam_cdm.h

@@ -378,6 +378,7 @@ enum cam_cdm_hw_process_intf_cmd {
 	CAM_CDM_HW_INTF_CMD_FLUSH_HW,
 	CAM_CDM_HW_INTF_CMD_HANDLE_ERROR,
 	CAM_CDM_HW_INTF_CMD_HANG_DETECT,
+	CAM_CDM_HW_INTF_DUMP_DBG_REGS,
 	CAM_CDM_HW_INTF_CMD_INVALID,
 };
 

drivers/cam_cdm/cam_cdm_core_common.c

@@ -822,6 +822,22 @@ int cam_cdm_process_cmd(void *hw_priv,
 		mutex_unlock(&cdm_hw->hw_mutex);
 		break;
 	}
+	case CAM_CDM_HW_INTF_DUMP_DBG_REGS:
+	{
+		uint32_t *handle = cmd_args;
+
+		if (sizeof(uint32_t) != arg_size) {
+			CAM_ERR(CAM_CDM,
+				"Invalid CDM cmd %d size=%x for handle=0x%x",
+				cmd, arg_size, *handle);
+				return -EINVAL;
+		}
+
+		mutex_lock(&cdm_hw->hw_mutex);
+		cam_hw_cdm_dump_core_debug_registers(cdm_hw, true);
+		mutex_unlock(&cdm_hw->hw_mutex);
+		break;
+	}
 	default:
 		CAM_ERR(CAM_CDM, "CDM HW intf command not valid =%d", cmd);
 		break;

drivers/cam_cdm/cam_cdm_core_common.h

@@ -56,5 +56,7 @@ struct cam_cdm_bl_cb_request_entry *cam_cdm_find_request_by_bl_tag(
 	uint32_t tag, struct list_head *bl_list);
 void cam_cdm_notify_clients(struct cam_hw_info *cdm_hw,
 	enum cam_cdm_cb_status status, void *data);
+void cam_hw_cdm_dump_core_debug_registers(
+	struct cam_hw_info *cdm_hw, bool pause_core);
 
 #endif /* _CAM_CDM_CORE_COMMON_H_ */

drivers/cam_cdm/cam_cdm_hw_core.c

@@ -169,14 +169,14 @@ static int cam_hw_cdm_pause_core(struct cam_hw_info *cdm_hw, bool pause)
 	return rc;
 }
 
-int cam_hw_cdm_enable_core_dbg(struct cam_hw_info *cdm_hw)
+int cam_hw_cdm_enable_core_dbg(struct cam_hw_info *cdm_hw, uint32_t value)
 {
 	int rc = 0;
 	struct cam_cdm *core = (struct cam_cdm *)cdm_hw->core_info;
 
 	if (cam_cdm_write_hw_reg(cdm_hw,
 			core->offsets->cmn_reg->core_debug,
-			0x10100)) {
+			value)) {
 		CAM_ERR(CAM_CDM, "Failed to Write CDM HW core debug");
 		rc = -EIO;
 	}
@@ -263,24 +263,7 @@ end:
 	return rc;
 }
 
-int cam_hw_cdm_enable_core_dbg_per_fifo(
-		struct cam_hw_info *cdm_hw,
-		uint32_t            fifo_idx)
-{
-	int rc = 0;
-	struct cam_cdm *core = (struct cam_cdm *)cdm_hw->core_info;
-
-	if (cam_cdm_write_hw_reg(cdm_hw,
-			core->offsets->cmn_reg->core_debug,
-			(0x10100 | fifo_idx << 20))) {
-		CAM_ERR(CAM_CDM, "Failed to Write CDM HW core debug");
-		rc = -EIO;
-	}
-
-	return rc;
-}
-
-void cam_hw_cdm_dump_bl_fifo_data(struct cam_hw_info *cdm_hw)
+static void cam_hw_cdm_dump_bl_fifo_data(struct cam_hw_info *cdm_hw)
 {
 	struct cam_cdm *core = (struct cam_cdm *)cdm_hw->core_info;
 	int i, j;
@@ -289,13 +272,7 @@ void cam_hw_cdm_dump_bl_fifo_data(struct cam_hw_info *cdm_hw)
 	for (i = 0; i < core->offsets->reg_data->num_bl_fifo; i++) {
 		cam_hw_cdm_bl_fifo_pending_bl_rb_in_fifo(cdm_hw,
 			i, &num_pending_req);
-
-		if (cam_hw_cdm_enable_core_dbg_per_fifo(cdm_hw, i)) {
-			CAM_ERR(CAM_CDM,
-				"Problem in selecting the fifo for readback");
-			continue;
-		}
-		for (j = 0 ; j < num_pending_req ; j++) {
+		for (j = 0; j < num_pending_req ; j++) {
 			cam_cdm_write_hw_reg(cdm_hw,
 				core->offsets->cmn_reg->bl_fifo_rb, j);
 			cam_cdm_read_hw_reg(cdm_hw,
@@ -316,83 +293,111 @@ void cam_hw_cdm_dump_bl_fifo_data(struct cam_hw_info *cdm_hw)
 	}
 }
 
-void cam_hw_cdm_dump_core_debug_registers(
-	struct cam_hw_info *cdm_hw)
+void cam_hw_cdm_dump_core_debug_registers(struct cam_hw_info *cdm_hw,
+	bool pause_core)
 {
-	uint32_t dump_reg, dump_reg1, dump_reg2, core_dbg;
+	uint32_t dump_reg, core_dbg = 0x100;
 	int i;
+	bool is_core_paused_already;
 	struct cam_cdm *core = (struct cam_cdm *)cdm_hw->core_info;
+	const struct cam_cdm_icl_regs *inv_cmd_log =
+		core->offsets->cmn_reg->icl_reg;
 
 	cam_cdm_read_hw_reg(cdm_hw, core->offsets->cmn_reg->core_en, &dump_reg);
-	CAM_INFO(CAM_CDM, "CDM HW core status=%x", dump_reg);
+	CAM_INFO(CAM_CDM, "CDM HW core status=0x%x", dump_reg);
+
+	if (pause_core) {
+		cam_hw_cdm_pause_core(cdm_hw, true);
+		usleep_range(1000, 1010);
+	}
+	cam_hw_cdm_enable_core_dbg(cdm_hw, core_dbg);
 
 	cam_cdm_read_hw_reg(cdm_hw, core->offsets->cmn_reg->usr_data,
 		&dump_reg);
 	CAM_INFO(CAM_CDM, "CDM HW core userdata=0x%x", dump_reg);
 
-	usleep_range(1000, 1010);
-
 	cam_cdm_read_hw_reg(cdm_hw,
 		core->offsets->cmn_reg->debug_status,
 		&dump_reg);
-	CAM_INFO(CAM_CDM, "CDM HW Debug status reg=%x", dump_reg);
-	cam_cdm_read_hw_reg(cdm_hw,
-		core->offsets->cmn_reg->core_debug,
-		&core_dbg);
+	CAM_INFO(CAM_CDM, "CDM HW Debug status reg=0x%x", dump_reg);
+
 	if (core_dbg & 0x100) {
 		cam_cdm_read_hw_reg(cdm_hw,
 			core->offsets->cmn_reg->last_ahb_addr,
 			&dump_reg);
-		CAM_INFO(CAM_CDM, "AHB dump reglastaddr=%x", dump_reg);
+		CAM_INFO(CAM_CDM, "AHB dump reglastaddr=0x%x", dump_reg);
 		cam_cdm_read_hw_reg(cdm_hw,
 			core->offsets->cmn_reg->last_ahb_data,
 			&dump_reg);
-		CAM_INFO(CAM_CDM, "AHB dump reglastdata=%x", dump_reg);
+		CAM_INFO(CAM_CDM, "AHB dump reglastdata=0x%x", dump_reg);
 	} else {
 		CAM_INFO(CAM_CDM, "CDM HW AHB dump not enable");
 	}
 
-	cam_cdm_read_hw_reg(cdm_hw,
-		core->offsets->cmn_reg->icl_reg->data_regs->icl_inv_data,
-		&dump_reg);
-	cam_cdm_read_hw_reg(cdm_hw,
-		core->offsets->cmn_reg->icl_reg->misc_regs->icl_inv_bl_addr,
-		&dump_reg1);
-	cam_cdm_read_hw_reg(cdm_hw,
-		core->offsets->cmn_reg->icl_reg->misc_regs->icl_status,
-		&dump_reg2);
-	CAM_INFO(CAM_CDM,
-		"Last Inv Cmd Log(ICL)Status: 0x%x, Last Inv cmd: 0x%x, Last Inv bl_addr: 0x%x",
-		dump_reg2, dump_reg, dump_reg1);
+	if (inv_cmd_log) {
+		if (inv_cmd_log->misc_regs) {
+			cam_cdm_read_hw_reg(cdm_hw,
+				inv_cmd_log->misc_regs->icl_status,
+				&dump_reg);
+			CAM_INFO(CAM_CDM,
+				"Last Inv Cmd Log(ICL)Status: 0x%x",
+				dump_reg);
+			cam_cdm_read_hw_reg(cdm_hw,
+				inv_cmd_log->misc_regs->icl_inv_bl_addr,
+				&dump_reg);
+			CAM_INFO(CAM_CDM,
+				"Last Inv bl_addr: 0x%x",
+				dump_reg);
+		}
+		if (inv_cmd_log->data_regs) {
+			cam_cdm_read_hw_reg(cdm_hw,
+				inv_cmd_log->data_regs->icl_inv_data,
+				&dump_reg);
+			CAM_INFO(CAM_CDM, "Last Inv cmd: 0x%x", dump_reg);
+		}
+	}
 
-	cam_hw_cdm_dump_bl_fifo_data(cdm_hw);
+	if (core_dbg & 0x10000) {
+		cam_cdm_read_hw_reg(cdm_hw,
+			core->offsets->cmn_reg->core_en, &dump_reg);
+		is_core_paused_already = (bool)(dump_reg & 0x20);
+		if (!is_core_paused_already) {
+			cam_hw_cdm_pause_core(cdm_hw, true);
+			usleep_range(1000, 1010);
+		}
+
+		cam_hw_cdm_dump_bl_fifo_data(cdm_hw);
+
+		if (!is_core_paused_already)
+			cam_hw_cdm_pause_core(cdm_hw, false);
+	}
 
 	CAM_INFO(CAM_CDM, "CDM HW default dump");
 	cam_cdm_read_hw_reg(cdm_hw,
 		core->offsets->cmn_reg->core_cfg, &dump_reg);
-	CAM_INFO(CAM_CDM, "CDM HW core cfg=%x", dump_reg);
+	CAM_INFO(CAM_CDM, "CDM HW core cfg=0x%x", dump_reg);
 
 	for (i = 0; i < core->offsets->reg_data->num_bl_fifo_irq; i++) {
 		cam_cdm_read_hw_reg(cdm_hw,
 			core->offsets->irq_reg[i]->irq_status, &dump_reg);
-		CAM_INFO(CAM_CDM, "CDM HW irq status%d=%x", i, dump_reg);
+		CAM_INFO(CAM_CDM, "CDM HW irq status%d=0x%x", i, dump_reg);
 
 		cam_cdm_read_hw_reg(cdm_hw,
 			core->offsets->irq_reg[i]->irq_set, &dump_reg);
-		CAM_INFO(CAM_CDM, "CDM HW irq set%d=%x", i, dump_reg);
+		CAM_INFO(CAM_CDM, "CDM HW irq set%d=0x%x", i, dump_reg);
 
 		cam_cdm_read_hw_reg(cdm_hw,
 			core->offsets->irq_reg[i]->irq_mask, &dump_reg);
-		CAM_INFO(CAM_CDM, "CDM HW irq mask%d=%x", i, dump_reg);
+		CAM_INFO(CAM_CDM, "CDM HW irq mask%d=0x%x", i, dump_reg);
 
 		cam_cdm_read_hw_reg(cdm_hw,
 			core->offsets->irq_reg[i]->irq_clear, &dump_reg);
-		CAM_INFO(CAM_CDM, "CDM HW irq clear%d=%x", i, dump_reg);
+		CAM_INFO(CAM_CDM, "CDM HW irq clear%d=0x%x", i, dump_reg);
 	}
 
 	cam_cdm_read_hw_reg(cdm_hw,
 		core->offsets->cmn_reg->current_bl_base, &dump_reg);
-	CAM_INFO(CAM_CDM, "CDM HW current BL base=%x", dump_reg);
+	CAM_INFO(CAM_CDM, "CDM HW current BL base=0x%x", dump_reg);
 
 	cam_cdm_read_hw_reg(cdm_hw,
 		core->offsets->cmn_reg->current_bl_len, &dump_reg);
@@ -408,8 +413,11 @@ void cam_hw_cdm_dump_core_debug_registers(
 
 	cam_cdm_read_hw_reg(cdm_hw,
 		core->offsets->cmn_reg->current_used_ahb_base, &dump_reg);
-	CAM_INFO(CAM_CDM, "CDM HW current AHB base=%x", dump_reg);
+	CAM_INFO(CAM_CDM, "CDM HW current AHB base=0x%x", dump_reg);
 
+	cam_hw_cdm_disable_core_dbg(cdm_hw);
+	if (pause_core)
+		cam_hw_cdm_pause_core(cdm_hw, false);
 }
 
 enum cam_cdm_arbitration cam_cdm_get_arbitration_type(
@@ -1209,14 +1217,7 @@ static void cam_hw_cdm_work(struct work_struct *work)
 			for (i = 0; i < core->offsets->reg_data->num_bl_fifo;
 					i++)
 				mutex_lock(&core->bl_fifo[i].fifo_lock);
-			/*
-			 * First pause CDM, If it fails still proceed
-			 * to dump debug info
-			 */
-			cam_hw_cdm_pause_core(cdm_hw, true);
-			cam_hw_cdm_dump_core_debug_registers(cdm_hw);
-			/* Resume CDM back */
-			cam_hw_cdm_pause_core(cdm_hw, false);
+			cam_hw_cdm_dump_core_debug_registers(cdm_hw, true);
 			for (i = 0; i < core->offsets->reg_data->num_bl_fifo;
 					i++)
 				mutex_unlock(&core->bl_fifo[i].fifo_lock);
@@ -1247,14 +1248,7 @@ static void cam_hw_cdm_iommu_fault_handler(struct cam_smmu_pf_info *pf_info)
 		for (i = 0; i < core->offsets->reg_data->num_bl_fifo; i++)
 			mutex_lock(&core->bl_fifo[i].fifo_lock);
 		if (cdm_hw->hw_state == CAM_HW_STATE_POWER_UP) {
-			/*
-			 * First pause CDM, If it fails still proceed
-			 * to dump debug info
-			 */
-			cam_hw_cdm_pause_core(cdm_hw, true);
-			cam_hw_cdm_dump_core_debug_registers(cdm_hw);
-			/* Resume CDM back */
-			cam_hw_cdm_pause_core(cdm_hw, false);
+			cam_hw_cdm_dump_core_debug_registers(cdm_hw, true);
 		} else
 			CAM_INFO(CAM_CDM, "CDM hw is power in off state");
 		for (i = 0; i < core->offsets->reg_data->num_bl_fifo; i++)
@@ -1574,7 +1568,7 @@ int cam_hw_cdm_handle_error_info(
 		current_fifo, current_tag);
 
 	/* dump cdm registers for further debug */
-	cam_hw_cdm_dump_core_debug_registers(cdm_hw);
+	cam_hw_cdm_dump_core_debug_registers(cdm_hw, false);
 
 	for (i = 0; i < cdm_core->offsets->reg_data->num_bl_fifo; i++) {
 		if (!cdm_core->bl_fifo[i].bl_depth)
@@ -1690,8 +1684,8 @@ int cam_hw_cdm_hang_detect(
 	for (i = 0; i < cdm_core->offsets->reg_data->num_bl_fifo; i++)
 		if (cdm_core->bl_fifo[i].work_record) {
 			CAM_WARN(CAM_CDM,
-				"workqueue got delayed, work_record :%u",
-				cdm_core->bl_fifo[i].work_record);
+				"workqueue got delayed, bl_fifo: %d, work_record :%u",
+				i, cdm_core->bl_fifo[i].work_record);
 			rc = 0;
 			break;
 		}

drivers/cam_cdm/cam_cdm_intf.c

@@ -507,6 +507,32 @@ int cam_cdm_detect_hang_error(uint32_t handle)
 }
 EXPORT_SYMBOL(cam_cdm_detect_hang_error);
 
+int cam_cdm_dump_debug_registers(uint32_t handle)
+{
+	uint32_t hw_index;
+	int rc = -EINVAL;
+	struct cam_hw_intf *hw;
+
+	if (get_cdm_mgr_refcount()) {
+		CAM_ERR(CAM_CDM, "CDM intf mgr get refcount failed");
+		rc = -EPERM;
+		return rc;
+	}
+
+	hw_index = CAM_CDM_GET_HW_IDX(handle);
+	if (hw_index < CAM_CDM_INTF_MGR_MAX_SUPPORTED_CDM) {
+		hw = cdm_mgr.nodes[hw_index].device;
+		if (hw && hw->hw_ops.process_cmd)
+			rc = hw->hw_ops.process_cmd(hw->hw_priv,
+				CAM_CDM_HW_INTF_DUMP_DBG_REGS,
+				&handle,
+				sizeof(handle));
+	}
+	put_cdm_mgr_refcount();
+
+	return rc;
+}
+
 int cam_cdm_intf_register_hw_cdm(struct cam_hw_intf *hw,
 	struct cam_cdm_private_dt_data *data, enum cam_cdm_type type,
 	uint32_t *index)

drivers/cam_cdm/cam_cdm_intf_api.h

@@ -290,4 +290,13 @@ struct cam_cdm_utils_ops *cam_cdm_publish_ops(void);
  * @return 0 on success
  */
 int cam_cdm_detect_hang_error(uint32_t handle);
+
+/**
+ * @brief : API to dump the CDM Debug registers
+ *
+ * @handle : Input handle of the CDM to dump the registers
+ *
+ * @return 0 on success
+ */
+int cam_cdm_dump_debug_registers(uint32_t handle);
 #endif /* _CAM_CDM_API_H_ */

drivers/cam_isp/cam_isp_context.c

@@ -5033,9 +5033,11 @@ static int __cam_isp_ctx_start_dev_in_ready(struct cam_context *ctx,
 		/* HW failure. user need to clean up the resource */
 		CAM_ERR(CAM_ISP, "Start HW failed");
 		ctx->state = CAM_CTX_READY;
-		trace_cam_context_state("ISP", ctx);
-		if (rc == -ETIMEDOUT)
+		if ((rc == -ETIMEDOUT) &&
+			(isp_ctx_debug.enable_cdm_cmd_buff_dump))
 			rc = cam_isp_ctx_dump_req(req_isp, 0, 0, NULL, false);
+
+		trace_cam_context_state("ISP", ctx);
 		list_del_init(&req->list);
 		list_add(&req->list, &ctx->pending_req_list);
 		goto end;
@@ -5628,7 +5630,7 @@ static int cam_isp_context_debug_register(void)
 
 	dbgfileptr = debugfs_create_dir("camera_isp_ctx", NULL);
 	if (!dbgfileptr) {
-		CAM_ERR(CAM_ICP,"DebugFS could not create directory!");
+		CAM_ERR(CAM_ISP, "DebugFS could not create directory!");
 		rc = -ENOENT;
 		goto end;
 	}
@@ -5637,9 +5639,11 @@ static int cam_isp_context_debug_register(void)
 
 	dbgfileptr = debugfs_create_u32("enable_state_monitor_dump", 0644,
 		isp_ctx_debug.dentry, &isp_ctx_debug.enable_state_monitor_dump);
+	dbgfileptr = debugfs_create_u8("enable_cdm_cmd_buffer_dump", 0644,
+		isp_ctx_debug.dentry, &isp_ctx_debug.enable_cdm_cmd_buff_dump);
 	if (IS_ERR(dbgfileptr)) {
 		if (PTR_ERR(dbgfileptr) == -ENODEV)
-			CAM_WARN(CAM_ICP, "DebugFS not enabled in kernel!");
+			CAM_WARN(CAM_ISP, "DebugFS not enabled in kernel!");
 		else
 			rc = PTR_ERR(dbgfileptr);
 	}

drivers/cam_isp/cam_isp_context.h

@@ -114,13 +114,15 @@ enum cam_isp_state_change_trigger {
 /**
  * struct cam_isp_ctx_debug -  Contains debug parameters
  *
- * @dentry:                    Debugfs entry
- * @enable_state_monitor_dump: Enable isp state monitor dump
+ * @dentry:                     Debugfs entry
+ * @enable_state_monitor_dump:  Enable isp state monitor dump
+ * @enable_cdm_cmd_buff_dump: Enable CDM Command buffer dump
  *
  */
 struct cam_isp_ctx_debug {
 	struct dentry  *dentry;
 	uint32_t        enable_state_monitor_dump;
+	uint8_t         enable_cdm_cmd_buff_dump;
 };
 
 /**

drivers/cam_isp/isp_hw_mgr/cam_ife_hw_mgr.c

@@ -3852,11 +3852,14 @@ static int cam_ife_mgr_config_hw(void *hw_mgr_priv,
 		if (cfg->init_packet) {
 			rem_jiffies = wait_for_completion_timeout(
 				&ctx->config_done_complete,
-				msecs_to_jiffies(30));
+				msecs_to_jiffies(60));
 			if (rem_jiffies == 0) {
 				CAM_ERR(CAM_ISP,
 					"config done completion timeout for req_id=%llu ctx_index %d",
 					cfg->request_id, ctx->ctx_index);
+				if (cam_cdm_detect_hang_error(ctx->cdm_handle))
+					cam_cdm_dump_debug_registers(
+						ctx->cdm_handle);
 				rc = -ETIMEDOUT;
 			} else
 				CAM_DBG(CAM_ISP,