video: driver: fix error handling sequence in msm_vidc_open

Currently msm_vidc_close is called directly for any failure
in msm_vidc_open(at any level). So i.e leading to free without
alloc & deinit without init problems.

Example:
if msm_vdec_inst_init is failed, msm_vidc_event_queue_deinit
will be called as part of msm_vidc_close and leading to crash
at v4l2_fh_del.

v4l2_fh_del should be called only if v4l2_fh_add is successful.

Change-Id: I5dd063c19327b881f533aa141f77b23ee7fad125
Signed-off-by: Govindaraj Rajagopal <grajagop@codeaurora.org>

driver/vidc/inc/msm_vidc_inst.h

@@ -138,6 +138,7 @@ struct msm_vidc_inst {
 	struct completion                  completions[MAX_SIGNAL];
 	bool                               active;
 	u64                                last_qbuf_time_ns;
+	bool                               vb2q_init;
 };
 
 #endif // _MSM_VIDC_INST_H_

driver/vidc/src/msm_vidc_control.c

@@ -502,7 +502,9 @@ int msm_vidc_ctrl_deinit(struct msm_vidc_inst *inst)
 	}
 	i_vpr_h(inst, "%s(): num ctrls %d\n", __func__, inst->num_ctrls);
 	v4l2_ctrl_handler_free(&inst->ctrl_handler);
+	memset(&inst->ctrl_handler, 0, sizeof(struct v4l2_ctrl_handler));
 	kfree(inst->ctrls);
+	inst->ctrls = NULL;
 
 	return 0;
 }
@@ -548,7 +550,7 @@ int msm_vidc_ctrl_init(struct msm_vidc_inst *inst)
 	if (rc) {
 		i_vpr_e(inst, "control handler init failed, %d\n",
 				inst->ctrl_handler.error);
-		return rc;
+		goto error;
 	}
 
 	for (idx = 0; idx < INST_CAP_MAX; idx++) {
@@ -562,7 +564,8 @@ int msm_vidc_ctrl_init(struct msm_vidc_inst *inst)
 				"%s: invalid ctrl %#x, max allowed %d\n",
 				__func__, capability->cap[idx].v4l2_id,
 				num_ctrls);
-			return -EINVAL;
+			rc = -EINVAL;
+			goto error;
 		}
 		i_vpr_h(inst,
 			"%s: cap idx %d, value %d min %d max %d step_or_mask %#x flags %#x v4l2_id %#x hfi_id %#x\n",
@@ -603,7 +606,8 @@ int msm_vidc_ctrl_init(struct msm_vidc_inst *inst)
 			if (!ctrl_cfg.name) {
 				i_vpr_e(inst, "%s: %#x ctrl name is null\n",
 					__func__, ctrl_cfg.id);
-				return -EINVAL;
+				rc = -EINVAL;
+				goto error;
 			}
 			ctrl = v4l2_ctrl_new_custom(&inst->ctrl_handler,
 					&ctrl_cfg, NULL);
@@ -629,7 +633,8 @@ int msm_vidc_ctrl_init(struct msm_vidc_inst *inst)
 		if (!ctrl) {
 			i_vpr_e(inst, "%s: invalid ctrl %#x\n", __func__,
 				capability->cap[idx].v4l2_id);
-			return -EINVAL;
+			rc = -EINVAL;
+			goto error;
 		}
 
 		rc = inst->ctrl_handler.error;
@@ -638,7 +643,7 @@ int msm_vidc_ctrl_init(struct msm_vidc_inst *inst)
 				"error adding ctrl (%#x) to ctrl handle, %d\n",
 				capability->cap[idx].v4l2_id,
 				inst->ctrl_handler.error);
-			return rc;
+			goto error;
 		}
 
 		/*
@@ -652,6 +657,10 @@ int msm_vidc_ctrl_init(struct msm_vidc_inst *inst)
 	inst->num_ctrls = num_ctrls;
 	i_vpr_h(inst, "%s(): num ctrls %d\n", __func__, inst->num_ctrls);
 
+	return 0;
+error:
+	msm_vidc_ctrl_deinit(inst);
+
 	return rc;
 }
 

driver/vidc/src/msm_vidc_driver.c

@@ -1977,26 +1977,40 @@ int msm_vidc_vb2_queue_init(struct msm_vidc_inst *inst)
 	int rc = 0;
 
 	if (!inst) {
-		d_vpr_e("%s: invalid params\n", __func__);
+		i_vpr_e(inst, "%s: invalid params\n", __func__);
 		return -EINVAL;
 	}
 
+	if (inst->vb2q_init) {
+		i_vpr_h(inst, "%s: vb2q already inited\n", __func__);
+		return 0;
+	}
+
 	rc = vb2q_init(inst, &inst->vb2q[INPUT_PORT], INPUT_MPLANE);
 	if (rc)
-		return rc;
+		goto exit;
 
 	rc = vb2q_init(inst, &inst->vb2q[OUTPUT_PORT], OUTPUT_MPLANE);
 	if (rc)
-		return rc;
+		goto fail_out_vb2q_init;
 
 	rc = vb2q_init(inst, &inst->vb2q[INPUT_META_PORT], INPUT_META_PLANE);
 	if (rc)
-		return rc;
+		goto fail_in_meta_vb2q_init;
 
 	rc = vb2q_init(inst, &inst->vb2q[OUTPUT_META_PORT], OUTPUT_META_PLANE);
 	if (rc)
-		return rc;
+		goto fail_out_meta_vb2q_init;
+	inst->vb2q_init = true;
 
+	return 0;
+fail_out_meta_vb2q_init:
+	vb2_queue_release(&inst->vb2q[INPUT_META_PORT]);
+fail_in_meta_vb2q_init:
+	vb2_queue_release(&inst->vb2q[OUTPUT_PORT]);
+fail_out_vb2q_init:
+	vb2_queue_release(&inst->vb2q[INPUT_PORT]);
+exit:
 	return rc;
 }
 
@@ -2008,10 +2022,16 @@ int msm_vidc_vb2_queue_deinit(struct msm_vidc_inst *inst)
 		d_vpr_e("%s: invalid params\n", __func__);
 		return -EINVAL;
 	}
+	if (!inst->vb2q_init) {
+		i_vpr_h(inst, "%s: vb2q already deinited\n", __func__);
+		return 0;
+	}
+
 	vb2_queue_release(&inst->vb2q[OUTPUT_META_PORT]);
 	vb2_queue_release(&inst->vb2q[INPUT_META_PORT]);
 	vb2_queue_release(&inst->vb2q[OUTPUT_PORT]);
 	vb2_queue_release(&inst->vb2q[INPUT_PORT]);
+	inst->vb2q_init = false;
 
 	return rc;
 }
@@ -2293,10 +2313,6 @@ int msm_vidc_get_inst_capability(struct msm_vidc_inst *inst)
 				sizeof(struct msm_vidc_inst_capability));
 		}
 	}
-	if (!inst->capabilities) {
-		i_vpr_e(inst, "%s: capabilities not found\n", __func__);
-		return -EINVAL;
-	}
 
 	return rc;
 }