video: driver: reject 7th VP9 decoder session

reject 7th VP9 decoder session.

Change-Id: I2b739be0aff5492bd10d00c3068aa44e0895ee09
Signed-off-by: Akshata Sahukar <[email protected]>

driver/vidc/inc/msm_vidc_internal.h

@@ -50,6 +50,7 @@
 #define BIT_DEPTH_10 (10 << 16 | 10)
 #define CODED_FRAMES_PROGRESSIVE 0x0
 #define CODED_FRAMES_INTERLACE 0x1
+#define MAX_VP9D_INST_COUNT     6
 /* TODO: move below macros to waipio.c */
 #define MAX_ENH_LAYER_HB        3
 #define MAX_HEVC_ENH_LAYER_SLIDING_WINDOW     5

driver/vidc/src/msm_vdec.c

@@ -91,6 +91,7 @@ static int msm_vdec_codec_change(struct msm_vidc_inst *inst, u32 v4l2_codec)
 		v4l2_pixelfmt_name(v4l2_codec));
 
 	inst->codec = v4l2_codec_to_driver(v4l2_codec, __func__);
+	inst->fmts[INPUT_PORT].fmt.pix_mp.pixelformat = v4l2_codec;
 	rc = msm_vidc_update_debug_str(inst);
 	if (rc)
 		goto exit;
@@ -2199,6 +2200,25 @@ int msm_vdec_try_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 	return rc;
 }
 
+static bool msm_vidc_check_max_sessions_vp9d(struct msm_vidc_core *core)
+{
+	u32 vp9d_instance_count = 0;
+	struct msm_vidc_inst *inst = NULL;
+
+	core_lock(core, __func__);
+	list_for_each_entry(inst, &core->instances, list) {
+		if (is_decode_session(inst) &&
+			inst->fmts[INPUT_PORT].fmt.pix_mp.pixelformat ==
+				V4L2_PIX_FMT_VP9)
+			vp9d_instance_count++;
+	}
+	core_unlock(core, __func__);
+
+	if (vp9d_instance_count > MAX_VP9D_INST_COUNT)
+		return true;
+	return false;
+}
+
 int msm_vdec_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 {
 	int rc = 0;
@@ -2219,6 +2239,17 @@ int msm_vdec_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 			if (rc)
 				goto err_invalid_fmt;
 		}
+
+		if (f->fmt.pix_mp.pixelformat == V4L2_PIX_FMT_VP9) {
+			if (msm_vidc_check_max_sessions_vp9d(inst->core)) {
+				i_vpr_e(inst,
+					"%s: vp9d sessions exceeded max limit %d\n",
+					__func__, MAX_VP9D_INST_COUNT);
+				rc = -ENOMEM;
+				goto err_invalid_fmt;
+			}
+		}
+
 		fmt = &inst->fmts[INPUT_PORT];
 		fmt->type = INPUT_MPLANE;
 
@@ -2226,7 +2257,6 @@ int msm_vdec_s_fmt(struct msm_vidc_inst *inst, struct v4l2_format *f)
 			V4L2_PIX_FMT_HEVC ? 32 : 16;
 		fmt->fmt.pix_mp.width = ALIGN(f->fmt.pix_mp.width, codec_align);
 		fmt->fmt.pix_mp.height = ALIGN(f->fmt.pix_mp.height, codec_align);
-		fmt->fmt.pix_mp.pixelformat = f->fmt.pix_mp.pixelformat;
 		fmt->fmt.pix_mp.num_planes = 1;
 		fmt->fmt.pix_mp.plane_fmt[0].bytesperline = 0;
 		fmt->fmt.pix_mp.plane_fmt[0].sizeimage = call_session_op(core,

driver/vidc/src/msm_venc.c

@@ -72,6 +72,7 @@ static int msm_venc_codec_change(struct msm_vidc_inst *inst, u32 v4l2_codec)
 		v4l2_pixelfmt_name(v4l2_codec));
 
 	inst->codec = v4l2_codec_to_driver(v4l2_codec, __func__);
+	inst->fmts[OUTPUT_PORT].fmt.pix_mp.pixelformat = v4l2_codec;
 	rc = msm_vidc_update_debug_str(inst);
 	if (rc)
 		goto exit;
@@ -1131,7 +1132,6 @@ int msm_venc_s_fmt_output(struct msm_vidc_inst *inst, struct v4l2_format *f)
 	/* use grid dimension for image session */
 	if (is_image_session(inst))
 		fmt->fmt.pix_mp.width = fmt->fmt.pix_mp.height = HEIC_GRID_DIMENSION;
-	fmt->fmt.pix_mp.pixelformat = f->fmt.pix_mp.pixelformat;
 	fmt->fmt.pix_mp.num_planes = 1;
 	fmt->fmt.pix_mp.plane_fmt[0].bytesperline = 0;
 	fmt->fmt.pix_mp.plane_fmt[0].sizeimage = call_session_op(core,