From 5086924ad4f219ad6498b2cb217a98637b0b55b9 Mon Sep 17 00:00:00 2001 From: Hans Verkuil Date: Mon, 6 Mar 2017 11:23:15 -0300 Subject: [PATCH] [media] vivid: fix try_fmt behavior vivid_try_fmt_vid_cap() called tpg_calc_line_width to calculate the sizeimage value, but that tpg function uses the current format, not the proposed (tried) format. Rewrote this code to calculate this correctly. The vivid_try_fmt_vid_out() code was completely wrong w.r.t. sizeimage, and neither did it take the vdownsampling[] factors into account. Signed-off-by: Hans Verkuil Tested-by: Vincent Abriou Signed-off-by: Mauro Carvalho Chehab --- drivers/media/platform/vivid/vivid-vid-cap.c | 13 +++++++++--- drivers/media/platform/vivid/vivid-vid-out.c | 22 +++++++++++++------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/drivers/media/platform/vivid/vivid-vid-cap.c b/drivers/media/platform/vivid/vivid-vid-cap.c index a18e6fec219b..01419455e545 100644 --- a/drivers/media/platform/vivid/vivid-vid-cap.c +++ b/drivers/media/platform/vivid/vivid-vid-cap.c @@ -616,7 +616,7 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv, /* This driver supports custom bytesperline values */ mp->num_planes = fmt->buffers; - for (p = 0; p < mp->num_planes; p++) { + for (p = 0; p < fmt->buffers; p++) { /* Calculate the minimum supported bytesperline value */ bytesperline = (mp->width * fmt->bit_depth[p]) >> 3; /* Calculate the maximum supported bytesperline value */ @@ -626,10 +626,17 @@ int vivid_try_fmt_vid_cap(struct file *file, void *priv, pfmt[p].bytesperline = max_bpl; if (pfmt[p].bytesperline < bytesperline) pfmt[p].bytesperline = bytesperline; - pfmt[p].sizeimage = tpg_calc_line_width(&dev->tpg, p, pfmt[p].bytesperline) * - mp->height + fmt->data_offset[p]; + + pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) / + fmt->vdownsampling[p] + fmt->data_offset[p]; + memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved)); } + for (p = fmt->buffers; p < fmt->planes; p++) + pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height * + (fmt->bit_depth[p] / fmt->vdownsampling[p])) / + (fmt->bit_depth[0] / fmt->vdownsampling[0]); + mp->colorspace = vivid_colorspace_cap(dev); if (fmt->color_enc == TGP_COLOR_ENC_HSV) mp->hsv_enc = vivid_hsv_enc_cap(dev); diff --git a/drivers/media/platform/vivid/vivid-vid-out.c b/drivers/media/platform/vivid/vivid-vid-out.c index 1a3373060954..0b1b6218ede8 100644 --- a/drivers/media/platform/vivid/vivid-vid-out.c +++ b/drivers/media/platform/vivid/vivid-vid-out.c @@ -390,22 +390,28 @@ int vivid_try_fmt_vid_out(struct file *file, void *priv, /* This driver supports custom bytesperline values */ - /* Calculate the minimum supported bytesperline value */ - bytesperline = (mp->width * fmt->bit_depth[0]) >> 3; - /* Calculate the maximum supported bytesperline value */ - max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[0]) >> 3; mp->num_planes = fmt->buffers; - for (p = 0; p < mp->num_planes; p++) { + for (p = 0; p < fmt->buffers; p++) { + /* Calculate the minimum supported bytesperline value */ + bytesperline = (mp->width * fmt->bit_depth[p]) >> 3; + /* Calculate the maximum supported bytesperline value */ + max_bpl = (MAX_ZOOM * MAX_WIDTH * fmt->bit_depth[p]) >> 3; + if (pfmt[p].bytesperline > max_bpl) pfmt[p].bytesperline = max_bpl; if (pfmt[p].bytesperline < bytesperline) pfmt[p].bytesperline = bytesperline; - pfmt[p].sizeimage = pfmt[p].bytesperline * mp->height; + + pfmt[p].sizeimage = (pfmt[p].bytesperline * mp->height) / + fmt->vdownsampling[p]; + memset(pfmt[p].reserved, 0, sizeof(pfmt[p].reserved)); } for (p = fmt->buffers; p < fmt->planes; p++) - pfmt[0].sizeimage += (pfmt[0].bytesperline * fmt->bit_depth[p]) / - (fmt->bit_depth[0] * fmt->vdownsampling[p]); + pfmt[0].sizeimage += (pfmt[0].bytesperline * mp->height * + (fmt->bit_depth[p] / fmt->vdownsampling[p])) / + (fmt->bit_depth[0] / fmt->vdownsampling[0]); + mp->xfer_func = V4L2_XFER_FUNC_DEFAULT; mp->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT; mp->quantization = V4L2_QUANTIZATION_DEFAULT; -- 2.20.1