bttv: Width must be a multiple of 16 when capturing planar formats

commit 5c915c68763889f0183a1cc61c84bb228b60124a upstream.

On my bttv card "Hauppauge WinTV [card=10]" capturing in YV12 fmt at max
size results in a solid green rectangle being captured (all colors 0 in
YUV).

This turns out to be caused by max-width (924) not being a multiple of 16.

We've likely never hit this problem before since normally xawtv / tvtime,
etc. will prefer packed pixel formats. But when using a video card which
is using xf86-video-modesetting + glamor, only planar XVideo fmts are
available, and xawtv will chose a matching capture format to avoid needing
to do conversion, triggering the solid green window problem.

Cc: stable@vger.kernel.org
Signed-off-by: Hans de Goede <hdegoede@redhat.com>
Acked-by: Hans Verkuil <hans.verkuil@cisco.com>
Signed-off-by: Mauro Carvalho Chehab <mchehab@osg.samsung.com>
Signed-off-by: Willy Tarreau <w@1wt.eu>

diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c
index e2633d9270b7514d9d0ba1d8ed27cb73f5e42724..aa4519e5cafa985f6c02e44322a6868c710a22b3 100644 (file)
--- a/drivers/media/pci/bt8xx/bttv-driver.c
+++ b/drivers/media/pci/bt8xx/bttv-driver.c
@@ -2376,6 +2376,19 @@ static int bttv_g_fmt_vid_overlay(struct file *file, void *priv,
        return 0;
 }
 
+static void bttv_get_width_mask_vid_cap(const struct bttv_format *fmt,
+                                       unsigned int *width_mask,
+                                       unsigned int *width_bias)
+{
+       if (fmt->flags & FORMAT_FLAGS_PLANAR) {
+               *width_mask = ~15; /* width must be a multiple of 16 pixels */
+               *width_bias = 8;   /* nearest */
+       } else {
+               *width_mask = ~3; /* width must be a multiple of 4 pixels */
+               *width_bias = 2;  /* nearest */
+       }
+}
+
 static int bttv_try_fmt_vid_cap(struct file *file, void *priv,
                                                struct v4l2_format *f)
 {
@@ -2385,6 +2398,7 @@ static int bttv_try_fmt_vid_cap(struct file *file, void *priv,
        enum v4l2_field field;
        __s32 width, height;
        __s32 height2;
+       unsigned int width_mask, width_bias;
        int rc;
 
        fmt = format_by_fourcc(f->fmt.pix.pixelformat);
@@ -2417,9 +2431,9 @@ static int bttv_try_fmt_vid_cap(struct file *file, void *priv,
        width = f->fmt.pix.width;
        height = f->fmt.pix.height;
 
+       bttv_get_width_mask_vid_cap(fmt, &width_mask, &width_bias);
        rc = limit_scaled_size_lock(fh, &width, &height, field,
-                              /* width_mask: 4 pixels */ ~3,
-                              /* width_bias: nearest */ 2,
+                              width_mask, width_bias,
                               /* adjust_size */ 1,
                               /* adjust_crop */ 0);
        if (0 != rc)
@@ -2452,6 +2466,7 @@ static int bttv_s_fmt_vid_cap(struct file *file, void *priv,
        struct bttv_fh *fh = priv;
        struct bttv *btv = fh->btv;
        __s32 width, height;
+       unsigned int width_mask, width_bias;
        enum v4l2_field field;
 
        retval = bttv_switch_type(fh, f->type);
@@ -2466,9 +2481,10 @@ static int bttv_s_fmt_vid_cap(struct file *file, void *priv,
        height = f->fmt.pix.height;
        field = f->fmt.pix.field;
 
+       fmt = format_by_fourcc(f->fmt.pix.pixelformat);
+       bttv_get_width_mask_vid_cap(fmt, &width_mask, &width_bias);
        retval = limit_scaled_size_lock(fh, &width, &height, f->fmt.pix.field,
-                              /* width_mask: 4 pixels */ ~3,
-                              /* width_bias: nearest */ 2,
+                              width_mask, width_bias,
                               /* adjust_size */ 1,
                               /* adjust_crop */ 1);
        if (0 != retval)
@@ -2476,8 +2492,6 @@ static int bttv_s_fmt_vid_cap(struct file *file, void *priv,
 
        f->fmt.pix.field = field;
 
-       fmt = format_by_fourcc(f->fmt.pix.pixelformat);
-
        /* update our state informations */
        fh->fmt              = fmt;
        fh->cap.field        = f->fmt.pix.field;