[FFmpeg-cvslog] avfilter/vidstab: check bytesPerPixel only for packed formats.

[Gyan Doshi]

ffmpeg | branch: master | Gyan Doshi <gyandoshi at gmail.com> | Sat Dec 23 16:32:25 2017 +0530| [e1e89c0695b430ca1f0f869ac8a2b6b46be9e2fa] | committer: Michael Niedermayer

avfilter/vidstab: check bytesPerPixel only for packed formats.

libvidstab introduced this variable only for packed formats but in
vf_vidstab*.c, it's checked for all inputs. So the filter errors out for YUV422/444P streams.

Fixes #6736.

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e1e89c0695b430ca1f0f869ac8a2b6b46be9e2fa
---

 libavfilter/vf_vidstabdetect.c    | 3 ++-
 libavfilter/vf_vidstabtransform.c | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/libavfilter/vf_vidstabdetect.c b/libavfilter/vf_vidstabdetect.c
index 63a178a0c2..fd7ff3be24 100644
--- a/libavfilter/vf_vidstabdetect.c
+++ b/libavfilter/vf_vidstabdetect.c
@@ -107,10 +107,11 @@ static int config_input(AVFilterLink *inlink)
     VSMotionDetect* md = &(s->md);
     VSFrameInfo fi;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int is_planar = desc->flags & AV_PIX_FMT_FLAG_PLANAR;
 
     vsFrameInfoInit(&fi, inlink->w, inlink->h,
                     ff_av2vs_pixfmt(ctx, inlink->format));
-    if (fi.bytesPerPixel != av_get_bits_per_pixel(desc)/8) {
+    if (!is_planar && fi.bytesPerPixel != av_get_bits_per_pixel(desc)/8) {
         av_log(ctx, AV_LOG_ERROR, "pixel-format error: wrong bits/per/pixel, please report a BUG");
         return AVERROR(EINVAL);
     }
diff --git a/libavfilter/vf_vidstabtransform.c b/libavfilter/vf_vidstabtransform.c
index 11a0e3d512..d1ec1391cb 100644
--- a/libavfilter/vf_vidstabtransform.c
+++ b/libavfilter/vf_vidstabtransform.c
@@ -146,6 +146,7 @@ static int config_input(AVFilterLink *inlink)
     FILE *f;
 
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int is_planar = desc->flags & AV_PIX_FMT_FLAG_PLANAR;
 
     VSTransformData *td = &(tc->td);
 
@@ -161,7 +162,7 @@ static int config_input(AVFilterLink *inlink)
         return AVERROR(EINVAL);
     }
 
-    if (fi_src.bytesPerPixel != av_get_bits_per_pixel(desc)/8 ||
+    if ((!is_planar && fi_src.bytesPerPixel != av_get_bits_per_pixel(desc)/8) ||
         fi_src.log2ChromaW != desc->log2_chroma_w ||
         fi_src.log2ChromaH != desc->log2_chroma_h) {
         av_log(ctx, AV_LOG_ERROR, "pixel-format error: bpp %i<>%i  ",