[FFmpeg-devel] [PATCH V3] lavc/vp8: Fix HWAccel VP8 decoder can't support resolution change.

[Jun Zhao]

V3: refine the code.
V2: fix the V1 lead to webp crash issue.
-------------- next part --------------
From f55a7e26214e066e9bf817ae04ca3d0c2d3e7a6d Mon Sep 17 00:00:00 2001
From: Jun Zhao <mypopydev at gmail.com>
Date: Thu, 30 Nov 2017 07:53:53 +0800
Subject: [PATCH V3] lavc/vp8: Fix HWAccel VP8 decoder can't support resolution
 change.

Use the following command to reproduce this issue:
make fate-vp8-size-change HWACCEL="vaapi -vaapi_device \
/dev/dri/renderD128 -hwaccel_output_format yuv420p"
SAMPLES=../fate-suite/.

At the same time, reconstruct the public logic as a function.

Signed-off-by: Yun Zhou <yunx.z.zhou at intel.com>
Signed-off-by: Jun Zhao <jun.zhao at intel.com>
Tested by: Mark Thompson <sw at jkqxz.net>
---
 libavcodec/vp8.c | 36 ++++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index 471c0bb89e..7f71a75e4b 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -167,6 +167,22 @@ static VP8Frame *vp8_find_free_buffer(VP8Context *s)
     return frame;
 }
 
+static enum AVPixelFormat get_pixel_format(VP8Context *s)
+{
+    enum AVPixelFormat pix_fmts[] = {
+#if CONFIG_VP8_VAAPI_HWACCEL
+        AV_PIX_FMT_VAAPI,
+#endif
+#if CONFIG_VP8_NVDEC_HWACCEL
+        AV_PIX_FMT_CUDA,
+#endif
+        AV_PIX_FMT_YUV420P,
+        AV_PIX_FMT_NONE,
+    };
+
+    return ff_get_format(s->avctx, pix_fmts);
+}
+
 static av_always_inline
 int update_dimensions(VP8Context *s, int width, int height, int is_vp7)
 {
@@ -182,6 +198,13 @@ int update_dimensions(VP8Context *s, int width, int height, int is_vp7)
             return ret;
     }
 
+    if (!s->actually_webp && !is_vp7) {
+        s->pix_fmt = get_pixel_format(s);
+        if (s->pix_fmt < 0)
+            return AVERROR(EINVAL);
+        avctx->pix_fmt = s->pix_fmt;
+    }
+
     s->mb_width  = (s->avctx->coded_width  + 15) / 16;
     s->mb_height = (s->avctx->coded_height + 15) / 16;
 
@@ -2598,18 +2621,7 @@ int vp78_decode_frame(AVCodecContext *avctx, void *data, int *got_frame,
     if (s->actually_webp) {
         // avctx->pix_fmt already set in caller.
     } else if (!is_vp7 && s->pix_fmt == AV_PIX_FMT_NONE) {
-        enum AVPixelFormat pix_fmts[] = {
-#if CONFIG_VP8_VAAPI_HWACCEL
-            AV_PIX_FMT_VAAPI,
-#endif
-#if CONFIG_VP8_NVDEC_HWACCEL
-            AV_PIX_FMT_CUDA,
-#endif
-            AV_PIX_FMT_YUV420P,
-            AV_PIX_FMT_NONE,
-        };
-
-        s->pix_fmt = ff_get_format(s->avctx, pix_fmts);
+        s->pix_fmt = get_pixel_format(s);
         if (s->pix_fmt < 0) {
             ret = AVERROR(EINVAL);
             goto err;
-- 
2.14.1