[FFmpeg-devel] [PATCH] vp8: check for too large dimensions

[Andreas Cadhalpun]

If the dimensions are too large, s->mb_width or s->mb_height can become
too large, leading to an int16_t overflow of s->mv_max.{x,y}.

Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com>
---
 libavcodec/vp8.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/libavcodec/vp8.c b/libavcodec/vp8.c
index dbba568..69cf138 100644
--- a/libavcodec/vp8.c
+++ b/libavcodec/vp8.c
@@ -145,6 +145,8 @@ static VP8Frame *vp8_find_free_buffer(VP8Context *s)
     return frame;
 }
 
+#define MARGIN (16 << 2)
+#define MAX_MB_SIZE (((INT16_MAX - MARGIN) >> 6) + 1)
 static av_always_inline
 int update_dimensions(VP8Context *s, int width, int height, int is_vp7)
 {
@@ -160,6 +162,13 @@ int update_dimensions(VP8Context *s, int width, int height, int is_vp7)
             return ret;
     }
 
+    if (s->avctx->coded_width  > MAX_MB_SIZE * 16 ||
+        s->avctx->coded_height > MAX_MB_SIZE * 16) {
+            av_log(s->avctx, AV_LOG_ERROR, "too large dimensions %dx%d\n",
+                   s->avctx->coded_width, s->avctx->coded_height);
+            return AVERROR_INVALIDDATA;
+        }
+
     s->mb_width  = (s->avctx->coded_width  + 15) / 16;
     s->mb_height = (s->avctx->coded_height + 15) / 16;
 
@@ -2177,7 +2186,6 @@ void filter_mb_simple(VP8Context *s, uint8_t *dst, VP8FilterStrength *f,
     }
 }
 
-#define MARGIN (16 << 2)
 static av_always_inline
 void vp78_decode_mv_mb_modes(AVCodecContext *avctx, VP8Frame *curframe,
                                     VP8Frame *prev_frame, int is_vp7)
-- 
2.1.4