[FFmpeg-devel] [PATCH] vp9: just disallow temporal or no=update segmentation on size-change.

[Ronald S. Bultje]

The spec doesn't describe how it should be decoded so this is probably
the safest thing to do. Fixes valgrind errors on fuzzed11.ivf and fixes
valgrind errors on fuzzed10.ivf differently.
---
 libavcodec/vp9.c | 21 ++++++++++++---------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/libavcodec/vp9.c b/libavcodec/vp9.c
index 10e26f9..ea2a17b 100644
--- a/libavcodec/vp9.c
+++ b/libavcodec/vp9.c
@@ -110,7 +110,6 @@ typedef struct VP9Context {
     uint8_t keyframe, last_keyframe;
     uint8_t invisible;
     uint8_t use_last_frame_mvs;
-    uint8_t use_last_frame_segmap;
     uint8_t errorres;
     uint8_t colorspace;
     uint8_t fullrange;
@@ -279,7 +278,7 @@ static int vp9_alloc_frame(AVCodecContext *ctx, VP9Frame *f)
 
     // retain segmentation map if it doesn't update
     if (s->segmentation.enabled && !s->segmentation.update_map &&
-        s->use_last_frame_segmap) {
+        !s->intraonly && !s->keyframe) {
         memcpy(f->segmentation_map, s->frames[LAST_FRAME].segmentation_map, sz);
     }
 
@@ -622,14 +621,19 @@ static int decode_frame_header(AVCodecContext *ctx,
             for (i = 0; i < 7; i++)
                 s->prob.seg[i] = get_bits1(&s->gb) ?
                                  get_bits(&s->gb, 8) : 255;
-            if ((s->segmentation.temporal = get_bits1(&s->gb)))
+            if ((s->segmentation.temporal = get_bits1(&s->gb))) {
                 for (i = 0; i < 3; i++)
                     s->prob.segpred[i] = get_bits1(&s->gb) ?
                                          get_bits(&s->gb, 8) : 255;
-        } else {
-            s->use_last_frame_segmap = !s->keyframe && !s->intraonly &&
-                s->frames[CUR_FRAME].tf.f->width == w &&
-                s->frames[CUR_FRAME].tf.f->height == h;
+            }
+        }
+        if ((!s->segmentation.update_map || s->segmentation.temporal) &&
+            (w != s->frames[CUR_FRAME].tf.f->width ||
+             h != s->frames[CUR_FRAME].tf.f->height)) {
+            av_log(ctx, AV_LOG_ERROR,
+                   "Reference segmap (temp=%d,update=%d) enabled on size-change!\n",
+                   s->segmentation.temporal, s->segmentation.update_map);
+            return AVERROR_INVALIDDATA;
         }
 
         if (get_bits1(&s->gb)) {
@@ -1331,8 +1335,7 @@ static void decode_mode(AVCodecContext *ctx)
     int have_a = row > 0, have_l = col > s->tiling.tile_col_start;
     int vref, filter_id;
 
-    if (!s->segmentation.enabled ||
-        (!s->segmentation.update_map && !s->use_last_frame_segmap)) {
+    if (!s->segmentation.enabled) {
         b->seg_id = 0;
     } else if (s->keyframe || s->intraonly) {
         b->seg_id = vp8_rac_get_tree(&s->c, vp9_segmentation_tree, s->prob.seg);
-- 
1.8.4