[FFmpeg-cvslog] avcodec/vvc/dec: Check ff_init_cabac_decoder() for failure

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sun Jun 30 19:35:01 2024 +0200| [6398242bb2b9bfdef9d7fd4614a3518ffb1fefec] | committer: Michael Niedermayer

avcodec/vvc/dec: Check ff_init_cabac_decoder() for failure

Fixes: signed integer overflow: 1107820800 + 1107820800 cannot be represented in type 'int'
Fixes: left shift of 1091059712 by 6 places cannot be represented in type 'int'
Fixes: 69910/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VVC_fuzzer-5162839971528704

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Nuo Mi <nuomi2021 at gmail.com>
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=6398242bb2b9bfdef9d7fd4614a3518ffb1fefec
---

 libavcodec/vvc/dec.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/libavcodec/vvc/dec.c b/libavcodec/vvc/dec.c
index a6b601b778..563f6fcf08 100644
--- a/libavcodec/vvc/dec.c
+++ b/libavcodec/vvc/dec.c
@@ -476,13 +476,14 @@ static int slices_realloc(VVCFrameContext *fc)
     return 0;
 }
 
-static void ep_init_cabac_decoder(SliceContext *sc, const int index,
+static int ep_init_cabac_decoder(SliceContext *sc, const int index,
     const H2645NAL *nal, GetBitContext *gb, const CodedBitstreamUnit *unit)
 {
     const H266RawSlice *slice     = unit->content_ref;
     const H266RawSliceHeader *rsh = sc->sh.r;
     EntryPoint *ep                = sc->eps + index;
     int size;
+    int ret;
 
     if (index < rsh->num_entry_points) {
         int skipped = 0;
@@ -501,8 +502,11 @@ static void ep_init_cabac_decoder(SliceContext *sc, const int index,
         size = get_bits_left(gb) / 8;
     }
     av_assert0(gb->buffer + get_bits_count(gb) / 8 + size <= gb->buffer_end);
-    ff_init_cabac_decoder (&ep->cc, gb->buffer + get_bits_count(gb) / 8, size);
+    ret = ff_init_cabac_decoder (&ep->cc, gb->buffer + get_bits_count(gb) / 8, size);
+    if (ret < 0)
+        return ret;
     skip_bits(gb, size * 8);
+    return 0;
 }
 
 static int slice_init_entry_points(SliceContext *sc,
@@ -538,7 +542,9 @@ static int slice_init_entry_points(SliceContext *sc,
             fc->tab.slice_idx[rs] = sc->slice_idx;
         }
 
-        ep_init_cabac_decoder(sc, i, nal, &gb, unit);
+        ret = ep_init_cabac_decoder(sc, i, nal, &gb, unit);
+        if (ret < 0)
+            return ret;
 
         if (i + 1 < sc->nb_eps)
             ctu_addr = sh->entry_point_start_ctu[i];