[FFmpeg-cvslog] h264: check sps.log2_max_frame_num for validity

[Janne Grunau]

ffmpeg | branch: release/0.10 | Janne Grunau <janne-libav at jannau.net> | Sun Nov 25 12:56:04 2012 +0100| [f620c12067a2a80af9fb63927665f82f583e18d7] | committer: Reinhard Tartler

h264: check sps.log2_max_frame_num for validity

Fixes infinite or long taking loop in frame num gap code in
the fuzzed sample bipbop234.ts_s223302.

CC: libav-stable at libav.org
(cherry picked from commit d7d6efe42b0d2057e67999b96b9a391f533d2333)

Signed-off-by: Reinhard Tartler <siretart at tauware.de>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=f620c12067a2a80af9fb63927665f82f583e18d7
---

 libavcodec/h264_ps.c |   16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c
index ff6103c..a468c96 100644
--- a/libavcodec/h264_ps.c
+++ b/libavcodec/h264_ps.c
@@ -37,6 +37,9 @@
 //#undef NDEBUG
 #include <assert.h>
 
+#define MAX_LOG2_MAX_FRAME_NUM    (12 + 4)
+#define MIN_LOG2_MAX_FRAME_NUM    4
+
 static const AVRational pixel_aspect[17]={
  {0, 1},
  {1, 1},
@@ -301,7 +304,7 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
     MpegEncContext * const s = &h->s;
     int profile_idc, level_idc, constraint_set_flags = 0;
     unsigned int sps_id;
-    int i;
+    int i, log2_max_frame_num_minus4;
     SPS *sps;
 
     profile_idc= get_bits(&s->gb, 8);
@@ -348,7 +351,16 @@ int ff_h264_decode_seq_parameter_set(H264Context *h){
         sps->bit_depth_chroma = 8;
     }
 
-    sps->log2_max_frame_num= get_ue_golomb(&s->gb) + 4;
+    log2_max_frame_num_minus4 = get_ue_golomb(&s->gb);
+    if (log2_max_frame_num_minus4 < MIN_LOG2_MAX_FRAME_NUM - 4 ||
+        log2_max_frame_num_minus4 > MAX_LOG2_MAX_FRAME_NUM - 4) {
+        av_log(h->s.avctx, AV_LOG_ERROR,
+               "log2_max_frame_num_minus4 out of range (0-12): %d\n",
+               log2_max_frame_num_minus4);
+        return AVERROR_INVALIDDATA;
+    }
+    sps->log2_max_frame_num = log2_max_frame_num_minus4 + 4;
+
     sps->poc_type= get_ue_golomb_31(&s->gb);
 
     if(sps->poc_type == 0){ //FIXME #define