[FFmpeg-cvslog] vorbis: fix overflows in floor1[] vector and inverse db table index.

[Ronald S. Bultje]

ffmpeg | branch: release/0.8 | Ronald S. Bultje <rsbultje at gmail.com> | Tue Jan 10 17:01:26 2012 -0800| [117b8b00cc12efb473402a8d6ab1e9013ea0a851] | committer: Reinhard Tartler

vorbis: fix overflows in floor1[] vector and inverse db table index.

(cherry picked from commit 24947d4988012f1f0fd467c83418615adc11c3e8)

Signed-off-by: Reinhard Tartler <siretart at tauware.de>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=117b8b00cc12efb473402a8d6ab1e9013ea0a851
---

 libavcodec/vorbis.c    |   19 +++++++++----------
 libavcodec/vorbisdec.c |   10 +++++-----
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/libavcodec/vorbis.c b/libavcodec/vorbis.c
index cc039be..20be707 100644
--- a/libavcodec/vorbis.c
+++ b/libavcodec/vorbis.c
@@ -150,7 +150,7 @@ void ff_vorbis_ready_floor1_list(vorbis_floor1_entry * list, int values)
     }
 }
 
-static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
+static inline void render_line_unrolled(intptr_t x, int y, int x1,
                                         intptr_t sy, int ady, int adx,
                                         float *buf)
 {
@@ -162,30 +162,30 @@ static inline void render_line_unrolled(intptr_t x, uint8_t y, int x1,
         if (err >= 0) {
             err += ady - adx;
             y   += sy;
-            buf[x++] = ff_vorbis_floor1_inverse_db_table[y];
+            buf[x++] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
         }
-        buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+        buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
     }
     if (x <= 0) {
         if (err + ady >= 0)
             y += sy;
-        buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+        buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
     }
 }
 
-static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
+static void render_line(int x0, int y0, int x1, int y1, float *buf)
 {
     int dy  = y1 - y0;
     int adx = x1 - x0;
     int ady = FFABS(dy);
     int sy  = dy < 0 ? -1 : 1;
-    buf[x0] = ff_vorbis_floor1_inverse_db_table[y0];
+    buf[x0] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y0)];
     if (ady*2 <= adx) { // optimized common case
         render_line_unrolled(x0, y0, x1, sy, ady, adx, buf);
     } else {
         int base  = dy / adx;
         int x     = x0;
-        uint8_t y = y0;
+        int y     = y0;
         int err   = -adx;
         ady -= FFABS(base) * adx;
         while (++x < x1) {
@@ -195,7 +195,7 @@ static void render_line(int x0, uint8_t y0, int x1, int y1, float *buf)
                 err -= adx;
                 y   += sy;
             }
-            buf[x] = ff_vorbis_floor1_inverse_db_table[y];
+            buf[x] = ff_vorbis_floor1_inverse_db_table[av_clip_uint8(y)];
         }
     }
 }
@@ -204,8 +204,7 @@ void ff_vorbis_floor1_render_list(vorbis_floor1_entry * list, int values,
                                   uint16_t *y_list, int *flag,
                                   int multiplier, float *out, int samples)
 {
-    int lx, i;
-    uint8_t ly;
+    int lx, ly, i;
     lx = 0;
     ly = y_list[0] * multiplier;
     for (i = 1; i < values; i++) {
diff --git a/libavcodec/vorbisdec.c b/libavcodec/vorbisdec.c
index 572e06e..d6850b7 100644
--- a/libavcodec/vorbisdec.c
+++ b/libavcodec/vorbisdec.c
@@ -1232,20 +1232,20 @@ static int vorbis_floor1_decode(vorbis_context *vc,
             floor1_flag[i]               = 1;
             if (val >= room) {
                 if (highroom > lowroom) {
-                    floor1_Y_final[i] = val - lowroom + predicted;
+                    floor1_Y_final[i] = av_clip_uint16(val - lowroom + predicted);
                 } else {
-                    floor1_Y_final[i] = predicted - val + highroom - 1;
+                    floor1_Y_final[i] = av_clip_uint16(predicted - val + highroom - 1);
                 }
             } else {
                 if (val & 1) {
-                    floor1_Y_final[i] = predicted - (val + 1) / 2;
+                    floor1_Y_final[i] = av_clip_uint16(predicted - (val + 1) / 2);
                 } else {
-                    floor1_Y_final[i] = predicted + val / 2;
+                    floor1_Y_final[i] = av_clip_uint16(predicted + val / 2);
                 }
             }
         } else {
             floor1_flag[i]    = 0;
-            floor1_Y_final[i] = predicted;
+            floor1_Y_final[i] = av_clip_uint16(predicted);
         }
 
         av_dlog(NULL, " Decoded floor(%d) = %u / val %u\n",