[FFmpeg-cvslog] sgidec: fix buffer size check in expand_rle_row()

[Anton Khirnov]

ffmpeg | branch: release/1.1 | Anton Khirnov <anton at khirnov.net> | Thu Jan  2 09:34:20 2014 +0100| [ea1806ce650f0502dd25939c335b9216fa4a955f] | committer: Sean McGovern

sgidec: fix buffer size check in expand_rle_row()

Right now it will spuriously fail if the linesize is exactly equal to
the data width.

CC:libav-stable at libav.org

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=ea1806ce650f0502dd25939c335b9216fa4a955f
---

 libavcodec/sgidec.c |   22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/libavcodec/sgidec.c b/libavcodec/sgidec.c
index 0e72751..94de9fd 100644
--- a/libavcodec/sgidec.c
+++ b/libavcodec/sgidec.c
@@ -26,6 +26,7 @@
 #include "sgi.h"
 
 typedef struct SgiState {
+    AVCodecContext *avctx;
     AVFrame picture;
     unsigned int width;
     unsigned int height;
@@ -39,12 +40,12 @@ typedef struct SgiState {
  * Expand an RLE row into a channel.
  * @param s the current image state
  * @param out_buf Points to one line after the output buffer.
- * @param out_end end of line in output buffer
+ * @param len length of out_buf in bytes
  * @param pixelstride pixel stride of input buffer
  * @return size of output in bytes, -1 if buffer overflows
  */
 static int expand_rle_row(SgiState *s, uint8_t *out_buf,
-                          uint8_t *out_end, int pixelstride)
+                          int len, int pixelstride)
 {
     unsigned char pixel, count;
     unsigned char *orig = out_buf;
@@ -58,7 +59,10 @@ static int expand_rle_row(SgiState *s, uint8_t *out_buf,
         }
 
         /* Check for buffer overflow. */
-        if(out_buf + pixelstride * count >= out_end) return -1;
+        if (pixelstride * (count - 1) >= len) {
+            av_log(s->avctx, AV_LOG_ERROR, "Invalid pixel count.\n");
+            return AVERROR_INVALIDDATA;
+        }
 
         if (pixel & 0x80) {
             while (count--) {
@@ -101,7 +105,7 @@ static int read_rle_sgi(uint8_t *out_buf, SgiState *s)
             dest_row -= s->linesize;
             start_offset = bytestream2_get_be32(&g_table);
             bytestream2_seek(&s->g, start_offset, SEEK_SET);
-            if (expand_rle_row(s, dest_row + z, dest_row + FFABS(s->linesize),
+            if (expand_rle_row(s, dest_row + z, FFABS(s->linesize) - z,
                                s->depth) != s->width) {
                 return AVERROR_INVALIDDATA;
             }
@@ -259,6 +263,15 @@ static av_cold int sgi_end(AVCodecContext *avctx)
     return 0;
 }
 
+static av_cold int sgi_decode_init(AVCodecContext *avctx)
+{
+    SgiState *s = avctx->priv_data;
+
+    s->avctx = avctx;
+
+    return 0;
+}
+
 AVCodec ff_sgi_decoder = {
     .name           = "sgi",
     .type           = AVMEDIA_TYPE_VIDEO,
@@ -267,6 +280,7 @@ AVCodec ff_sgi_decoder = {
     .init           = sgi_init,
     .close          = sgi_end,
     .decode         = decode_frame,
+    .init           = sgi_decode_init,
     .long_name      = NULL_IF_CONFIG_SMALL("SGI image"),
     .capabilities   = CODEC_CAP_DR1,
 };