[FFmpeg-cvslog] avcodec/lcldec: Check that dimensions are a multiple of the subsample factors

[Michael Niedermayer]

ffmpeg | branch: release/1.0 | Michael Niedermayer <michaelni at gmx.at> | Mon May 13 18:09:04 2013 +0200| [e32acc455db2e45e638a45b68984931321864956] | committer: Michael Niedermayer

avcodec/lcldec: Check that dimensions are a multiple of the subsample factors

Other dimensions would not work correctly currently,
also ask for a sample for files that fail this check.

This fixes an integer overflow leading to out of array
accesses.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni at gmx.at>
(cherry picked from commit 1e00bbb10cbde3da03a1e744265ce6def9ae4c56)

Conflicts:

	libavcodec/lcldec.c

Signed-off-by: Michael Niedermayer <michaelni at gmx.at>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=e32acc455db2e45e638a45b68984931321864956
---

 libavcodec/lcldec.c |   12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libavcodec/lcldec.c b/libavcodec/lcldec.c
index f180373..2a4d61d 100644
--- a/libavcodec/lcldec.c
+++ b/libavcodec/lcldec.c
@@ -41,6 +41,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include "libavutil/pixdesc.h"
 #include "avcodec.h"
 #include "bytestream.h"
 #include "lcl.h"
@@ -484,6 +485,7 @@ static av_cold int decode_init(AVCodecContext *avctx)
     unsigned int basesize = avctx->width * avctx->height;
     unsigned int max_basesize = FFALIGN(avctx->width, 4) * FFALIGN(avctx->height, 4) + AV_LZO_OUTPUT_PADDING;
     unsigned int max_decomp_size;
+    int subsample_h, subsample_v;
 
     avcodec_get_frame_defaults(&c->pic);
     if (avctx->extradata_size < 8) {
@@ -510,6 +512,10 @@ static av_cold int decode_init(AVCodecContext *avctx)
         max_decomp_size = max_basesize * 2;
         avctx->pix_fmt = PIX_FMT_YUV422P;
         av_log(avctx, AV_LOG_DEBUG, "Image type is YUV 4:2:2.\n");
+        if (avctx->width % 4) {
+            av_log_ask_for_sample(avctx, "Unsupported dimensions\n");
+            return AVERROR_INVALIDDATA;
+        }
         break;
     case IMGTYPE_RGB24:
         c->decomp_size = basesize * 3;
@@ -540,6 +546,12 @@ static av_cold int decode_init(AVCodecContext *avctx)
         return AVERROR_INVALIDDATA;
     }
 
+    avcodec_get_chroma_sub_sample(avctx->pix_fmt, &subsample_h, &subsample_v);
+    if (avctx->width % (1<<subsample_h) || avctx->height % (1<<subsample_v)) {
+        av_log_ask_for_sample(avctx, "Unsupported dimensions\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     /* Detect compression method */
     c->compression = (int8_t)avctx->extradata[5];
     switch (avctx->codec_id) {