[FFmpeg-devel] [PATCH] exr: rle decompression

[Paul B Mahol]

Signed-off-by: Paul B Mahol <onemda at gmail.com>
---
 libavcodec/exr.c |   45 ++++++++++++++++++++++++++++++++++++++++++++-
 1 files changed, 44 insertions(+), 1 deletions(-)

diff --git a/libavcodec/exr.c b/libavcodec/exr.c
index aa77b8e..893d9a2 100644
--- a/libavcodec/exr.c
+++ b/libavcodec/exr.c
@@ -34,6 +34,7 @@
 
 #include "avcodec.h"
 #include "bytestream.h"
+#include "mathops.h"
 #include "libavutil/imgutils.h"
 
 enum ExrCompr {
@@ -183,6 +184,40 @@ static void reorder_pixels(uint8_t *src, uint8_t *dst, int size)
     }
 }
 
+static int rle_uncompress(const uint8_t *src, int ssize, uint8_t *dst, int dsize)
+{
+    uint8_t *dst_end = dst + dsize;
+    int count;
+
+    while (ssize > 0) {
+        count = sign_extend(*src++, 8);
+
+        if (count < 0) {
+            count = -count;
+
+            if ((dsize -= count    ) < 0 ||
+                (ssize -= count + 1) < 0)
+                return -1;
+
+            while (count--)
+                *dst++ = *src++;
+        } else {
+            count++;
+
+            if ((dsize -= count) < 0 ||
+                (ssize -= 2    ) < 0)
+                return -1;
+
+            while (count-- > 0)
+                *dst++ = *src;
+
+            src++;
+        }
+    }
+
+    return dst_end != dst;
+}
+
 static int decode_frame(AVCodecContext *avctx,
                         void *data,
                         int *data_size,
@@ -366,10 +401,10 @@ static int decode_frame(AVCodecContext *avctx,
             s->compr = *buf;
             switch (s->compr) {
             case EXR_RAW:
+            case EXR_RLE:
             case EXR_ZIP1:
             case EXR_ZIP16:
                 break;
-            case EXR_RLE:
             case EXR_PIZ:
             case EXR_B44:
             default:
@@ -431,6 +466,7 @@ static int decode_frame(AVCodecContext *avctx,
 
     switch (s->compr) {
     case EXR_RAW:
+    case EXR_RLE:
     case EXR_ZIP1:
         scan_lines_per_block = 1;
         break;
@@ -508,7 +544,14 @@ static int decode_frame(AVCodecContext *avctx,
                         av_log(avctx, AV_LOG_ERROR, "error during zlib decompression\n");
                         return AVERROR(EINVAL);
                     }
+                } else if (s->compr == EXR_RLE && data_size < uncompressed_size) {
+                    if (rle_uncompress(avpkt->data + line_offset, data_size, s->tmp, uncompressed_size)) {
+                        av_log(avctx, AV_LOG_ERROR, "error during rle decompression\n");
+                        return AVERROR(EINVAL);
+                    }
+                }
 
+                if (s->compr != EXR_RAW && data_size < uncompressed_size) {
                     predictor(s->tmp, uncompressed_size);
                     reorder_pixels(s->tmp, s->uncompressed_data, uncompressed_size);
 
-- 
1.7.7