[FFmpeg-cvslog] avcodec/flicvideo: fix decoding with AVFrame's negative linesize
Paul B Mahol
git at videolan.org
Sat Sep 23 16:33:05 EEST 2023
ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Sat Sep 23 15:23:38 2023 +0200| [c7f8d42c12582b0626ea38117df6c9aea9fcf5b1] | committer: Paul B Mahol
avcodec/flicvideo: fix decoding with AVFrame's negative linesize
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=c7f8d42c12582b0626ea38117df6c9aea9fcf5b1
---
libavcodec/flicvideo.c | 80 ++++++++++++++++++++++++++++++--------------------
1 file changed, 48 insertions(+), 32 deletions(-)
diff --git a/libavcodec/flicvideo.c b/libavcodec/flicvideo.c
index 8531eb715c..f0d32bab38 100644
--- a/libavcodec/flicvideo.c
+++ b/libavcodec/flicvideo.c
@@ -60,12 +60,28 @@
#define FLC_DTA_TYPE_CODE (0xAF44) /* Marks an "Extended FLC" comes from Dave's Targa Animator (DTA) */
#define FLC_MAGIC_CARPET_SYNTHETIC_TYPE_CODE (0xAF13)
+static inline int check_pixel_ptr(ptrdiff_t ptr, int n,
+ ptrdiff_t limit, int direction)
+{
+ if (( direction && ptr + n > limit) ||
+ (!direction && ptr + n < limit))
+ return AVERROR_INVALIDDATA;
+ return 0;
+}
+
#define CHECK_PIXEL_PTR(n) \
- if (pixel_ptr + n > pixel_limit) { \
- av_log (s->avctx, AV_LOG_ERROR, "Invalid pixel_ptr = %d > pixel_limit = %d\n", \
- pixel_ptr + n, pixel_limit); \
- return AVERROR_INVALIDDATA; \
- } \
+{ \
+ ret = check_pixel_ptr(pixel_ptr, (n), pixel_limit, direction); \
+ if (ret < 0) \
+ return ret; \
+}
+
+#define CHECK_Y_PTR() \
+{ \
+ ret = check_pixel_ptr(y_ptr, 0, pixel_limit, direction); \
+ if (ret < 0) \
+ return ret; \
+}
typedef struct FlicDecodeContext {
AVCodecContext *avctx;
@@ -153,7 +169,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
FlicDecodeContext *s = avctx->priv_data;
GetByteContext g2;
- int pixel_ptr;
+ ptrdiff_t pixel_ptr;
int palette_ptr;
unsigned char palette_idx1;
unsigned char palette_idx2;
@@ -164,7 +180,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
unsigned int chunk_size;
int chunk_type;
- int i, j, ret;
+ int i, j, ret, direction;
int color_packets;
int color_changes;
@@ -175,18 +191,19 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
int compressed_lines;
int starting_line;
int line_packets;
- int y_ptr;
+ ptrdiff_t y_ptr;
int byte_run;
int pixel_skip;
int pixel_countdown;
unsigned char *pixels;
- unsigned int pixel_limit;
+ ptrdiff_t pixel_limit;
bytestream2_init(&g2, buf, buf_size);
if ((ret = ff_reget_buffer(avctx, s->frame, 0)) < 0)
return ret;
+ direction = s->frame->linesize[0] > 0;
pixels = s->frame->data[0];
pixel_limit = s->avctx->height * s->frame->linesize[0];
if (buf_size < 16 || buf_size > INT_MAX - (3 * 256 + AV_INPUT_BUFFER_PADDING_SIZE))
@@ -271,8 +288,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
while (compressed_lines > 0) {
if (bytestream2_tell(&g2) + 2 > stream_ptr_after_chunk)
break;
- if (y_ptr > pixel_limit)
- return AVERROR_INVALIDDATA;
+ CHECK_Y_PTR()
line_packets = sign_extend(bytestream2_get_le16(&g2), 16);
if ((line_packets & 0xC000) == 0xC000) {
// line skip opcode
@@ -374,8 +390,8 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
case FLI_BLACK:
/* set the whole frame to color 0 (which is usually black) */
- memset(pixels, 0,
- s->frame->linesize[0] * s->avctx->height);
+ for (int y = 0; y < s->avctx->height; y++)
+ memset(pixels + y * s->frame->linesize[0], 0, s->avctx->width);
break;
case FLI_BRUN:
@@ -433,7 +449,7 @@ static int flic_decode_frame_8BPP(AVCodecContext *avctx,
"has incorrect size, skipping chunk\n", chunk_size - 6);
bytestream2_skip(&g2, chunk_size - 6);
} else {
- for (y_ptr = 0; y_ptr < s->frame->linesize[0] * s->avctx->height;
+ for (y_ptr = 0; check_pixel_ptr(y_ptr, 0, pixel_limit, direction) == 0;
y_ptr += s->frame->linesize[0]) {
bytestream2_get_buffer(&g2, &pixels[y_ptr],
s->avctx->width);
@@ -498,7 +514,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
FlicDecodeContext *s = avctx->priv_data;
GetByteContext g2;
- int pixel_ptr;
+ ptrdiff_t pixel_ptr;
unsigned char palette_idx1;
unsigned int frame_size;
@@ -507,24 +523,25 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
unsigned int chunk_size;
int chunk_type;
- int i, j, ret;
+ int i, j, ret, direction;
int lines;
int compressed_lines;
int line_packets;
- int y_ptr;
+ ptrdiff_t y_ptr;
int byte_run;
int pixel_skip;
int pixel_countdown;
unsigned char *pixels;
int pixel;
- unsigned int pixel_limit;
+ ptrdiff_t pixel_limit;
bytestream2_init(&g2, buf, buf_size);
if ((ret = ff_reget_buffer(avctx, s->frame, 0)) < 0)
return ret;
+ direction = s->frame->linesize[0] > 0;
pixels = s->frame->data[0];
pixel_limit = s->avctx->height * s->frame->linesize[0];
@@ -573,8 +590,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
while (compressed_lines > 0) {
if (bytestream2_tell(&g2) + 2 > stream_ptr_after_chunk)
break;
- if (y_ptr > pixel_limit)
- return AVERROR_INVALIDDATA;
+ CHECK_Y_PTR()
line_packets = sign_extend(bytestream2_get_le16(&g2), 16);
if (line_packets < 0) {
line_packets = -line_packets;
@@ -625,8 +641,8 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
case FLI_BLACK:
/* set the whole frame to 0x0000 which is black in both 15Bpp and 16Bpp modes. */
- memset(pixels, 0x0000,
- s->frame->linesize[0] * s->avctx->height);
+ for (int y = 0; y < s->avctx->height; y++)
+ memset(pixels + y * s->frame->linesize[0], 0, s->avctx->width * 2);
break;
case FLI_BRUN:
@@ -740,7 +756,7 @@ static int flic_decode_frame_15_16BPP(AVCodecContext *avctx,
if (bytestream2_get_bytes_left(&g2) < 2 * s->avctx->width * s->avctx->height )
return AVERROR_INVALIDDATA;
- for (y_ptr = 0; y_ptr < s->frame->linesize[0] * s->avctx->height;
+ for (y_ptr = 0; check_pixel_ptr(y_ptr, 0, pixel_limit, direction) == 0;
y_ptr += s->frame->linesize[0]) {
pixel_countdown = s->avctx->width;
@@ -798,7 +814,7 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
FlicDecodeContext *s = avctx->priv_data;
GetByteContext g2;
- int pixel_ptr;
+ ptrdiff_t pixel_ptr;
unsigned char palette_idx1;
unsigned int frame_size;
@@ -807,24 +823,25 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
unsigned int chunk_size;
int chunk_type;
- int i, j, ret;
+ int i, j, ret, direction;
int lines;
int compressed_lines;
int line_packets;
- int y_ptr;
+ ptrdiff_t y_ptr;
int byte_run;
int pixel_skip;
int pixel_countdown;
unsigned char *pixels;
int pixel;
- unsigned int pixel_limit;
+ ptrdiff_t pixel_limit;
bytestream2_init(&g2, buf, buf_size);
if ((ret = ff_reget_buffer(avctx, s->frame, 0)) < 0)
return ret;
+ direction = s->frame->linesize[0] > 0;
pixels = s->frame->data[0];
pixel_limit = s->avctx->height * s->frame->linesize[0];
@@ -873,8 +890,7 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
while (compressed_lines > 0) {
if (bytestream2_tell(&g2) + 2 > stream_ptr_after_chunk)
break;
- if (y_ptr > pixel_limit)
- return AVERROR_INVALIDDATA;
+ CHECK_Y_PTR()
line_packets = sign_extend(bytestream2_get_le16(&g2), 16);
if (line_packets < 0) {
line_packets = -line_packets;
@@ -926,8 +942,8 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
case FLI_BLACK:
/* set the whole frame to 0x00 which is black for 24 bit mode. */
- memset(pixels, 0x00,
- s->frame->linesize[0] * s->avctx->height);
+ for (int y = 0; y < s->avctx->height; y++)
+ memset(pixels + y * s->frame->linesize[0], 0, s->avctx->width * 3);
break;
case FLI_BRUN:
@@ -1026,7 +1042,7 @@ static int flic_decode_frame_24BPP(AVCodecContext *avctx,
"bigger than image, skipping chunk\n", chunk_size - 6);
bytestream2_skip(&g2, chunk_size - 6);
} else {
- for (y_ptr = 0; y_ptr < s->frame->linesize[0] * s->avctx->height;
+ for (y_ptr = 0; check_pixel_ptr(y_ptr, 0, pixel_limit, direction) == 0;
y_ptr += s->frame->linesize[0]) {
bytestream2_get_buffer(&g2, pixels + y_ptr, 3*s->avctx->width);
More information about the ffmpeg-cvslog
mailing list