[FFmpeg-devel] [PATCH 3/4] proresenc_kostya: realloc if buffer too small

[Christophe Gisquet]

The buffer allocation may be incorrect (e.g. with an alpha plane),
and currently causes the buffer to be set to NULL by init_put_bits,
later on causing crashing.

So, detect that situation, and if detected, reallocate the buffer
and ask a sample if it happens.

Fixes ticket #2760
---
 libavcodec/proresenc_kostya.c | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/libavcodec/proresenc_kostya.c b/libavcodec/proresenc_kostya.c
index a70ae3c..2cd33ce 100644
--- a/libavcodec/proresenc_kostya.c
+++ b/libavcodec/proresenc_kostya.c
@@ -1023,6 +1023,30 @@ static int encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                 bytestream_put_byte(&buf, slice_hdr_size << 3);
                 slice_hdr = buf;
                 buf += slice_hdr_size - 1;
+                if (pkt_size <= buf - orig_buf) {
+                    static int warn = 0;
+                    uint8_t *start = pkt->data;
+                    int ret, delta = buf - orig_buf; // double the size
+
+                    if (!warn) {
+                        avpriv_request_sample(avctx,
+                                              "Packet too small (%i/%i)",
+                                              pkt_size, delta);
+                    }
+                    ctx->frame_size_upper_bound += delta;
+                    ret = av_grow_packet(pkt, delta);
+                    if (ret < 0)
+                        return AVERROR(ENOMEM);
+
+                    pkt_size += delta;
+                    // restore pointers
+                    orig_buf = pkt->data + (orig_buf - start);
+                    buf = pkt->data + (buf - start);
+                    picture_size_pos = pkt->data + (picture_size_pos - start);
+                    slice_sizes = pkt->data + (slice_sizes - start);
+                    slice_hdr = pkt->data + (slice_hdr - start);
+                    tmp = pkt->data + (tmp - start);
+                }
                 init_put_bits(&pb, buf, (pkt_size - (buf - orig_buf)) * 8);
                 ret = encode_slice(avctx, pic, &pb, sizes, x, y, q, mbs_per_slice);
                 if (ret < 0)
-- 
1.9.2.msysgit.0