[FFmpeg-devel] [PATCH]Fail on oom in vdpau
Carl Eugen Hoyos
cehoyos at ag.or.at
Tue Jul 9 01:44:25 CEST 2013
Hi!
Attached patch intends to fix a theoretical issue, reported by Gwenole ages
ago.
Please review, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/h264.c b/libavcodec/h264.c
index 6a6a68c..e582c09 100644
--- a/libavcodec/h264.c
+++ b/libavcodec/h264.c
@@ -4728,10 +4728,20 @@ again:
h->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU) {
static const uint8_t start_code[] = {
0x00, 0x00, 0x01 };
+ int ret =
ff_vdpau_add_data_chunk(h->cur_pic_ptr->f.data[0], start_code,
sizeof(start_code));
+ if (ret < 0) {
+ buf_index = ret;
+ goto end;
+ }
+ ret =
ff_vdpau_add_data_chunk(h->cur_pic_ptr->f.data[0], &buf[buf_index - consumed],
consumed);
+ if (ret < 0) {
+ buf_index = ret;
+ goto end;
+ }
} else
context_count++;
}
diff --git a/libavcodec/vdpau.c b/libavcodec/vdpau.c
index 04355b0..2f7ff39 100644
--- a/libavcodec/vdpau.c
+++ b/libavcodec/vdpau.c
@@ -157,21 +157,26 @@ void ff_vdpau_h264_set_reference_frames(H264Context *h)
}
}
-void ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf, int buf_size)
+int ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf, int buf_size)
{
+ VdpBitstreamBuffer *new_buffers;
struct vdpau_render_state *render = (struct vdpau_render_state*)data;
assert(render);
- render->bitstream_buffers= av_fast_realloc(
+ new_buffers = av_fast_realloc(
render->bitstream_buffers,
&render->bitstream_buffers_allocated,
sizeof(*render->bitstream_buffers)*(render->bitstream_buffers_used + 1)
);
+ if (!new_buffers)
+ return AVERROR(ENOMEM);
+ render->bitstream_buffers = new_buffers;
render->bitstream_buffers[render->bitstream_buffers_used].struct_version = VDP_BITSTREAM_BUFFER_VERSION;
render->bitstream_buffers[render->bitstream_buffers_used].bitstream = buf;
render->bitstream_buffers[render->bitstream_buffers_used].bitstream_bytes = buf_size;
render->bitstream_buffers_used++;
+ return 0;
}
#if CONFIG_H264_VDPAU_DECODER
diff --git a/libavcodec/vdpau_internal.h b/libavcodec/vdpau_internal.h
index 790b3ef..b2f9f3e 100644
--- a/libavcodec/vdpau_internal.h
+++ b/libavcodec/vdpau_internal.h
@@ -41,7 +41,7 @@ int ff_vdpau_add_buffer(AVCodecContext *avctx,
const uint8_t *buf, uint32_t buf_size);
-void ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf,
+int ff_vdpau_add_data_chunk(uint8_t *data, const uint8_t *buf,
int buf_size);
void ff_vdpau_mpeg_picture_complete(MpegEncContext *s, const uint8_t *buf,
More information about the ffmpeg-devel
mailing list