[FFmpeg-devel] [PATCH 2/2] avcodec/vda_h264{, _dec}: fix leak of buffers

[Xidorn Quan]

CVPixelBuffers in vda_h264_end_frame may never be got by outside, so it
should be managed inside the hwaccel. This patch will break applications
which use this hwaccel because of double releasing, but it is impossible
to keep compatibility and fix this leak at the same time.
---
 libavcodec/vda_h264.c     | 26 ++++++++++++++++++++++++--
 libavcodec/vda_h264_dec.c |  7 +++++--
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/libavcodec/vda_h264.c b/libavcodec/vda_h264.c
index d0237c2..66d48b8 100644
--- a/libavcodec/vda_h264.c
+++ b/libavcodec/vda_h264.c
@@ -28,6 +28,9 @@
 #include "libavutil/avutil.h"
 #include "h264.h"
 
+struct vda_buffer {
+    CVPixelBufferRef cv_buffer;
+};
 
 /* Decoder callback that adds the vda frame to the queue in display order. */
 static void vda_decoder_callback (void *vda_hw_ctx,
@@ -108,22 +111,41 @@ static int vda_h264_decode_slice(AVCodecContext *avctx,
     return 0;
 }
 
+static void vda_h264_release_buffer(void *opaque, uint8_t *data)
+{
+    struct vda_buffer *context = opaque;
+    CVPixelBufferRelease(context->cv_buffer);
+    av_free(context);
+}
+
 static int vda_h264_end_frame(AVCodecContext *avctx)
 {
     H264Context *h                      = avctx->priv_data;
     struct vda_context *vda_ctx         = avctx->hwaccel_context;
     AVFrame *frame                      = &h->cur_pic_ptr->f;
+    struct vda_buffer *context;
+    AVBufferRef *buffer;
     int status;
 
     if (!vda_ctx->decoder || !vda_ctx->priv_bitstream)
         return -1;
 
     status = vda_sync_decode(vda_ctx);
-    frame->data[3] = (void*)vda_ctx->cv_buffer;
-
     if (status)
         av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
 
+    context = av_mallocz(sizeof(*context));
+    buffer = av_buffer_create(NULL, 0, vda_h264_release_buffer, context, 0);
+    if (!context || !buffer) {
+        CVPixelBufferRelease(vda_ctx->cv_buffer);
+        av_free(context);
+        return -1;
+    }
+
+    context->cv_buffer = vda_ctx->cv_buffer;
+    frame->buf[3] = buffer;
+    frame->data[3] = (void*)vda_ctx->cv_buffer;
+
     return status;
 }
 
diff --git a/libavcodec/vda_h264_dec.c b/libavcodec/vda_h264_dec.c
index 4e60de0..39069dc 100644
--- a/libavcodec/vda_h264_dec.c
+++ b/libavcodec/vda_h264_dec.c
@@ -65,6 +65,7 @@ static enum AVPixelFormat get_format(struct AVCodecContext *avctx,
 }
 
 typedef struct {
+    AVBufferRef *buffer;
     CVPixelBufferRef cv_buffer;
 } VDABufferContext;
 
@@ -72,7 +73,7 @@ static void release_buffer(void *opaque, uint8_t *data)
 {
     VDABufferContext *context = opaque;
     CVPixelBufferUnlockBaseAddress(context->cv_buffer, 0);
-    CVPixelBufferRelease(context->cv_buffer);
+    av_buffer_unref(&context->buffer);
     av_free(context);
 }
 
@@ -102,8 +103,10 @@ static int vdadec_decode(AVCodecContext *avctx,
         AVBufferRef *buffer = pic->buf[0];
         VDABufferContext *context = av_buffer_get_opaque(buffer);
         CVPixelBufferRef cv_buffer = (CVPixelBufferRef)pic->data[3];
-        CVPixelBufferLockBaseAddress(cv_buffer, 0);
+
+        context->buffer = av_buffer_ref(pic->buf[3]);
         context->cv_buffer = cv_buffer;
+        CVPixelBufferLockBaseAddress(cv_buffer, 0);
         pic->format = ctx->pix_fmt;
         if (CVPixelBufferIsPlanar(cv_buffer)) {
             int i, count = CVPixelBufferGetPlaneCount(cv_buffer);
-- 
1.8.2.2