[FFmpeg-devel] [PATCH 10/13] avformat/mux: Fix memleaks on errors I

[Andreas Rheinhardt]

If writing an uncoded frame fails at the preparatory steps of
av_[interleaved_]write_frame, the frame would be either not freed
at all in case of av_write_frame or would leak when the fake packet
would be unreferenced like an ordinary packet.
There is also a memleak when the output format is not suited for
writing uncoded frames as the documentation states that ownership of the
frame passes to av_[interleaved_]write_uncoded frame. Both of these
memleaks have been fixed.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at gmail.com>
---
 libavformat/mux.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/libavformat/mux.c b/libavformat/mux.c
index 5b67a793ac..c524c0886c 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -1241,7 +1241,11 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
             return s->pb->error;
     }
 fail:
-    av_packet_unref(pkt);
+    // This is a deviation from the usual behaviour
+    // of av_interleaved_write_frame: We leave cleaning
+    // up uncoded frames to write_uncoded_frame_internal.
+    if (!(pkt->flags & AV_PKT_FLAG_UNCODED_FRAME))
+        av_packet_unref(pkt);
     return ret;
 }
 
@@ -1334,10 +1338,13 @@ static int write_uncoded_frame_internal(AVFormatContext *s, int stream_index,
                                         AVFrame *frame, int interleaved)
 {
     AVPacket pkt, *pktp;
+    int ret;
 
     av_assert0(s->oformat);
-    if (!s->oformat->write_uncoded_frame)
-        return AVERROR(ENOSYS);
+    if (!s->oformat->write_uncoded_frame) {
+        ret = AVERROR(ENOSYS);
+        goto free;
+    }
 
     if (!frame) {
         pktp = NULL;
@@ -1353,8 +1360,14 @@ static int write_uncoded_frame_internal(AVFormatContext *s, int stream_index,
         pkt.flags |= AV_PKT_FLAG_UNCODED_FRAME;
     }
 
-    return interleaved ? av_interleaved_write_frame(s, pktp) :
-                         av_write_frame(s, pktp);
+    ret = interleaved ? av_interleaved_write_frame(s, pktp) :
+                        av_write_frame(s, pktp);
+    if (ret < 0 && pktp && pktp->data) {
+        frame = (AVFrame*)pktp->data;
+    free:
+        av_frame_free(&frame);
+    }
+    return ret;
 }
 
 int av_write_uncoded_frame(AVFormatContext *s, int stream_index,
-- 
2.21.0