[FFmpeg-devel] [PATCH v2 6/7] avcodec/avcodec: Use avcodec_close() on avcodec_open2() failure

[Andreas Rheinhardt]

Compared to the earlier behaviour the following changes:
a) AVCodecInternal.byte_buffer is freed.
b) The last_pkt_props FIFO is emptied before freeing it.
c) If set AVCodecContext.hwaccel is uninitialized and its private data
is freed; hw_frames_ctx and hw_device_ctx are also unreferenced.
d) coded_side_data is freed.
e) active_thread_type is reset.
a), b), d) should be no-ops as the buffer/fifo should be empty and
no coded_side_data should exist at any point of avcodec_open2().
e) is obviously not bad.
c) is in accordance with the documentation of hw_(frames|device)_ctx
which states that libacodec takes over ownership of these references.
At least in the case of VC-1 it is possible for the hw acceleration to
be set during init and in this case freeing it actually fixes a memleak.

avcodec_close() needed only minor adjustments to make it work with
a potentially not fully initialized codec.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>
---
Now including minor necessary adjustments to make avcodec_close() work
with a not properly-allocated AVCodecContext.

 libavcodec/avcodec.c | 59 +++++++-------------------------------------
 1 file changed, 9 insertions(+), 50 deletions(-)

diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c
index c7a8001608..7f9c9c1c87 100644
--- a/libavcodec/avcodec.c
+++ b/libavcodec/avcodec.c
@@ -395,51 +395,8 @@ end:
 
     return ret;
 free_and_end:
-    if (avci->needs_close && avctx->codec->close)
-        avctx->codec->close(avctx);
-
-    if (CONFIG_FRAME_THREAD_ENCODER && avci->frame_thread_encoder)
-        ff_frame_thread_encoder_free(avctx);
-    if (HAVE_THREADS && avci->thread_ctx)
-        ff_thread_free(avctx);
-
-    if (codec->priv_class && avctx->priv_data)
-        av_opt_free(avctx->priv_data);
-    av_opt_free(avctx);
-
-    if (av_codec_is_encoder(avctx->codec)) {
-#if FF_API_CODED_FRAME
-FF_DISABLE_DEPRECATION_WARNINGS
-        av_frame_free(&avctx->coded_frame);
-FF_ENABLE_DEPRECATION_WARNINGS
-#endif
-        av_freep(&avctx->extradata);
-        avctx->extradata_size = 0;
-    }
-
+    avcodec_close(avctx);
     av_dict_free(&tmp);
-    av_freep(&avctx->priv_data);
-    if (av_codec_is_decoder(avctx->codec))
-        av_freep(&avctx->subtitle_header);
-
-#if FF_API_OLD_ENCDEC
-    av_frame_free(&avci->to_free);
-    av_frame_free(&avci->compat_decode_frame);
-    av_packet_free(&avci->compat_encode_packet);
-#endif
-    av_frame_free(&avci->buffer_frame);
-    av_packet_free(&avci->buffer_pkt);
-    av_packet_free(&avci->last_pkt_props);
-    av_fifo_freep(&avci->pkt_props);
-
-    av_packet_free(&avci->ds.in_pkt);
-    av_frame_free(&avci->es.in_frame);
-    av_bsf_free(&avci->bsf);
-
-    av_buffer_unref(&avci->pool);
-    av_freep(&avci);
-    avctx->internal = NULL;
-    avctx->codec = NULL;
     goto end;
 }
 
@@ -549,14 +506,15 @@ av_cold int avcodec_close(AVCodecContext *avctx)
 #endif
         av_frame_free(&avci->buffer_frame);
         av_packet_free(&avci->buffer_pkt);
-        av_packet_unref(avci->last_pkt_props);
-        while (av_fifo_size(avci->pkt_props) >= sizeof(*avci->last_pkt_props)) {
-            av_fifo_generic_read(avci->pkt_props, avci->last_pkt_props,
-                                 sizeof(*avci->last_pkt_props), NULL);
-            av_packet_unref(avci->last_pkt_props);
+        if (avci->pkt_props) {
+            while (av_fifo_size(avci->pkt_props) >= sizeof(*avci->last_pkt_props)) {
+                av_packet_unref(avci->last_pkt_props);
+                av_fifo_generic_read(avci->pkt_props, avci->last_pkt_props,
+                                     sizeof(*avci->last_pkt_props), NULL);
+            }
+            av_fifo_freep(&avci->pkt_props);
         }
         av_packet_free(&avci->last_pkt_props);
-        av_fifo_freep(&avci->pkt_props);
 
         av_packet_free(&avci->ds.in_pkt);
         av_frame_free(&avci->es.in_frame);
@@ -586,6 +544,7 @@ av_cold int avcodec_close(AVCodecContext *avctx)
     av_freep(&avctx->priv_data);
     if (av_codec_is_encoder(avctx->codec)) {
         av_freep(&avctx->extradata);
+        avctx->extradata_size = 0;
 #if FF_API_CODED_FRAME
 FF_DISABLE_DEPRECATION_WARNINGS
         av_frame_free(&avctx->coded_frame);
-- 
2.27.0