[FFmpeg-devel] [PATCH] avcodec/nvenc: fix flushing for encoder-reuse

[Jai Luthra]

Recent encode API restructure (827d6fe73d) removed some state - which broke 
the API for flushing without closing the encoder.

This functionality was originally added in 3ea7057677 and is useful for 
segmented video, where we don't want to do expensive re-init of HW sessions 
for every segment.

---
 libavcodec/nvenc.c | 16 +++++++++++-----
 libavcodec/nvenc.h |  2 ++
 2 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index dddee8cac1..8eeb955b59 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -2177,6 +2177,11 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
         return AVERROR(EINVAL);
 
     if (frame && frame->buf[0]) {
+        if (ctx->encoder_flushing) {
+            ctx->encoder_flushing = 0;
+            av_fifo_reset(ctx->timestamp_list);
+        }
+
         in_surf = get_free_frame(ctx);
         if (!in_surf)
             return AVERROR(EAGAIN);
@@ -2256,6 +2261,7 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
         nvenc_codec_specific_pic_params(avctx, &pic_params, sei_data, sei_count);
     } else {
         pic_params.encodePicFlags = NV_ENC_PIC_FLAG_EOS;
+        ctx->encoder_flushing = 1;
     }
 
     res = nvenc_push_context(avctx);
@@ -2305,8 +2311,11 @@ int ff_nvenc_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
 
     if (!frame->buf[0]) {
         res = ff_encode_get_frame(avctx, frame);
-        if (res < 0 && res != AVERROR_EOF)
+        if (res == AVERROR_EOF || (ctx->encoder_flushing && res == AVERROR(EAGAIN))) {
+            // flushing mode, continue to send packets
+        } else if (res < 0) {
             return res;
+        }
     }
 
     res = nvenc_send_frame(avctx, frame);
@@ -2316,7 +2325,7 @@ int ff_nvenc_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
     } else
         av_frame_unref(frame);
 
-    if (output_ready(avctx, avctx->internal->draining)) {
+    if (output_ready(avctx, ctx->encoder_flushing)) {
         av_fifo_generic_read(ctx->output_surface_ready_queue, &tmp_out_surf, sizeof(tmp_out_surf), NULL);
 
         res = nvenc_push_context(avctx);
@@ -2344,8 +2353,5 @@ int ff_nvenc_receive_packet(AVCodecContext *avctx, AVPacket *pkt)
 
 av_cold void ff_nvenc_encode_flush(AVCodecContext *avctx)
 {
-    NvencContext *ctx = avctx->priv_data;
-
     nvenc_send_frame(avctx, NULL);
-    av_fifo_reset(ctx->timestamp_list);
 }
diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
index fefc5f7f0b..be8194424e 100644
--- a/libavcodec/nvenc.h
+++ b/libavcodec/nvenc.h
@@ -166,6 +166,8 @@ typedef struct NvencContext
     AVFifoBuffer *output_surface_ready_queue;
     AVFifoBuffer *timestamp_list;
 
+    int encoder_flushing;
+
     struct {
         void *ptr;
         int ptr_index;
-- 
2.32.0