[FFmpeg-devel] [PATCH] lavf/mux: bypass interleave delay early when waiting on subtitle streams

[rcombs]

This avoids long delays when converting live streams that have sparse subtitles
---
 libavformat/avformat.h      |  9 +++++++++
 libavformat/mux.c           | 25 +++++++++++++++++++++----
 libavformat/options_table.h |  1 +
 libavformat/version.h       |  2 +-
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 9b9b634ec3..da7e5755e8 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -1957,6 +1957,15 @@ typedef struct AVFormatContext {
      * - decoding: set by user
      */
     int max_probe_packets;
+
+    /**
+     * Maximum buffering duration for interleaving sparse streams.
+     *
+     * @see max_interleave_delta
+     *
+     * Applies only to subtitle and data streams.
+     */
+    int64_t max_sparse_interleave_delta;
 } AVFormatContext;
 
 #if FF_API_FORMAT_GET_SET
diff --git a/libavformat/mux.c b/libavformat/mux.c
index d88746e8c5..f2f272cf65 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -1033,6 +1033,7 @@ int ff_interleave_packet_per_dts(AVFormatContext *s, AVPacket *out,
     AVPacketList *pktl;
     int stream_count = 0;
     int noninterleaved_count = 0;
+    int sparse_count = 0;
     int i, ret;
     int eof = flush;
 
@@ -1044,6 +1045,9 @@ int ff_interleave_packet_per_dts(AVFormatContext *s, AVPacket *out,
     for (i = 0; i < s->nb_streams; i++) {
         if (s->streams[i]->last_in_packet_buffer) {
             ++stream_count;
+        } else if (s->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE ||
+                   s->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_DATA) {
+            ++sparse_count;
         } else if (s->streams[i]->codecpar->codec_type != AVMEDIA_TYPE_ATTACHMENT &&
                    s->streams[i]->codecpar->codec_id != AV_CODEC_ID_VP8 &&
                    s->streams[i]->codecpar->codec_id != AV_CODEC_ID_VP9) {
@@ -1054,10 +1058,12 @@ int ff_interleave_packet_per_dts(AVFormatContext *s, AVPacket *out,
     if (s->internal->nb_interleaved_streams == stream_count)
         flush = 1;
 
-    if (s->max_interleave_delta > 0 &&
-        s->internal->packet_buffer &&
+    if (s->internal->packet_buffer &&
         !flush &&
-        s->internal->nb_interleaved_streams == stream_count+noninterleaved_count
+        ((s->max_interleave_delta > 0 &&
+          s->internal->nb_interleaved_streams == stream_count+noninterleaved_count+sparse_count) ||
+         (s->max_sparse_interleave_delta > 0 &&
+          s->internal->nb_interleaved_streams == stream_count+sparse_count))
     ) {
         AVPacket *top_pkt = &s->internal->packet_buffer->pkt;
         int64_t delta_dts = INT64_MIN;
@@ -1078,12 +1084,23 @@ int ff_interleave_packet_per_dts(AVFormatContext *s, AVPacket *out,
             delta_dts = FFMAX(delta_dts, last_dts - top_dts);
         }
 
-        if (delta_dts > s->max_interleave_delta) {
+        if (s->max_interleave_delta > 0 &&
+            delta_dts > s->max_interleave_delta &&
+            s->internal->nb_interleaved_streams == stream_count+noninterleaved_count+sparse_count) {
             av_log(s, AV_LOG_DEBUG,
                    "Delay between the first packet and last packet in the "
                    "muxing queue is %"PRId64" > %"PRId64": forcing output\n",
                    delta_dts, s->max_interleave_delta);
             flush = 1;
+        } else if (s->max_sparse_interleave_delta > 0 &&
+                   delta_dts > s->max_sparse_interleave_delta &&
+                   s->internal->nb_interleaved_streams == stream_count+sparse_count) {
+            av_log(s, AV_LOG_DEBUG,
+                   "Delay between the first packet and last packet in the "
+                   "muxing queue is %"PRId64" > %"PRId64" and all delayed "
+                   "streams are sparse: forcing output\n",
+                   delta_dts, s->max_sparse_interleave_delta);
+            flush = 1;
         }
     }
 
diff --git a/libavformat/options_table.h b/libavformat/options_table.h
index e26b512440..db04f970e2 100644
--- a/libavformat/options_table.h
+++ b/libavformat/options_table.h
@@ -91,6 +91,7 @@ static const AVOption avformat_options[] = {
 {"metadata_header_padding", "set number of bytes to be written as padding in a metadata header", OFFSET(metadata_header_padding), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, E},
 {"output_ts_offset", "set output timestamp offset", OFFSET(output_ts_offset), AV_OPT_TYPE_DURATION, {.i64 = 0}, -INT64_MAX, INT64_MAX, E},
 {"max_interleave_delta", "maximum buffering duration for interleaving", OFFSET(max_interleave_delta), AV_OPT_TYPE_INT64, { .i64 = 10000000 }, 0, INT64_MAX, E },
+{"max_sparse_interleave_delta", "maximum buffering duration for interleaving sparse streams", OFFSET(max_sparse_interleave_delta), AV_OPT_TYPE_INT64, { .i64 = 1000000 }, 0, INT64_MAX, E },
 {"f_strict", "how strictly to follow the standards (deprecated; use strict, save via avconv)", OFFSET(strict_std_compliance), AV_OPT_TYPE_INT, {.i64 = DEFAULT }, INT_MIN, INT_MAX, D|E, "strict"},
 {"strict", "how strictly to follow the standards", OFFSET(strict_std_compliance), AV_OPT_TYPE_INT, {.i64 = DEFAULT }, INT_MIN, INT_MAX, D|E, "strict"},
 {"very", "strictly conform to a older more strict version of the spec or reference software", 0, AV_OPT_TYPE_CONST, {.i64 = FF_COMPLIANCE_VERY_STRICT }, INT_MIN, INT_MAX, D|E, "strict"},
diff --git a/libavformat/version.h b/libavformat/version.h
index e815c1f3c4..18c2f5fec2 100644
--- a/libavformat/version.h
+++ b/libavformat/version.h
@@ -32,7 +32,7 @@
 // Major bumping may affect Ticket5467, 5421, 5451(compatibility with Chromium)
 // Also please add any ticket numbers that you believe might be affected here
 #define LIBAVFORMAT_VERSION_MAJOR  58
-#define LIBAVFORMAT_VERSION_MINOR  41
+#define LIBAVFORMAT_VERSION_MINOR  42
 #define LIBAVFORMAT_VERSION_MICRO 100
 
 #define LIBAVFORMAT_VERSION_INT AV_VERSION_INT(LIBAVFORMAT_VERSION_MAJOR, \
-- 
2.24.1