[FFmpeg-devel] [PATCH] libavformat/mpegtsenc.c -- correctly re-emit extradata ahead of IDR pictures

[John Coiner]

This replaces and obsoletes the earlier version of this patch, which had a bug. Thanks to Marton Balint for spotting it.

This fixes https://trac.ffmpeg.org/ticket/10148, it ensures that each IDR frame is prefixed with SPS/PPS.

Tested manually, with inputs containing AUDs (x264's aud=1 mode) and also without (aud=0) to confirm it still inserts AUDs when necessary, and not otherwise.

---
 libavformat/mpegtsenc.c | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/libavformat/mpegtsenc.c b/libavformat/mpegtsenc.c
index 48d39e6a7d..64948f4d75 100644
--- a/libavformat/mpegtsenc.c
+++ b/libavformat/mpegtsenc.c
@@ -1877,6 +1877,7 @@ static int mpegts_write_packet_internal(AVFormatContext *s, AVPacket *pkt)
 
     if (st->codecpar->codec_id == AV_CODEC_ID_H264) {
         const uint8_t *p = buf, *buf_end = p + size;
+        uint8_t found_aud = 0;
         uint32_t state = -1;
         int extradd = (pkt->flags & AV_PKT_FLAG_KEY) ? st->codecpar->extradata_size : 0;
         int ret = ff_check_h264_startcode(s, st, pkt);
@@ -1886,17 +1887,25 @@ static int mpegts_write_packet_internal(AVFormatContext *s, AVPacket *pkt)
         if (extradd && AV_RB24(st->codecpar->extradata) > 1)
             extradd = 0;
 
+        // Ensure that all pictures are prefixed with an AUD, and that
+        // IDR pictures are also prefixed with SPS and PPS. SPS and PPS
+        // are assumed to be available in 'extradata' if not found in-band.
         do {
             p = avpriv_find_start_code(p, buf_end, &state);
             av_log(s, AV_LOG_TRACE, "nal %"PRId32"\n", state & 0x1f);
-            if ((state & 0x1f) == 7)
+            if ((state & 0x1f) == 7)  // SPS NAL
                 extradd = 0;
-        } while (p < buf_end && (state & 0x1f) != 9 &&
-                 (state & 0x1f) != 5 && (state & 0x1f) != 1);
-
-        if ((state & 0x1f) != 5)
+            if ((state & 0x1f) == 9)  // AUD
+                found_aud = 1;
+        } while (p < buf_end
+                 && (state & 0x1f) != 5  // IDR picture
+                 && (state & 0x1f) != 1  // non-IDR picture
+                 && (extradd > 0 || !found_aud));
+        if ((state & 0x1f) != 5) {
+            // Did not find IDR picture; do not emit extradata.
             extradd = 0;
-        if ((state & 0x1f) != 9) { // AUD NAL
+        }
+        if (extradd > 0 || !found_aud) {
             data = av_malloc(pkt->size + 6 + extradd);
             if (!data)
                 return AVERROR(ENOMEM);
-- 
2.39.1.456.gfc5497dd1b-goog