[FFmpeg-devel] [PATCH] hevc_mp4toannexb: Do not duplicate parameter sets

[Andriy Gelman]

From: Andriy Gelman <andriy.gelman at gmail.com>

Fixes #7799

Currently, the mp4toannexb filter always inserts extradata at the start
of each IRAP unit. This can lead to duplication of parameter sets if the
demuxed packet from mdat atom already contains a version of the
parameters.

As in ticket #7799 this can also lead to decoding errors when the
parameter sets of the IRAP frames are different from the ones stored in
extradata.

This commit avoids duplicating the parameter sets if they are already
present in the demuxed packet.

This commit also makes an update to the hevc-bsf-mp4toannexb fate
test since the result before this patch contained duplicate vps/sps/pps
nal units.
---
 libavcodec/hevc_mp4toannexb_bsf.c | 9 ++++++++-
 tests/fate/hevc.mak               | 2 +-
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/libavcodec/hevc_mp4toannexb_bsf.c b/libavcodec/hevc_mp4toannexb_bsf.c
index 09bce5b34c..5c27306b09 100644
--- a/libavcodec/hevc_mp4toannexb_bsf.c
+++ b/libavcodec/hevc_mp4toannexb_bsf.c
@@ -123,6 +123,7 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
 
     int got_irap = 0;
     int i, ret = 0;
+    int vps_detected, sps_detected, pps_detected = 0;
 
     ret = ff_bsf_get_packet(ctx, &in);
     if (ret < 0)
@@ -146,9 +147,15 @@ static int hevc_mp4toannexb_filter(AVBSFContext *ctx, AVPacket *out)
 
         nalu_type = (bytestream2_peek_byte(&gb) >> 1) & 0x3f;
 
+        switch (nalu_type) {
+          case HEVC_NAL_VPS: vps_detected = 1; break;
+          case HEVC_NAL_SPS: sps_detected = 1; break;
+          case HEVC_NAL_PPS: pps_detected = 1; break;
+        }
+
         /* prepend extradata to IRAP frames */
         is_irap       = nalu_type >= 16 && nalu_type <= 23;
-        add_extradata = is_irap && !got_irap;
+        add_extradata = is_irap && !got_irap && !(vps_detected && sps_detected && pps_detected);
         extra_size    = add_extradata * ctx->par_out->extradata_size;
         got_irap     |= is_irap;
 
diff --git a/tests/fate/hevc.mak b/tests/fate/hevc.mak
index 559c3898bc..4f812b0834 100644
--- a/tests/fate/hevc.mak
+++ b/tests/fate/hevc.mak
@@ -238,7 +238,7 @@ FATE_HEVC-$(call ALLYES, HEVC_DEMUXER MOV_DEMUXER HEVC_MP4TOANNEXB_BSF MOV_MUXER
 fate-hevc-bsf-mp4toannexb: tests/data/hevc-mp4.mov
 fate-hevc-bsf-mp4toannexb: CMD = md5 -i $(TARGET_PATH)/tests/data/hevc-mp4.mov -c:v copy -fflags +bitexact -f hevc
 fate-hevc-bsf-mp4toannexb: CMP = oneline
-fate-hevc-bsf-mp4toannexb: REF = 1873662a3af1848c37e4eb25722c8df9
+fate-hevc-bsf-mp4toannexb: REF = 3c9d998a3aa2b9e0fb1c1f434952bf8b
 
 fate-hevc-skiploopfilter: CMD = framemd5 -skip_loop_filter nokey -i $(TARGET_SAMPLES)/hevc-conformance/SAO_D_Samsung_5.bit -sws_flags bitexact
 FATE_HEVC += fate-hevc-skiploopfilter
-- 
2.22.0