[FFmpeg-devel] [PATCH] lavc/qsvdec: update HDR side data on output AVFrame

[Xiang, Haihao]

From: Haihao Xiang <haihao.xiang at intel.com>

The SDK may provides HDR metadata for HDR streams via mfxExtBuffer
attached on output mfxFrameSurface1

Signed-off-by: Haihao Xiang <haihao.xiang at intel.com>
---
 libavcodec/qsv_internal.h |  6 ++++
 libavcodec/qsvdec.c       | 73 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)

diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index f6e739a686..5119ef4dff 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -85,6 +85,12 @@ typedef struct QSVFrame {
 #if QSV_VERSION_ATLEAST(1, 34)
     mfxExtAV1FilmGrainParam av1_film_grain_param;
 #endif
+
+#if QSV_VERSION_ATLEAST(1, 35)
+    mfxExtMasteringDisplayColourVolume mdcv;
+    mfxExtContentLightLevelInfo clli;
+#endif
+
     mfxExtBuffer *ext_param[QSV_MAX_FRAME_EXT_PARAMS];
     int num_ext_params;
 
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 73405b5747..92bfea196e 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -41,6 +41,7 @@
 #include "libavutil/time.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/film_grain_params.h"
+#include "libavutil/mastering_display_metadata.h"
 
 #include "avcodec.h"
 #include "codec_internal.h"
@@ -492,6 +493,22 @@ static int alloc_frame(AVCodecContext *avctx, QSVContext *q, QSVFrame *frame)
     }
 #endif
 
+#if QSV_VERSION_ATLEAST(1, 35)
+    if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) {
+        frame->mdcv.Header.BufferId = MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
+        frame->mdcv.Header.BufferSz = sizeof(frame->mdcv);
+        // The data in mdcv is valid when this flag is 1
+        frame->mdcv.InsertPayloadToggle = 0;
+        ff_qsv_frame_add_ext_param(avctx, frame, (mfxExtBuffer *)&frame->mdcv);
+
+        frame->clli.Header.BufferId = MFX_EXTBUFF_CONTENT_LIGHT_LEVEL_INFO;
+        frame->clli.Header.BufferSz = sizeof(frame->clli);
+        // The data in clli is valid when this flag is 1
+        frame->clli.InsertPayloadToggle = 0;
+        ff_qsv_frame_add_ext_param(avctx, frame, (mfxExtBuffer *)&frame->clli);
+    }
+#endif
+
     frame->used = 1;
 
     return 0;
@@ -628,6 +645,53 @@ static int qsv_export_film_grain(AVCodecContext *avctx, mfxExtAV1FilmGrainParam
 }
 #endif
 
+#if QSV_VERSION_ATLEAST(1, 35)
+static int qsv_export_hdr_side_data(AVCodecContext *avctx, mfxExtMasteringDisplayColourVolume *mdcv,
+                                    mfxExtContentLightLevelInfo *clli, AVFrame *frame)
+{
+    // The SDK re-uses this flag for HDR SEI parsing
+    if (mdcv->InsertPayloadToggle) {
+        AVMasteringDisplayMetadata *mastering = av_mastering_display_metadata_create_side_data(frame);
+        const int mapping[3] = {2, 0, 1};
+        const int chroma_den = 50000;
+        const int luma_den = 10000;
+        int i;
+
+        if (!mastering)
+            return AVERROR(ENOMEM);
+
+        for (i = 0; i < 3; i++) {
+            const int j = mapping[i];
+            mastering->display_primaries[i][0] = av_make_q(mdcv->DisplayPrimariesX[j], chroma_den);
+            mastering->display_primaries[i][1] = av_make_q(mdcv->DisplayPrimariesY[j], chroma_den);
+        }
+
+        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
+        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
+
+        mastering->max_luminance = av_make_q(mdcv->MaxDisplayMasteringLuminance, luma_den);
+        mastering->min_luminance = av_make_q(mdcv->MinDisplayMasteringLuminance, luma_den);
+
+        mastering->has_luminance = 1;
+        mastering->has_primaries = 1;
+    }
+
+    // The SDK re-uses this flag for HDR SEI parsing
+    if (clli->InsertPayloadToggle) {
+        AVContentLightMetadata *light = av_content_light_metadata_create_side_data(frame);
+
+        if (!light)
+            return AVERROR(ENOMEM);
+
+        light->MaxCLL  = clli->MaxContentLightLevel;
+        light->MaxFALL = clli->MaxPicAverageLightLevel;
+    }
+
+    return 0;
+}
+
+#endif
+
 static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
                       AVFrame *frame, int *got_frame,
                       const AVPacket *avpkt)
@@ -749,6 +813,15 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
         }
 #endif
 
+#if QSV_VERSION_ATLEAST(1, 35)
+        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id == AV_CODEC_ID_HEVC) {
+            ret = qsv_export_hdr_side_data(avctx, &aframe.frame->mdcv, &aframe.frame->clli, frame);
+
+            if (ret < 0)
+                return ret;
+        }
+#endif
+
         frame->repeat_pict =
             outsurf->Info.PicStruct & MFX_PICSTRUCT_FRAME_TRIPLING ? 4 :
             outsurf->Info.PicStruct & MFX_PICSTRUCT_FRAME_DOUBLING ? 2 :
-- 
2.25.1