[FFmpeg-devel] [PATCH] lavc/qsvdec: update HDR side data on output AVFrame

[Xiang, Haihao]

On Mon, 2022-11-21 at 10:31 +0800, Xiang, Haihao wrote:
> From: Haihao Xiang <haihao.xiang at intel.com>
> 
> The SDK may provides HDR metadata for HDR streams via mfxExtBuffer
> attached on output mfxFrameSurface1
> 
> Signed-off-by: Haihao Xiang <haihao.xiang at intel.com>
> ---
>  libavcodec/qsv_internal.h |  6 ++++
>  libavcodec/qsvdec.c       | 73 +++++++++++++++++++++++++++++++++++++++
>  2 files changed, 79 insertions(+)
> 
> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
> index f6e739a686..5119ef4dff 100644
> --- a/libavcodec/qsv_internal.h
> +++ b/libavcodec/qsv_internal.h
> @@ -85,6 +85,12 @@ typedef struct QSVFrame {
>  #if QSV_VERSION_ATLEAST(1, 34)
>      mfxExtAV1FilmGrainParam av1_film_grain_param;
>  #endif
> +
> +#if QSV_VERSION_ATLEAST(1, 35)
> +    mfxExtMasteringDisplayColourVolume mdcv;
> +    mfxExtContentLightLevelInfo clli;
> +#endif
> +
>      mfxExtBuffer *ext_param[QSV_MAX_FRAME_EXT_PARAMS];
>      int num_ext_params;
>  
> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
> index 73405b5747..92bfea196e 100644
> --- a/libavcodec/qsvdec.c
> +++ b/libavcodec/qsvdec.c
> @@ -41,6 +41,7 @@
>  #include "libavutil/time.h"
>  #include "libavutil/imgutils.h"
>  #include "libavutil/film_grain_params.h"
> +#include "libavutil/mastering_display_metadata.h"
>  
>  #include "avcodec.h"
>  #include "codec_internal.h"
> @@ -492,6 +493,22 @@ static int alloc_frame(AVCodecContext *avctx, QSVContext
> *q, QSVFrame *frame)
>      }
>  #endif
>  
> +#if QSV_VERSION_ATLEAST(1, 35)
> +    if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id ==
> AV_CODEC_ID_HEVC) {
> +        frame->mdcv.Header.BufferId =
> MFX_EXTBUFF_MASTERING_DISPLAY_COLOUR_VOLUME;
> +        frame->mdcv.Header.BufferSz = sizeof(frame->mdcv);
> +        // The data in mdcv is valid when this flag is 1
> +        frame->mdcv.InsertPayloadToggle = 0;
> +        ff_qsv_frame_add_ext_param(avctx, frame, (mfxExtBuffer *)&frame-
> >mdcv);
> +
> +        frame->clli.Header.BufferId = MFX_EXTBUFF_CONTENT_LIGHT_LEVEL_INFO;
> +        frame->clli.Header.BufferSz = sizeof(frame->clli);
> +        // The data in clli is valid when this flag is 1
> +        frame->clli.InsertPayloadToggle = 0;
> +        ff_qsv_frame_add_ext_param(avctx, frame, (mfxExtBuffer *)&frame-
> >clli);
> +    }
> +#endif
> +
>      frame->used = 1;
>  
>      return 0;
> @@ -628,6 +645,53 @@ static int qsv_export_film_grain(AVCodecContext *avctx,
> mfxExtAV1FilmGrainParam
>  }
>  #endif
>  
> +#if QSV_VERSION_ATLEAST(1, 35)
> +static int qsv_export_hdr_side_data(AVCodecContext *avctx,
> mfxExtMasteringDisplayColourVolume *mdcv,
> +                                    mfxExtContentLightLevelInfo *clli,
> AVFrame *frame)
> +{
> +    // The SDK re-uses this flag for HDR SEI parsing
> +    if (mdcv->InsertPayloadToggle) {
> +        AVMasteringDisplayMetadata *mastering =
> av_mastering_display_metadata_create_side_data(frame);
> +        const int mapping[3] = {2, 0, 1};
> +        const int chroma_den = 50000;
> +        const int luma_den = 10000;
> +        int i;
> +
> +        if (!mastering)
> +            return AVERROR(ENOMEM);
> +
> +        for (i = 0; i < 3; i++) {
> +            const int j = mapping[i];
> +            mastering->display_primaries[i][0] = av_make_q(mdcv-
> >DisplayPrimariesX[j], chroma_den);
> +            mastering->display_primaries[i][1] = av_make_q(mdcv-
> >DisplayPrimariesY[j], chroma_den);
> +        }
> +
> +        mastering->white_point[0] = av_make_q(mdcv->WhitePointX, chroma_den);
> +        mastering->white_point[1] = av_make_q(mdcv->WhitePointY, chroma_den);
> +
> +        mastering->max_luminance = av_make_q(mdcv-
> >MaxDisplayMasteringLuminance, luma_den);
> +        mastering->min_luminance = av_make_q(mdcv-
> >MinDisplayMasteringLuminance, luma_den);
> +
> +        mastering->has_luminance = 1;
> +        mastering->has_primaries = 1;
> +    }
> +
> +    // The SDK re-uses this flag for HDR SEI parsing
> +    if (clli->InsertPayloadToggle) {
> +        AVContentLightMetadata *light =
> av_content_light_metadata_create_side_data(frame);
> +
> +        if (!light)
> +            return AVERROR(ENOMEM);
> +
> +        light->MaxCLL  = clli->MaxContentLightLevel;
> +        light->MaxFALL = clli->MaxPicAverageLightLevel;
> +    }
> +
> +    return 0;
> +}
> +
> +#endif
> +
>  static int qsv_decode(AVCodecContext *avctx, QSVContext *q,
>                        AVFrame *frame, int *got_frame,
>                        const AVPacket *avpkt)
> @@ -749,6 +813,15 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext
> *q,
>          }
>  #endif
>  
> +#if QSV_VERSION_ATLEAST(1, 35)
> +        if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 1, 35) && avctx->codec_id ==
> AV_CODEC_ID_HEVC) {
> +            ret = qsv_export_hdr_side_data(avctx, &aframe.frame->mdcv,
> &aframe.frame->clli, frame);
> +
> +            if (ret < 0)
> +                return ret;
> +        }
> +#endif
> +
>          frame->repeat_pict =
>              outsurf->Info.PicStruct & MFX_PICSTRUCT_FRAME_TRIPLING ? 4 :
>              outsurf->Info.PicStruct & MFX_PICSTRUCT_FRAME_DOUBLING ? 2 :

Will apply, 

-Haihao