[FFmpeg-devel] [PATCH 1/3] avcodec: move mastering display colour volume SEI handling to h2645_sei
Leo Izen
leo.izen at gmail.com
Thu Jul 20 04:05:53 EEST 2023
On 7/12/23 14:32, Jan Ekström wrote:
> This allows this common H.274 SEI to be parsed from both H.264
> as well as HEVC, as well as probably from VVC in the future.
>
> Generally attempts to keep the original code as similar as possible.
>
> FATE test refererence changes only change the order of side data
> export within a single frame. Nothing else seems to have changed.
> ---
> libavcodec/h2645_sei.c | 79 +++++++++++++++++++++++++
> libavcodec/h2645_sei.h | 9 +++
> libavcodec/h264_slice.c | 1 +
> libavcodec/hevc_sei.c | 31 ----------
> libavcodec/hevc_sei.h | 9 ---
> libavcodec/hevcdec.c | 50 +---------------
> tests/ref/fate/hevc-hdr-vivid-metadata | 16 ++---
> tests/ref/fate/hevc-hdr10-plus-metadata | 10 ++--
> 8 files changed, 105 insertions(+), 100 deletions(-)
>
> diff --git a/libavcodec/h2645_sei.c b/libavcodec/h2645_sei.c
> index 63ab711bc9..ea01e75405 100644
> --- a/libavcodec/h2645_sei.c
> +++ b/libavcodec/h2645_sei.c
> @@ -26,6 +26,7 @@
> #include "config_components.h"
>
> #include "libavutil/ambient_viewing_environment.h"
> +#include "libavutil/mastering_display_metadata.h"
Nitpick: This isn't alphabetized.
> #include "libavutil/display.h"
> #include "libavutil/hdr_dynamic_metadata.h"
> #include "libavutil/film_grain_params.h"
> @@ -392,6 +393,35 @@ static int decode_film_grain_characteristics(H2645SEIFilmGrainCharacteristics *h
> return 0;
> }
>
> +static int decode_nal_sei_mastering_display_info(H2645SEIMasteringDisplay *s,
> + GetByteContext *gb)
> +{
> + int i;
It seems a bit clearer to me if you inline the declaration into the for
loop, the only place it's used.
> +
> + if (bytestream2_get_bytes_left(gb) < 24)
> + return AVERROR_INVALIDDATA;
> +
> + // Mastering primaries
> + for (i = 0; i < 3; i++) {
> + s->display_primaries[i][0] = bytestream2_get_be16u(gb);
> + s->display_primaries[i][1] = bytestream2_get_be16u(gb);
> + }
> + // White point (x, y)
> + s->white_point[0] = bytestream2_get_be16u(gb);
> + s->white_point[1] = bytestream2_get_be16u(gb);
> +
> + // Max and min luminance of mastering display
> + s->max_luminance = bytestream2_get_be32u(gb);
> + s->min_luminance = bytestream2_get_be32u(gb);
> +
> + // As this SEI message comes before the first frame that references it,
> + // initialize the flag to 2 and decrement on IRAP access unit so it
> + // persists for the coded video sequence (e.g., between two IRAPs)
> + s->present = 2;
> +
> + return 0;
> +}
> +
> int ff_h2645_sei_message_decode(H2645SEI *h, enum SEIType type,
> enum AVCodecID codec_id, GetBitContext *gb,
> GetByteContext *gbyte, void *logctx)
> @@ -412,6 +442,9 @@ int ff_h2645_sei_message_decode(H2645SEI *h, enum SEIType type,
> case SEI_TYPE_AMBIENT_VIEWING_ENVIRONMENT:
> return decode_ambient_viewing_environment(&h->ambient_viewing_environment,
> gbyte);
> + case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> + return decode_nal_sei_mastering_display_info(&h->mastering_display,
> + gbyte);
> default:
> return FF_H2645_SEI_MESSAGE_UNHANDLED;
> }
> @@ -652,6 +685,51 @@ int ff_h2645_sei_to_frame(AVFrame *frame, H2645SEI *sei,
> dst_env->ambient_light_y = av_make_q(env->ambient_light_y, 50000);
> }
>
> + if (sei->mastering_display.present) {
> + // HEVC uses a g,b,r ordering, which we convert to a more natural r,g,b
> + const int mapping[3] = {2, 0, 1};
> + const int chroma_den = 50000;
> + const int luma_den = 10000;
> + int i;
> + AVMasteringDisplayMetadata *metadata =
> + av_mastering_display_metadata_create_side_data(frame);
> + if (!metadata)
> + return AVERROR(ENOMEM);
> +
> + for (i = 0; i < 3; i++) {
> + const int j = mapping[i];
> + metadata->display_primaries[i][0].num = sei->mastering_display.display_primaries[j][0];
> + metadata->display_primaries[i][0].den = chroma_den;
> + metadata->display_primaries[i][1].num = sei->mastering_display.display_primaries[j][1];
> + metadata->display_primaries[i][1].den = chroma_den;
> + }
> + metadata->white_point[0].num = sei->mastering_display.white_point[0];
> + metadata->white_point[0].den = chroma_den;
> + metadata->white_point[1].num = sei->mastering_display.white_point[1];
> + metadata->white_point[1].den = chroma_den;
> +
> + metadata->max_luminance.num = sei->mastering_display.max_luminance;
> + metadata->max_luminance.den = luma_den;
> + metadata->min_luminance.num = sei->mastering_display.min_luminance;
> + metadata->min_luminance.den = luma_den;
> + metadata->has_luminance = 1;
> + metadata->has_primaries = 1;
> +
> + av_log(avctx, AV_LOG_DEBUG, "Mastering Display Metadata:\n");
> + av_log(avctx, AV_LOG_DEBUG,
> + "r(%5.4f,%5.4f) g(%5.4f,%5.4f) b(%5.4f %5.4f) wp(%5.4f, %5.4f)\n",
> + av_q2d(metadata->display_primaries[0][0]),
> + av_q2d(metadata->display_primaries[0][1]),
> + av_q2d(metadata->display_primaries[1][0]),
> + av_q2d(metadata->display_primaries[1][1]),
> + av_q2d(metadata->display_primaries[2][0]),
> + av_q2d(metadata->display_primaries[2][1]),
> + av_q2d(metadata->white_point[0]), av_q2d(metadata->white_point[1]));
> + av_log(avctx, AV_LOG_DEBUG,
> + "min_luminance=%f, max_luminance=%f\n",
> + av_q2d(metadata->min_luminance), av_q2d(metadata->max_luminance));
Is there any particular reason these are %f but the ones above are
%5.4f? Not a big deal, just curious.
> + }
> +
> return 0;
> }
>
> @@ -667,4 +745,5 @@ void ff_h2645_sei_reset(H2645SEI *s)
> av_buffer_unref(&s->dynamic_hdr_vivid.info);
>
> s->ambient_viewing_environment.present = 0;
> + s->mastering_display.present = 0;
> }
> diff --git a/libavcodec/h2645_sei.h b/libavcodec/h2645_sei.h
> index e07ae10376..83e1b2ec16 100644
> --- a/libavcodec/h2645_sei.h
> +++ b/libavcodec/h2645_sei.h
> @@ -105,6 +105,14 @@ typedef struct H2645SEIFilmGrainCharacteristics {
> int persistence_flag; //< HEVC only
> } H2645SEIFilmGrainCharacteristics;
>
> +typedef struct H2645SEIMasteringDisplay {
> + int present;
> + uint16_t display_primaries[3][2];
> + uint16_t white_point[2];
> + uint32_t max_luminance;
> + uint32_t min_luminance;
> +} H2645SEIMasteringDisplay;
> +
> typedef struct H2645SEI {
> H2645SEIA53Caption a53_caption;
> H2645SEIAFD afd;
> @@ -116,6 +124,7 @@ typedef struct H2645SEI {
> H2645SEIAlternativeTransfer alternative_transfer;
> H2645SEIFilmGrainCharacteristics film_grain_characteristics;
> H2645SEIAmbientViewingEnvironment ambient_viewing_environment;
> + H2645SEIMasteringDisplay mastering_display;
> } H2645SEI;
>
> enum {
> diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c
> index 41bf30eefc..586ce20bba 100644
> --- a/libavcodec/h264_slice.c
> +++ b/libavcodec/h264_slice.c
> @@ -439,6 +439,7 @@ int ff_h264_update_thread_context(AVCodecContext *dst,
> return ret;
>
> h->sei.common.unregistered.x264_build = h1->sei.common.unregistered.x264_build;
> + h->sei.common.mastering_display = h1->sei.common.mastering_display;
>
> if (!h->cur_pic_ptr)
> return 0;
> diff --git a/libavcodec/hevc_sei.c b/libavcodec/hevc_sei.c
> index 3c6bde1b62..b7b77d4d0c 100644
> --- a/libavcodec/hevc_sei.c
> +++ b/libavcodec/hevc_sei.c
> @@ -49,35 +49,6 @@ static int decode_nal_sei_decoded_picture_hash(HEVCSEIPictureHash *s,
> return 0;
> }
>
> -static int decode_nal_sei_mastering_display_info(HEVCSEIMasteringDisplay *s,
> - GetByteContext *gb)
> -{
> - int i;
> -
> - if (bytestream2_get_bytes_left(gb) < 24)
> - return AVERROR_INVALIDDATA;
> -
> - // Mastering primaries
> - for (i = 0; i < 3; i++) {
> - s->display_primaries[i][0] = bytestream2_get_be16u(gb);
> - s->display_primaries[i][1] = bytestream2_get_be16u(gb);
> - }
> - // White point (x, y)
> - s->white_point[0] = bytestream2_get_be16u(gb);
> - s->white_point[1] = bytestream2_get_be16u(gb);
> -
> - // Max and min luminance of mastering display
> - s->max_luminance = bytestream2_get_be32u(gb);
> - s->min_luminance = bytestream2_get_be32u(gb);
> -
> - // As this SEI message comes before the first frame that references it,
> - // initialize the flag to 2 and decrement on IRAP access unit so it
> - // persists for the coded video sequence (e.g., between two IRAPs)
> - s->present = 2;
> -
> - return 0;
> -}
> -
> static int decode_nal_sei_content_light_info(HEVCSEIContentLight *s,
> GetByteContext *gb)
> {
> @@ -206,8 +177,6 @@ static int decode_nal_sei_prefix(GetBitContext *gb, GetByteContext *gbyte,
> return decode_nal_sei_decoded_picture_hash(&s->picture_hash, gbyte);
> case SEI_TYPE_PIC_TIMING:
> return decode_nal_sei_pic_timing(s, gb, ps, logctx);
> - case SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME:
> - return decode_nal_sei_mastering_display_info(&s->mastering_display, gbyte);
> case SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO:
> return decode_nal_sei_content_light_info(&s->content_light, gbyte);
> case SEI_TYPE_ACTIVE_PARAMETER_SETS:
> diff --git a/libavcodec/hevc_sei.h b/libavcodec/hevc_sei.h
> index 4189f5e6f7..077abdc74a 100644
> --- a/libavcodec/hevc_sei.h
> +++ b/libavcodec/hevc_sei.h
> @@ -53,14 +53,6 @@ typedef struct HEVCSEIPictureTiming {
> int picture_struct;
> } HEVCSEIPictureTiming;
>
> -typedef struct HEVCSEIMasteringDisplay {
> - int present;
> - uint16_t display_primaries[3][2];
> - uint16_t white_point[2];
> - uint32_t max_luminance;
> - uint32_t min_luminance;
> -} HEVCSEIMasteringDisplay;
> -
> typedef struct HEVCSEIContentLight {
> int present;
> uint16_t max_content_light_level;
> @@ -96,7 +88,6 @@ typedef struct HEVCSEI {
> H2645SEI common;
> HEVCSEIPictureHash picture_hash;
> HEVCSEIPictureTiming picture_timing;
> - HEVCSEIMasteringDisplay mastering_display;
> HEVCSEIContentLight content_light;
> int active_seq_parameter_set_id;
> HEVCSEITimeCode timecode;
> diff --git a/libavcodec/hevcdec.c b/libavcodec/hevcdec.c
> index fcf19b4eb6..434750965b 100644
> --- a/libavcodec/hevcdec.c
> +++ b/libavcodec/hevcdec.c
> @@ -2763,53 +2763,9 @@ static int set_side_data(HEVCContext *s)
>
> // Decrement the mastering display flag when IRAP frame has no_rasl_output_flag=1
> // so the side data persists for the entire coded video sequence.
> - if (s->sei.mastering_display.present > 0 &&
> + if (s->sei.common.mastering_display.present > 0 &&
> IS_IRAP(s) && s->no_rasl_output_flag) {
> - s->sei.mastering_display.present--;
> - }
> - if (s->sei.mastering_display.present) {
> - // HEVC uses a g,b,r ordering, which we convert to a more natural r,g,b
> - const int mapping[3] = {2, 0, 1};
> - const int chroma_den = 50000;
> - const int luma_den = 10000;
> - int i;
> - AVMasteringDisplayMetadata *metadata =
> - av_mastering_display_metadata_create_side_data(out);
> - if (!metadata)
> - return AVERROR(ENOMEM);
> -
> - for (i = 0; i < 3; i++) {
> - const int j = mapping[i];
> - metadata->display_primaries[i][0].num = s->sei.mastering_display.display_primaries[j][0];
> - metadata->display_primaries[i][0].den = chroma_den;
> - metadata->display_primaries[i][1].num = s->sei.mastering_display.display_primaries[j][1];
> - metadata->display_primaries[i][1].den = chroma_den;
> - }
> - metadata->white_point[0].num = s->sei.mastering_display.white_point[0];
> - metadata->white_point[0].den = chroma_den;
> - metadata->white_point[1].num = s->sei.mastering_display.white_point[1];
> - metadata->white_point[1].den = chroma_den;
> -
> - metadata->max_luminance.num = s->sei.mastering_display.max_luminance;
> - metadata->max_luminance.den = luma_den;
> - metadata->min_luminance.num = s->sei.mastering_display.min_luminance;
> - metadata->min_luminance.den = luma_den;
> - metadata->has_luminance = 1;
> - metadata->has_primaries = 1;
> -
> - av_log(s->avctx, AV_LOG_DEBUG, "Mastering Display Metadata:\n");
> - av_log(s->avctx, AV_LOG_DEBUG,
> - "r(%5.4f,%5.4f) g(%5.4f,%5.4f) b(%5.4f %5.4f) wp(%5.4f, %5.4f)\n",
> - av_q2d(metadata->display_primaries[0][0]),
> - av_q2d(metadata->display_primaries[0][1]),
> - av_q2d(metadata->display_primaries[1][0]),
> - av_q2d(metadata->display_primaries[1][1]),
> - av_q2d(metadata->display_primaries[2][0]),
> - av_q2d(metadata->display_primaries[2][1]),
> - av_q2d(metadata->white_point[0]), av_q2d(metadata->white_point[1]));
> - av_log(s->avctx, AV_LOG_DEBUG,
> - "min_luminance=%f, max_luminance=%f\n",
> - av_q2d(metadata->min_luminance), av_q2d(metadata->max_luminance));
> + s->sei.common.mastering_display.present--;
> }
> // Decrement the mastering display flag when IRAP frame has no_rasl_output_flag=1
> // so the side data persists for the entire coded video sequence.
> @@ -3667,7 +3623,7 @@ static int hevc_update_thread_context(AVCodecContext *dst,
> s->sei.common.frame_packing = s0->sei.common.frame_packing;
> s->sei.common.display_orientation = s0->sei.common.display_orientation;
> s->sei.common.alternative_transfer = s0->sei.common.alternative_transfer;
> - s->sei.mastering_display = s0->sei.mastering_display;
> + s->sei.common.mastering_display = s0->sei.common.mastering_display;
> s->sei.content_light = s0->sei.content_light;
>
> ret = export_stream_params_from_sei(s);
> diff --git a/tests/ref/fate/hevc-hdr-vivid-metadata b/tests/ref/fate/hevc-hdr-vivid-metadata
> index 5f69973098..cb5db4557f 100644
> --- a/tests/ref/fate/hevc-hdr-vivid-metadata
> +++ b/tests/ref/fate/hevc-hdr-vivid-metadata
> @@ -1,5 +1,13 @@
> [FRAME]
> [SIDE_DATA]
> +side_data_type=Content light level metadata
> +max_content=0
> +max_average=0
> +[/SIDE_DATA]
> +[SIDE_DATA]
> +side_data_type=H.26[45] User Data Unregistered SEI message
> +[/SIDE_DATA]
> +[SIDE_DATA]
> side_data_type=Mastering display metadata
> red_x=34000/50000
> red_y=16000/50000
> @@ -13,14 +21,6 @@ min_luminance=50/10000
> max_luminance=40000000/10000
> [/SIDE_DATA]
> [SIDE_DATA]
> -side_data_type=Content light level metadata
> -max_content=0
> -max_average=0
> -[/SIDE_DATA]
> -[SIDE_DATA]
> -side_data_type=H.26[45] User Data Unregistered SEI message
> -[/SIDE_DATA]
> -[SIDE_DATA]
> side_data_type=HDR Dynamic Metadata CUVA 005.1 2021 (Vivid)
> system_start_code=1
> num_windows=1
> diff --git a/tests/ref/fate/hevc-hdr10-plus-metadata b/tests/ref/fate/hevc-hdr10-plus-metadata
> index f226cd8c7b..cdf308b96a 100644
> --- a/tests/ref/fate/hevc-hdr10-plus-metadata
> +++ b/tests/ref/fate/hevc-hdr10-plus-metadata
> @@ -1,5 +1,10 @@
> [FRAME]
> [SIDE_DATA]
> +side_data_type=Content light level metadata
> +max_content=1000
> +max_average=200
> +[/SIDE_DATA]
> +[SIDE_DATA]
> side_data_type=Mastering display metadata
> red_x=13250/50000
> red_y=34500/50000
> @@ -13,11 +18,6 @@ min_luminance=50/10000
> max_luminance=10000000/10000
> [/SIDE_DATA]
> [SIDE_DATA]
> -side_data_type=Content light level metadata
> -max_content=1000
> -max_average=200
> -[/SIDE_DATA]
> -[SIDE_DATA]
> side_data_type=HDR Dynamic Metadata SMPTE2094-40 (HDR10+)
> application version=1
> num_windows=1
More information about the ffmpeg-devel
mailing list