[FFmpeg-devel] [PATCH 2/2] avformat/flvdec: support enhanced flv PacketTypeMetadata
朱鹏飞
411294962 at qq.com
Wed Dec 13 15:41:47 EET 2023
> 2023年12月5日 05:32，James Almer <jamrial at gmail.com> 写道：
>
> On 11/15/2023 11:40 AM, zhupengfei via ffmpeg-devel wrote:
>> From: Zhu Pengfei <411294962 at qq.com>
>> Signed-off-by: Zhu Pengfei <411294962 at qq.com>
>> ---
>> libavformat/flvdec.c | 171 ++++++++++++++++++++++++++++++++++++++++++-
>> 1 file changed, 170 insertions(+), 1 deletion(-)
>> diff --git a/libavformat/flvdec.c b/libavformat/flvdec.c
>> index e25b5bd163..46bb0825ca 100644
>> --- a/libavformat/flvdec.c
>> +++ b/libavformat/flvdec.c
>> @@ -34,6 +34,7 @@
>> #include "libavutil/intfloat.h"
>> #include "libavutil/intreadwrite.h"
>> #include "libavutil/mathematics.h"
>> +#include "libavutil/mastering_display_metadata.h"
>> #include "avformat.h"
>> #include "demux.h"
>> #include "internal.h"
>> @@ -45,6 +46,28 @@
>> #define MAX_DEPTH 16 ///< arbitrary limit to prevent unbounded recursion
>> +typedef struct FLVMasteringMeta {
>> + double r_x;
>> + double r_y;
>> + double g_x;
>> + double g_y;
>> + double b_x;
>> + double b_y;
>> + double white_x;
>> + double white_y;
>> + double max_luminance;
>> + double min_luminance;
>> +} FLVMasteringMeta;
>> +
>> +typedef struct FLVMetaVideoColor {
>> + uint64_t matrix_coefficients;
>> + uint64_t transfer_characteristics;
>> + uint64_t primaries;
>> + uint64_t max_cll;
>> + uint64_t max_fall;
>> + FLVMasteringMeta mastering_meta;
>> +} FLVMetaVideoColor;
>> +
>> typedef struct FLVContext {
>> const AVClass *class; ///< Class for private options.
>> int trust_metadata; ///< configure streams according onMetaData
>> @@ -80,6 +103,8 @@ typedef struct FLVContext {
>> int64_t time_offset;
>> int64_t time_pos;
>> + FLVMetaVideoColor *metaVideoColor;
>> + int meta_color_info_flag;
>> } FLVContext;
>> /* AMF date type */
>> @@ -524,6 +549,7 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream,
>> FLVContext *flv = s->priv_data;
>> AVIOContext *ioc;
>> AMFDataType amf_type;
>> + FLVMetaVideoColor *meta_video_color = flv->metaVideoColor;
>> char str_val[1024];
>> double num_val;
>> amf_date date;
>> @@ -655,6 +681,36 @@ static int amf_parse_object(AVFormatContext *s, AVStream *astream,
>> } else if (!strcmp(key, "height") && vpar) {
>> vpar->height = num_val;
>> }
>> + } else if (!strcmp(key, "colorPrimaries") && meta_video_color) {
>
> You should put this inside an "else if (meta_video_color)" block, instead of checking for meta_video_color with every single string.
>
> See how it's done for flv->trust_metadata above.
>
>> + meta_video_color->primaries = num_val;
>> + } else if (!strcmp(key, "transferCharacteristics") && meta_video_color) {
>> + meta_video_color->transfer_characteristics = num_val;
>> + } else if (!strcmp(key, "matrixCoefficients") && meta_video_color) {
>> + meta_video_color->matrix_coefficients = num_val;
>> + } else if (!strcmp(key, "maxFall") && meta_video_color) {
>> + meta_video_color->max_fall = num_val;
>> + } else if (!strcmp(key, "maxCLL") && meta_video_color) {
>> + meta_video_color->max_cll = num_val;
>> + } else if (!strcmp(key, "redX") && meta_video_color) {
>> + meta_video_color->mastering_meta.r_x = num_val;
>> + } else if (!strcmp(key, "redY") && meta_video_color) {
>> + meta_video_color->mastering_meta.r_y = num_val;
>> + } else if (!strcmp(key, "greenX") && meta_video_color) {
>> + meta_video_color->mastering_meta.g_x = num_val;
>> + } else if (!strcmp(key, "greenY") && meta_video_color) {
>> + meta_video_color->mastering_meta.g_y = num_val;
>> + } else if (!strcmp(key, "blueX") && meta_video_color) {
>> + meta_video_color->mastering_meta.b_x = num_val;
>> + } else if (!strcmp(key, "blueY") && meta_video_color) {
>> + meta_video_color->mastering_meta.b_y = num_val;
>> + } else if (!strcmp(key, "whitePointX") && meta_video_color) {
>> + meta_video_color->mastering_meta.white_x = num_val;
>> + } else if (!strcmp(key, "whitePointY") && meta_video_color) {
>> + meta_video_color->mastering_meta.white_y = num_val;
>> + } else if (!strcmp(key, "maxLuminance") && meta_video_color) {
>> + meta_video_color->mastering_meta.max_luminance = num_val;
>> + } else if (!strcmp(key, "minLuminance") && meta_video_color) {
>> + meta_video_color->mastering_meta.min_luminance = num_val;
>> }
>> }
>> if (amf_type == AMF_DATA_TYPE_STRING) {
>> @@ -824,6 +880,7 @@ static int flv_read_close(AVFormatContext *s)
>> av_freep(&flv->new_extradata[i]);
>> av_freep(&flv->keyframe_times);
>> av_freep(&flv->keyframe_filepositions);
>> + av_freep(&flv->metaVideoColor);
>> return 0;
>> }
>> @@ -1028,6 +1085,104 @@ static int resync(AVFormatContext *s)
>> return AVERROR_EOF;
>> }
>> +static int flv_parse_video_color_info(AVFormatContext *s, AVStream *st, int64_t next_pos)
>> +{
>> + FLVContext *flv = s->priv_data;
>> + AMFDataType type;
>> + AVIOContext *ioc;
>> + char buffer[32];
>> + ioc = s->pb;
>> +
>> + // first object needs to be "colorInfo" string
>> + type = avio_r8(ioc);
>> + if (type != AMF_DATA_TYPE_STRING ||
>> + amf_get_string(ioc, buffer, sizeof(buffer)) < 0)
>> + return TYPE_UNKNOWN;
>> +
>> + if (strcmp(buffer, "colorInfo")) {
>> + av_log(s, AV_LOG_DEBUG, "Unknown type %s\n", buffer);
>> + return TYPE_UNKNOWN;
>> + }
>> +
>> + flv->metaVideoColor = av_mallocz(sizeof(FLVMetaVideoColor));
>> + if (!flv->metaVideoColor) {
>> + return AVERROR(ENOMEM);
>> + }
>> + flv->meta_color_info_flag = 1;
>> + amf_parse_object(s, NULL, NULL, buffer, next_pos, 0); // parse metadata
>> + return 0;
>> +}
>> +
>> +static int flv_update_video_color_info(AVFormatContext *s, AVStream *st)
>> +{
>> + FLVContext *flv = s->priv_data;
>> + const FLVMetaVideoColor* meta_video_color = flv->metaVideoColor;
>> + const FLVMasteringMeta *mastering_meta = &meta_video_color->mastering_meta;
>> +
>> + int has_mastering_primaries, has_mastering_luminance;
>> + // Mastering primaries are CIE 1931 coords, and must be > 0.
>> + has_mastering_primaries =
>> + mastering_meta->r_x > 0 && mastering_meta->r_y > 0 &&
>> + mastering_meta->g_x > 0 && mastering_meta->g_y > 0 &&
>> + mastering_meta->b_x > 0 && mastering_meta->b_y > 0 &&
>> + mastering_meta->white_x > 0 && mastering_meta->white_y > 0;
>> + has_mastering_luminance = mastering_meta->max_luminance >= 0 && mastering_meta->min_luminance >= 0;
>> +
>> + if (meta_video_color->matrix_coefficients != AVCOL_SPC_RESERVED)
>> + st->codecpar->color_space = meta_video_color->matrix_coefficients;
>> + if (meta_video_color->primaries != AVCOL_PRI_RESERVED &&
>> + meta_video_color->primaries != AVCOL_PRI_RESERVED0)
>> + st->codecpar->color_primaries = meta_video_color->primaries;
>> + if (meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED &&
>> + meta_video_color->transfer_characteristics != AVCOL_TRC_RESERVED0)
>> + st->codecpar->color_trc = meta_video_color->transfer_characteristics;
>> +
>> + if (meta_video_color->max_cll && meta_video_color->max_fall) {
>> + size_t size = 0;
>> + AVContentLightMetadata *metadata = av_content_light_metadata_alloc(&size);
>> + if (!metadata)
>> + return AVERROR(ENOMEM);
>> + if (!av_packet_side_data_add(&st->codecpar->coded_side_data, &st->codecpar->nb_coded_side_data,
>> + AV_PKT_DATA_CONTENT_LIGHT_LEVEL, metadata, size, 0)) {
>> + av_freep(&metadata);
>> + return AVERROR(ENOMEM);
>> + }
>> + metadata->MaxCLL = meta_video_color->max_cll;
>> + metadata->MaxFALL = meta_video_color->max_fall;
>> + }
>> +
>> + if (has_mastering_primaries || has_mastering_luminance) {
>> + AVMasteringDisplayMetadata *metadata;
>> + AVPacketSideData *sd = av_packet_side_data_new(&st->codecpar->coded_side_data,
>> + &st->codecpar->nb_coded_side_data,
>> + AV_PKT_DATA_MASTERING_DISPLAY_METADATA,
>> + sizeof(AVMasteringDisplayMetadata), 0);
>> + if (!sd)
>> + return AVERROR(ENOMEM);
>> + metadata = (AVMasteringDisplayMetadata*)sd->data;
>> + memset(metadata, 0, sizeof(AVMasteringDisplayMetadata));
>> + // hdrCll
>> + if (has_mastering_luminance) {
>> + metadata->max_luminance = av_d2q(mastering_meta->max_luminance, INT_MAX);
>> + metadata->min_luminance = av_d2q(mastering_meta->min_luminance, INT_MAX);
>> + metadata->has_luminance = 1;
>> + }
>> + // hdrMdcv
>> + if (has_mastering_primaries) {
>> + metadata->display_primaries[0][0] = av_d2q(mastering_meta->r_x, INT_MAX);
>> + metadata->display_primaries[0][1] = av_d2q(mastering_meta->r_y, INT_MAX);
>> + metadata->display_primaries[1][0] = av_d2q(mastering_meta->g_x, INT_MAX);
>> + metadata->display_primaries[1][1] = av_d2q(mastering_meta->g_y, INT_MAX);
>> + metadata->display_primaries[2][0] = av_d2q(mastering_meta->b_x, INT_MAX);
>> + metadata->display_primaries[2][1] = av_d2q(mastering_meta->b_y, INT_MAX);
>> + metadata->white_point[0] = av_d2q(mastering_meta->white_x, INT_MAX);
>> + metadata->white_point[1] = av_d2q(mastering_meta->white_y, INT_MAX);
>> + metadata->has_primaries = 1;
>> + }
>> + }
>> + return 0;
>> +}
>> +
>> static int flv_read_packet(AVFormatContext *s, AVPacket *pkt)
>> {
>> FLVContext *flv = s->priv_data;
>> @@ -1100,8 +1255,17 @@ retry:
>> video_codec_id = avio_rb32(s->pb);
>> size -= 4;
>> }
>> - if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD)
>> +
>> + if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && (flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
>> + type = flags & 0x0F;
>> + if (type == PacketTypeMetadata) {
>> + int ret = flv_parse_video_color_info(s, st, next);
>> + av_log(s, AV_LOG_INFO, "enhanced flv parse metadata ret %d and skip\n", ret);
>> + }
>> + goto skip;
>> + } else if ((flags & FLV_VIDEO_FRAMETYPE_MASK) == FLV_FRAME_VIDEO_INFO_CMD) {
>> goto skip;
>> + }
>> } else if (type == FLV_TAG_TYPE_META) {
>> stream_type=FLV_STREAM_TYPE_SUBTITLE;
>> if (size > 13 + 1 + 4) { // Header-type metadata stuff
>> @@ -1287,6 +1451,11 @@ retry_duration:
>> goto leave;
>> }
>> + if (enhanced_flv && stream_type == FLV_STREAM_TYPE_VIDEO && flv->meta_color_info_flag) {
>> + flv_update_video_color_info(s, st); // update av packet side data
>> + flv->meta_color_info_flag = 0;
>> + }
>> +
>> if (st->codecpar->codec_id == AV_CODEC_ID_H264 || st->codecpar->codec_id == AV_CODEC_ID_MPEG4 ||
>> (st->codecpar->codec_id == AV_CODEC_ID_HEVC && type == PacketTypeCodedFrames)) {
>> // sign extension
>
> A test for this would be nice. Can be one that creates a file and then demuxes it.
> See the fate-lavf-fate-* tests to remux an existing sample.
Thank you very much for your suggestion. All other comments have been modified as per your suggestions locally.
But about fate test, I have some questions that would like to ask for advice:
1. During the development stage, can I use ffmpeg with the mov/mkv muxer to write side data to construct an hdr flv file (ffmpeg -i *** -metadata:s:v **.mkv)
2. Referring to fate-lavf-fate-* tests, I did not find sample file with metadata in the official fate suite. If test-fate uses codec copy, it cannot obtain sidedata and therefore cannot be constructed flv file.
3. Do you have any suggestions? Besides creating a new fate suite file, are there any other methods?
Thanks
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org <mailto:ffmpeg-devel at ffmpeg.org>
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org <mailto:ffmpeg-devel-request at ffmpeg.org> with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list