[FFmpeg-devel] [PATCH 03/27] cbs_h2645: Merge SEI message handling in common between codecs
Mark Thompson
sw at jkqxz.net
Wed Jan 6 20:57:51 EET 2021
On 04/01/2021 15:42, Nuo Mi wrote:
> On Sat, Jan 2, 2021 at 5:44 AM Mark Thompson <sw at jkqxz.net> wrote:
>
>> ---
>> libavcodec/Makefile | 4 +-
>> libavcodec/cbs_h264.h | 50 +---
>> libavcodec/cbs_h2645.c | 300 +++++++++++----------
>> libavcodec/cbs_h264_syntax_template.c | 173 +-----------
>> libavcodec/cbs_h265.h | 33 +--
>> libavcodec/cbs_h265_syntax_template.c | 269 +++----------------
>> libavcodec/cbs_sei.c | 369 ++++++++++++++++++++++++++
>> libavcodec/cbs_sei.h | 255 ++++++++++++++++++
>> libavcodec/cbs_sei_syntax_template.c | 215 +++++++++++++--
>> libavcodec/h264_metadata_bsf.c | 113 ++++----
>> libavcodec/vaapi_encode_h264.c | 51 ++--
>> libavcodec/vaapi_encode_h265.c | 38 +--
>> 12 files changed, 1107 insertions(+), 763 deletions(-)
>> create mode 100644 libavcodec/cbs_sei.c
>>
>> ...
>> diff --git a/libavcodec/cbs_sei.c b/libavcodec/cbs_sei.c
>> new file mode 100644
>> index 0000000000..323997b600
>> --- /dev/null
>> +++ b/libavcodec/cbs_sei.c
>> @@ -0,0 +1,369 @@
>> +/*
>> + * This file is part of FFmpeg.
>> + *
>> + * FFmpeg is free software; you can redistribute it and/or
>> + * modify it under the terms of the GNU Lesser General Public
>> + * License as published by the Free Software Foundation; either
>> + * version 2.1 of the License, or (at your option) any later version.
>> + *
>> + * FFmpeg is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> + * Lesser General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU Lesser General Public
>> + * License along with FFmpeg; if not, write to the Free Software
>> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
>> 02110-1301 USA
>> + */
>> +
>> +#include "cbs.h"
>> +#include "cbs_internal.h"
>> +#include "cbs_h264.h"
>> +#include "cbs_h265.h"
>> +#include "cbs_sei.h"
>> +
>> +static void cbs_free_user_data_registered(void *opaque, uint8_t *data)
>> +{
>> + SEIRawUserDataRegistered *udr = (SEIRawUserDataRegistered*)data;
>> + av_buffer_unref(&udr->data_ref);
>> + av_free(udr);
>> +}
>> +
>> +static void cbs_free_user_data_unregistered(void *opaque, uint8_t *data)
>> +{
>> + SEIRawUserDataUnregistered *udu = (SEIRawUserDataUnregistered*)data;
>> + av_buffer_unref(&udu->data_ref);
>> + av_free(udu);
>> +}
>> +
>> +int ff_cbs_sei_alloc_message_payload(SEIRawMessage *message,
>> + const SEIMessageTypeDescriptor *desc)
>> +{
>> + void (*free_func)(void*, uint8_t*);
>> +
>> + av_assert0(message->payload == NULL &&
>> + message->payload_ref == NULL);
>> + message->payload_type = desc->type;
>> +
>> + if (desc->type == SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35)
>> + free_func = &cbs_free_user_data_registered;
>> + else if (desc->type == SEI_TYPE_USER_DATA_UNREGISTERED)
>> + free_func = &cbs_free_user_data_unregistered;
>> + else
>> + free_func = NULL;
>> +
>> + if (free_func) {
>> + message->payload = av_mallocz(desc->size);
>> + if (!message->payload)
>> + return AVERROR(ENOMEM);
>> + message->payload_ref =
>> + av_buffer_create(message->payload, desc->size,
>> + free_func, NULL, 0);
>>
> leak message->payload if av_buffer_create returns error
Fixed.
>> + } else {
>> + message->payload_ref = av_buffer_alloc(desc->size);
>> + }
>> + if (!message->payload_ref)
>> + return AVERROR(ENOMEM);
>> + message->payload = message->payload_ref->data;
>> +
>> + return 0;
>> +}
>> +
>> +int ff_cbs_sei_list_add(SEIRawMessageList *list)
>> +{
>> + void *ptr;
>> + int old_count = list->nb_messages_allocated;
>> +
>> + av_assert0(list->nb_messages <= old_count);
>> + if (list->nb_messages + 1 > old_count) {
>> + int new_count = 2 * old_count + 1;
>> +
>> + ptr = av_realloc_array(list->messages,
>> + new_count, sizeof(*list->messages));
>> + if (!ptr)
>> + return AVERROR(ENOMEM);
>> +
>> + list->messages = ptr;
>> + list->nb_messages_allocated = new_count;
>> +
>> + // Zero the newly-added entries.
>> + memset(list->messages + old_count, 0,
>> + (new_count - old_count) * sizeof(*list->messages));
>> + }
>> + ++list->nb_messages;
>> + return 0;
>> +}
>> +
>> +void ff_cbs_sei_free_message_list(SEIRawMessageList *list)
>> +{
>> + for (int i = 0; i < list->nb_messages; i++) {
>> + SEIRawMessage *message = &list->messages[i];
>> + av_buffer_unref(&message->payload_ref);
>> + av_buffer_unref(&message->extension_data_ref);
>> + }
>> + av_free(list->messages);
>> +}
>> +
>> +static int cbs_sei_get_unit(CodedBitstreamContext *ctx,
>> + CodedBitstreamFragment *au,
>> + int prefix,
>> + CodedBitstreamUnit **sei_unit)
>> +{
>> + CodedBitstreamUnit *unit;
>> + int sei_type, highest_vcl_type, err, i, position;
>> +
>> + switch (ctx->codec->codec_id) {
>> + case AV_CODEC_ID_H264:
>> + // (We can ignore auxiliary slices because we only have prefix
>> + // SEI in H.264 and an auxiliary picture must always follow a
>> + // primary picture.)
>> + highest_vcl_type = H264_NAL_IDR_SLICE;
>> + if (prefix)
>> + sei_type = H264_NAL_SEI;
>> + else
>> + return AVERROR(EINVAL);
>> + break;
>> + case AV_CODEC_ID_H265:
>> + highest_vcl_type = HEVC_NAL_RSV_VCL31;
>> + if (prefix)
>> + sei_type = HEVC_NAL_SEI_PREFIX;
>> + else
>> + sei_type = HEVC_NAL_SEI_SUFFIX;
>> + break;
>> + default:
>> + return AVERROR(EINVAL);
>> + }
>> +
>> + // Find an existing SEI NAL unit of the right type.
>> + unit = NULL;
>> + for (i = 0; i < au->nb_units; i++) {
>> + if (au->units[i].type == sei_type) {
>> + unit = &au->units[i];
>> + break;
>> + }
>> + }
>> +
>> + if (unit) {
>> + *sei_unit = unit;
>> + return 0;
>> + }
>> +
>> + // Need to add a new SEI NAL unit ...
>> + if (prefix) {
>> + // ... before the first VCL NAL unit.
>> + for (i = 0; i < au->nb_units; i++) {
>> + if (au->units[i].type < highest_vcl_type)
>> + break;
>> + }
>> + position = i;
>> + } else {
>> + // ... after the last VCL NAL unit.
>> + for (i = au->nb_units; i >= 0; i--) {
>>
> This will access au->units[au->nb_units], will it read beyond the boundary?
Yes, fixed.
(This code is currently unreachable, because it never tries to insert anything which would be suffix SEI. I'll try hacking something in temporarily to test it.)
>> + if (au->units[i].type < highest_vcl_type)
>> + break;
>> + }
>> + if (i < 0) {
>> + // No VCL units; just put it at the end.
>> + position = -1;
>> + } else {
>> + position = i + 1;
>> + }
>> + }
>> +
>> + err = ff_cbs_insert_unit_content(au, position, sei_type,
>> + NULL, NULL);
>> + if (err < 0)
>> + return err;
>> + unit = &au->units[position];
>> + unit->type = sei_type;
>> +
>> + err = ff_cbs_alloc_unit_content2(ctx, unit);
>> + if (err < 0)
>> + return err;
>> +
>> + switch (ctx->codec->codec_id) {
>> + case AV_CODEC_ID_H264:
>> + {
>> + H264RawSEI sei = {
>> + .nal_unit_header = {
>> + .nal_ref_idc = 0,
>> + .nal_unit_type = sei_type,
>> + },
>> + };
>> + memcpy(unit->content, &sei, sizeof(sei));
>> + }
>> + break;
>> + case AV_CODEC_ID_H265:
>> + {
>> + H265RawSEI sei = {
>> + .nal_unit_header = {
>> + .nal_unit_type = sei_type,
>> + .nuh_layer_id = 0,
>> + .nuh_temporal_id_plus1 = 1,
>> + },
>> + };
>> + memcpy(unit->content, &sei, sizeof(sei));
>> + }
>> + break;
>> + default:
>> + av_assert0(0);
>> + }
>> +
>> + *sei_unit = unit;
>> + return 0;
>> +}
>> +
>> ...
>> +
>> +void ff_cbs_sei_delete_message_type(CodedBitstreamContext *ctx,
>> + CodedBitstreamFragment *au,
>> + uint32_t payload_type)
>> +{
>> + int err, i, j;
>> +
>> + for (i = 0; i < au->nb_units; i++) {
>> + CodedBitstreamUnit *unit = &au->units[i];
>> + SEIRawMessageList *list;
>> +
>> + err = cbs_sei_get_message_list(ctx, unit, &list);
>> + if (err < 0)
>> + continue;
>> +
>> + for (j = 0; j < list->nb_messages;) {
>> + if (list->messages[j].payload_type == payload_type)
>> + cbs_sei_delete_message(list, j);
>> + else
>> + ++j;
>> + }
>>
> loop from end to start may be better from performance and code size aspect.
Not really performance sensitive since it's unlikely there is more than one of any message type, but it also doesn't cost anything so changed.
>> + }
>> +}
>> ...
>> +
>> +// SEI payload types form a common namespace between the H.264, H.265
>> +// and H.266 standards. A given payload type always has the same
>> +// meaning, but some names have different payload types in different
>> +// standards (e.g. scalable-nesting is 30 in H.264 but 133 in H.265).
>> +// The content of the payload data depends on the standard, though
>> +// many generic parts have the same interpretation everywhere (such as
>> +// mastering-display-colour-volume and user-data-unregistered).
>> +enum {
>> + SEI_TYPE_BUFFERING_PERIOD = 0,
>> + SEI_TYPE_PIC_TIMING = 1,
>> + SEI_TYPE_PAN_SCAN_RECT = 2,
>> + SEI_TYPE_FILLER_PAYLOAD = 3,
>> + SEI_TYPE_USER_DATA_REGISTERED_ITU_T_T35 = 4,
>> + SEI_TYPE_USER_DATA_UNREGISTERED = 5,
>> + SEI_TYPE_RECOVERY_POINT = 6,
>> + SEI_TYPE_DEC_REF_PIC_MARKING_REPETITION = 7,
>> + SEI_TYPE_SPARE_PIC = 8,
>> + SEI_TYPE_SCENE_INFO = 9,
>> + SEI_TYPE_SUB_SEQ_INFO = 10,
>> + SEI_TYPE_SUB_SEQ_LAYER_CHARACTERISTICS = 11,
>> + SEI_TYPE_SUB_SEQ_CHARACTERISTICS = 12,
>> + SEI_TYPE_FULL_FRAME_FREEZE = 13,
>> + SEI_TYPE_FULL_FRAME_FREEZE_RELEASE = 14,
>> + SEI_TYPE_FULL_FRAME_SNAPSHOT = 15,
>> + SEI_TYPE_PROGRESSIVE_REFINEMENT_SEGMENT_START = 16,
>> + SEI_TYPE_PROGRESSIVE_REFINEMENT_SEGMENT_END = 17,
>> + SEI_TYPE_MOTION_CONSTRAINED_SLICE_GROUP_SET = 18,
>> + SEI_TYPE_FILM_GRAIN_CHARACTERISTICS = 19,
>> + SEI_TYPE_DEBLOCKING_FILTER_DISPLAY_PREFERENCE = 20,
>> + SEI_TYPE_STEREO_VIDEO_INFO = 21,
>> + SEI_TYPE_POST_FILTER_HINT = 22,
>> + SEI_TYPE_TONE_MAPPING_INFO = 23,
>> + SEI_TYPE_SCALABILITY_INFO = 24,
>> + SEI_TYPE_SUB_PIC_SCALABLE_LAYER = 25,
>> + SEI_TYPE_NON_REQUIRED_LAYER_REP = 26,
>> + SEI_TYPE_PRIORITY_LAYER_INFO = 27,
>> + SEI_TYPE_LAYERS_NOT_PRESENT_4 = 28,
>> + SEI_TYPE_LAYER_DEPENDENCY_CHANGE = 29,
>> + SEI_TYPE_SCALABLE_NESTING_4 = 30,
>> + SEI_TYPE_BASE_LAYER_TEMPORAL_HRD = 31,
>> + SEI_TYPE_QUALITY_LAYER_INTEGRITY_CHECK = 32,
>> + SEI_TYPE_REDUNDANT_PIC_PROPERTY = 33,
>> + SEI_TYPE_TL0_DEP_REP_INDEX = 34,
>> + SEI_TYPE_TL_SWITCHING_POINT = 35,
>> + SEI_TYPE_PARALLEL_DECODING_INFO = 36,
>> + SEI_TYPE_MVC_SCALABLE_NESTING = 37,
>> + SEI_TYPE_VIEW_SCALABILITY_INFO = 38,
>> + SEI_TYPE_MULTIVIEW_SCENE_INFO_4 = 39,
>> + SEI_TYPE_MULTIVIEW_ACQUISITION_INFO_4 = 40,
>> + SEI_TYPE_NON_REQUIRED_VIEW_COMPONENT = 41,
>> + SEI_TYPE_VIEW_DEPENDENCY_CHANGE = 42,
>> + SEI_TYPE_OPERATION_POINTS_NOT_PRESENT = 43,
>> + SEI_TYPE_BASE_VIEW_TEMPORAL_HRD = 44,
>> + SEI_TYPE_FRAME_PACKING_ARRANGEMENT = 45,
>> + SEI_TYPE_MULTIVIEW_VIEW_POSITION_4 = 46,
>> + SEI_TYPE_DISPLAY_ORIENTATION = 47,
>> + SEI_TYPE_MVCD_SCALABLE_NESTING = 48,
>> + SEI_TYPE_MVCD_VIEW_SCALABILITY_INFO = 49,
>> + SEI_TYPE_DEPTH_REPRESENTATION_INFO_4 = 50,
>> + SEI_TYPE_THREE_DIMENSIONAL_REFERENCE_DISPLAYS_INFO_4 = 51,
>> + SEI_TYPE_DEPTH_TIMING = 52,
>> + SEI_TYPE_DEPTH_SAMPLING_INFO = 53,
>> + SEI_TYPE_CONSTRAINED_DEPTH_PARAMETER_SET_IDENTIFIER = 54,
>> + SEI_TYPE_GREEN_METADATA = 56,
>> + SEI_TYPE_STRUCTURE_OF_PICTURES_INFO = 128,
>> + SEI_TYPE_ACTIVE_PARAMETER_SETS = 129,
>
> + SEI_TYPE_DECODING_UNIT_INFO = 130,
Um, it's already there? (And in <https://lists.ffmpeg.org/pipermail/ffmpeg-devel/2021-January/274208.html> - maybe you accidentally deleted it in your reply?)
>> + SEI_TYPE_TEMPORAL_SUB_LAYER_ZERO_IDX = 131,
>> + SEI_TYPE_DECODED_PICTURE_HASH = 132,
>> + SEI_TYPE_SCALABLE_NESTING_5 = 133,
>> + SEI_TYPE_REGION_REFRESH_INFO = 134,
>> + SEI_TYPE_NO_DISPLAY = 135,
>> + SEI_TYPE_TIME_CODE = 136,
>> + SEI_TYPE_MASTERING_DISPLAY_COLOUR_VOLUME = 137,
>> + SEI_TYPE_SEGMENTED_RECT_FRAME_PACKING_ARRANGEMENT = 138,
>> + SEI_TYPE_TEMPORAL_MOTION_CONSTRAINED_TILE_SETS = 139,
>> + SEI_TYPE_CHROMA_RESAMPLING_FILTER_HINT = 140,
>> + SEI_TYPE_KNEE_FUNCTION_INFO = 141,
>> + SEI_TYPE_COLOUR_REMAPPING_INFO = 142,
>> + SEI_TYPE_DEINTERLACED_FIELD_IDENTIFICATION = 143,
>> + SEI_TYPE_CONTENT_LIGHT_LEVEL_INFO = 144,
>> + SEI_TYPE_DEPENDENT_RAP_INDICATION = 145,
>> + SEI_TYPE_CODED_REGION_COMPLETION = 146,
>> + SEI_TYPE_ALTERNATIVE_TRANSFER_CHARACTERISTICS = 147,
>> + SEI_TYPE_AMBIENT_VIEWING_ENVIRONMENT = 148,
>> + SEI_TYPE_CONTENT_COLOUR_VOLUME = 149,
>> + SEI_TYPE_EQUIRECTANGULAR_PROJECTION = 150,
>> + SEI_TYPE_CUBEMAP_PROJECTION = 151,
>> + SEI_TYPE_FISHEYE_VIDEO_INFO = 152,
>> + SEI_TYPE_SPHERE_ROTATION = 154,
>> + SEI_TYPE_REGIONWISE_PACKING = 155,
>> + SEI_TYPE_OMNI_VIEWPORT = 156,
>> + SEI_TYPE_REGIONAL_NESTING = 157,
>> + SEI_TYPE_MCTS_EXTRACTION_INFO_SETS = 158,
>> + SEI_TYPE_MCTS_EXTRACTION_INFO_NESTING = 159,
>> + SEI_TYPE_LAYERS_NOT_PRESENT_5 = 160,
>> + SEI_TYPE_INTER_LAYER_CONSTRAINED_TILE_SETS = 161,
>> + SEI_TYPE_BSP_NESTING = 162,
>> + SEI_TYPE_BSP_INITIAL_ARRIVAL_TIME = 163,
>> + SEI_TYPE_SUB_BITSTREAM_PROPERTY = 164,
>> + SEI_TYPE_ALPHA_CHANNEL_INFO = 165,
>> + SEI_TYPE_OVERLAY_INFO = 166,
>> + SEI_TYPE_TEMPORAL_MV_PREDICTION_CONSTRAINTS = 167,
>> + SEI_TYPE_FRAME_FIELD_INFO = 168,
>> + SEI_TYPE_THREE_DIMENSIONAL_REFERENCE_DISPLAYS_INFO = 176,
>> + SEI_TYPE_DEPTH_REPRESENTATION_INFO_5 = 177,
>> + SEI_TYPE_MULTIVIEW_SCENE_INFO_5 = 178,
>> + SEI_TYPE_MULTIVIEW_ACQUISITION_INFO_5 = 179,
>> + SEI_TYPE_MULTIVIEW_VIEW_POSITION_5 = 180,
>> + SEI_TYPE_ALTERNATIVE_DEPTH_INFO = 181,
>> + SEI_TYPE_SEI_MANIFEST = 200,
>> + SEI_TYPE_SEI_PREFIX_INDICATION = 201,
>> + SEI_TYPE_ANNOTATED_REGIONS = 202,
>> + SEI_TYPE_SUBPIC_LEVEL_INFO = 203,
>> + SEI_TYPE_SAMPLE_ASPECT_RATIO_INFO = 204,
>> +};
>> +
>> ...
On reviewing, it really helps to trim irrelevant context from replies so that we aren't searching through long messages looking for comments. (And apologies if I missed anything.)
Thanks,
- Mark
More information about the ffmpeg-devel
mailing list