[FFmpeg-devel] [PATCH] rtp enc/dec update for vvc
Frank Plowman
post at frankplowman.com
Fri Jun 14 21:38:39 EEST 2024
Hi,
Thanks for the patch. Unfortunately it looks to be corrupted and does
not apply. Also, it looks as though you submitted five near-identical
patches. I would suggest you try directing patches to your own mailbox
and re-applying them while debugging the formatting issues, rather than
sending lots of corrupted patches to the ML.
A couple more comments inline.
Cheers,
Frank
On 14/06/2024 03:35, ftaft2000 wrote:
> Signed-off-by: ftaft2000 <ftaft2000 at msn.com>
> ---
> .gitignore | 1 +
> configure | 4 +
> libavcodec/Makefile | 1 +
> libavcodec/allcodecs.c | 1 +
> libavcodec/libvvenc.c | 566 +++++++++++++++++++++++++++++++++
> libavformat/Makefile | 1 +
> libavformat/rtpdec.c | 1 +
> libavformat/rtpdec_formats.h | 1 +
> libavformat/rtpdec_vvc.c | 349 ++++++++++++++++++++
> libavformat/rtpenc.c | 2 +
> libavformat/rtpenc_h264_hevc.c | 94 +++++-
> libavformat/sdp.c | 182 +++++++++++
> 12 files changed, 1197 insertions(+), 6 deletions(-)
> create mode 100644 libavcodec/libvvenc.c
> create mode 100644 libavformat/rtpdec_vvc.c
>
> diff --git a/.gitignore b/.gitignore
> index e810d11107..d7441a6cdc 100644
> --- a/.gitignore
> +++ b/.gitignore
> @@ -41,3 +41,4 @@
> /src
> /mapfile
> /tools/python/__pycache__/
> +/build
I don't think this should be here - this is specific to your
environment. If you want to ignore something locally, you can use
.git/info/exclude
> \ No newline at end of file
> diff --git a/configure b/configure
> index 83284427df..d331688eb4 100755
> --- a/configure
> +++ b/configure
> @@ -296,6 +296,7 @@ External library support:
> --enable-libwebp enable WebP encoding via libwebp [no]
> --enable-libx264 enable H.264 encoding via x264 [no]
> --enable-libx265 enable HEVC encoding via x265 [no]
> + --enable-libvvenc enable H.266/VVC encoding via vvenc [no]
This looks like you had the VVenC patchset applied when you created this
commit. If your patch depends on the VVenC patchset, it will have to
wait until that is applied (which could well be in the next day or two).
> --enable-libxeve enable EVC encoding via libxeve [no]
> --enable-libxevd enable EVC decoding via libxevd [no]
> --enable-libxavs enable AVS encoding via xavs [no]
> @@ -1867,6 +1868,7 @@ EXTERNAL_LIBRARY_GPL_LIST="
> libvidstab
> libx264
> libx265
> + libvvenc
> libxavs
> libxavs2
> libxvid
> @@ -3569,6 +3571,7 @@ libx264rgb_encoder_deps="libx264"
> libx264rgb_encoder_select="libx264_encoder"
> libx265_encoder_deps="libx265"
> libx265_encoder_select="atsc_a53 dovi_rpuenc"
> +libvvenc_encoder_deps="libvvenc"
> libxavs_encoder_deps="libxavs"
> libxavs2_encoder_deps="libxavs2"
> libxevd_decoder_deps="libxevd"
> @@ -7041,6 +7044,7 @@ enabled libx264 && require_pkg_config
> libx264 x264 "stdint.h x264.h" x
> check_cpp_condition libx262 x264.h
> "X264_MPEG2"
> enabled libx265 && require_pkg_config libx265 x265 x265.h
> x265_api_get &&
> require_cpp_condition libx265 x265.h
> "X265_BUILD >= 89"
> +enabled libvvenc && require_pkg_config libvvenc "libvvenc >=
> 1.6.1" "vvenc/vvenc.h" vvenc_get_version
> enabled libxavs && require libxavs "stdint.h xavs.h"
> xavs_encoder_encode "-lxavs $pthreads_extralibs $libm_extralibs"
> enabled libxavs2 && require_pkg_config libxavs2 "xavs2 >=
> 1.3.0" "stdint.h xavs2.h" xavs2_api_get
> enabled libxevd && require_pkg_config libxevd "xevd >= 0.4.1"
> "xevd.h" xevd_decode
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 1a44352906..2e98e1c72f 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -1155,6 +1155,7 @@ OBJS-$(CONFIG_LIBWEBP_ANIM_ENCODER) +=
> libwebpenc_common.o libwebpenc_anim
> OBJS-$(CONFIG_LIBX262_ENCODER) += libx264.o
> OBJS-$(CONFIG_LIBX264_ENCODER) += libx264.o
> OBJS-$(CONFIG_LIBX265_ENCODER) += libx265.o
> +OBJS-$(CONFIG_LIBVVENC_ENCODER) += libvvenc.o
> OBJS-$(CONFIG_LIBXAVS_ENCODER) += libxavs.o
> OBJS-$(CONFIG_LIBXAVS2_ENCODER) += libxavs2.o
> OBJS-$(CONFIG_LIBXEVD_DECODER) += libxevd.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index b102a8069e..7650abebe4 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -807,6 +807,7 @@ extern const FFCodec ff_libx262_encoder;
> extern const FFCodec ff_libx264_encoder;
> extern const FFCodec ff_libx264rgb_encoder;
> extern FFCodec ff_libx265_encoder;
> +extern const FFCodec ff_libvvenc_encoder;
> extern const FFCodec ff_libxeve_encoder;
> extern const FFCodec ff_libxevd_decoder;
> extern const FFCodec ff_libxavs_encoder;
> diff --git a/libavcodec/libvvenc.c b/libavcodec/libvvenc.c
> new file mode 100644
> index 0000000000..78d4f55a2a
> --- /dev/null
> +++ b/libavcodec/libvvenc.c
> @@ -0,0 +1,566 @@
> +/*
> + * H.266 encoding using the VVenC library
> + *
> + * Copyright (C) 2022, Thomas Siedel
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +#include "config_components.h"
> +
> +#include <vvenc/vvenc.h>
> +#include <vvenc/vvencCfg.h>
> +#include <vvenc/version.h>
> +
> +#include "avcodec.h"
> +#include "codec_internal.h"
> +#include "encode.h"
> +#include "internal.h"
> +#include "packet_internal.h"
> +#include "profiles.h"
> +
> +#include "libavutil/avutil.h"
> +#include "libavutil/mem.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/common.h"
> +#include "libavutil/imgutils.h"
> +#include "libavutil/frame.h"
> +#include "libavutil/log.h"
> +
> +typedef struct VVenCOptions {
> + int preset; // preset 0: faster 4: slower
> + int qp; // quantization parameter 0-63
> + int subjectiveOptimization; // perceptually motivated QP
> adaptation, XPSNR based
> + int flag8bitCoding; // encode in 8bit instead of 10bit
> + int intraRefreshSec; // intra period/refresh in seconds
> + int levelIdc; // vvc level_idc
> + int tier; // vvc tier
> + AVDictionary *vvenc_opts;
> +} VVenCOptions;
> +
> +typedef struct VVenCContext {
> + AVClass *av_class;
> + VVenCOptions options; // encoder options
> + vvencEncoder *vvencEnc;
> + vvencAccessUnit *pAU;
> + bool encodeDone;
> +} VVenCContext;
> +
> +
> +static av_cold void ff_vvenc_log_callback(void *ctx, int level,
> + const char *fmt, va_list args)
> +{
> + vvenc_config params;
> + vvencEncoder *vvencEnc = (vvencEncoder *)ctx;
> + if (vvencEnc){
> + vvenc_config_default(¶ms);
> + vvenc_get_config(vvencEnc, ¶ms);
> + if ((int)params.m_verbosity >= level)
> + vfprintf(level == 1 ? stderr : stdout, fmt, args);
> + }
> +}
> +
> +static void ff_vvenc_set_verbository(vvenc_config* params )
> +{
> + params->m_verbosity = VVENC_VERBOSE;
> + if (av_log_get_level() >= AV_LOG_DEBUG)
> + params->m_verbosity = VVENC_DETAILS;
> + else if (av_log_get_level() >= AV_LOG_VERBOSE)
> + params->m_verbosity = VVENC_NOTICE; // output per picture
> info
> + else if (av_log_get_level() >= AV_LOG_INFO)
> + params->m_verbosity = VVENC_WARNING; // ffmpeg default
> ffmpeg loglevel
> + else
> + params->m_verbosity = VVENC_SILENT;
> +}
> +
> +static int ff_vvenc_set_pic_format(AVCodecContext *avctx, vvenc_config*
> params )
> +{
> + VVenCContext *s =(VVenCContext *) avctx->priv_data;
> +
> + params->m_internChromaFormat = VVENC_CHROMA_420;
> + params->m_inputBitDepth[0] = 10;
> +
> + if (avctx->pix_fmt != AV_PIX_FMT_YUV420P10LE){
> + av_log(avctx, AV_LOG_ERROR,
> + "unsupported pixel format %s, currently only support for
> yuv420p10le\n",
> + av_get_pix_fmt_name(avctx->pix_fmt));
> + return AVERROR(EINVAL);
> + }
> +
> + if (s->options.flag8bitCoding) {
> +#if VVENC_VERSION_MAJOR > 1 || (VVENC_VERSION_MAJOR == 1 &&
> VVENC_VERSION_MINOR > 9) || (VVENC_VERSION_MAJOR == 1 &&
> VVENC_VERSION_MINOR >= 9 && VVENC_VERSION_PATCH >= 1)
> + params->m_internalBitDepth[0] = 8;
> +#else
> + av_log(avctx, AV_LOG_ERROR,
> + "unsupported 8bit coding mode. 8bit coding needs at
> least vvenc version >= 1.9.1 "
> + "(current version %s)\n", vvenc_get_version() );
> + return AVERROR(EINVAL);
> +#endif
> + }
> + return 0;
> +}
> +
> +static void ff_vvenc_set_color_format(AVCodecContext *avctx,
> vvenc_config* params )
> +{
> + if (avctx->color_primaries != AVCOL_PRI_UNSPECIFIED)
> + params->m_colourPrimaries = (int) avctx->color_primaries;
> + if (avctx->colorspace != AVCOL_SPC_UNSPECIFIED)
> + params->m_matrixCoefficients = (int) avctx->colorspace;
> + if (avctx->color_trc != AVCOL_TRC_UNSPECIFIED) {
> + params->m_transferCharacteristics = (int) avctx->color_trc;
> +
> + if (avctx->color_trc == AVCOL_TRC_SMPTE2084)
> + params->m_HdrMode = (avctx->color_primaries ==
> AVCOL_PRI_BT2020) ?
> + VVENC_HDR_PQ_BT2020 : VVENC_HDR_PQ;
> + else if (avctx->color_trc == AVCOL_TRC_BT2020_10
> + || avctx->color_trc == AVCOL_TRC_ARIB_STD_B67)
> + params->m_HdrMode = (avctx->color_trc ==
> AVCOL_TRC_BT2020_10 ||
> + avctx->color_primaries ==
> AVCOL_PRI_BT2020 ||
> + avctx->colorspace ==
> AVCOL_SPC_BT2020_NCL ||
> + avctx->colorspace ==
> AVCOL_SPC_BT2020_CL) ?
> + VVENC_HDR_HLG_BT2020 : VVENC_HDR_HLG;
> + }
> +
> + if (params->m_HdrMode == VVENC_HDR_OFF
> + && (avctx->color_primaries != AVCOL_PRI_UNSPECIFIED
> + || avctx->colorspace != AVCOL_SPC_UNSPECIFIED)) {
> + params->m_vuiParametersPresent = 1;
> + params->m_colourDescriptionPresent = true;
> + }
> +}
> +
> +static void ff_vvenc_set_framerate(AVCodecContext *avctx, vvenc_config*
> params )
> +{
> + params->m_FrameRate = avctx->time_base.den;
> + params->m_FrameScale = avctx->time_base.num;
> +
> +FF_DISABLE_DEPRECATION_WARNINGS
> +
> +#if FF_API_TICKS_PER_FRAME
> + if (avctx->ticks_per_frame == 1) {
> +#endif
> + params->m_TicksPerSecond = -1; // auto mode for ticks per
> frame = 1
> +#if FF_API_TICKS_PER_FRAME
> + } else {
> + params->m_TicksPerSecond =
> + ceil((avctx->time_base.den / (double) avctx->time_base.num) *
> + (double) avctx->ticks_per_frame);
> + }
> +#endif
> +FF_ENABLE_DEPRECATION_WARNINGS
> +}
> +
> +static int ff_vvenc_parse_vvenc_params(AVCodecContext *avctx,
> vvenc_config* params, char* statsfile )
> +{
> + int parse_ret, ret;
> + VVenCContext *s;
> + AVDictionaryEntry *en = NULL;
> + s =(VVenCContext *) avctx->priv_data;
> + ret = 0;
> +
> + while ((en = av_dict_get(s->options.vvenc_opts, "", en,
> + AV_DICT_IGNORE_SUFFIX))) {
> + av_log(avctx, AV_LOG_DEBUG, "vvenc_set_param: '%s:%s'\n", en->key,
> + en->value);
> + parse_ret = vvenc_set_param(params, en->key, en->value);
> + switch (parse_ret) {
> + case VVENC_PARAM_BAD_NAME:
> + av_log(avctx, AV_LOG_ERROR, "Unknown vvenc option: %s.\n",
> + en->key);
> + ret = AVERROR(EINVAL);
> + break;
> + case VVENC_PARAM_BAD_VALUE:
> + av_log(avctx, AV_LOG_ERROR,
> + "Invalid vvenc value for %s: %s.\n", en->key,
> en->value);
> + ret = AVERROR(EINVAL);
> + break;
> + default:
> + break;
> + }
> +
> + if (memcmp(en->key, "rcstatsfile", 11) == 0 ||
> + memcmp(en->key, "RCStatsFile", 11) == 0) {
> + strncpy(statsfile, en->value, strlen(statsfile));
> + statsfile[strlen(statsfile)] = '\0';
> + }
> + }
> + return ret;
> +}
> +
> +static int ff_vvenc_set_rc_mode(AVCodecContext *avctx, vvenc_config*
> params)
> +{
> + if (params->m_RCPass != -1 && params->m_RCNumPasses == 1)
> + params->m_RCNumPasses = 2; /* enable 2pass mode */
> +
> + if(avctx->rc_max_rate) {
> +#if VVENC_VERSION_MAJOR > 1 || (VVENC_VERSION_MAJOR == 1 &&
> VVENC_VERSION_MINOR > 8)
> + params->m_RCMaxBitrate = avctx->rc_max_rate;
> +#endif
> +
> +#if VVENC_VERSION_MAJOR == 1 && VVENC_VERSION_MINOR < 11
> + /* rc_max_rate without a bit_rate enables capped CQF mode.
> + (QP + subj. optimization + max. bitrate) */
> + if(!avctx->bit_rate) {
> + av_log( avctx, AV_LOG_ERROR,
> + "Capped Constant Quality Factor mode (capped CQF) needs
> at "
> + "least vvenc version >= 1.11.0 (current version %s)\n",
> + vvenc_get_version());
> + return AVERROR(EINVAL);
> + }
> +#endif
> + }
> + return 0;
> +}
> +
> +static int ff_vvenc_init_extradata(AVCodecContext *avctx, VVenCContext *s)
> +{
> + int ret;
> + if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
> + ret = vvenc_get_headers(s->vvencEnc, s->pAU);
> + if (0 != ret) {
> + av_log(avctx, AV_LOG_ERROR,
> + "cannot get headers (SPS,PPS) from vvc
> encoder(vvenc): %s\n",
> + vvenc_get_last_error(s->vvencEnc));
> + vvenc_encoder_close(s->vvencEnc);
> + return AVERROR(EINVAL);
> + }
> +
> + if (s->pAU->payloadUsedSize <= 0) {
> + vvenc_encoder_close(s->vvencEnc);
> + return AVERROR_INVALIDDATA;
> + }
> +
> + avctx->extradata_size = s->pAU->payloadUsedSize;
> + avctx->extradata =
> + av_mallocz(avctx->extradata_size +
> AV_INPUT_BUFFER_PADDING_SIZE);
> + if (!avctx->extradata) {
> + av_log(avctx, AV_LOG_ERROR,
> + "Cannot allocate VVC header of size %d.\n",
> + avctx->extradata_size);
> + vvenc_encoder_close(s->vvencEnc);
> + return AVERROR(ENOMEM);
> + }
> +
> + memcpy(avctx->extradata, s->pAU->payload, avctx->extradata_size);
> + memset(avctx->extradata + avctx->extradata_size, 0,
> + AV_INPUT_BUFFER_PADDING_SIZE);
> + }
> + return 0;
> +}
> +
> +static av_cold int ff_vvenc_encode_init(AVCodecContext *avctx)
> +{
> + int ret;
> + int framerate, qp;
> + VVenCContext *s;
> + vvenc_config params;
> + vvencPresetMode preset;
> + char statsfile[1024] = "vvenc-rcstats.json";
> +
> + s = (VVenCContext *) avctx->priv_data;
> + qp = s->options.qp;
> + preset = (vvencPresetMode) s->options.preset;
> +
> + if (avctx->flags & AV_CODEC_FLAG_INTERLACED_DCT) {
> + av_log(avctx, AV_LOG_ERROR,
> + "ff_vvenc_encode_init::init() interlaced encoding not
> supported yet\n");
> + return AVERROR_INVALIDDATA;
> + }
> +
> + vvenc_config_default(¶ms);
> +
> + framerate = avctx->time_base.den / avctx->time_base.num;
> + vvenc_init_default(¶ms, avctx->width, avctx->height, framerate,
> + (qp >= 0) ? 0 : avctx->bit_rate, (qp < 0) ? 32 :
> qp, preset);
> +
> + ff_vvenc_set_verbository(¶ms);
> +
> + if (avctx->thread_count > 0)
> + params.m_numThreads = avctx->thread_count;
> +
> + /* GOP settings (IDR/CRA) */
> + if (avctx->flags & AV_CODEC_FLAG_CLOSED_GOP)
> + params.m_DecodingRefreshType = VVENC_DRT_IDR;
> +
> + if (avctx->gop_size == 1) {
> + params.m_GOPSize = 1;
> + params.m_IntraPeriod = 1;
> + } else {
> + params.m_IntraPeriodSec = s->options.intraRefreshSec;
> + }
> +
> + params.m_AccessUnitDelimiter = true;
> + params.m_RCNumPasses = 1;
> +
> + params.m_usePerceptQPA = s->options.subjectiveOptimization;
> + params.m_level = (vvencLevel) s->options.levelIdc;
> + params.m_levelTier = (vvencTier) s->options.tier;
> +
> + ff_vvenc_set_framerate(avctx, ¶ms);
> +
> + ret = ff_vvenc_set_pic_format(avctx, ¶ms);
> + if( ret != 0 )
> + return ret;
> +
> + ff_vvenc_set_color_format(avctx, ¶ms);
> +
> + ret = ff_vvenc_parse_vvenc_params(avctx, ¶ms, &statsfile[0]);
> + if( ret != 0 )
> + return ret;
> +
> +
> + ret = ff_vvenc_set_rc_mode(avctx, ¶ms);
> + if( ret != 0 )
> + return ret;
> +
> + s->vvencEnc = vvenc_encoder_create();
> + if (NULL == s->vvencEnc) {
> + av_log(avctx, AV_LOG_ERROR, "cannot create vvc encoder
> (vvenc)\n");
> + return AVERROR(ENOMEM);
> + }
> +
> + vvenc_set_msg_callback(¶ms, s->vvencEnc, ff_vvenc_log_callback);
> + ret = vvenc_encoder_open(s->vvencEnc, ¶ms);
> + if (0 != ret) {
> + av_log(avctx, AV_LOG_ERROR, "cannot open vvc encoder (vvenc):
> %s\n",
> + vvenc_get_last_error(s->vvencEnc));
> + vvenc_encoder_close(s->vvencEnc);
> + return AVERROR(EINVAL);
> + }
> +
> + vvenc_get_config(s->vvencEnc, ¶ms); /* get the adapted
> config */
> +
> + av_log(avctx, av_log_get_level(), "vvenc version: %s\n",
> vvenc_get_version());
> + av_log(avctx, av_log_get_level(), "%s\n",
> + vvenc_get_config_as_string(¶ms, params.m_verbosity));
> +
> + if (params.m_RCNumPasses == 2) {
> + ret = vvenc_init_pass(s->vvencEnc, params.m_RCPass - 1,
> &statsfile[0]);
> + if (0 != ret) {
> + av_log(avctx, AV_LOG_ERROR,
> + "cannot init pass %d for vvc encoder (vvenc): %s\n",
> + params.m_RCPass, vvenc_get_last_error(s->vvencEnc));
> + vvenc_encoder_close(s->vvencEnc);
> + return AVERROR(EINVAL);
> + }
> + }
> +
> + s->pAU = vvenc_accessUnit_alloc();
> + if( !s->pAU ){
> + av_log(avctx, AV_LOG_FATAL, "cannot allocate memory for AU
> payload\n");
> + return AVERROR(ENOMEM);
> + }
> + vvenc_accessUnit_alloc_payload(s->pAU, avctx->width * avctx->height);
> + if( !s->pAU ){
> + av_log(avctx, AV_LOG_FATAL, "cannot allocate payload memory of
> size %d\n",
> + avctx->width * avctx->height );
> + return AVERROR(ENOMEM);
> + }
> +
> + ret = ff_vvenc_init_extradata(avctx, s);
> + if( ret != 0 )
> + return ret;
> +
> + s->encodeDone = false;
> + return 0;
> +}
> +
> +static av_cold int ff_vvenc_encode_close(AVCodecContext * avctx)
> +{
> + VVenCContext *s = (VVenCContext *) avctx->priv_data;
> + if (s->vvencEnc) {
> + if (av_log_get_level() >= AV_LOG_VERBOSE)
> + vvenc_print_summary(s->vvencEnc);
> +
> + if (0 != vvenc_encoder_close(s->vvencEnc)) {
> + av_log(avctx, AV_LOG_ERROR, "cannot close vvenc\n");
> + return -1;
> + }
> + }
> +
> + vvenc_accessUnit_free(s->pAU, true);
> +
> + return 0;
> +}
> +
> +static av_cold int ff_vvenc_encode_frame(AVCodecContext *avctx,
> AVPacket *pkt,
> + const AVFrame *frame, int
> *got_packet)
> +{
> + VVenCContext *s = (VVenCContext *) avctx->priv_data;
> + vvencYUVBuffer *pyuvbuf;
> + vvencYUVBuffer yuvbuf;
> + int pict_type;
> + int ret;
> +
> + pyuvbuf = NULL;
> + if (frame) {
> + if (avctx->pix_fmt == AV_PIX_FMT_YUV420P10LE) {
> + vvenc_YUVBuffer_default(&yuvbuf);
> + yuvbuf.planes[0].ptr = (int16_t *) frame->data[0];
> + yuvbuf.planes[1].ptr = (int16_t *) frame->data[1];
> + yuvbuf.planes[2].ptr = (int16_t *) frame->data[2];
> +
> + yuvbuf.planes[0].width = frame->width;
> + yuvbuf.planes[0].height = frame->height;
> + /* stride is used in 16bitsamples (16bit) in vvenc, ffmpeg
> uses stride in bytes */
> + yuvbuf.planes[0].stride = frame->linesize[0] >> 1;
> +
> + yuvbuf.planes[1].width = frame->width >> 1;
> + yuvbuf.planes[1].height = frame->height >> 1;
> + yuvbuf.planes[1].stride = frame->linesize[1] >> 1;
> +
> + yuvbuf.planes[2].width = frame->width >> 1;
> + yuvbuf.planes[2].height = frame->height >> 1;
> + yuvbuf.planes[2].stride = frame->linesize[2] >> 1;
> +
> + yuvbuf.cts = frame->pts;
> + yuvbuf.ctsValid = true;
> + pyuvbuf = &yuvbuf;
> + } else {
> + av_log(avctx, AV_LOG_ERROR,
> + "unsupported input colorspace! input must be
> yuv420p10le");
> + return AVERROR(EINVAL);
> + }
> + }
> +
> + if (!s->encodeDone) {
> + ret = vvenc_encode(s->vvencEnc, pyuvbuf, s->pAU, &s->encodeDone);
> + if (ret != 0) {
> + av_log(avctx, AV_LOG_ERROR, "error in vvenc::encode -
> ret:%d\n",
> + ret);
> + return AVERROR(EINVAL);
> + }
> + } else {
> + *got_packet = 0;
> + return 0;
> + }
> +
> + if (s->pAU->payloadUsedSize > 0) {
> + ret = ff_get_encode_buffer(avctx, pkt, s->pAU->payloadUsedSize,
> 0);
> + if (ret < 0) {
> + av_log(avctx, AV_LOG_ERROR, "Error getting output packet.\n");
> + return ret;
> + }
> +
> + memcpy(pkt->data, s->pAU->payload, s->pAU->payloadUsedSize);
> +
> + if (s->pAU->ctsValid)
> + pkt->pts = s->pAU->cts;
> + if (s->pAU->dtsValid)
> + pkt->dts = s->pAU->dts;
> + pkt->flags |= AV_PKT_FLAG_KEY * s->pAU->rap;
> +
> + switch (s->pAU->sliceType) {
> + case VVENC_I_SLICE:
> + pict_type = AV_PICTURE_TYPE_I;
> + break;
> + case VVENC_P_SLICE:
> + pict_type = AV_PICTURE_TYPE_P;
> + break;
> + case VVENC_B_SLICE:
> + pict_type = AV_PICTURE_TYPE_B;
> + break;
> + default:
> + av_log(avctx, AV_LOG_ERROR, "Unknown picture type
> encountered.\n");
> + return AVERROR_EXTERNAL;
> + }
> +
> + ff_side_data_set_encoder_stats(pkt, 0, NULL, 0, pict_type);
> +
> + *got_packet = 1;
> +
> + return 0;
> + } else {
> + *got_packet = 0;
> + return 0;
> + }
> +
> + return 0;
> +}
> +
> +static const enum AVPixelFormat pix_fmts_vvenc[] = {
> + AV_PIX_FMT_YUV420P10LE,
> + AV_PIX_FMT_NONE
> +};
> +
> +#define OFFSET(x) offsetof(VVenCContext, x)
> +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
> +static const AVOption libvvenc_options[] = {
> + {"preset", "set encoding preset(0: faster - 4: slower", OFFSET(
> options.preset), AV_OPT_TYPE_INT, {.i64 = 2} , 0 , 4 , VE, "preset"},
> + { "faster", "0", 0, AV_OPT_TYPE_CONST, {.i64 = VVENC_FASTER},
> INT_MIN, INT_MAX, VE, "preset" },
> + { "fast", "1", 0, AV_OPT_TYPE_CONST, {.i64 = VVENC_FAST},
> INT_MIN, INT_MAX, VE, "preset" },
> + { "medium", "2", 0, AV_OPT_TYPE_CONST, {.i64 = VVENC_MEDIUM},
> INT_MIN, INT_MAX, VE, "preset" },
> + { "slow", "3", 0, AV_OPT_TYPE_CONST, {.i64 = VVENC_SLOW},
> INT_MIN, INT_MAX, VE, "preset" },
> + { "slower", "4", 0, AV_OPT_TYPE_CONST, {.i64 = VVENC_SLOWER},
> INT_MIN, INT_MAX, VE, "preset" },
> + { "qp" , "set quantization", OFFSET(options.qp),
> AV_OPT_TYPE_INT, {.i64 = -1}, -1 , 63 ,VE, "qp_mode" },
> + { "period" , "set (intra) refresh period in seconds",
> OFFSET(options.intraRefreshSec), AV_OPT_TYPE_INT, {.i64 = 1}, 1 ,
> INT_MAX ,VE,"irefreshsec" },
> + { "subjopt", "set subjective (perceptually motivated)
> optimization", OFFSET(options.subjectiveOptimization), AV_OPT_TYPE_BOOL,
> {.i64 = 1}, 0 , 1, VE},
> + { "bitdepth8", "set 8bit coding mode",
> OFFSET(options.flag8bitCoding), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0 , 1, VE},
> + { "vvenc-params", "set the vvenc configuration using a :-separated
> list of key=value parameters", OFFSET(options.vvenc_opts),
> AV_OPT_TYPE_DICT, { 0 }, 0, 0, VE },
> + { "levelidc", "vvc level_idc", OFFSET( options.levelIdc),
> AV_OPT_TYPE_INT, {.i64 = 0}, 0, 105, VE, "levelidc"},
> + { "0", "auto", 0, AV_OPT_TYPE_CONST, {.i64 = 0}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "1", "1" , 0, AV_OPT_TYPE_CONST, {.i64 = 16}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "2", "2" , 0, AV_OPT_TYPE_CONST, {.i64 = 32}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "2.1", "2.1" , 0, AV_OPT_TYPE_CONST, {.i64 = 35}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "3", "3" , 0, AV_OPT_TYPE_CONST, {.i64 = 48}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "3.1", "3.1" , 0, AV_OPT_TYPE_CONST, {.i64 = 51}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "4", "4" , 0, AV_OPT_TYPE_CONST, {.i64 = 64}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "4.1", "4.1" , 0, AV_OPT_TYPE_CONST, {.i64 = 67}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "5", "5" , 0, AV_OPT_TYPE_CONST, {.i64 = 80}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "5.1", "5.1" , 0, AV_OPT_TYPE_CONST, {.i64 = 83}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "5.2", "5.2" , 0, AV_OPT_TYPE_CONST, {.i64 = 86}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "6", "6" , 0, AV_OPT_TYPE_CONST, {.i64 = 96}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "6.1", "6.1" , 0, AV_OPT_TYPE_CONST, {.i64 = 99}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "6.2", "6.2" , 0, AV_OPT_TYPE_CONST, {.i64 = 102}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "6.3", "6.3" , 0, AV_OPT_TYPE_CONST, {.i64 = 105}, INT_MIN,
> INT_MAX, VE, "levelidc"},
> + { "tier", "set vvc tier", OFFSET( options.tier), AV_OPT_TYPE_INT,
> {.i64 = 0}, 0 , 1 , VE, "tier"},
> + { "main", "main", 0, AV_OPT_TYPE_CONST, {.i64 = 0}, INT_MIN,
> INT_MAX, VE, "tier"},
> + { "high", "high", 0, AV_OPT_TYPE_CONST, {.i64 = 1}, INT_MIN,
> INT_MAX, VE, "tier"},
> + {NULL}
> +};
> +
> +static const AVClass class_libvvenc = {
> + .class_name = "libvvenc-vvc encoder",
> + .item_name = av_default_item_name,
> + .option = libvvenc_options,
> + .version = LIBAVUTIL_VERSION_INT,
> +};
> +
> +static const FFCodecDefault vvenc_defaults[] = {
> + { "b", "0" },
> + { "g", "-1" },
> + { NULL },
> +};
> +
> +FFCodec ff_libvvenc_encoder = {
> + .p.name = "libvvenc",
> + CODEC_LONG_NAME("H.266 / VVC Encoder VVenC"),
> + .p.type = AVMEDIA_TYPE_VIDEO,
> + .p.id = AV_CODEC_ID_VVC,
> + .p.capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_OTHER_THREADS,
> + .p.profiles = NULL_IF_CONFIG_SMALL(ff_vvc_profiles),
> + .p.priv_class = &class_libvvenc,
> + .p.wrapper_name = "libvvenc",
> + .priv_data_size = sizeof(VVenCContext),
> + .p.pix_fmts = pix_fmts_vvenc,
> + .init = ff_vvenc_encode_init,
> + FF_CODEC_ENCODE_CB(ff_vvenc_encode_frame),
> + .close = ff_vvenc_encode_close,
> + .defaults = vvenc_defaults,
> + .caps_internal = FF_CODEC_CAP_AUTO_THREADS,
> +};
> diff --git a/libavformat/Makefile b/libavformat/Makefile
> index af31d6f795..7f6db3d6d6 100644
> --- a/libavformat/Makefile
> +++ b/libavformat/Makefile
> @@ -54,6 +54,7 @@ OBJS-$(CONFIG_RTPDEC) += rdt.o
> \
> rtpdec_h263_rfc2190.o \
> rtpdec_h264.o \
> rtpdec_hevc.o \
> + rtpdec_vvc.o \
> rtpdec_ilbc.o \
> rtpdec_jpeg.o \
> rtpdec_latm.o \
> diff --git a/libavformat/rtpdec.c b/libavformat/rtpdec.c
> index 729bf83685..34ef92a2a0 100644
> --- a/libavformat/rtpdec.c
> +++ b/libavformat/rtpdec.c
> @@ -98,6 +98,7 @@ static const RTPDynamicProtocolHandler *const
> rtp_dynamic_protocol_handler_list[
> &ff_h263_rfc2190_dynamic_handler,
> &ff_h264_dynamic_handler,
> &ff_hevc_dynamic_handler,
> + &ff_vvc_dynamic_handler,
> &ff_ilbc_dynamic_handler,
> &ff_jpeg_dynamic_handler,
> &ff_mp4a_latm_dynamic_handler,
> diff --git a/libavformat/rtpdec_formats.h b/libavformat/rtpdec_formats.h
> index dad2b8ac1b..af2e17fb99 100644
> --- a/libavformat/rtpdec_formats.h
> +++ b/libavformat/rtpdec_formats.h
> @@ -65,6 +65,7 @@ extern const RTPDynamicProtocolHandler
> ff_h263_2000_dynamic_handler;
> extern const RTPDynamicProtocolHandler ff_h263_rfc2190_dynamic_handler;
> extern const RTPDynamicProtocolHandler ff_h264_dynamic_handler;
> extern const RTPDynamicProtocolHandler ff_hevc_dynamic_handler;
> +extern const RTPDynamicProtocolHandler ff_vvc_dynamic_handler;
> extern const RTPDynamicProtocolHandler ff_ilbc_dynamic_handler;
> extern const RTPDynamicProtocolHandler ff_jpeg_dynamic_handler;
> extern const RTPDynamicProtocolHandler ff_mp4a_latm_dynamic_handler;
> diff --git a/libavformat/rtpdec_vvc.c b/libavformat/rtpdec_vvc.c
> new file mode 100644
> index 0000000000..86f0182b01
> --- /dev/null
> +++ b/libavformat/rtpdec_vvc.c
> @@ -0,0 +1,349 @@
> +/*
> + * RTP parser for VVC/H.266 payload format (draft version 0.1)
> + * Copyright (c) 2024 RobinShi <shiqifeng at gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> 02110-1301 USA
> + */
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/mem.h"
> +
> +#include "avformat.h"
> +#include "internal.h"
> +#include "rtpdec.h"
> +#include "rtpdec_formats.h"
> +
> +#define RTP_VVC_PAYLOAD_HEADER_SIZE 2
> +#define RTP_VVC_FU_HEADER_SIZE 1
> +#define RTP_VVC_DONL_FIELD_SIZE 2
> +#define RTP_VVC_DOND_FIELD_SIZE 1
> +#define RTP_VVC_AP_NALU_LENGTH_FIELD_SIZE 2
> +#define VVC_SPECIFIED_NAL_UNIT_TYPES 28
> +
> +/* SDP out-of-band signaling data */
> +struct PayloadContext {
> + int using_donl_field;
> + int profile_id;
> + uint8_t *sps, *pps, *vps, *sei;
> + int sps_size, pps_size, vps_size, sei_size;
> +};
> +
> +static const uint8_t start_sequence[] = { 0x00, 0x00, 0x00, 0x01 };
> +
> +static av_cold int vvc_sdp_parse_fmtp_config(AVFormatContext *s,
> + AVStream *stream,
> + PayloadContext *vvc_data,
> + const char *attr, const
> char *value)
> +{
> + /* profile-space: 0-3 */
> + /* profile-id: 0-31 */
> + if (!strcmp(attr, "profile-id")) {
> + vvc_data->profile_id = atoi(value);
> + av_log(s, AV_LOG_TRACE, "SDP: found profile-id: %d\n",
> vvc_data->profile_id);
> + }
> +
> + /* tier-flag: 0-1 */
> + /* level-id: 0-255 */
> + /* interop-constraints: [base16] */
> + /* profile-compatibility-indicator: [base16] */
> + /* sprop-sub-layer-id: 0-6, defines highest possible value for TID,
> default: 6 */
> + /* recv-sub-layer-id: 0-6 */
> + /* max-recv-level-id: 0-255 */
> + /* tx-mode: MSM,SSM */
> + /* sprop-vps: [base64] */
> + /* sprop-sps: [base64] */
> + /* sprop-pps: [base64] */
> + /* sprop-sei: [base64] */
> + if (!strcmp(attr, "sprop-vps") || !strcmp(attr, "sprop-sps") ||
> + !strcmp(attr, "sprop-pps") || !strcmp(attr, "sprop-sei")) {
> + uint8_t **data_ptr = NULL;
> + int *size_ptr = NULL;
> + if (!strcmp(attr, "sprop-vps")) {
> + data_ptr = &vvc_data->vps;
> + size_ptr = &vvc_data->vps_size;
> + } else if (!strcmp(attr, "sprop-sps")) {
> + data_ptr = &vvc_data->sps;
> + size_ptr = &vvc_data->sps_size;
> + } else if (!strcmp(attr, "sprop-pps")) {
> + data_ptr = &vvc_data->pps;
> + size_ptr = &vvc_data->pps_size;
> + } else if (!strcmp(attr, "sprop-sei")) {
> + data_ptr = &vvc_data->sei;
> + size_ptr = &vvc_data->sei_size;
> + } else
> + av_assert0(0);
> +
> + ff_h264_parse_sprop_parameter_sets(s, data_ptr,
> + size_ptr, value);
> + }
> +
> + /* max-lsr, max-lps, max-cpb, max-dpb, max-br, max-tr, max-tc */
> + /* max-fps */
> +
> + /* sprop-max-don-diff: 0-32767
> +
> + When the RTP stream depends on one or more other RTP
> + streams (in this case tx-mode MUST be equal to "MSM" and
> + MSM is in use), this parameter MUST be present and the
> + value MUST be greater than 0.
> + */
> + if (!strcmp(attr, "sprop-max-don-diff")) {
> + if (atoi(value) > 0)
> + vvc_data->using_donl_field = 1;
> + av_log(s, AV_LOG_TRACE, "Found sprop-max-don-diff in SDP, DON
> field usage is: %d\n",
> + vvc_data->using_donl_field);
> + }
> + // VVC dont have sprop-depack-buf-nalus setting according to
> rfc9328 + /* sprop-depack-buf-bytes: 0-4294967295 */
> + /* depack-buf-cap */
> + /* sprop-segmentation-id: 0-3 */
> + /* sprop-spatial-segmentation-idc: [base16] */
> + /* dec-parallel-ca: */
> + /* include-dph */
> +
> + return 0;
> +}
> +
> +static av_cold int vvc_parse_sdp_line(AVFormatContext *ctx, int st_index,
> + PayloadContext *vvc_data, const
> char *line)
> +{
> + AVStream *current_stream;
> + AVCodecParameters *par;
> + const char *sdp_line_ptr = line;
> +
> + if (st_index < 0)
> + return 0;
> +
> + current_stream = ctx->streams[st_index];
> + par = current_stream->codecpar;
> +
> + if (av_strstart(sdp_line_ptr, "framesize:", &sdp_line_ptr)) {
> + ff_h264_parse_framesize(par, sdp_line_ptr);
> + } else if (av_strstart(sdp_line_ptr, "fmtp:", &sdp_line_ptr)) {
> + int ret = ff_parse_fmtp(ctx, current_stream, vvc_data,
> sdp_line_ptr,
> + vvc_sdp_parse_fmtp_config);
> + if (vvc_data->vps_size || vvc_data->sps_size ||
> + vvc_data->pps_size || vvc_data->sei_size) {
> + par->extradata_size = vvc_data->vps_size +
> vvc_data->sps_size +
> + vvc_data->pps_size + vvc_data->sei_size;
> + if ((ret = ff_alloc_extradata(par, par->extradata_size)) >=
> 0) {
> + int pos = 0;
> + memcpy(par->extradata + pos, vvc_data->vps,
> vvc_data->vps_size);
> + pos += vvc_data->vps_size;
> + memcpy(par->extradata + pos, vvc_data->sps,
> vvc_data->sps_size);
> + pos += vvc_data->sps_size;
> + memcpy(par->extradata + pos, vvc_data->pps,
> vvc_data->pps_size);
> + pos += vvc_data->pps_size;
> + memcpy(par->extradata + pos, vvc_data->sei,
> vvc_data->sei_size);
> + }
> +
> + av_freep(&vvc_data->vps);
> + av_freep(&vvc_data->sps);
> + av_freep(&vvc_data->pps);
> + av_freep(&vvc_data->sei);
> + vvc_data->vps_size = 0;
> + vvc_data->sps_size = 0;
> + vvc_data->pps_size = 0;
> + vvc_data->sei_size = 0;
> + }
> + return ret;
> + }
> +
> + return 0;
> +}
> +
> +static int vvc_handle_packet(AVFormatContext *ctx, PayloadContext
> *rtp_vvc_ctx,
> + AVStream *st, AVPacket *pkt, uint32_t
> *timestamp,
> + const uint8_t *buf, int len, uint16_t seq,
> + int flags)
> +{
> + const uint8_t *rtp_pl = buf;
> + int tid, lid, nal_type;
> + int first_fragment, last_fragment, pic_fragment, fu_type;
> + uint8_t new_nal_header[2];
> + int res = 0;
> +
> + /* sanity check for size of input packet: 1 byte payload at least */
> + if (len < RTP_VVC_PAYLOAD_HEADER_SIZE + 1) {
> + av_log(ctx, AV_LOG_ERROR, "Too short RTP/VVC packet, got %d
> bytes\n", len);
> + return AVERROR_INVALIDDATA;
> + }
> +
> + /*
> + * decode the VVC payload header according to section 4 of draft
> version 6:
> + *
> + * +---------------+---------------+
> + * |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
> + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
> + * |F|Z| LayerID | Type | TID |
> + * +---------------+---------------+
> + *
> + * Forbidden zero (F): 1 bit
> + * Reserved (Z): 1 bits
> + * NUH layer ID (LayerId): 6 bits
> + * NAL unit type (Type): 5 bits
> + * NUH temporal ID plus 1 (TID): 3 bits
> + */
> + nal_type = (buf[1] >> 3) & 0x1f;
> + lid = buf[0] & 0x3f;
> + tid = buf[1] & 0x07;
> +
> + /* sanity check for correct layer ID */
> + if (lid) {
> + /* future scalable or 3D video coding extensions */
> + avpriv_report_missing_feature(ctx, "Multi-layer VVC coding");
> + return AVERROR_PATCHWELCOME;
> + }
> +
> + /* sanity check for correct temporal ID */
> + if (!tid) {
> + av_log(ctx, AV_LOG_ERROR, "Illegal temporal ID in RTP/VVC
> packet\n");
> + return AVERROR_INVALIDDATA;
> + }
> +
> + /* sanity check for correct NAL unit type */
> + if (nal_type > 31) {
> + av_log(ctx, AV_LOG_ERROR, "Unsupported (VVC) NAL type (%d)\n",
> nal_type);
> + return AVERROR_INVALIDDATA;
> + }
> +
> + switch (nal_type) {
> + /* video parameter set (VPS) */
> + case 14:
> + /* sequence parameter set (SPS) */
> + case 15:
> + /* picture parameter set (PPS) */
> + case 16:
> + /* supplemental enhancement information (SEI) */
> + case 23:
> + case 24:
> + /* single NAL unit packet */
> + default:
> + /* create A/V packet */
> + if ((res = av_new_packet(pkt, sizeof(start_sequence) + len)) < 0)
> + return res;
> + /* A/V packet: copy start sequence */
> + memcpy(pkt->data, start_sequence, sizeof(start_sequence));
> + /* A/V packet: copy NAL unit data */
> + memcpy(pkt->data + sizeof(start_sequence), buf, len);
> +
> + break;
> + /* aggregated packet (AP) - with two or more NAL units */
> + case 28:
> + /* pass the VVC payload header */
> + buf += RTP_VVC_PAYLOAD_HEADER_SIZE;
> + len -= RTP_VVC_PAYLOAD_HEADER_SIZE;
> +
> + /* pass the VVC DONL field */
> + if (rtp_vvc_ctx->using_donl_field) {
> + buf += RTP_VVC_DONL_FIELD_SIZE;
> + len -= RTP_VVC_DONL_FIELD_SIZE;
> + }
> +
> + res = ff_h264_handle_aggregated_packet(ctx, rtp_vvc_ctx, pkt,
> buf, len,
> + rtp_vvc_ctx->using_donl_field ?
> + RTP_VVC_DOND_FIELD_SIZE
> : 0,
> + NULL, 0);
> + if (res < 0)
> + return res;
> + break;
> + /* fragmentation unit (FU) */
> + case 29:
> + /* pass the VVC payload header */
> + buf += RTP_VVC_PAYLOAD_HEADER_SIZE;
> + len -= RTP_VVC_PAYLOAD_HEADER_SIZE;
> +
> + /*
> + * create the FU header
> + *
> + * 0 1 2 3 4 5 6 7
> + * +---------------+
> + * |0|1|2|3|4|5|6|7|
> + * +-+-+-+-+-+-+-+-+
> + * |S|E|P| FuType |
> + * +---------------+
> + * S = variable
> + * E = variable
> + * P = variable
> + * FuType = NAL unit type
> + */
> + first_fragment = buf[0] & 0x80;
> + last_fragment = buf[0] & 0x40;
> + pic_fragment = buf[0] & 0x10;
> + fu_type = buf[0] & 0x3f;
> +
> + /* pass the VVC FU header */
> + buf += RTP_VVC_FU_HEADER_SIZE;
> + len -= RTP_VVC_FU_HEADER_SIZE;
> +
> + /* pass the VVC DONL field */
> + if (rtp_vvc_ctx->using_donl_field) {
> + buf += RTP_VVC_DONL_FIELD_SIZE;
> + len -= RTP_VVC_DONL_FIELD_SIZE;
> + }
> +
> + av_log(ctx, AV_LOG_TRACE, " FU type %d with %d bytes\n",
> fu_type, len);
> +
> + /* sanity check for size of input packet: 1 byte payload at
> least */
> + if (len <= 0) {
> + if (len < 0) {
> + av_log(ctx, AV_LOG_ERROR,
> + "Too short RTP/VVC packet, got %d bytes of NAL
> unit type %d\n",
> + len, nal_type);
> + return AVERROR_INVALIDDATA;
> + } else {
> + return AVERROR(EAGAIN);
> + }
> + }
> +
> + if (first_fragment && last_fragment) {
> + av_log(ctx, AV_LOG_ERROR, "Illegal combination of S and E
> bit in RTP/VVC packet\n");
> + return AVERROR_INVALIDDATA;
> + }
> +
> + new_nal_header[0] = rtp_pl[0];
> + new_nal_header[1] = rtp_pl[1];
> +
> + new_nal_header[1] = (new_nal_header[1] & 0x07) | (fu_type << 3);
> +
> + res = ff_h264_handle_frag_packet(pkt, buf, len, first_fragment,
> + new_nal_header,
> sizeof(new_nal_header));
> +
> + break;
> + /* PACI packet */
> + case 30:
> + /* Temporal scalability control information (TSCI) */
> + avpriv_report_missing_feature(ctx, "PACI packets for RTP/VVC");
> + res = AVERROR_PATCHWELCOME;
> + break;
> + }
> +
> + pkt->stream_index = st->index;
> +
> + return res;
> +}
> +
> +const RTPDynamicProtocolHandler ff_vvc_dynamic_handler = {
> + .enc_name = "H266",
> + .codec_type = AVMEDIA_TYPE_VIDEO,
> + .codec_id = AV_CODEC_ID_VVC,
> + .need_parsing = AVSTREAM_PARSE_FULL,
> + .priv_data_size = sizeof(PayloadContext),
> + .parse_sdp_a_line = vvc_parse_sdp_line,
> + .parse_packet = vvc_handle_packet,
> +};
> diff --git a/libavformat/rtpenc.c b/libavformat/rtpenc.c
> index 7b4ae37d13..848b245ad7 100644
> --- a/libavformat/rtpenc.c
> +++ b/libavformat/rtpenc.c
> @@ -57,6 +57,7 @@ static int is_supported(enum AVCodecID id)
> case AV_CODEC_ID_H263P:
> case AV_CODEC_ID_H264:
> case AV_CODEC_ID_HEVC:
> + case AV_CODEC_ID_VVC:
> case AV_CODEC_ID_MPEG1VIDEO:
> case AV_CODEC_ID_MPEG2VIDEO:
> case AV_CODEC_ID_MPEG4:
> @@ -605,6 +606,7 @@ static int rtp_write_packet(AVFormatContext *s1,
> AVPacket *pkt)
> ff_rtp_send_h263(s1, pkt->data, size);
> break;
> case AV_CODEC_ID_HEVC:
> + case AV_CODEC_ID_VVC:
> ff_rtp_send_h264_hevc(s1, pkt->data, size);
> break;
> case AV_CODEC_ID_VORBIS:
> diff --git a/libavformat/rtpenc_h264_hevc.c
> b/libavformat/rtpenc_h264_hevc.c
> index 0c88fc2a23..7dbd9f3e11 100644
> --- a/libavformat/rtpenc_h264_hevc.c
> +++ b/libavformat/rtpenc_h264_hevc.c
> @@ -84,9 +84,38 @@ static void nal_send(AVFormatContext *s1, const
> uint8_t *buf, int size, int last
> if (buffered_size == 0) {
> if (codec == AV_CODEC_ID_H264) {
> *s->buf_ptr++ = 24;
> - } else {
> + } else if (codec == AV_CODEC_ID_HEVC) {
> *s->buf_ptr++ = 48 << 1;
> *s->buf_ptr++ = 1;
> + } else {
> + // VVC case, rfc9328, page 20
> + uint8_t first_nal_type = buf[0];
> + uint8_t last_nal_type = buf[1];
> + first_nal_type &= ~(1<<7); // clear the F bit,
> initially
> + last_nal_type = (last_nal_type & 0x07) | (28 << 3);
> + *s->buf_ptr++ = first_nal_type;
> + *s->buf_ptr++ = last_nal_type;
> + }
> + } else if (codec == AV_CODEC_ID_VVC) {
> + // VVC case, rfc9328, page 20, use lowest layerid/tid
> + uint8_t first_nal_type = s->buf[0];
> + uint8_t last_nal_type = s->buf[1];
> + uint8_t new_first_nal_type = buf[0];
> + uint8_t new_last_nal_type = buf[1];
> + int cur_f = first_nal_type >> 7;
> + int new_f = new_first_nal_type >> 7;
> + if (!cur_f && new_f) {
> + s->buf[0] |= 1 << 7;
> + }
> + int cur_lid = first_nal_type & 0x3f;
> + int new_lid = new_first_nal_type & 0x3f;
> + if (cur_lid > new_lid) {
> + s->buf[0] |= new_lid;
> + }
> + int cur_tid = last_nal_type & 0x07;
> + int new_tid = new_last_nal_type & 0x07;
> + if (cur_tid > new_tid) {
> + s->buf[1] |= new_tid;
> }
> }
> AV_WB16(s->buf_ptr, size);
> @@ -100,6 +129,7 @@ static void nal_send(AVFormatContext *s1, const
> uint8_t *buf, int size, int last
> }
> } else {
> int flag_byte, header_size;
> + uint8_t nal_type;
> flush_buffered(s1, 0);
> if (codec == AV_CODEC_ID_H264 && (s->flags &
> FF_RTP_FLAG_H264_MODE0)) {
> av_log(s1, AV_LOG_ERROR,
> @@ -109,20 +139,20 @@ static void nal_send(AVFormatContext *s1, const
> uint8_t *buf, int size, int last
> }
> av_log(s1, AV_LOG_DEBUG, "NAL size %d > %d\n", size,
> s->max_payload_size);
> if (codec == AV_CODEC_ID_H264) {
> - uint8_t type = buf[0] & 0x1F;
> + nal_type = buf[0] & 0x1F;
> uint8_t nri = buf[0] & 0x60;
> s->buf[0] = 28; /* FU Indicator; Type = 28 --->
> FU-A */
> s->buf[0] |= nri;
> - s->buf[1] = type;
> + s->buf[1] = nal_type;
> s->buf[1] |= 1 << 7;
> buf += 1;
> size -= 1;
> flag_byte = 1;
> header_size = 2;
> - } else {
> - uint8_t nal_type = (buf[0] >> 1) & 0x3F;
> + } else if (codec == AV_CODEC_ID_HEVC) {
> + nal_type = (buf[0] >> 1) & 0x3F;
> /*
> * create the HEVC payload header and transmit the buffer
> as fragmentation units (FU)
> *
> @@ -154,8 +184,54 @@ static void nal_send(AVFormatContext *s1, const
> uint8_t *buf, int size, int last
> */
> s->buf[2] = nal_type;
> /* set the S bit: mark as start fragment */
> - s->buf[2] |= 1 << 7;
> + s->buf[1] = (s->buf[1] & 0x07) | (29 << 3);
> +
> + /* pass the original NAL header */
> + buf += 2;
> + size -= 2;
> +
> + flag_byte = 2;
> + header_size = 3;
> + } else {
> + s->buf[0] = buf[0];
> + s->buf[1] = buf[1];
> + /*
> + * create the VVC payload header and transmit the buffer as
> fragmentation units (FU)
> + *
> + * +---------------+---------------+
> + * |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7|
> + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
> + * |F|Z| LayerID | Type | TID |
> + * +---------------+---------------+
> + * F = 0
> + * Type = 29 (fragmentation unit (FU))
> + * LayerId = 0
> + * TID = 1
> + */
> + nal_type = (buf[1] >> 3) & 0x1F;
> + s->buf[1] = (s->buf[1] & 0x07) | (29 << 3);
> + /*
> + * create the FU header
> + *
> + * 0 1 2 3 4 5 6 7
> + * +---------------+
> + * |0|1|2|3|4|5|6|7|
> + * +-+-+-+-+-+-+-+-+
> + * |S|E|P| FuType |
> + * +---------------+
> + * S = variable
> + * E = variable
> + * P = variable
> + * FuType = NAL unit type
> + */
> + s->buf[2] = nal_type;
> + /* set the S bit: mark as start fragment */
> + s->buf[2] |= 1 << 7;
> + // Set the P bit if it's the last fragment and if it's a
> VCL NAL unit
> + if (nal_type <= 11) {
> + s->buf[2] &= ~(1 << 5); // Clear the P bit otherwise
> + }
> /* pass the original NAL header */
> buf += 2;
> size -= 2;
> @@ -170,8 +246,14 @@ static void nal_send(AVFormatContext *s1, const
> uint8_t *buf, int size, int last
> buf += s->max_payload_size - header_size;
> size -= s->max_payload_size - header_size;
> s->buf[flag_byte] &= ~(1 << 7);
> + if (codec == AV_CODEC_ID_VVC && nal_type <= 11) {
> + s->buf[flag_byte] &= ~(1 << 5); // Clear the P bit
> otherwise
> + }
> }
> s->buf[flag_byte] |= 1 << 6;
> + if (codec == AV_CODEC_ID_VVC && nal_type <= 11) {
> + s->buf[flag_byte] |= 1 << 5; // Set the P bit
> + }
> memcpy(&s->buf[header_size], buf, size);
> ff_rtp_send_data(s1, s->buf, size + header_size, last);
> }
> diff --git a/libavformat/sdp.c b/libavformat/sdp.c
> index ccfaa8aff5..2fbf173e41 100644
> --- a/libavformat/sdp.c
> +++ b/libavformat/sdp.c
> @@ -33,6 +33,7 @@
> #include "internal.h"
> #include "avc.h"
> #include "hevc.h"
> +#include "vvc.h"
> #include "rtp.h"
> #include "version.h"
> #if CONFIG_NETWORK
> @@ -347,6 +348,176 @@ err:
> return ret;
> }
> +
> +static int calculate_ptl_size(uint8_t *extradata) {
> + int offset = 0;
> + /*
> + * unsigned int(9) ols_idx;
> + * unsigned int(3) num_sublayers;
> + * unsigned int(2) constant_frame_rate;
> + * unsigned int(2) chroma_format_idc; */
> + uint8_t num_sublayers = (AV_RB16(&extradata[offset]) >> 4) & 0x07;
> + offset += 2;
> + /* unsigned int(3) bit_depth_minus8;
> + bit(5) reserved = ‘11111’b; */
> + offset += 1;
> + /* bit(2) reserved = ‘00’b;
> + unsigned int (6) num_bytes_constraint_info */
> + uint8_t num_bytes_constraint_info = AV_RB8(&extradata[offset]) & 0x3f;
> + offset += 1;
> + /* unsigned int (7) general_profile_idc
> + unsigned int (1) general_tier_flag */
> + offset += 1;
> + /* unsigned int (8) general_level_idc */
> + offset += 1;
> + /*
> + * unsigned int (1) ptl_frame_only_constraint_flag
> + * unsigned int (1) ptl_multilayer_enabled_flag
> + * unsigned int (8*num_bytes_constraint_info -2)
> general_constraint_info */
> + offset += num_bytes_constraint_info;
> +
> + if (num_sublayers > 1) {
> + // ptl_sublayer_level_present_flags
> + uint8_t ptl_sublayer_level_present_flags = extradata[offset];
> + offset += 1;
> + // Calculate the number of sublayer_level_idc bytes
> + for (int i = 0; i < num_sublayers - 1; i++) {
> + if (ptl_sublayer_level_present_flags & (1 << i)) {
> + offset += 1; // Add a byte for each sublayer_level_idc
> + }
> + }
> + }
> + /* unsigned int(8) num_sub_profiles; */
> + uint8_t ptl_num_sub_profiles = extradata[offset];
> + offset += 1;
> + /* unsigned int(32) general_sub_profile_idc[j]; */
> + offset += 4 * ptl_num_sub_profiles;
> + //End of VvcPTLRecord
> + /*
> + * unsigned int(16) max_picture_width;*/
> + offset += 2;
> + /*
> + * unsigned int(16) max_picture_height;*/
> + offset += 2;
> + /*
> + * unsigned int(16) avg_frame_rate; */
> + offset += 2;
> + return offset;
> +}
> +
> +static int extradata2psets_vvcc(const AVCodecParameters *par, char **out)
> +{
> + char *psets;
> + uint8_t *extradata = par->extradata;
> + int extradata_size = par->extradata_size;
> + uint8_t *tmpbuf = NULL;
> + int ps_pos[3] = { 0 };
> + static const char * const ps_names[3] = { "vps", "sps", "pps" };
> + int num_arrays, num_nalus;
> + int pos, i, j, ret = 0;
> +
> + *out = NULL;
> +
> + if (par->extradata[0] != 1) {
> + AVIOContext *pb;
> +
> + ret = avio_open_dyn_buf(&pb);
> + if (ret < 0)
> + return ret;
> +
> + ret = ff_isom_write_vvcc(pb, par->extradata,
> par->extradata_size, 0);
> + if (ret < 0) {
> + avio_close_dyn_buf(pb, &tmpbuf);
> + goto err;
> + }
> + extradata_size = avio_close_dyn_buf(pb, &extradata);
> + tmpbuf = extradata;
> + }
> +
> + int ptl_present_flag = (extradata[0] & 0x01);
> +
> + pos = 1; // Start after the initial configuration byte
> + if (ptl_present_flag) {
> + // PTL is present, so parse PTL fields (the number of bytes
> depends on PTL structure)
> + pos += calculate_ptl_size(extradata + 1);
> + }
> + if (pos + 1 > extradata_size)
> + goto err;
> + num_arrays = extradata[pos++];
> + for (i = 0; i < num_arrays; i++) {
> + int num_nalus, nalu_type;
> + if (pos + 3 > extradata_size)
> + goto err;
> + nalu_type = extradata[pos] & 0x1f;
> + if (nalu_type == 14) // VPS
> + ps_pos[0] = pos;
> + else if (nalu_type == 15) // SPS
> + ps_pos[1] = pos;
> + else if (nalu_type == 16) // PPS
> + ps_pos[2] = pos;
> + num_nalus = AV_RB16(&extradata[pos + 1]);
> + pos += 3;
> + for (j = 0; j < num_nalus; j++) {
> + int len;
> + if (pos + 2 > extradata_size)
> + goto err;
> + len = AV_RB16(&extradata[pos]);
> + pos += 2;
> + if (pos + len > extradata_size)
> + goto err;
> + pos += len;
> + }
> + }
> + if (!ps_pos[1] || !ps_pos[2])
> + goto err;
> + psets = av_mallocz(MAX_PSET_SIZE);
> + if (!psets) {
> + ret = AVERROR(ENOMEM);
> + goto err;
> + }
> +
> + psets[0] = '\0';
> +
> + for (i = 0; i < 3; i++) {
> + if (!ps_pos[i]) {
> + continue;
> + }
> + pos = ps_pos[i];
> +
> + if (i > 0 && ps_pos[i-1])
> + av_strlcat(psets, "; ", MAX_PSET_SIZE);
> + av_strlcatf(psets, MAX_PSET_SIZE, "sprop-%s=", ps_names[i]);
> +
> + // Skipping boundary checks in the input here; we've already
> traversed
> + // the whole hvcc structure above without issues
> + num_nalus = AV_RB16(&extradata[pos + 1]);
> + pos += 3;
> + for (j = 0; j < num_nalus; j++) {
> + int len = AV_RB16(&extradata[pos]);
> + int strpos;
> + pos += 2;
> + if (j > 0)
> + av_strlcat(psets, ",", MAX_PSET_SIZE);
> + strpos = strlen(psets);
> + if (!av_base64_encode(psets + strpos, MAX_PSET_SIZE - strpos,
> + &extradata[pos], len)) {
> + av_free(psets);
> + goto err;
> + }
> + pos += len;
> + }
> + }
> + av_free(tmpbuf);
> +
> + *out = psets;
> + return 0;
> +err:
> + if (ret >= 0)
> + ret = AVERROR_INVALIDDATA;
> + av_free(tmpbuf);
> + return ret;
> +}
> +
> static int extradata2config(AVFormatContext *s, const AVCodecParameters
> *par,
> char **out)
> {
> @@ -579,6 +750,17 @@ static int sdp_write_media_attributes(char *buff,
> int size, const AVStream *st,
> av_strlcatf(buff, size, "a=fmtp:%d %s\r\n",
> payload_type, config);
> break;
> + case AV_CODEC_ID_VVC:
> + if (p->extradata_size) {
> + ret = extradata2psets_vvcc(p, &config);
> + if (ret < 0)
> + return ret;
> + }
> + av_strlcatf(buff, size, "a=rtpmap:%d H266/90000\r\n",
> payload_type);
> + if (config)
> + av_strlcatf(buff, size, "a=fmtp:%d %s\r\n",
> + payload_type, config);
> + break;
> case AV_CODEC_ID_MPEG4:
> if (p->extradata_size) {
> ret = extradata2config(fmt, p, &config);
More information about the ffmpeg-devel
mailing list