[FFmpeg-devel] [PATCH v3 01/14] vvcdec: add vvc decoder stub
Andreas Rheinhardt
andreas.rheinhardt at outlook.com
Sun Oct 15 12:36:48 EEST 2023
Nuo Mi:
> ---
> configure | 1 +
> libavcodec/Makefile | 1 +
> libavcodec/allcodecs.c | 1 +
> libavcodec/vvc/Makefile | 4 +
> libavcodec/vvc/vvcdec.c | 84 +++++++++++
> libavcodec/vvc/vvcdec.h | 307 ++++++++++++++++++++++++++++++++++++++++
> 6 files changed, 398 insertions(+)
> create mode 100644 libavcodec/vvc/Makefile
> create mode 100644 libavcodec/vvc/vvcdec.c
> create mode 100644 libavcodec/vvc/vvcdec.h
>
> diff --git a/configure b/configure
> index 8a1902810a..442c439c3e 100755
> --- a/configure
> +++ b/configure
> @@ -3025,6 +3025,7 @@ vp6f_decoder_select="vp6_decoder"
> vp7_decoder_select="h264pred videodsp vp8dsp"
> vp8_decoder_select="h264pred videodsp vp8dsp"
> vp9_decoder_select="videodsp vp9_parser vp9_superframe_split_bsf"
> +vvc_decoder_select="cabac golomb videodsp"
> wcmv_decoder_select="inflate_wrapper"
> webp_decoder_select="vp8_decoder exif"
> wmalossless_decoder_select="llauddsp"
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index 08fd151619..fb2738d8eb 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -62,6 +62,7 @@ OBJS = ac3_parser.o \
> xiph.o \
>
> # subsystems
> +include $(SRC_PATH)/libavcodec/vvc/Makefile
I don't see why vvc should have a subfolder of its own.
> OBJS-$(CONFIG_AANDCTTABLES) += aandcttab.o
> OBJS-$(CONFIG_AC3DSP) += ac3dsp.o ac3.o ac3tab.o
> OBJS-$(CONFIG_ADTS_HEADER) += adts_header.o mpeg4audio_sample_rates.o
> diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
> index 6e95ca5636..c16d80cb85 100644
> --- a/libavcodec/allcodecs.c
> +++ b/libavcodec/allcodecs.c
> @@ -388,6 +388,7 @@ extern const FFCodec ff_vp9_rkmpp_decoder;
> extern const FFCodec ff_vp9_v4l2m2m_decoder;
> extern const FFCodec ff_vqa_decoder;
> extern const FFCodec ff_vqc_decoder;
> +extern const FFCodec ff_vvc_decoder;
> extern const FFCodec ff_wbmp_decoder;
> extern const FFCodec ff_wbmp_encoder;
> extern const FFCodec ff_webp_decoder;
> diff --git a/libavcodec/vvc/Makefile b/libavcodec/vvc/Makefile
> new file mode 100644
> index 0000000000..bd14dbc1df
> --- /dev/null
> +++ b/libavcodec/vvc/Makefile
> @@ -0,0 +1,4 @@
> +clean::
> + $(RM) $(CLEANSUFFIXES:%=libavcodec/vvc/%)
> +
> +OBJS-$(CONFIG_VVC_DECODER) += vvc/vvcdec.o \
> diff --git a/libavcodec/vvc/vvcdec.c b/libavcodec/vvc/vvcdec.c
> new file mode 100644
> index 0000000000..8d027af0b9
> --- /dev/null
> +++ b/libavcodec/vvc/vvcdec.c
> @@ -0,0 +1,84 @@
> +/*
> + * VVC video decoder
> + *
> + * Copyright (C) 2021 Nuo Mi
> + * Copyright (C) 2022 Xu Mu
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +#include "config_components.h"
What do you need this for?
> +
> +#include "libavcodec/codec_internal.h"
> +#include "libavcodec/decode.h"
> +#include "libavcodec/golomb.h"
> +#include "libavcodec/profiles.h"
> +#include "libavcodec/vvc.h"
> +
> +#include "libavutil/cpu.h"
Why would you ever need cpu.h in the decoder?
Even if a later patch needs it, it should be added in the later patch.
> +
> +#include "vvcdec.h"
> +
> +static int vvc_decode_frame(AVCodecContext *avctx, AVFrame *output,
> + int *got_output, AVPacket *avpkt)
> +{
> + return avpkt->size;
> +}
> +
> +static void vvc_decode_flush(AVCodecContext *avctx)
> +{
> +}
> +
> +static av_cold int vvc_decode_free(AVCodecContext *avctx)
> +{
> + return 0;
> +}
> +
> +static av_cold int vvc_decode_init(AVCodecContext *avctx)
> +{
> + return 0;
> +}
> +
> +#define OFFSET(x) offsetof(VVCContext, x)
> +#define PAR (AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
> +
> +static const AVOption options[] = {
> + { NULL },
> +};
> +
> +static const AVClass vvc_decoder_class = {
> + .class_name = "vvc decoder",
> + .item_name = av_default_item_name,
> + .option = options,
> + .version = LIBAVUTIL_VERSION_INT,
> +};
This seems to be based upon H.264, but the H.264 decoder has options,
whereas this decoder doesn't, so this AVClass and the options are
unnecessary.
> +
> +const FFCodec ff_vvc_decoder = {
> + .p.name = "vvc",
> + .p.long_name = NULL_IF_CONFIG_SMALL("VVC (Versatile Video Coding)"),
> + .p.type = AVMEDIA_TYPE_VIDEO,
> + .p.id = AV_CODEC_ID_VVC,
> + .priv_data_size = sizeof(VVCContext),
> + .p.priv_class = &vvc_decoder_class,
> + .init = vvc_decode_init,
> + .close = vvc_decode_free,
> + FF_CODEC_DECODE_CB(vvc_decode_frame),
> + .flush = vvc_decode_flush,
> + .p.capabilities = AV_CODEC_CAP_DR1 | AV_CODEC_CAP_DELAY | AV_CODEC_CAP_OTHER_THREADS,
> + .caps_internal = FF_CODEC_CAP_EXPORTS_CROPPING | FF_CODEC_CAP_INIT_CLEANUP |
> + FF_CODEC_CAP_AUTO_THREADS,
> + .p.profiles = NULL_IF_CONFIG_SMALL(ff_vvc_profiles),
> +};
> diff --git a/libavcodec/vvc/vvcdec.h b/libavcodec/vvc/vvcdec.h
> new file mode 100644
> index 0000000000..698ab89b9d
> --- /dev/null
> +++ b/libavcodec/vvc/vvcdec.h
> @@ -0,0 +1,307 @@
> +/*
> + * VVC video decoder
> + *
> + * Copyright (C) 2021 Nuo Mi
> + * Copyright (C) 2022 Xu Mu
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVCODEC_VVCDEC_H
> +#define AVCODEC_VVCDEC_H
> +
> +#include "libavcodec/cbs.h"
> +#include "libavcodec/cbs_h266.h"
> +#include "libavcodec/h2645_parse.h"
> +#include "libavcodec/threadframe.h"
> +#include "libavcodec/videodsp.h"
> +#include "libavcodec/vvc.h"
> +#include "libavutil/executor.h"
> +
> +#define LUMA 0
> +#define CHROMA 1
> +#define CB 1
> +#define CR 2
> +#define JCBCR 3
> +
> +#define MAX_CTU_SIZE 128
> +
> +#define MAX_CU_SIZE MAX_CTU_SIZE
> +#define MIN_CU_SIZE 4
> +#define MIN_CU_LOG2 2
> +#define MAX_CU_DEPTH 7
> +
> +#define MIN_PU_SIZE 4
> +#define MIN_PU_LOG2 2
> +
> +#define MAX_TB_SIZE 64
> +#define MIN_TU_LOG2 2 ///< MinTbLog2SizeY
> +#define MIN_TU_SIZE 4
> +#define MAX_TUS_IN_CU 64
> +
> +#define MAX_PARTS_IN_CTU ((MAX_CTU_SIZE >> MIN_CU_LOG2) * (MAX_CTU_SIZE >> MIN_CU_LOG2))
> +
> +#define MAX_CONTROL_POINTS 3
> +
> +#define MRG_MAX_NUM_CANDS 6
> +#define MAX_NUM_HMVP_CANDS 5
> +
> +#define L0 0
> +#define L1 1
> +
> +#define CHROMA_EXTRA_BEFORE 1
> +#define CHROMA_EXTRA_AFTER 2
> +#define CHROMA_EXTRA 3
> +#define LUMA_EXTRA_BEFORE 3
> +#define LUMA_EXTRA_AFTER 4
> +#define LUMA_EXTRA 7
> +#define BILINEAR_EXTRA_BEFORE 0
> +#define BILINEAR_EXTRA_AFTER 1
> +#define BILINEAR_EXTRA 1
> +
> +#define MAX_QP 63
> +#define DEFAULT_INTRA_TC_OFFSET 2
> +
> +#define SAO_PADDING_SIZE 1
> +
> +#define ALF_PADDING_SIZE 8
> +#define ALF_BLOCK_SIZE 4
> +
> +#define ALF_BORDER_LUMA 3
> +#define ALF_BORDER_CHROMA 2
> +
> +#define ALF_VB_POS_ABOVE_LUMA 4
> +#define ALF_VB_POS_ABOVE_CHROMA 2
> +
> +#define ALF_GRADIENT_STEP 2
> +#define ALF_GRADIENT_BORDER 2
> +#define ALF_GRADIENT_SIZE ((MAX_CU_SIZE + ALF_GRADIENT_BORDER * 2) / ALF_GRADIENT_STEP)
> +#define ALF_NUM_DIR 4
> +
> +#define MAX_PB_SIZE 128
> +#define EDGE_EMU_BUFFER_STRIDE (MAX_PB_SIZE + 32)
> +
> +#define AFFINE_MIN_BLOCK_SIZE 4
> +#define PROF_BORDER_EXT 1
> +#define PROF_BLOCK_SIZE (AFFINE_MIN_BLOCK_SIZE + PROF_BORDER_EXT * 2)
> +#define BDOF_BORDER_EXT 1
> +
> +#define BDOF_PADDED_SIZE (16 + BDOF_BORDER_EXT * 2)
> +#define BDOF_BLOCK_SIZE 4
> +#define BDOF_GRADIENT_SIZE (BDOF_BLOCK_SIZE + BDOF_BORDER_EXT * 2)
> +
> +/**
> + * Value of the luma sample at position (x, y) in the 2D array tab.
> + */
> +#define SAMPLE(tab, x, y) ((tab)[(y) * s->sps->width + (x)])
> +#define SAMPLE_CTB(tab, x, y) ((tab)[(y) * min_cb_width + (x)])
> +#define CTB(tab, x, y) ((tab)[(y) * fc->ps.pps->ctb_width + (x)])
> +
> +typedef struct VVCLocalContext VVCLocalContext;
> +typedef struct SliceContext SliceContext;
> +typedef struct VVCFrameContext VVCFrameContext;
> +typedef struct VVCFrameThread VVCFrameThread;
> +typedef struct EntryPoint EntryPoint;
> +typedef struct VVCTask VVCTask;
> +typedef struct Mv Mv;
> +typedef struct MvField MvField;
> +typedef struct DMVRInfo DMVRInfo;
> +typedef struct CTU CTU;
> +typedef struct SAOParams SAOParams;
> +typedef struct ALFParams ALFParams;
> +
> +typedef struct RefPicList {
> + struct VVCFrame *ref[VVC_MAX_REF_ENTRIES];
> + int list[VVC_MAX_REF_ENTRIES];
> + int isLongTerm[VVC_MAX_REF_ENTRIES];
> + int nb_refs;
> +} RefPicList;
> +
> +typedef struct RefPicListTab {
> + RefPicList refPicList[2];
> +} RefPicListTab;
> +
> +typedef struct VVCFrame {
> + AVFrame *frame;
> + ThreadFrame tf;
You are not using the ThreadFrame API at all; you can simply remove this
by switching to ff_thread_get_buffer() instead of
ff_thread_get_ext_buffer().
> +
> + MvField *tab_dmvr_mvf;
> + RefPicListTab **rpl_tab;
> +
> + int ctb_count;
> +
> + int poc;
> +
> + struct VVCFrame *collocated_ref;
> +
> + AVBufferRef *tab_dmvr_mvf_buf;
> + AVBufferRef *rpl_tab_buf;
> + AVBufferRef *rpl_buf;
> + AVBufferRef *progress_buf;
> +
> + /**
> + * A sequence counter, so that old frames are output first
> + * after a POC reset
> + */
> + uint16_t sequence;
> + /**
> + * A combination of VVC_FRAME_FLAG_*
> + */
> + uint8_t flags;
> +} VVCFrame;
> +
> +struct SliceContext {
> + int slice_idx;
> + EntryPoint *eps;
> + int nb_eps;
> + RefPicList *rpl;
> +};
> +
> +struct VVCFrameContext {
> + AVCodecContext *avctx;
Lots of files of the H.264 and HEVC decoder actually don't need
AVCodecContext as complete type at all, they can work with a forward
declaration. It is probably the same for this one, so could you use
struct AVCodecContext* in the header?
> +
> + // +1 for the current frame
> + VVCFrame DPB[VVC_MAX_DPB_SIZE + 1];
> +
> + AVFrame *frame;
> + AVFrame *output_frame;
> +
> + SliceContext **slices;
> + int nb_slices;
> + int nb_slices_allocated;
> +
> + VVCFrame *ref;
> +
> + VideoDSPContext vdsp;
> +
> + VVCFrameThread *frame_thread;
> +
> + uint64_t decode_order;
> +
> + AVBufferPool *tab_dmvr_mvf_pool;
> + AVBufferPool *rpl_tab_pool;
> +
> + AVBufferPool *cu_pool;
> + AVBufferPool *tu_pool;
> +
> + struct {
> + int16_t *slice_idx;
> +
> + DMVRInfo *dmvr;
> +
> + int *cb_pos_x[2]; ///< CbPosX[][][]
> + int *cb_pos_y[2]; ///< CbPosY[][][]
> + uint8_t *cb_width[2]; ///< CbWidth[][][]
> + uint8_t *cb_height[2]; ///< CbHeight[][][]
> + uint8_t *cqt_depth[2]; ///< CqtDepth[][][]
> + int8_t *qp[VVC_MAX_SAMPLE_ARRAYS];
> +
> + uint8_t *skip; ///< CuSkipFlag[][]
> + uint8_t *ispmf; ///< intra_sub_partitions_mode_flag
> + uint8_t *msm[2]; ///< MttSplitMode[][][] in 32 pixels
> + uint8_t *imf; ///< IntraMipFlag[][]
> + uint8_t *imtf; ///< intra_mip_transposed_flag[][]
> + uint8_t *imm; ///< intra_mip_mode[][]
> + uint8_t *ipm; ///< IntraPredModeY[][]
> + uint8_t *cpm[2]; ///< CuPredMode[][][]
> + uint8_t *msf; ///< MergeSubblockFlag[][]
> + uint8_t *iaf; ///< InterAffineFlag[][]
> + uint8_t *mmi; ///< MotionModelIdc[][]
> + Mv *cp_mv[2]; ///< CpMvLX[][][][MAX_CONTROL_POINTS];
> + MvField *mvf; ///< MvDmvrL0, MvDmvrL1
> +
> + uint8_t *tu_coded_flag[VVC_MAX_SAMPLE_ARRAYS]; ///< tu_y_coded_flag[][], tu_cb_coded_flag[][], tu_cr_coded_flag[][]
> + uint8_t *tu_joint_cbcr_residual_flag; ///< tu_joint_cbcr_residual_flag[][]
> + int *tb_pos_x0[2];
> + int *tb_pos_y0[2];
> + uint8_t *tb_width[2];
> + uint8_t *tb_height[2];
> + uint8_t *pcmf[2];
> +
> + uint8_t *horizontal_bs[VVC_MAX_SAMPLE_ARRAYS];
> + uint8_t *vertical_bs[VVC_MAX_SAMPLE_ARRAYS];
> + uint8_t *horizontal_p; ///< horizontal maxFilterLengthPs for luma
> + uint8_t *horizontal_q; ///< horizontal maxFilterLengthPs for luma
> + uint8_t *vertical_p; ///< vertical maxFilterLengthQs for luma
> + uint8_t *vertical_q; ///< vertical maxFilterLengthQs for luma
> +
> + uint8_t *sao_pixel_buffer_h[VVC_MAX_SAMPLE_ARRAYS];
> + uint8_t *sao_pixel_buffer_v[VVC_MAX_SAMPLE_ARRAYS];
> + uint8_t *alf_pixel_buffer_h[VVC_MAX_SAMPLE_ARRAYS][2];
> + uint8_t *alf_pixel_buffer_v[VVC_MAX_SAMPLE_ARRAYS][2];
> +
> + int *coeffs;
> + CTU *ctus;
> +
> + //used in arrays_init only
> + int ctu_count;
> + int ctu_size;
> + int pic_size_in_min_cb;
> + int pic_size_in_min_pu;
> + int pic_size_in_min_tu;
> + int ctu_width;
> + int ctu_height;
> + int width;
> + int height;
> + int chroma_format_idc;
> + int pixel_shift;
> + int bs_width;
> + int bs_height;
> + } tab;
> +} ;
> +
> +typedef struct VVCContext {
> + const AVClass *c; // needed by private avoptions
> + AVCodecContext *avctx;
> +
> + CodedBitstreamContext *cbc;
> + CodedBitstreamFragment current_frame;
> +
> + int temporal_id; ///< temporal_id_plus1 - 1
> + int poc_tid0;
> +
> + int eos; ///< current packet contains an EOS/EOB NAL
> + int last_eos; ///< last packet contains an EOS/EOB NAL
> +
> +
> + enum VVCNALUnitType vcl_unit_type;
> + int no_output_before_recovery_flag; ///< NoOutputBeforeRecoveryFlag
> + int gdr_recovery_point_poc; ///< recoveryPointPocVal
> +
> + /**
> + * Sequence counters for decoded and output frames, so that old
> + * frames are output first after a POC reset
> + */
> + uint16_t seq_decode;
> + uint16_t seq_output;
> +
> + int is_nalff; ///< this flag is != 0 if bitstream is encapsulated
> + ///< as a format defined in 14496-15
> +
> + int apply_defdispwin;
> + int nal_length_size; ///< Number of bytes used for nal length (1, 2 or 4)
> +
> + AVExecutor *executor;
Same as above with AVCodecContext: It seems only vvc_thread.c will ever
need to have executor.h included, so by using struct AVExecutor you can
avoid the inclusion.
> +
> + VVCFrameContext *fcs;
> + int nb_fcs;
> +
> + uint64_t nb_frames; ///< processed frames
> + int nb_delayed; ///< delayed frames
> +} VVCContext ;
> +
> +#endif /* AVCODEC_VVCDEC_H */
More information about the ffmpeg-devel
mailing list