[FFmpeg-devel] [PATCH] Optimization: support for libx264's mb_info
Carotti, Elias
eliascrt at amazon.it
Mon Jun 5 18:32:35 EEST 2023
Hi,
please find attached the patch which I updated according to your
suggestions.
Best,
Elias
On Sun, 2023-06-04 at 17:29 +0200, Stefano Sabatini wrote:
> CAUTION: This email originated from outside of the organization. Do
> not click links or open attachments unless you can confirm the sender
> and know the content is safe.
>
>
>
> On date Monday 2023-05-29 17:56:55 +0000, Carotti, Elias wrote:
> [...]
> > From 7cb97ee977197e310a932b2d7a53bf5c6e99990e Mon Sep 17 00:00:00
> > 2001
> > From: Elias Carotti <eliascrt at amazon.it>
> > Date: Wed, 19 Apr 2023 11:49:39 +0200
> > Subject: [PATCH] Add support for libx264's MB_INFO
> >
> > libx264's x264_image_properties_t, which is passed to the encoding
> > function,
> > contains a field to pass down information on the portions of the
> > frame which
> > changed with respect to the previous one (used for prediction) to
> > mark
> > unchanged macroblocks P_SKIP.
> > ---
> > libavcodec/libx264.c | 81 +++++++++++++++++++++++++++++++++
> > libavutil/Makefile | 4 ++
> > libavutil/frame.h | 10 +++++
> > libavutil/video_hint_info.c | 89
> > +++++++++++++++++++++++++++++++++++++
> > libavutil/video_hint_info.h | 87
> > ++++++++++++++++++++++++++++++++++++
> > 5 files changed, 271 insertions(+)
> > create mode 100644 libavutil/video_hint_info.c
> > create mode 100644 libavutil/video_hint_info.h
> >
> > diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
> > index 5736f1efa7..32fa80d0a1 100644
> > --- a/libavcodec/libx264.c
> > +++ b/libavcodec/libx264.c
> > @@ -30,6 +30,7 @@
> > #include "libavutil/stereo3d.h"
> > #include "libavutil/time.h"
> > #include "libavutil/intreadwrite.h"
> > +#include "libavutil/video_hint_info.h"
> > #include "avcodec.h"
> > #include "codec_internal.h"
> > #include "encode.h"
> > @@ -48,6 +49,9 @@
> > // from x264.h, for quant_offsets, Macroblocks are 16x16
> > // blocks of pixels (with respect to the luma plane)
> > #define MB_SIZE 16
> > +#define MB_LSIZE 4
> > +#define MB_FLOOR(x) ((x) >> (MB_LSIZE))
> > +#define MB_CEIL(x) MB_FLOOR((x) + (MB_SIZE - 1))
> >
> > typedef struct X264Opaque {
> > #if FF_API_REORDERED_OPAQUE
> > @@ -123,6 +127,8 @@ typedef struct X264Context {
> > * encounter a frame with ROI side data.
> > */
> > int roi_warned;
> > +
> > + int mb_info;
> > } X264Context;
> >
> > static void X264_log(void *p, int level, const char *fmt, va_list
> > args)
> > @@ -295,6 +301,7 @@ static void free_picture(x264_picture_t *pic)
> > av_free(pic->extra_sei.payloads[i].payload);
> > av_freep(&pic->extra_sei.payloads);
> > av_freep(&pic->prop.quant_offsets);
> > + av_freep(&pic->prop.mb_info);
> > pic->extra_sei.num_payloads = 0;
> > }
> >
> > @@ -320,6 +327,64 @@ static enum AVPixelFormat csp_to_pixfmt(int
> > csp)
> > return AV_PIX_FMT_NONE;
> > }
> >
> > +static int setup_mb_info(AVCodecContext *ctx, x264_picture_t *pic,
> > + const AVFrame *frame,
> > + const AVVideoHint *info)
> > +{
>
> > + int mb_width = (frame->width + MB_SIZE - 1) / MB_SIZE;
> > + int mb_height = (frame->height + MB_SIZE - 1) / MB_SIZE;
>
>
> > +
> > + const AVVideoRect *mbinfo_rects;
> > + int nb_rects;
> > + uint8_t *mbinfo;
> > +
> > + mbinfo_rects = (const AVVideoRect *)av_video_hint_rects(info);
> > + nb_rects = info->nb_rects;
> > +
> > + mbinfo = av_calloc(mb_width * mb_height, sizeof(*mbinfo));
> > + if (!mbinfo)
> > + return AVERROR(ENOMEM);
> > +
>
> > + if (info->type == AV_VIDEO_HINT_CHANGED) {
> > + /* Sets the default as constant, i.e. P_SKIP-able, then
> > selectively resets the flag */
> > + memset(mbinfo, X264_MBINFO_CONSTANT, sizeof(*mbinfo) *
> > mb_width * mb_height);
> > +
> > + for (int i = 0; i < nb_rects; i++) {
> > + int min_y = MB_FLOOR(mbinfo_rects->y);
> > + int max_y = MB_CEIL(mbinfo_rects->y + mbinfo_rects-
> > >height);
> > + int min_x = MB_FLOOR(mbinfo_rects->x);
> > + int max_x = MB_CEIL(mbinfo_rects->x + mbinfo_rects-
> > >width);
> > +
> > + for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> > + memset(mbinfo + mb_y * mb_width + min_x, 0, max_x
> > - min_x);
> > + }
> > +
> > + mbinfo_rects++;
> > + }
> > + } else {
> > + /* Sets the default as changed, i.e. *not* P_SKIP-able,
> > then selectively sets the flag */
> > + memset(mbinfo, 0, sizeof(*mbinfo) * mb_width * mb_height);
> > +
> > + for (int i = 0; i < nb_rects; i++) {
> > + int min_y = MB_CEIL(mbinfo_rects->y);
> > + int max_y = MB_FLOOR(mbinfo_rects->y + mbinfo_rects-
> > >height);
> > + int min_x = MB_CEIL(mbinfo_rects->x);
> > + int max_x = MB_FLOOR(mbinfo_rects->x + mbinfo_rects-
> > >width);
> > +
> > + for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> > + memset(mbinfo + mb_y * mb_width + min_x,
> > X264_MBINFO_CONSTANT, max_x - min_x);
> > + }
> > +
> > + mbinfo_rects++;
>
> you can merge the two loops by adding an inline function to compute
> the mins and maxs, and then setting:
>
> mbinfo_filler = AV_VIDEO_HINT_CHANGED ? X264_MBINFO_CONSTANT : 0;
> mbinfo_marker = AV_VIDEO_HINT_CHANGED ? 0 : X264_MBINFO_CONSTANT;
> compute_coords_fn = AV_VIDEO_HINT_CHANGED ? compute_changed_coords :
> compute_constant_coords;
>
> memset(mbinfo, mbinfo_filler, sizeof(*mbinfo) * mb_width *
> mb_height);
> for (int i = 0; i < nb_rects; i++) {
> compute_coords_fn...
> for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> memset(mbinfo + mb_y * mb_width + min_x, mbinfo_marker, max_x
> - min_x);
> }
>
> mbinfo_rects++;
> }
>
> > + }
> > + }
> > +
> > + pic->prop.mb_info = mbinfo;
> > + pic->prop.mb_info_free = av_free;
> > +
> > + return 0;
> > +}
> > +
> > static int setup_roi(AVCodecContext *ctx, x264_picture_t *pic, int
> > bit_depth,
> > const AVFrame *frame, const uint8_t *data,
> > size_t size)
> > {
> > @@ -404,6 +469,7 @@ static int setup_frame(AVCodecContext *ctx,
> > const AVFrame *frame,
> > int64_t wallclock = 0;
> > int bit_depth, ret;
> > AVFrameSideData *sd;
> > + AVFrameSideData *mbinfo_sd;
> >
> > *ppic = NULL;
> > if (!frame)
> > @@ -499,6 +565,17 @@ FF_ENABLE_DEPRECATION_WARNINGS
> > goto fail;
> > }
> >
> > + mbinfo_sd = av_frame_get_side_data(frame,
> > AV_FRAME_DATA_VIDEO_HINT);
> > + if (mbinfo_sd) {
> > + int ret = setup_mb_info(ctx, pic, frame, (const
> > AVVideoHint *)mbinfo_sd->data);
> > + if (ret < 0) {
> > + /* No need to fail here, this is not fatal. We just
> > proceed with no
> > + * mb_info and log a message */
> > +
> > + av_log(ctx, AV_LOG_WARNING, "mb_info setup
> > failure\n");
> > + }
> > + }
> > +
> > if (x4->udu_sei) {
> > for (int j = 0; j < frame->nb_side_data; j++) {
> > AVFrameSideData *side_data = frame->side_data[j];
> > @@ -1102,6 +1179,9 @@ FF_ENABLE_DEPRECATION_WARNINGS
> > }
> > }
> >
> > + x4->params.analyse.b_mb_info = x4->mb_info;
> > + x4->params.analyse.b_fast_pskip = 1;
> > +
> > // update AVCodecContext with x264 parameters
> > avctx->has_b_frames = x4->params.i_bframe ?
> > x4->params.i_bframe_pyramid ? 2 : 1 : 0;
> > @@ -1311,6 +1391,7 @@ static const AVOption options[] = {
> > { "noise_reduction", "Noise
> > reduction", OFFSET(noise_reduction),
> > AV_OPT_TYPE_INT, { .i64 = -1 }, INT_MIN, INT_MAX, VE },
> > { "udu_sei", "Use user data unregistered SEI if
> > available", OFFSET(udu_sei), AV_OPT_TYPE_BOOL, { .i64 = 0
> > }, 0, 1, VE },
> > { "x264-params", "Override the x264 configuration using a :-
> > separated list of key=value parameters", OFFSET(x264_params),
> > AV_OPT_TYPE_DICT, { 0 }, 0, 0, VE },
> > + { "mb_info", "Set mb_info data through AVSideData, only
> > useful when used from the API", OFFSET(mb_info), AV_OPT_TYPE_BOOL,
> > { .i64 = 0 }, 0, 1, VE },
> > { NULL },
> > };
> >
> > diff --git a/libavutil/Makefile b/libavutil/Makefile
> > index dc9012f9a8..bb5ecc3235 100644
> > --- a/libavutil/Makefile
> > +++ b/libavutil/Makefile
> > @@ -91,6 +91,7 @@ HEADERS =
> > adler32.h \
> >
> > tea.h \
> >
> > tx.h \
> >
> > film_grain_params.h \
> > +
> > video_hint_info.h
> > \
>
> >
> > ARCH_HEADERS =
> > bswap.h \
> >
> > intmath.h \
> > @@ -196,6 +197,7 @@ OBJS-$(CONFIG_VAAPI) +=
> > hwcontext_vaapi.o
> > OBJS-$(CONFIG_VIDEOTOOLBOX) +=
> > hwcontext_videotoolbox.o
> > OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o
> > OBJS-$(CONFIG_VULKAN) += hwcontext_vulkan.o
> > +OBJS-$(CONFIG_LIBX264) += video_hint_info.o
>
> order
>
> >
> > OBJS-$(!CONFIG_VULKAN) += hwcontext_stub.o
> >
> > @@ -219,6 +221,8 @@ SKIPHEADERS-$(CONFIG_VULKAN) +=
> > hwcontext_vulkan.h vulkan.h \
> >
> > vulkan_functions.h \
> > vulkan_loader.h
> >
> > +SKIPHEADERS-$(CONFIG_LIBX264) += video_hint_info.h
> > +
> > TESTPROGS =
> > adler32 \
> >
> > aes \
> >
> > aes_ctr \
> > diff --git a/libavutil/frame.h b/libavutil/frame.h
> > index a491315f25..0e765e5499 100644
> > --- a/libavutil/frame.h
> > +++ b/libavutil/frame.h
> > @@ -214,6 +214,16 @@ enum AVFrameSideDataType {
> > * Ambient viewing environment metadata, as defined by H.274.
> > */
> > AV_FRAME_DATA_AMBIENT_VIEWING_ENVIRONMENT,
> > +
> > + /**
> > + * Provide macro block encoder-specific hinting information
> > for the encoder
> > + * processing. It can be used to pass information about which
> > macroblock
> > + * can be skipped because it hasn't changed from the
> > corresponding one in
> > + * the previous frame. This is useful for applications which
> > know in
> > + * advance this information to speed up real-time encoding.
> > Currently only
> > + * used by libx264.
> > + */
> > + AV_FRAME_DATA_VIDEO_HINT,
> > };
> >
> > enum AVActiveFormatDescription {
> > diff --git a/libavutil/video_hint_info.c
> > b/libavutil/video_hint_info.c
> > new file mode 100644
> > index 0000000000..0a763962da
> > --- /dev/null
> > +++ b/libavutil/video_hint_info.c
> > @@ -0,0 +1,89 @@
> > +/*
> > + * Copyright 2023 Elias Carotti <eliascrt at amazon dot it>
> > + *
> > + * This file is part of FFmpeg.
> > + *
> > + * FFmpeg is free software; you can redistribute it and/or
> > + * modify it under the terms of the GNU Lesser General Public
> > + * License as published by the Free Software Foundation; either
> > + * version 2.1 of the License, or (at your option) any later
> > version.
> > + *
> > + * FFmpeg is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> > GNU
> > + * Lesser General Public License for more details.
> > + *
> > + * You should have received a copy of the GNU Lesser General
> > Public
> > + * License along with FFmpeg; if not, write to the Free Software
> > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> > 02110-1301 USA
> > + */
> > +
> > +#include <string.h>
> > +
> > +#include "avstring.h"
> > +#include "frame.h"
> > +#include "macros.h"
> > +#include "mem.h"
> > +#include "video_hint_info.h"
> > +
> > +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects,
>
> > + size_t nb_rects,
> > + VideoHintType type,
> > + size_t* out_size)
>
> align to (
>
> > +{
> > + struct TestStruct {
> > + AVVideoHint p;
> > + AVVideoRect b;
> > + };
>
> nit: weird align (also use more meaningful names - e.g. hint and
> rect)
>
> > + const size_t blocks_offset = offsetof(struct TestStruct, b);
> > + size_t size = blocks_offset;
>
> > + AVVideoHint *par;
>
> AVVideoHint *hint?
>
> > +
> > + *out_size = 0;
>
> > + if (nb_rects > (SIZE_MAX - size) / sizeof(AVVideoRect))
>
> why SIZE_MAX - size?
>
> > + return NULL;
>
> > + size += sizeof(AVVideoRect) * nb_rects;
> > +
> > + par = av_mallocz(size);
> > + if (!par)
> > + return NULL;
> > +
> > + par->type = type;
> > + par->nb_rects = nb_rects;
> > + par->blocks_offset = blocks_offset;
> > +
> > + /* Just copies the rects over the newly allocated buffer */
> > + memcpy((uint8_t *)par + blocks_offset, rects,
> > sizeof(AVVideoRect) * nb_rects);
> > +
> > + *out_size = size;
> > +
> > + return par;
> > +}
> > +
> > +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame,
> > + AVVideoRect *rects,
> > + size_t num_rects,
> > + VideoHintType type)
> > +{
>
> > + AVVideoHint *par;
>
> use a more meaningful name (I think "par" is a leftover of an older
> API)
>
> > + AVBufferRef *buf;
> > + size_t size = 0;
> > +
> > + par = av_video_hint_alloc(rects, num_rects, type, &size);
> > + if (!par)
> > + return NULL;
> > +
> > + buf = av_buffer_create((uint8_t *)par, size, NULL, NULL, 0);
> > + if (!buf) {
> > + av_freep(&par);
> > + return NULL;
> > + }
> > +
> > + if (!av_frame_new_side_data_from_buf(frame,
> > AV_FRAME_DATA_VIDEO_HINT, buf)) {
> > + av_buffer_unref(&buf);
> > + return NULL;
> > + }
> > +
> > + return par;
> > +}
> > +
> > diff --git a/libavutil/video_hint_info.h
> > b/libavutil/video_hint_info.h
> > new file mode 100644
> > index 0000000000..3b04e0c40e
> > --- /dev/null
> > +++ b/libavutil/video_hint_info.h
> > @@ -0,0 +1,87 @@
> > +/**
> > + * Copyright 2023 Elias Carotti <eliascrt at amazon dot it>
> > + *
> > + * This file is part of FFmpeg.
> > + *
> > + * FFmpeg is free software; you can redistribute it and/or
> > + * modify it under the terms of the GNU Lesser General Public
> > + * License as published by the Free Software Foundation; either
> > + * version 2.1 of the License, or (at your option) any later
> > version.
> > + *
> > + * FFmpeg is distributed in the hope that it will be useful,
> > + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> > + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> > GNU
> > + * Lesser General Public License for more details.
> > + *
> > + * You should have received a copy of the GNU Lesser General
> > Public
> > + * License along with FFmpeg; if not, write to the Free Software
> > + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
> > 02110-1301 USA
> > + */
> > +
> > +#ifndef AVUTIL_VIDEO_HINT_INFO_H
> > +#define AVUTIL_VIDEO_HINT_INFO_H
> > +
> > +#include <stddef.h>
> > +#include <stdint.h>
> > +#include "libavutil/avassert.h"
> > +#include "libavutil/frame.h"
> > +
> > +typedef struct AVVideoRect {
> > + uint32_t x, y;
> > + uint32_t width, height;
> > +} AVVideoRect;
> > +
>
> > +typedef enum VideoHintType {
> > + /* blocks delimit the constant areas (unchanged), default is
> > changed */
> > + AV_VIDEO_HINT_CONSTANT,
> > +
> > + /* blocks delimit the constant areas (changed), default is not
> > changed */
> > + AV_VIDEO_HINT_CHANGED,
> > +} VideoHintType;
>
> missing AV prefix
>
> > +
> > +typedef struct AVVideoHint {
> > + /**
> > + * Number of blocks in the array.
> > + *
> > + * May be 0, in which case no per-block information is
> > present. In this case
> > + * the values of blocks_offset / block_size are unspecified
> > and should not
> > + * be accessed.
> > + */
> > + int nb_rects;
> > +
> > + /**
> > + * Offset in bytes from the beginning of this structure at
> > which the array
> > + * of blocks starts.
> > + */
> > + size_t blocks_offset;
> > +
> > + VideoHintType type;
> > +} AVVideoHint;
> > +
> > +static av_always_inline AVVideoRect*
> > +av_video_hint_rects(const AVVideoHint *par)
> > +{
> > + return (AVVideoRect *)((uint8_t *)par + par->blocks_offset);
> > +}
> > +
> > +/**
> > + * Allocate memory for a vector of AVVideoRect in the given
> > AVFrame
> > + * {@code frame} as AVFrameSideData of type
> > AV_FRAME_DATA_VIDEO_HINT_INFO.
> > + * The side data contains a list of rectangles for the portions of
> > the frame
> > + * which changed from the last encoded one (and the remainder are
> > assumed to be
> > + * changed), or, alternately (depending on the type parameter) the
> > unchanged
> > + * ones (and the remanining ones are those which changed).
> > + * Macroblocks will thus be hinted either to be P_SKIP-ped or go
> > through the
> > + * regular encoding procedure.
> > + */
> > +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame,
> > + AVVideoRect *rects,
> > + size_t num_rects,
> > + VideoHintType type);
> > +
> > +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects,
> > + size_t nb_rects,
> > + VideoHintType type,
> > + size_t *out_size);
> > +
>
> > +#endif /* AVUTIL__VIDEO_HINT_INFO_H */
>
> AVUTIL_VIDEO...
>
> > --
> > 2.34.1
> >
>
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel at ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
NICE SRL, viale Monte Grappa 3/5, 20124 Milano, Italia, Registro delle Imprese di Milano Monza Brianza Lodi REA n. 2096882, Capitale Sociale: 10.329,14 EUR i.v., Cod. Fisc. e P.IVA 01133050052, Societa con Socio Unico
-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0001-Add-support-for-libx264-s-MB_INFO.patch
Type: text/x-patch
Size: 15595 bytes
Desc: 0001-Add-support-for-libx264-s-MB_INFO.patch
URL: <https://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20230605/6638868c/attachment.bin>
More information about the ffmpeg-devel
mailing list