[FFmpeg-devel] [PATCH] Optimization: support for libx264's mb_info
Stefano Sabatini
stefasab at gmail.com
Sun Jun 4 18:29:12 EEST 2023
On date Monday 2023-05-29 17:56:55 +0000, Carotti, Elias wrote:
[...]
> From 7cb97ee977197e310a932b2d7a53bf5c6e99990e Mon Sep 17 00:00:00 2001
> From: Elias Carotti <eliascrt at amazon.it>
> Date: Wed, 19 Apr 2023 11:49:39 +0200
> Subject: [PATCH] Add support for libx264's MB_INFO
>
> libx264's x264_image_properties_t, which is passed to the encoding function,
> contains a field to pass down information on the portions of the frame which
> changed with respect to the previous one (used for prediction) to mark
> unchanged macroblocks P_SKIP.
> ---
> libavcodec/libx264.c | 81 +++++++++++++++++++++++++++++++++
> libavutil/Makefile | 4 ++
> libavutil/frame.h | 10 +++++
> libavutil/video_hint_info.c | 89 +++++++++++++++++++++++++++++++++++++
> libavutil/video_hint_info.h | 87 ++++++++++++++++++++++++++++++++++++
> 5 files changed, 271 insertions(+)
> create mode 100644 libavutil/video_hint_info.c
> create mode 100644 libavutil/video_hint_info.h
>
> diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
> index 5736f1efa7..32fa80d0a1 100644
> --- a/libavcodec/libx264.c
> +++ b/libavcodec/libx264.c
> @@ -30,6 +30,7 @@
> #include "libavutil/stereo3d.h"
> #include "libavutil/time.h"
> #include "libavutil/intreadwrite.h"
> +#include "libavutil/video_hint_info.h"
> #include "avcodec.h"
> #include "codec_internal.h"
> #include "encode.h"
> @@ -48,6 +49,9 @@
> // from x264.h, for quant_offsets, Macroblocks are 16x16
> // blocks of pixels (with respect to the luma plane)
> #define MB_SIZE 16
> +#define MB_LSIZE 4
> +#define MB_FLOOR(x) ((x) >> (MB_LSIZE))
> +#define MB_CEIL(x) MB_FLOOR((x) + (MB_SIZE - 1))
>
> typedef struct X264Opaque {
> #if FF_API_REORDERED_OPAQUE
> @@ -123,6 +127,8 @@ typedef struct X264Context {
> * encounter a frame with ROI side data.
> */
> int roi_warned;
> +
> + int mb_info;
> } X264Context;
>
> static void X264_log(void *p, int level, const char *fmt, va_list args)
> @@ -295,6 +301,7 @@ static void free_picture(x264_picture_t *pic)
> av_free(pic->extra_sei.payloads[i].payload);
> av_freep(&pic->extra_sei.payloads);
> av_freep(&pic->prop.quant_offsets);
> + av_freep(&pic->prop.mb_info);
> pic->extra_sei.num_payloads = 0;
> }
>
> @@ -320,6 +327,64 @@ static enum AVPixelFormat csp_to_pixfmt(int csp)
> return AV_PIX_FMT_NONE;
> }
>
> +static int setup_mb_info(AVCodecContext *ctx, x264_picture_t *pic,
> + const AVFrame *frame,
> + const AVVideoHint *info)
> +{
> + int mb_width = (frame->width + MB_SIZE - 1) / MB_SIZE;
> + int mb_height = (frame->height + MB_SIZE - 1) / MB_SIZE;
> +
> + const AVVideoRect *mbinfo_rects;
> + int nb_rects;
> + uint8_t *mbinfo;
> +
> + mbinfo_rects = (const AVVideoRect *)av_video_hint_rects(info);
> + nb_rects = info->nb_rects;
> +
> + mbinfo = av_calloc(mb_width * mb_height, sizeof(*mbinfo));
> + if (!mbinfo)
> + return AVERROR(ENOMEM);
> +
> + if (info->type == AV_VIDEO_HINT_CHANGED) {
> + /* Sets the default as constant, i.e. P_SKIP-able, then selectively resets the flag */
> + memset(mbinfo, X264_MBINFO_CONSTANT, sizeof(*mbinfo) * mb_width * mb_height);
> +
> + for (int i = 0; i < nb_rects; i++) {
> + int min_y = MB_FLOOR(mbinfo_rects->y);
> + int max_y = MB_CEIL(mbinfo_rects->y + mbinfo_rects->height);
> + int min_x = MB_FLOOR(mbinfo_rects->x);
> + int max_x = MB_CEIL(mbinfo_rects->x + mbinfo_rects->width);
> +
> + for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> + memset(mbinfo + mb_y * mb_width + min_x, 0, max_x - min_x);
> + }
> +
> + mbinfo_rects++;
> + }
> + } else {
> + /* Sets the default as changed, i.e. *not* P_SKIP-able, then selectively sets the flag */
> + memset(mbinfo, 0, sizeof(*mbinfo) * mb_width * mb_height);
> +
> + for (int i = 0; i < nb_rects; i++) {
> + int min_y = MB_CEIL(mbinfo_rects->y);
> + int max_y = MB_FLOOR(mbinfo_rects->y + mbinfo_rects->height);
> + int min_x = MB_CEIL(mbinfo_rects->x);
> + int max_x = MB_FLOOR(mbinfo_rects->x + mbinfo_rects->width);
> +
> + for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
> + memset(mbinfo + mb_y * mb_width + min_x, X264_MBINFO_CONSTANT, max_x - min_x);
> + }
> +
> + mbinfo_rects++;
you can merge the two loops by adding an inline function to compute
the mins and maxs, and then setting:
mbinfo_filler = AV_VIDEO_HINT_CHANGED ? X264_MBINFO_CONSTANT : 0;
mbinfo_marker = AV_VIDEO_HINT_CHANGED ? 0 : X264_MBINFO_CONSTANT;
compute_coords_fn = AV_VIDEO_HINT_CHANGED ? compute_changed_coords : compute_constant_coords;
memset(mbinfo, mbinfo_filler, sizeof(*mbinfo) * mb_width * mb_height);
for (int i = 0; i < nb_rects; i++) {
compute_coords_fn...
for (int mb_y = min_y; mb_y < max_y; ++mb_y) {
memset(mbinfo + mb_y * mb_width + min_x, mbinfo_marker, max_x - min_x);
}
mbinfo_rects++;
}
> + }
> + }
> +
> + pic->prop.mb_info = mbinfo;
> + pic->prop.mb_info_free = av_free;
> +
> + return 0;
> +}
> +
> static int setup_roi(AVCodecContext *ctx, x264_picture_t *pic, int bit_depth,
> const AVFrame *frame, const uint8_t *data, size_t size)
> {
> @@ -404,6 +469,7 @@ static int setup_frame(AVCodecContext *ctx, const AVFrame *frame,
> int64_t wallclock = 0;
> int bit_depth, ret;
> AVFrameSideData *sd;
> + AVFrameSideData *mbinfo_sd;
>
> *ppic = NULL;
> if (!frame)
> @@ -499,6 +565,17 @@ FF_ENABLE_DEPRECATION_WARNINGS
> goto fail;
> }
>
> + mbinfo_sd = av_frame_get_side_data(frame, AV_FRAME_DATA_VIDEO_HINT);
> + if (mbinfo_sd) {
> + int ret = setup_mb_info(ctx, pic, frame, (const AVVideoHint *)mbinfo_sd->data);
> + if (ret < 0) {
> + /* No need to fail here, this is not fatal. We just proceed with no
> + * mb_info and log a message */
> +
> + av_log(ctx, AV_LOG_WARNING, "mb_info setup failure\n");
> + }
> + }
> +
> if (x4->udu_sei) {
> for (int j = 0; j < frame->nb_side_data; j++) {
> AVFrameSideData *side_data = frame->side_data[j];
> @@ -1102,6 +1179,9 @@ FF_ENABLE_DEPRECATION_WARNINGS
> }
> }
>
> + x4->params.analyse.b_mb_info = x4->mb_info;
> + x4->params.analyse.b_fast_pskip = 1;
> +
> // update AVCodecContext with x264 parameters
> avctx->has_b_frames = x4->params.i_bframe ?
> x4->params.i_bframe_pyramid ? 2 : 1 : 0;
> @@ -1311,6 +1391,7 @@ static const AVOption options[] = {
> { "noise_reduction", "Noise reduction", OFFSET(noise_reduction), AV_OPT_TYPE_INT, { .i64 = -1 }, INT_MIN, INT_MAX, VE },
> { "udu_sei", "Use user data unregistered SEI if available", OFFSET(udu_sei), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
> { "x264-params", "Override the x264 configuration using a :-separated list of key=value parameters", OFFSET(x264_params), AV_OPT_TYPE_DICT, { 0 }, 0, 0, VE },
> + { "mb_info", "Set mb_info data through AVSideData, only useful when used from the API", OFFSET(mb_info), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
> { NULL },
> };
>
> diff --git a/libavutil/Makefile b/libavutil/Makefile
> index dc9012f9a8..bb5ecc3235 100644
> --- a/libavutil/Makefile
> +++ b/libavutil/Makefile
> @@ -91,6 +91,7 @@ HEADERS = adler32.h \
> tea.h \
> tx.h \
> film_grain_params.h \
> + video_hint_info.h \
>
> ARCH_HEADERS = bswap.h \
> intmath.h \
> @@ -196,6 +197,7 @@ OBJS-$(CONFIG_VAAPI) += hwcontext_vaapi.o
> OBJS-$(CONFIG_VIDEOTOOLBOX) += hwcontext_videotoolbox.o
> OBJS-$(CONFIG_VDPAU) += hwcontext_vdpau.o
> OBJS-$(CONFIG_VULKAN) += hwcontext_vulkan.o
> +OBJS-$(CONFIG_LIBX264) += video_hint_info.o
order
>
> OBJS-$(!CONFIG_VULKAN) += hwcontext_stub.o
>
> @@ -219,6 +221,8 @@ SKIPHEADERS-$(CONFIG_VULKAN) += hwcontext_vulkan.h vulkan.h \
> vulkan_functions.h \
> vulkan_loader.h
>
> +SKIPHEADERS-$(CONFIG_LIBX264) += video_hint_info.h
> +
> TESTPROGS = adler32 \
> aes \
> aes_ctr \
> diff --git a/libavutil/frame.h b/libavutil/frame.h
> index a491315f25..0e765e5499 100644
> --- a/libavutil/frame.h
> +++ b/libavutil/frame.h
> @@ -214,6 +214,16 @@ enum AVFrameSideDataType {
> * Ambient viewing environment metadata, as defined by H.274.
> */
> AV_FRAME_DATA_AMBIENT_VIEWING_ENVIRONMENT,
> +
> + /**
> + * Provide macro block encoder-specific hinting information for the encoder
> + * processing. It can be used to pass information about which macroblock
> + * can be skipped because it hasn't changed from the corresponding one in
> + * the previous frame. This is useful for applications which know in
> + * advance this information to speed up real-time encoding. Currently only
> + * used by libx264.
> + */
> + AV_FRAME_DATA_VIDEO_HINT,
> };
>
> enum AVActiveFormatDescription {
> diff --git a/libavutil/video_hint_info.c b/libavutil/video_hint_info.c
> new file mode 100644
> index 0000000000..0a763962da
> --- /dev/null
> +++ b/libavutil/video_hint_info.c
> @@ -0,0 +1,89 @@
> +/*
> + * Copyright 2023 Elias Carotti <eliascrt at amazon dot it>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include <string.h>
> +
> +#include "avstring.h"
> +#include "frame.h"
> +#include "macros.h"
> +#include "mem.h"
> +#include "video_hint_info.h"
> +
> +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects,
> + size_t nb_rects,
> + VideoHintType type,
> + size_t* out_size)
align to (
> +{
> + struct TestStruct {
> + AVVideoHint p;
> + AVVideoRect b;
> + };
nit: weird align (also use more meaningful names - e.g. hint and rect)
> + const size_t blocks_offset = offsetof(struct TestStruct, b);
> + size_t size = blocks_offset;
> + AVVideoHint *par;
AVVideoHint *hint?
> +
> + *out_size = 0;
> + if (nb_rects > (SIZE_MAX - size) / sizeof(AVVideoRect))
why SIZE_MAX - size?
> + return NULL;
> + size += sizeof(AVVideoRect) * nb_rects;
> +
> + par = av_mallocz(size);
> + if (!par)
> + return NULL;
> +
> + par->type = type;
> + par->nb_rects = nb_rects;
> + par->blocks_offset = blocks_offset;
> +
> + /* Just copies the rects over the newly allocated buffer */
> + memcpy((uint8_t *)par + blocks_offset, rects, sizeof(AVVideoRect) * nb_rects);
> +
> + *out_size = size;
> +
> + return par;
> +}
> +
> +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame,
> + AVVideoRect *rects,
> + size_t num_rects,
> + VideoHintType type)
> +{
> + AVVideoHint *par;
use a more meaningful name (I think "par" is a leftover of an older
API)
> + AVBufferRef *buf;
> + size_t size = 0;
> +
> + par = av_video_hint_alloc(rects, num_rects, type, &size);
> + if (!par)
> + return NULL;
> +
> + buf = av_buffer_create((uint8_t *)par, size, NULL, NULL, 0);
> + if (!buf) {
> + av_freep(&par);
> + return NULL;
> + }
> +
> + if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_VIDEO_HINT, buf)) {
> + av_buffer_unref(&buf);
> + return NULL;
> + }
> +
> + return par;
> +}
> +
> diff --git a/libavutil/video_hint_info.h b/libavutil/video_hint_info.h
> new file mode 100644
> index 0000000000..3b04e0c40e
> --- /dev/null
> +++ b/libavutil/video_hint_info.h
> @@ -0,0 +1,87 @@
> +/**
> + * Copyright 2023 Elias Carotti <eliascrt at amazon dot it>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVUTIL_VIDEO_HINT_INFO_H
> +#define AVUTIL_VIDEO_HINT_INFO_H
> +
> +#include <stddef.h>
> +#include <stdint.h>
> +#include "libavutil/avassert.h"
> +#include "libavutil/frame.h"
> +
> +typedef struct AVVideoRect {
> + uint32_t x, y;
> + uint32_t width, height;
> +} AVVideoRect;
> +
> +typedef enum VideoHintType {
> + /* blocks delimit the constant areas (unchanged), default is changed */
> + AV_VIDEO_HINT_CONSTANT,
> +
> + /* blocks delimit the constant areas (changed), default is not changed */
> + AV_VIDEO_HINT_CHANGED,
> +} VideoHintType;
missing AV prefix
> +
> +typedef struct AVVideoHint {
> + /**
> + * Number of blocks in the array.
> + *
> + * May be 0, in which case no per-block information is present. In this case
> + * the values of blocks_offset / block_size are unspecified and should not
> + * be accessed.
> + */
> + int nb_rects;
> +
> + /**
> + * Offset in bytes from the beginning of this structure at which the array
> + * of blocks starts.
> + */
> + size_t blocks_offset;
> +
> + VideoHintType type;
> +} AVVideoHint;
> +
> +static av_always_inline AVVideoRect*
> +av_video_hint_rects(const AVVideoHint *par)
> +{
> + return (AVVideoRect *)((uint8_t *)par + par->blocks_offset);
> +}
> +
> +/**
> + * Allocate memory for a vector of AVVideoRect in the given AVFrame
> + * {@code frame} as AVFrameSideData of type AV_FRAME_DATA_VIDEO_HINT_INFO.
> + * The side data contains a list of rectangles for the portions of the frame
> + * which changed from the last encoded one (and the remainder are assumed to be
> + * changed), or, alternately (depending on the type parameter) the unchanged
> + * ones (and the remanining ones are those which changed).
> + * Macroblocks will thus be hinted either to be P_SKIP-ped or go through the
> + * regular encoding procedure.
> + */
> +AVVideoHint *av_video_hint_create_side_data(AVFrame *frame,
> + AVVideoRect *rects,
> + size_t num_rects,
> + VideoHintType type);
> +
> +AVVideoHint *av_video_hint_alloc(AVVideoRect *rects,
> + size_t nb_rects,
> + VideoHintType type,
> + size_t *out_size);
> +
> +#endif /* AVUTIL__VIDEO_HINT_INFO_H */
AVUTIL_VIDEO...
> --
> 2.34.1
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list