[FFmpeg-devel] [PATCH v6 1/2] lavc/vaapi_encode: add support for maxframesize
Mark Thompson
sw at jkqxz.net
Sat Apr 30 17:55:55 EEST 2022
On 29/04/2022 08:31, Fei Wang wrote:
> From: Linjie Fu <linjie.fu at intel.com>
>
> Add support for max frame size:
> - max_frame_size (bytes) to indicate the max allowed size for frame.
>
> If the frame size exceeds the limitation, encoder will to control the frame
> size by adjusting QP value.
>
> ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -f rawvideo \
> -v verbose -s:v 352x288 -i ./input.yuv -vf format=nv12,hwupload \
> -c:v h264_vaapi -profile:v main -g 30 -rc_mode VBR -b:v 500k \
> -bf 3 -max_frame_size 40000 -vframes 100 -y ./max_frame_size.h264
Can you give some more explanation of the circumstances in which this is expected to work in?
For example, I tried a simple test I gave the Intel (gen9, iHD) H.264 encoder white noise.
At QP 51 I get:
$ ./ffmpeg_g -v 55 -y -f rawvideo -video_size 1280x720 -pixel_format nv12 -framerate 30 -i /dev/urandom -an -init_hw_device vaapi:/dev/dri/renderD128 -vf hwupload -c:v h264_vaapi -g 0 -qp 51 -f null - 2>&1 | grep 'Output buffer:'
[h264_vaapi @ 0x557de0809640] Output buffer: 190975 bytes (status 01000033).
[h264_vaapi @ 0x557de0809640] Output buffer: 191265 bytes (status 01000033).
[h264_vaapi @ 0x557de0809640] Output buffer: 191825 bytes (status 01000033).
[h264_vaapi @ 0x557de0809640] Output buffer: 191893 bytes (status 01000033).
[h264_vaapi @ 0x557de0809640] Output buffer: 191746 bytes (status 01000033).
So I should be able to set the max_frame_size to 200000 (with a corresponding bitrate target of 30*200000*8 = 48M) and it will respect that?
$ ./ffmpeg_g -v 55 -y -f rawvideo -video_size 1280x720 -pixel_format nv12 -framerate 30 -i /dev/urandom -an -init_hw_device vaapi:/dev/dri/renderD128 -vf hwupload -c:v h264_vaapi -g 0 -b:v 48M -max_frame_size 200000 -f null - 2>&1 | grep 'Output buffer:'
[h264_vaapi @ 0x5566f49b7580] Output buffer: 204360 bytes (status 04000033).
[h264_vaapi @ 0x5566f49b7580] Output buffer: 201899 bytes (status 04000033).
[h264_vaapi @ 0x5566f49b7580] Output buffer: 194869 bytes (status 02000033).
[h264_vaapi @ 0x5566f49b7580] Output buffer: 191128 bytes (status 01000033).
[h264_vaapi @ 0x5566f49b7580] Output buffer: 191348 bytes (status 01000033).
But it still has some convergence delay, which really isn't what I expect from an absolute option like max_frame_size.
(The 04000000 in 04000033 is VA_CODED_BUF_STATUS_NUMBER_PASSES_MASK (0xf000000), indicating that it took four (!) passes across the frame and still didn't achieve the target we know is possible from the previous test.)
For a less artificial setup, I tried some 4K test streams. For example:
$ ./ffmpeg_g -v 55 -y -i ~/video/test/LG\ New\ York\ HDR\ UHD\ 4K\ Demo.ts -an -init_hw_device vaapi:/dev/dri/renderD128 -vf 'format=nv12,hwupload' -c:v h264_vaapi -b:v 24M -max_frame_size 100000 out.mp4 2>&1 | grep 'Output buffer:' | nl | tee out
...
$ cat out | awk '{ if ($7 > 100000) { print } }'
308 [h264_vaapi @ 0x5653126613c0] Output buffer: 118390 bytes (status 04000033).
361 [h264_vaapi @ 0x5653126613c0] Output buffer: 139404 bytes (status 0400002d).
481 [h264_vaapi @ 0x5653126613c0] Output buffer: 117728 bytes (status 04000033).
601 [h264_vaapi @ 0x5653126613c0] Output buffer: 117328 bytes (status 04000032).
721 [h264_vaapi @ 0x5653126613c0] Output buffer: 124737 bytes (status 0400002d).
841 [h264_vaapi @ 0x5653126613c0] Output buffer: 109479 bytes (status 0400002a).
961 [h264_vaapi @ 0x5653126613c0] Output buffer: 128128 bytes (status 0400002b).
1081 [h264_vaapi @ 0x5653126613c0] Output buffer: 103071 bytes (status 04000031).
1201 [h264_vaapi @ 0x5653126613c0] Output buffer: 115270 bytes (status 04000030).
1321 [h264_vaapi @ 0x5653126613c0] Output buffer: 154613 bytes (status 04000030).
1406 [h264_vaapi @ 0x5653126613c0] Output buffer: 114548 bytes (status 04000032).
1441 [h264_vaapi @ 0x5653126613c0] Output buffer: 133374 bytes (status 04000033).
1561 [h264_vaapi @ 0x5653126613c0] Output buffer: 150675 bytes (status 0400002f).
1681 [h264_vaapi @ 0x5653126613c0] Output buffer: 119443 bytes (status 0400002a).
(With average QPs there indicating that it's usually nowhere near 51 when it misses the target. Most of those failures are on GOP-start intra frames (default GOP size is 120), but not all of them.)
> Max frame size was enabled since VA-API version (0, 33, 0), but query is available
> since (1, 5, 0). It will be passed as a parameter in picParam and should be set
> for each frame.
>
> Signed-off-by: Linjie Fu <linjie.fu at intel.com>
> Signed-off-by: Fei Wang <fei.w.wang at intel.com>
> ---
> update:
> Set ctx->max_frame_size to 0 in CQP mode to avoid to pass
> VAEncMiscParameterTypeMaxFrameSize type VABuffer.
>
> libavcodec/vaapi_encode.c | 72 +++++++++++++++++++++++++++++++++++++++
> libavcodec/vaapi_encode.h | 10 +++++-
> 2 files changed, 81 insertions(+), 1 deletion(-)
>
> diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
> index 0e2f5ed447..3b96d4c671 100644
> --- a/libavcodec/vaapi_encode.c
> +++ b/libavcodec/vaapi_encode.c
> @@ -365,6 +365,17 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
> goto fail;
> }
>
> +#if VA_CHECK_VERSION(1, 5, 0)
> + if (ctx->max_frame_size) {
> + err = vaapi_encode_make_misc_param_buffer(avctx, pic,
> + VAEncMiscParameterTypeMaxFrameSize,
> + &ctx->mfs_params,
> + sizeof(ctx->mfs_params));
> + if (err < 0)
> + goto fail;
> + }
> +#endif
> +
> if (pic->type == PICTURE_TYPE_IDR) {
> if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE &&
> ctx->codec->write_sequence_header) {
> @@ -1869,6 +1880,61 @@ rc_mode_found:
> return 0;
> }
>
> +static av_cold int vaapi_encode_init_max_frame_size(AVCodecContext *avctx)
> +{
> +#if VA_CHECK_VERSION(1, 5, 0)
> + VAAPIEncodeContext *ctx = avctx->priv_data;
> + VAConfigAttrib attr = { VAConfigAttribMaxFrameSize };
> + VAStatus vas;
> +
> + if (ctx->va_rc_mode == VA_RC_CQP) {
> + ctx->max_frame_size = 0;
> + av_log(avctx, AV_LOG_WARNING, "Max frame size is invalid in CQP rate "
> + "control mode.\n");
> + return 0;
> + }
> +
> + vas = vaGetConfigAttributes(ctx->hwctx->display,
> + ctx->va_profile,
> + ctx->va_entrypoint,
> + &attr, 1);
> + if (vas != VA_STATUS_SUCCESS) {
> + ctx->max_frame_size = 0;
> + av_log(avctx, AV_LOG_ERROR, "Failed to query max frame size "
> + "config attribute: %d (%s).\n", vas, vaErrorStr(vas));
> + return AVERROR_EXTERNAL;
> + }
> +
> + if (attr.value == VA_ATTRIB_NOT_SUPPORTED) {
> + ctx->max_frame_size = 0;
> + av_log(avctx, AV_LOG_WARNING, "Max frame size attribute "
> + "is not supported.\n");
I'm not sure this should be ignored. If the user has requested this option then presumably it is because they have some real constraint on frame sizes, so maybe this should fail? (Like a request for slices does.)
> + } else {
> + VAConfigAttribValMaxFrameSize attr_mfs;
> + attr_mfs.value = attr.value;
> + // Prefer to use VAEncMiscParameterTypeMaxFrameSize for max frame size.
> + if (!attr_mfs.bits.max_frame_size && attr_mfs.bits.multiple_pass) {
> + ctx->max_frame_size = 0;
> + av_log(avctx, AV_LOG_WARNING, "Driver only support multiple pass "
> + "max frame size which have not been implemented in FFmpeg.\n");
"only supports", "which has not been".
> + return 0;
> + }
> +
> + ctx->mfs_params = (VAEncMiscParameterBufferMaxFrameSize){
> + .max_frame_size = ctx->max_frame_size * 8,
> + };
> +
> + av_log(avctx, AV_LOG_VERBOSE, "Max Frame Size: %d bytes.\n",
> + ctx->max_frame_size);
> + }
> +#else
> + av_log(avctx, AV_LOG_WARNING, "Max Frame Size is not support with "
Maybe:
"The max frame size option is not supported with this VAAPI version.\n");
to match the other similar messages.
> + "this VA version.\n");
> +#endif
> +
> + return 0;
> +}
> +
> static av_cold int vaapi_encode_init_gop_structure(AVCodecContext *avctx)
> {
> VAAPIEncodeContext *ctx = avctx->priv_data;
> @@ -2548,6 +2614,12 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
> goto fail;
> }
>
> + if (ctx->max_frame_size) {
> + err = vaapi_encode_init_max_frame_size(avctx);
> + if (err < 0)
> + goto fail;
> + }
> +
> vas = vaCreateConfig(ctx->hwctx->display,
> ctx->va_profile, ctx->va_entrypoint,
> ctx->config_attributes, ctx->nb_config_attributes,
> diff --git a/libavcodec/vaapi_encode.h b/libavcodec/vaapi_encode.h
> index af0588c30b..359f954fff 100644
> --- a/libavcodec/vaapi_encode.h
> +++ b/libavcodec/vaapi_encode.h
> @@ -191,6 +191,9 @@ typedef struct VAAPIEncodeContext {
> // Desired B frame reference depth.
> int desired_b_depth;
>
> + // Max Frame Size
> + int max_frame_size;
> +
> // Explicitly set RC mode (otherwise attempt to pick from
> // available modes).
> int explicit_rc_mode;
> @@ -268,6 +271,7 @@ typedef struct VAAPIEncodeContext {
> VAEncMiscParameterRateControl rc_params;
> VAEncMiscParameterHRD hrd_params;
> VAEncMiscParameterFrameRate fr_params;
> + VAEncMiscParameterBufferMaxFrameSize mfs_params;
Should this have a version check too?
> #if VA_CHECK_VERSION(0, 36, 0)
> VAEncMiscParameterBufferQualityLevel quality_params;
> #endif
> @@ -478,7 +482,11 @@ int ff_vaapi_encode_close(AVCodecContext *avctx);
> "Increase this to improve single channel performance. This option " \
> "doesn't work if driver doesn't implement vaSyncBuffer function.", \
> OFFSET(common.async_depth), AV_OPT_TYPE_INT, \
> - { .i64 = 2 }, 1, MAX_ASYNC_DEPTH, FLAGS }
> + { .i64 = 2 }, 1, MAX_ASYNC_DEPTH, FLAGS }, \
> + { "max_frame_size", \
> + "Maximum frame size (in bytes)",\
> + OFFSET(common.max_frame_size), AV_OPT_TYPE_INT, \
> + { .i64 = 0 }, 0, INT_MAX, FLAGS }
>
> #define VAAPI_ENCODE_RC_MODE(name, desc) \
> { #name, desc, 0, AV_OPT_TYPE_CONST, { .i64 = RC_MODE_ ## name }, \
- Mark
More information about the ffmpeg-devel
mailing list