[FFmpeg-devel] [PATCH] avcodec/amfenc: DX12 Reference-only feature support
Benjamin Cheng
ben at bcheng.me
Fri Jan 31 03:22:00 EET 2025
On 2025-01-30 9:09 a.m., Araz Iusubov wrote:
> The Reference-Only feature in DirectX 12 is a memory optimization
> technique designed for video decoding scenarios.
> This feature requires that reference resources must be allocated with
> the D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY resource flag.
> Reference textures must also be separated from output textures.
> This feature is not supported in the current version of ffmpeg.
> Since AMD GPU uses this feature in Direct 12 decoder,
> ffmpeg does not support AMD GPU Direct 12 decoding.
> To properly support the Reference-Only feature,
> two parallel resource pools must be configured and managed:
> General Resource Pool:
> Contains resources used for output decoded frames.
> Defined in AVHWFramesContext and manages the final decoded textures.
> Reference-Only Resource Pool:
> Intended for storing reference frame resources.
> Resources created with the
> D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY flag
> are allocated to AVBufferPool.
>
> ---
> libavcodec/d3d12va_decode.c | 58 ++++++++++++++++++++++++++++---
> libavutil/hwcontext_d3d12va.c | 65 ++++++++++++++++++++++++++++++++---
> 2 files changed, 115 insertions(+), 8 deletions(-)
This patch only affects d3d12va, why is the commit message amfenc?
>
> diff --git a/libavcodec/d3d12va_decode.c b/libavcodec/d3d12va_decode.c
> index 3b8978635e..8916f94d10 100644
> --- a/libavcodec/d3d12va_decode.c
> +++ b/libavcodec/d3d12va_decode.c
> @@ -51,11 +51,19 @@ unsigned ff_d3d12va_get_surface_index(const AVCodecContext *avctx,
> D3D12VADecodeContext *ctx, const AVFrame *frame,
> int curr)
> {
> + AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);
> + AVD3D12VAFramesContext *frames_hwctx = frames_ctx->hwctx;
> +
> AVD3D12VAFrame *f;
> ID3D12Resource *res;
> unsigned i;
>
> - f = (AVD3D12VAFrame *)frame->data[0];
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + f = (AVD3D12VAFrame*)frame->data[1];
> + } else {
> + f = (AVD3D12VAFrame*)frame->data[0];
> + }
> +
> if (!f)
> goto fail;
>
> @@ -250,6 +258,11 @@ static int d3d12va_create_decoder(AVCodecContext *avctx)
> return AVERROR_PATCHWELCOME;
> }
>
> + if (feature.ConfigurationFlags & D3D12_VIDEO_DECODE_CONFIGURATION_FLAG_REFERENCE_ONLY_ALLOCATIONS_REQUIRED) {
> + frames_hwctx->flags |= (D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY | D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE);
> + av_log(avctx, AV_LOG_INFO, "Reference-Only Allocations are required for this configuration.\n");
> + }
> +
> desc = (D3D12_VIDEO_DECODER_DESC) {
> .NodeMask = 0,
> .Configuration = ctx->cfg,
> @@ -440,8 +453,19 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
> D3D12VADecodeContext *ctx = D3D12VA_DECODE_CONTEXT(avctx);
> ID3D12Resource *buffer = NULL;
> ID3D12CommandAllocator *command_allocator = NULL;
> - AVD3D12VAFrame *f = (AVD3D12VAFrame *)frame->data[0];
> - ID3D12Resource *resource = (ID3D12Resource *)f->texture;
> + AVHWFramesContext *frames_ctx = D3D12VA_FRAMES_CONTEXT(avctx);
> + AVD3D12VAFramesContext *frames_hwctx = frames_ctx->hwctx;
> + AVD3D12VAFrame *f = NULL;
> + AVD3D12VAFrame *output_data = NULL;
> +
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + f = (AVD3D12VAFrame*)frame->data[1];
> + output_data = (AVD3D12VAFrame*)frame->data[0];
> + } else {
> + f = (AVD3D12VAFrame*)frame->data[0];
> + }
> +
> + ID3D12Resource* resource = (ID3D12Resource*)f->texture;
>
> ID3D12VideoDecodeCommandList *cmd_list = ctx->command_list;
> D3D12_RESOURCE_BARRIER barriers[32] = { 0 };
> @@ -469,6 +493,14 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
> .pOutputTexture2D = resource,
> };
>
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + output_args.pOutputTexture2D = output_data->texture;
> +
> + output_args.ConversionArguments.Enable = 1;
> + output_args.ConversionArguments.pReferenceTexture2D = resource;
> + output_args.ConversionArguments.ReferenceSubresource = 0;
> + }
> +
> UINT num_barrier = 1;
> barriers[0] = (D3D12_RESOURCE_BARRIER) {
> .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
> @@ -481,6 +513,20 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
> },
> };
>
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + barriers[1] = (D3D12_RESOURCE_BARRIER) {
> + .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION,
> + .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE,
> + .Transition = {
> + .pResource = output_data->texture,
> + .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES,
> + .StateBefore = D3D12_RESOURCE_STATE_COMMON,
> + .StateAfter = D3D12_RESOURCE_STATE_VIDEO_DECODE_WRITE,
> + },
> + };
> + num_barrier++;
> + }
> +
> memset(ctx->ref_subresources, 0, sizeof(UINT) * ctx->max_num_ref);
> input_args.ReferenceFrames.NumTexture2Ds = ctx->max_num_ref;
> input_args.ReferenceFrames.ppTexture2Ds = ctx->ref_resources;
> @@ -505,7 +551,7 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
>
> DX_CHECK(ID3D12VideoDecodeCommandList_Reset(cmd_list, command_allocator));
>
> - num_barrier += d3d12va_update_reference_frames_state(avctx, &barriers[1], resource, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_VIDEO_DECODE_READ);
> + num_barrier += d3d12va_update_reference_frames_state(avctx, &barriers[num_barrier], resource, D3D12_RESOURCE_STATE_COMMON, D3D12_RESOURCE_STATE_VIDEO_DECODE_READ);
>
You could optimize these barriers since reference-only resources don't
need to be transitioned to COMMON, and can remain in
VIDEO_DECODE_{READ,WRITE}.
I propose the following:
- Transition all reference texture to VIDEO_DECODE_READ at creation time.
- When preparing resources for input to DecodeFrame(), transition only
the texture for reference output to VIDEO_DECODE_WRITE
- After DecodeFrame(), transition the reference output texture to
VIDEO_DECODE_READ. This is already implicitly handled by the barrier SWAP.
All-in-all, for the cost of an initial transition at creation time, you
decrease the number of barriers in each frame to just 2.
> ID3D12VideoDecodeCommandList_ResourceBarrier(cmd_list, num_barrier, barriers);
>
> @@ -522,6 +568,10 @@ int ff_d3d12va_common_end_frame(AVCodecContext *avctx, AVFrame *frame,
>
> DX_CHECK(ID3D12CommandQueue_Signal(ctx->command_queue, f->sync_ctx.fence, ++f->sync_ctx.fence_value));
>
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + DX_CHECK(ID3D12CommandQueue_Signal(ctx->command_queue, output_data->sync_ctx.fence, ++output_data->sync_ctx.fence_value));
> + }
> +
> DX_CHECK(ID3D12CommandQueue_Signal(ctx->command_queue, ctx->sync_ctx.fence, ++ctx->sync_ctx.fence_value));
>
> ret = d3d12va_discard_helper_objects(avctx, command_allocator, buffer, ctx->sync_ctx.fence_value);
> diff --git a/libavutil/hwcontext_d3d12va.c b/libavutil/hwcontext_d3d12va.c
> index 6507cf69c1..328827b040 100644
> --- a/libavutil/hwcontext_d3d12va.c
> +++ b/libavutil/hwcontext_d3d12va.c
> @@ -49,6 +49,24 @@ typedef struct D3D12VAFramesContext {
> ID3D12GraphicsCommandList *command_list;
> AVD3D12VASyncContext sync_ctx;
> UINT luma_component_size;
> +
> + /**
> + * The Reference-Only feature in DirectX 12 is a memory optimization
> + * technique designed for video decoding/encoding scenarios.
> + * This feature requires that reference resources must be allocated
> + * with the `D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY` resource flag.
> + * Reference textures must also be separated from output textures.
> + * To correctly support the Reference-Only feature, two parallel resource
> + * pools must be configured and managed:
> + * 1. General Resource Pool:
> + * - Contains resources used for outputting decoded frames.
> + * - Defined in `AVHWFramesContext` and manages the final decoded textures.
> + * 2. Reference-Only Resource Pool:
> + * - Dedicated to storing reference frame resources.
> + * - Resources created with the `D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY`
> + * flag are allocated to this pool.
> + */
> + AVBufferPool *pool_reference_only;
> } D3D12VAFramesContext;
>
> typedef struct D3D12VADevicePriv {
> @@ -174,7 +192,8 @@ fail:
>
> static void d3d12va_frames_uninit(AVHWFramesContext *ctx)
> {
> - D3D12VAFramesContext *s = ctx->hwctx;
> + D3D12VAFramesContext *s = ctx->hwctx;
> + AVD3D12VAFramesContext *frames_hwctx = &s->p;
>
> D3D12_OBJECT_RELEASE(s->sync_ctx.fence);
> if (s->sync_ctx.event)
> @@ -185,6 +204,11 @@ static void d3d12va_frames_uninit(AVHWFramesContext *ctx)
> D3D12_OBJECT_RELEASE(s->command_allocator);
> D3D12_OBJECT_RELEASE(s->command_list);
> D3D12_OBJECT_RELEASE(s->command_queue);
> +
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + if (s->pool_reference_only)
> + av_buffer_pool_uninit(&s->pool_reference_only);
> + }
> }
>
> static int d3d12va_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
> @@ -281,6 +305,7 @@ fail:
> static int d3d12va_frames_init(AVHWFramesContext *ctx)
> {
> AVD3D12VAFramesContext *hwctx = ctx->hwctx;
> + D3D12VAFramesContext *s = ctx->hwctx;
> int i;
>
> for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
> @@ -304,16 +329,43 @@ static int d3d12va_frames_init(AVHWFramesContext *ctx)
> if (!ffhwframesctx(ctx)->pool_internal)
> return AVERROR(ENOMEM);
>
> + s->pool_reference_only = NULL;
> +
> return 0;
> }
>
> static int d3d12va_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
> {
> int ret;
> + D3D12VAFramesContext *s = ctx->hwctx;
> + AVD3D12VAFramesContext *frames_hwctx = &s->p;
>
> - frame->buf[0] = av_buffer_pool_get(ctx->pool);
> - if (!frame->buf[0])
> - return AVERROR(ENOMEM);
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + /*
> + * for the output texture, temporarily unset D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY
> + * and D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE
> + */
> + D3D12_RESOURCE_FLAGS temp_flags = frames_hwctx->flags;
> + frames_hwctx->flags &= ~(D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY | D3D12_RESOURCE_FLAG_DENY_SHADER_RESOURCE);
> +
> + frame->buf[0] = av_buffer_pool_get(ctx->pool);
> + if (!frame->buf[0])
> + return AVERROR(ENOMEM);
> +
> + if (s->pool_reference_only == NULL) {
> + s->pool_reference_only = av_buffer_pool_init2(sizeof(AVD3D12VAFrame),
> + ctx, d3d12va_pool_alloc, NULL);
> + }
> +
> + frames_hwctx->flags = temp_flags;
> + frame->buf[1] = av_buffer_pool_get(s->pool_reference_only);
> + if (!frame->buf[1])
> + return AVERROR(ENOMEM);
> + } else {
> + frame->buf[0] = av_buffer_pool_get(ctx->pool);
> + if (!frame->buf[0])
> + return AVERROR(ENOMEM);
> + }
>
> ret = av_image_fill_arrays(frame->data, frame->linesize, NULL,
> ctx->sw_format, ctx->width, ctx->height,
> @@ -322,6 +374,11 @@ static int d3d12va_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
> return ret;
>
> frame->data[0] = frame->buf[0]->data;
> +
> + if (frames_hwctx->flags & D3D12_RESOURCE_FLAG_VIDEO_DECODE_REFERENCE_ONLY) {
> + frame->data[1] = frame->buf[1]->data;
> + }
> +
> frame->format = AV_PIX_FMT_D3D12;
> frame->width = ctx->width;
> frame->height = ctx->height;
More information about the ffmpeg-devel
mailing list