[FFmpeg-devel] [PATCH V5 3/5] libavutil/hwcontext_vulkan: Allocate vkFrame in one memory
Chen, Wenbin
wenbin.chen at intel.com
Thu Dec 2 03:53:00 EET 2021
> The vaapi can import external frame, but the planes of the external
> frames should be in the same drm object. A new option "contiguous_planes"
> is added to device. This flag tells device to allocate places in one
> memory. When device is derived from vaapi this flag will be enabled.
> A new flag frame_flag is also added to AVVulkanFramesContext. User
> can use this flag to force enable or disable this behaviour.
> A new variable "offset "is added to AVVKFrame. It describe describe the
> offset from the memory currently bound to the VkImage.
>
> Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> ---
> libavutil/hwcontext_vulkan.c | 68
> +++++++++++++++++++++++++++++++++++-
> libavutil/hwcontext_vulkan.h | 24 +++++++++++++
> 2 files changed, 91 insertions(+), 1 deletion(-)
>
> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> index a0437c9661..eef9009ae1 100644
> --- a/libavutil/hwcontext_vulkan.c
> +++ b/libavutil/hwcontext_vulkan.c
> @@ -103,8 +103,14 @@ typedef struct VulkanDevicePriv {
> /* Settings */
> int use_linear_images;
>
> + /* allocate planes in a contiguous memory */
> + int contiguous_planes;
> +
> /* Nvidia */
> int dev_is_nvidia;
> +
> + /* Intel */
> + int dev_is_intel;
> } VulkanDevicePriv;
>
> typedef struct VulkanFramesPriv {
> @@ -153,6 +159,8 @@ typedef struct AVVkFrameInternal {
> av_free((void *)props); \
> }
>
> +#define VKF_FLAG(x, f) (((x) & (~AV_VK_FRAME_FLAG_NONE)) & (f))
> +
> static const struct {
> enum AVPixelFormat pixfmt;
> const VkFormat vkfmts[4];
> @@ -1374,6 +1382,13 @@ static int
> vulkan_device_create_internal(AVHWDeviceContext *ctx,
> if (opt_d)
> p->use_linear_images = strtol(opt_d->value, NULL, 10);
>
> + opt_d = av_dict_get(opts, "contiguous_planes", NULL, 0);
> + if (opt_d)
> + p->contiguous_planes = strtol(opt_d->value, NULL, 10);
> + else
> + p->contiguous_planes = -1;
> +
> +
> hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
> hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
>
> @@ -1425,6 +1440,8 @@ static int vulkan_device_init(AVHWDeviceContext
> *ctx)
>
> p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de);
>
> + p->dev_is_intel = (p->props.properties.vendorID == 0x8086);
> +
> vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev,
> &queue_num, NULL);
> if (!queue_num) {
> av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
> @@ -1742,8 +1759,12 @@ static int alloc_bind_mem(AVHWFramesContext
> *hwfc, AVVkFrame *f,
> AVHWDeviceContext *ctx = hwfc->device_ctx;
> VulkanDevicePriv *p = ctx->internal->priv;
> FFVulkanFunctions *vk = &p->vkfn;
> + AVVulkanFramesContext *hwfctx = hwfc->hwctx;
> const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
> VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
> + VkMemoryRequirements memory_requirements = { 0 };
> + int mem_size = 0;
> + int mem_size_list[AV_NUM_DATA_POINTERS] = { 0 };
>
> AVVulkanDeviceContext *hwctx = ctx->hwctx;
>
> @@ -1771,6 +1792,19 @@ static int alloc_bind_mem(AVHWFramesContext
> *hwfc, AVVkFrame *f,
> req.memoryRequirements.size =
> FFALIGN(req.memoryRequirements.size,
> p-
> >props.properties.limits.minMemoryMapAlignment);
>
> + if (VKF_FLAG(hwfctx->flags,
> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) {
> + if (memory_requirements.size == 0) {
> + memory_requirements = req.memoryRequirements;
> + } else if (memory_requirements.memoryTypeBits !=
> req.memoryRequirements.memoryTypeBits) {
> + av_log(hwfc, AV_LOG_ERROR, "the param for each planes are not
> the same\n");
> + return AVERROR(EINVAL);
> + }
> +
> + mem_size_list[i] = req.memoryRequirements.size;
> + mem_size += mem_size_list[i];
> + continue;
> + }
> +
> /* In case the implementation prefers/requires dedicated allocation */
> use_ded_mem = ded_req.prefersDedicatedAllocation |
> ded_req.requiresDedicatedAllocation;
> @@ -1792,6 +1826,29 @@ static int alloc_bind_mem(AVHWFramesContext
> *hwfc, AVVkFrame *f,
> bind_info[i].memory = f->mem[i];
> }
>
> + if (VKF_FLAG(hwfctx->flags,
> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)) {
> + memory_requirements.size = mem_size;
> +
> + /* Allocate memory */
> + if ((err = alloc_mem(ctx, &memory_requirements,
> + f->tiling == VK_IMAGE_TILING_LINEAR ?
> + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
> + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
> + (void *)(((uint8_t *)alloc_pnext)),
> + &f->flags, &f->mem[0])))
> + return err;
> +
> + f->size[0] = memory_requirements.size;
> +
> + for (int i = 0; i < planes; i++) {
> + bind_info[i].sType =
> VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
> + bind_info[i].image = f->img[i];
> + bind_info[i].memory = f->mem[0];
> + bind_info[i].memoryOffset = i == 0 ? 0 : mem_size_list[i-1];
> + f->offset[i] = bind_info[i].memoryOffset;
> + }
> + }
> +
> /* Bind the allocated memory to the images */
> ret = vk->BindImageMemory2(hwctx->act_dev, planes, bind_info);
> if (ret != VK_SUCCESS) {
> @@ -2154,6 +2211,12 @@ static int
> vulkan_frames_init(AVHWFramesContext *hwfc)
> if (!hwctx->usage)
> hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;
>
> + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE)) {
> + if (p->contiguous_planes == 1 ||
> + ((p->contiguous_planes == -1) && p->dev_is_intel))
> + hwctx->flags |= AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY;
> + }
> +
> err = create_exec_ctx(hwfc, &fp->conv_ctx,
> dev_hwctx->queue_family_comp_index,
> dev_hwctx->nb_comp_queues);
> @@ -3074,6 +3137,7 @@ static int
> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
> FFVulkanFunctions *vk = &p->vkfn;
> VulkanFramesPriv *fp = hwfc->internal->priv;
> AVVulkanDeviceContext *hwctx = hwfc->device_ctx->hwctx;
> + AVVulkanFramesContext *hwfctx = hwfc->hwctx;
> const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
> VkImageDrmFormatModifierPropertiesEXT drm_mod = {
> .sType =
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
> @@ -3142,7 +3206,9 @@ static int
> vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
> continue;
>
> vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub,
> &layout);
> - drm_desc->layers[i].planes[0].offset = layout.offset;
> + drm_desc->layers[i].planes[0].offset = layout.offset +
> + VKF_FLAG(hwfctx->flags,
> AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY) ?
> + f->offset[i] : 0;
> drm_desc->layers[i].planes[0].pitch = layout.rowPitch;
> }
>
> diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h
> index fdf2a60156..acb4073e3d 100644
> --- a/libavutil/hwcontext_vulkan.h
> +++ b/libavutil/hwcontext_vulkan.h
> @@ -35,6 +35,17 @@
> * with the data pointer set to an AVVkFrame.
> */
>
> +/**
> + * Defines the behaviour of frame allocation
> + * AV_VK_FRAME_FLAG_NONE: planes will be allocated in separte memory
> + * AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY: planes will be allocated in
> a
> + * contiguous memory.
> + */
> +typedef enum {
> + AV_VK_FRAME_FLAG_NONE = (1ULL << 0),
> + AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY = (1ULL << 1) | 1ULL
> +} AVVkFrameFlags;
> +
> /**
> * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
> * All of these can be set before init to change what the context uses
> @@ -165,6 +176,14 @@ typedef struct AVVulkanFramesContext {
> * extensions are present in enabled_dev_extensions.
> */
> void *alloc_pnext[AV_NUM_DATA_POINTERS];
> +
> + /**
> + * Is a combination of AVVkFrameFlags. Defines the behaviour of frame
> + * allocation.
> + * If no flag is set, then the flags are automatically determined
> + * based on the device.
> + */
> + int flags;
> } AVVulkanFramesContext;
>
> /*
> @@ -230,6 +249,11 @@ typedef struct AVVkFrame {
> * Internal data.
> */
> struct AVVkFrameInternal *internal;
> +
> + /**
> + * Describe the offset from the memory currently bound to the VkImage.
> + */
> + size_t offset[AV_NUM_DATA_POINTERS];
> } AVVkFrame;
>
> /**
> --
> 2.25.1
Hi Lynn:
(I move my comment from patch V4 to here, in case you miss it)
I made a mistake. I should use "layout.offset + f->offset[i]". I resubmit
the patchset. Let's discuss under this patch.
There are two offsets.
The first offset relates to the base address of the memory. This offset is the
memoryOffset we pass to vulkan when we bind memory.
The second offset relates to the base address of the image. This offset is set by
vulkan when we create vkImage.
The offset we get from vkGetImageSubresourceLayout is the second offset.
The offset drm_object need is the first offset.
If one image is bind to one memory, these two offsets should be the same, because the
start of the image is the start of the memory. If we bind multiple image to one memory,
these two offsets are different.
I don't find an API to get the second kind of offset, so I create a new variable to store them.
In practice, these two offsets are different. When I allocate planes to one memory. The
offsets I get from vkGetImageSubresourceLayout are all 0. However, I need to set it to 2088960
to make command works. 2088960 is the memoryOffset value I pass to vulkan when I bind
memory.
Regards
Wenbin
>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
>
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
More information about the ffmpeg-devel
mailing list