[FFmpeg-devel] [PATCH 7/7] libavutil/hwcontext_vulkan: specify the modifier to create VKImage
Lynne
dev at lynne.ee
Fri Nov 12 09:10:05 EET 2021
9 Nov 2021, 10:18 by wenbin.chen at intel.com:
> When vulkan image exports to drm, the tilling need to be
> VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to create vulkan
> image using this format.
>
> Now the following command line works:
>
> ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format \
> vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan, \
> scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v h264_vaapi output.264
>
> Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> ---
> libavutil/hwcontext_vulkan.c | 76 +++++++++++++++++++++++++++++++++---
> libavutil/hwcontext_vulkan.h | 5 +++
> 2 files changed, 75 insertions(+), 6 deletions(-)
>
> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> index 29ade94b7f..e252c2177e 100644
> --- a/libavutil/hwcontext_vulkan.c
> +++ b/libavutil/hwcontext_vulkan.c
> @@ -1919,6 +1919,7 @@ static void try_export_flags(AVHWFramesContext *hwfc,
> AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> FFVulkanFunctions *vk = &p->vkfn;
> + const int has_modifiers = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
> VkExternalImageFormatProperties eprops = {
> .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
> };
> @@ -1926,9 +1927,18 @@ static void try_export_flags(AVHWFramesContext *hwfc,
> .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
> .pNext = &eprops,
> };
> + VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
> + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
> + .pNext = NULL,
> + .pQueueFamilyIndices = p->qfs,
> + .queueFamilyIndexCount = p->num_qfs,
> + .sharingMode = p->num_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
> + VK_SHARING_MODE_EXCLUSIVE,
> + };
> VkPhysicalDeviceExternalImageFormatInfo enext = {
> .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
> .handleType = exp,
> + .pNext = has_modifiers ? &phy_dev_mod_info : NULL,
> };
> VkPhysicalDeviceImageFormatInfo2 pinfo = {
> .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
> @@ -1940,11 +1950,15 @@ static void try_export_flags(AVHWFramesContext *hwfc,
> .flags = VK_IMAGE_CREATE_ALIAS_BIT,
> };
>
> - ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
> - &pinfo, &props);
> - if (ret == VK_SUCCESS) {
> - *iexp |= exp;
> - *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
> + for (int i = 0; i < (has_modifiers ? hwctx->modifier_count : 1); i++) {
> + if (has_modifiers && hwctx->modifier_count)
> + phy_dev_mod_info.drmFormatModifier = hwctx->modifiers[i];
> + ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
> + &pinfo, &props);
> + if (ret == VK_SUCCESS) {
> + *iexp |= exp;
> + *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
> + }
> }
> }
>
> @@ -2007,6 +2021,7 @@ fail:
> static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
> {
> VulkanFramesPriv *fp = hwfc->internal->priv;
> + AVVulkanFramesContext *hwctx = hwfc->hwctx;
>
> free_exec_ctx(hwfc, &fp->conv_ctx);
> free_exec_ctx(hwfc, &fp->upload_ctx);
> @@ -2021,11 +2036,60 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc)
> VulkanFramesPriv *fp = hwfc->internal->priv;
> AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> + const int has_modifiers = !!(p->extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS);
>
> /* Default pool flags */
> - hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
> + hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
> + VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p->use_linear_images ?
> VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
>
> + /* get the supported modifier */
> + if (has_modifiers) {
> + const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
> + FFVulkanFunctions *vk = &p->vkfn;
> + VkDrmFormatModifierPropertiesEXT mod_props[MAX_VULKAN_MODIFIERS];
> +
> + VkDrmFormatModifierPropertiesListEXT mod_props_list = {
> + .sType = VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
> + .pNext = NULL,
> + .drmFormatModifierCount = 0,
> + .pDrmFormatModifierProperties = NULL,
> + };
> + VkFormatProperties2 prop = {
> + .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
> + .pNext = &mod_props_list,
> + };
> +
> + vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
> + if (!mod_props_list.drmFormatModifierCount) {
> + av_log(hwfc, AV_LOG_ERROR, "There are not supported modifiers for sw_format\n");
> + return AVERROR(EINVAL);
> + }
> + mod_props_list.drmFormatModifierCount =
> + FFMIN(mod_props_list.drmFormatModifierCount, MAX_VULKAN_MODIFIERS);
> +
> + mod_props_list.pDrmFormatModifierProperties = mod_props;
> + vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev, fmt[0], &prop);
> +
> + hwctx->modifier_count = 0;
> + for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
> + if (!(mod_props[i].drmFormatModifierTilingFeatures & DEFAULT_USAGE_FLAGS))
> + continue;
> + hwctx->modifiers[hwctx->modifier_count++] = mod_props[i].drmFormatModifier;
> + }
> + if (!hwctx->modifier_count) {
> + av_log(hwfc, AV_LOG_ERROR, "The supported modifiers doesn't support"
> + "default usage\n");
> + return AVERROR(EINVAL);
> + }
> +
> + hwctx->create_pnext = &hwctx->modifier_info;
> + hwctx->modifier_info.pNext = NULL;
> + hwctx->modifier_info.sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT;
> + hwctx->modifier_info.drmFormatModifierCount = hwctx->modifier_count;
> + hwctx->modifier_info.pDrmFormatModifiers = hwctx->modifiers;
> + }
> +
> if (!hwctx->usage)
> hwctx->usage = DEFAULT_USAGE_FLAGS;
>
> diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h
> index efb602ef27..113448bb51 100644
> --- a/libavutil/hwcontext_vulkan.h
> +++ b/libavutil/hwcontext_vulkan.h
> @@ -32,6 +32,7 @@
> * with the data pointer set to an AVVkFrame.
> */
>
> +#define MAX_VULKAN_MODIFIERS 16
> /**
> * Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
> * All of these can be set before init to change what the context uses
> @@ -149,6 +150,10 @@ typedef struct AVVulkanFramesContext {
> */
> void *create_pnext;
>
> + VkImageDrmFormatModifierListCreateInfoEXT modifier_info;
> + uint64_t modifiers[MAX_VULKAN_MODIFIERS];
> + int modifier_count;
>
I don't like having a platform-specific variable in the context, but I
understand why it's necessary. Wouldn't a single modifier be sufficient,
though, rather than exposing all supported modifiers? Since all images
that are allocated by the frames context should all have the same
modifier.
More information about the ffmpeg-devel
mailing list