[FFmpeg-devel] [PATCH V5 5/5] libavutil/hwcontext_vulkan: specify the modifier to create VKImage

Chen, Wenbin wenbin.chen at intel.com
Mon Dec 6 07:48:42 EET 2021


> 2 Dec 2021, 02:43 by wenbin.chen at intel.com:
> 
> > When vulkan image exports to drm, the tilling need to be
> > VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT. Now add code to
> create vulkan
> > image using this format.
> >
> > Now the following command line works:
> >
> > ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -
> hwaccel_output_format \
> > vaapi -i input_1080p.264 -vf "hwmap=derive_device=vulkan,format=vulkan,
> \
> > scale_vulkan=1920:1080,hwmap=derive_device=vaapi,format=vaapi" -c:v
> h264_vaapi output.264
> >
> > Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> > ---
> >  libavutil/hwcontext_vulkan.c | 127
> +++++++++++++++++++++++++++++++++--
> >  1 file changed, 121 insertions(+), 6 deletions(-)
> >
> > diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> > index f980b72720..8224c0d4e4 100644
> > --- a/libavutil/hwcontext_vulkan.c
> > +++ b/libavutil/hwcontext_vulkan.c
> > @@ -120,6 +120,9 @@ typedef struct VulkanFramesPriv {
> >  /* Image transfers */
> >  VulkanExecCtx upload_ctx;
> >  VulkanExecCtx download_ctx;
> > +
> > +    /*modifier info*/
> > +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> >  } VulkanFramesPriv;
> >
> >  typedef struct AVVkFrameInternal {
> > @@ -242,6 +245,28 @@ const VkFormat *av_vkfmt_from_pixfmt(enum
> AVPixelFormat p)
> >  return NULL;
> >  }
> >
> > +static void *find_in_structure_list(VkBaseOutStructure *stru_list,
> VkStructureType sType) {
> > +    if (!stru_list)
> > +        return NULL;
> > +
> > +    for(;stru_list;stru_list = stru_list->pNext)
> > +        if (stru_list->sType == sType)
> > +            return stru_list;
> > +
> > +    return NULL;
> > +}
> > +
> > +static void append_to_structure_list(VkBaseOutStructure **stru_list,
> VkBaseOutStructure *added_stru) {
> > +    VkBaseOutStructure *p;
> > +    if (!*stru_list) {
> > +        *stru_list = added_stru;
> > +        return;
> > +    }
> > +    for(p = *stru_list; p->pNext; p = p->pNext);
> > +    p->pNext = added_stru;
> > +    return;
> > +}
> >
> 
> Could you tidy these 2 functions up? Also, code style issues, as usual.
> Take a look at libplacebo's functions or copy them, they're much
> neater: https://0x1.st/tx.txt
> 
> 
> >  static int pixfmt_is_supported(AVHWDeviceContext *dev_ctx, enum
> AVPixelFormat p,
> >  int linear)
> >  {
> > @@ -2094,6 +2119,10 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> >  AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> >  VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> >  FFVulkanFunctions *vk = &p->vkfn;
> > +    const int has_modifiers = hwctx->tiling ==
> VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
> > +    int loop_count;
> > +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info =
> find_in_structure_list(hwctx->create_pnext,
> > +
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT);
> >  VkExternalImageFormatProperties eprops = {
> >  .sType =
> VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
> >  };
> > @@ -2101,9 +2130,18 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> >  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
> >  .pNext = &eprops,
> >  };
> > +    VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info
> = {
> > +        .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER
> _INFO_EXT,
> > +        .pNext = NULL,
> > +        .pQueueFamilyIndices   = p->qfs,
> > +        .queueFamilyIndexCount = p->num_qfs,
> > +        .sharingMode           = p->num_qfs > 1 ?
> VK_SHARING_MODE_CONCURRENT :
> > +                                                  VK_SHARING_MODE_EXCLUSIVE,
> > +    };
> >  VkPhysicalDeviceExternalImageFormatInfo enext = {
> >  .sType =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
> >  .handleType = exp,
> > +        .pNext = has_modifiers && modifier_info ? &phy_dev_mod_info :
> NULL,
> >  };
> >  VkPhysicalDeviceImageFormatInfo2 pinfo = {
> >  .sType  =
> VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
> > @@ -2115,11 +2153,16 @@ static void
> try_export_flags(AVHWFramesContext *hwfc,
> >  .flags  = VK_IMAGE_CREATE_ALIAS_BIT,
> >  };
> >
> > -    ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx-
> >phys_dev,
> > -                                                      &pinfo, &props);
> > -    if (ret == VK_SUCCESS) {
> > -        *iexp |= exp;
> > -        *comp_handle_types |=
> eprops.externalMemoryProperties.compatibleHandleTypes;
> > +    loop_count = has_modifiers && modifier_info ? modifier_info-
> >drmFormatModifierCount : 1;
> > +    for (int i = 0; i < loop_count; i++) {
> > +        if (has_modifiers && modifier_info)
> > +            phy_dev_mod_info.drmFormatModifier = modifier_info-
> >pDrmFormatModifiers[i];
> > +        ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx-
> >phys_dev,
> > +                                                        &pinfo, &props);
> > +        if (ret == VK_SUCCESS) {
> > +            *iexp |= exp;
> > +            *comp_handle_types |=
> eprops.externalMemoryProperties.compatibleHandleTypes;
> > +        }
> >  }
> >  }
> >
> > @@ -2190,6 +2233,12 @@ static void
> vulkan_frames_uninit(AVHWFramesContext *hwfc)
> >  {
> >  VulkanFramesPriv *fp = hwfc->internal->priv;
> >
> > +    if (fp->modifier_info) {
> > +        if (fp->modifier_info->pDrmFormatModifiers)
> > +            av_freep(&fp->modifier_info->pDrmFormatModifiers);
> > +        av_freep(&fp->modifier_info);
> > +    }
> > +
> >  free_exec_ctx(hwfc, &fp->conv_ctx);
> >  free_exec_ctx(hwfc, &fp->upload_ctx);
> >  free_exec_ctx(hwfc, &fp->download_ctx);
> > @@ -2203,11 +2252,77 @@ static int
> vulkan_frames_init(AVHWFramesContext *hwfc)
> >  VulkanFramesPriv *fp = hwfc->internal->priv;
> >  AVVulkanDeviceContext *dev_hwctx = hwfc->device_ctx->hwctx;
> >  VulkanDevicePriv *p = hwfc->device_ctx->internal->priv;
> > +    VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
> > +    const int has_modifiers = !!(p->extensions &
> FF_VK_EXT_DRM_MODIFIER_FLAGS);
> >
> >  /* Default pool flags */
> > -    hwctx->tiling = hwctx->tiling ? hwctx->tiling : p->use_linear_images ?
> > +    hwctx->tiling = hwctx->tiling ? hwctx->tiling : has_modifiers ?
> > +                    VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT : p-
> >use_linear_images ?
> >  VK_IMAGE_TILING_LINEAR : VK_IMAGE_TILING_OPTIMAL;
> >
> > +    modifier_info = find_in_structure_list(hwctx->create_pnext,
> > +
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT);
> > +    /* get the supported modifier */
> > +    if (has_modifiers && !modifier_info) {
> > +        const VkFormat *fmt = av_vkfmt_from_pixfmt(hwfc->sw_format);
> > +        FFVulkanFunctions *vk = &p->vkfn;
> > +        VkDrmFormatModifierPropertiesEXT *mod_props;
> > +        uint64_t *modifiers;
> > +        int modifier_count = 0;
> > +
> > +        VkDrmFormatModifierPropertiesListEXT mod_props_list = {
> > +            .sType =
> VK_STRUCTURE_TYPE_DRM_FORMAT_MODIFIER_PROPERTIES_LIST_EXT,
> > +            .pNext = NULL,
> > +            .drmFormatModifierCount = 0,
> > +            .pDrmFormatModifierProperties = NULL,
> > +        };
> > +        VkFormatProperties2 prop = {
> > +            .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
> > +            .pNext = &mod_props_list,
> > +        };
> > +        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev,
> fmt[0], &prop);
> > +        if (!mod_props_list.drmFormatModifierCount) {
> > +            av_log(hwfc, AV_LOG_ERROR, "There are not supported modifiers
> for sw_format\n");
> >
> 
> Grammar issues: should be something like
> "There are no supported modifiers for given sw_format\n".
> 
> 
> > +            return AVERROR(EINVAL);
> > +        }
> > +
> > +        modifier_info = av_malloc(sizeof(*modifier_info));
> > +        if (!modifier_info)
> > +            return AVERROR(ENOMEM);
> > +        append_to_structure_list((VkBaseOutStructure **)&hwctx-
> >create_pnext,
> > +                                 (VkBaseOutStructure *)modifier_info);
> > +        fp->modifier_info = modifier_info;
> > +        modifier_info->pNext = NULL;
> > +        modifier_info->sType =
> VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INF
> O_EXT;
> > +
> > +        modifiers = av_malloc(mod_props_list.drmFormatModifierCount *
> > +                              sizeof(*modifiers));
> > +        if (!modifiers)
> > +            return AVERROR(ENOMEM);
> > +        modifier_info->pDrmFormatModifiers = modifiers;
> > +
> > +        mod_props = av_malloc(mod_props_list.drmFormatModifierCount *
> > +                              sizeof(*mod_props));
> > +        if (!mod_props)
> > +            return AVERROR(ENOMEM);
> > +        mod_props_list.pDrmFormatModifierProperties = mod_props;
> > +        vk->GetPhysicalDeviceFormatProperties2(dev_hwctx->phys_dev,
> fmt[0], &prop);
> > +        for (int i = 0; i < mod_props_list.drmFormatModifierCount; i++) {
> > +            if (!(mod_props[i].drmFormatModifierTilingFeatures &
> FF_VK_DEFAULT_USAGE_FLAGS))
> >
> 
> You should use the usage flags given by the user in AVVulkanFramesContext,
> not the default ones (unless none were specified by the user, but in that case,
> that field is already set to the default usage flags).
> 
> 
> > +                continue;
> > +            modifiers[modifier_count++] = mod_props[i].drmFormatModifier;
> > +        }
> > +        if (!modifier_count) {
> > +            av_log(hwfc, AV_LOG_ERROR, "The supported modifiers doesn't
> support "
> > +                                       "default usage\n");
> >
> 
> Same, should be something like:
> "None of the given modifiers supports the usage flags!"
> 
> 
> > +            av_freep(&mod_props);
> > +            return AVERROR(EINVAL);
> > +        }
> > +        modifier_info->drmFormatModifierCount = modifier_count;
> > +        av_freep(&mod_props);
> > +    }
> > +
> > +
> >  if (!hwctx->usage)
> >  hwctx->usage = FF_VK_DEFAULT_USAGE_FLAGS;
> >
> > --
> > 2.25.1
> >

Thanks for review
I will fix this problem in next patchset.

> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel at ffmpeg.org
> > https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
> > To unsubscribe, visit link above, or email
> > ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".
> >
> 
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> https://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> 
> To unsubscribe, visit link above, or email
> ffmpeg-devel-request at ffmpeg.org with subject "unsubscribe".


More information about the ffmpeg-devel mailing list