[FFmpeg-devel] [PATCH 1/2] libavutil/hwcontext_vulkan: Fix VK_FORMAT_R8G8_UNORM and VK_FORMAT_R16G16_UNORM map problem on Vulkan

[Xiang, Haihao]

On Wed, 2022-03-30 at 11:41 +0800, Wenbin Chen wrote:
> Vulkan map both DRM_FORMAT_RG88 and DRM_FORMAT_GR88 to
> VK_FORMAT_R8G8_UNORM. This cannot distinguish nv12/nv24 and nv21/nv42.
> Vulkan also map both DRM_FORMAT_RG1616 and DRM_FORMAT_GR1616 to
> VK_FORMAT_R16G16_UNORM. This causes issue when map back.
> VK_FORMAT_R16G16 will be mapped to DRM_FORMAT_GR1616, while p010 need
> DRM_FORMAT_RG1616.
> Add sw_format check to vulkan_fmt_to_drm() to fix this problem.
> 
> Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> ---
>  libavutil/hwcontext_vulkan.c | 13 +++++++++++--
>  1 file changed, 11 insertions(+), 2 deletions(-)
> 
> diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
> index 237caa4bc0..791649001f 100644
> --- a/libavutil/hwcontext_vulkan.c
> +++ b/libavutil/hwcontext_vulkan.c
> @@ -3290,8 +3290,17 @@ static void vulkan_unmap_to_drm(AVHWFramesContext
> *hwfc, HWMapDescriptor *hwmap)
>      av_free(drm_desc);
>  }
>  
> -static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
> +static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt,
> +                                         enum AVPixelFormat sw_format)
>  {
> +    if (vkfmt == VK_FORMAT_R8G8_UNORM &&
> +       (sw_format == AV_PIX_FMT_NV12 || sw_format == AV_PIX_FMT_NV24))
> +            return DRM_FORMAT_RG88;
> +
> +    if (vkfmt == VK_FORMAT_R16G16_UNORM &&
> +       (sw_format == AV_PIX_FMT_P010 || sw_format == AV_PIX_FMT_P016))
> +            return DRM_FORMAT_RG1616;
> +
>      for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
>          if (vulkan_drm_format_map[i].vk_format == vkfmt)
>              return vulkan_drm_format_map[i].drm_fourcc;
> @@ -3373,7 +3382,7 @@ static int vulkan_map_to_drm(AVHWFramesContext *hwfc,
> AVFrame *dst,
>          };
>          VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
>  
> -        drm_desc->layers[i].format    = vulkan_fmt_to_drm(plane_vkfmt);
> +        drm_desc->layers[i].format    = vulkan_fmt_to_drm(plane_vkfmt, hwfc-
> >sw_format);
>          drm_desc->layers[i].nb_planes = 1;
>  
>          if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {

Patchset LGTM

-Haihao