[FFmpeg-devel] [PATCH 1/2] avutil/hwcontext_vulkan: cuda doesn't belong to valid_sw_formats

[Philip Langdale]

On Fri, 27 Oct 2023 23:37:00 +0800
Zhao Zhili <quinkblack at foxmail.com> wrote:

> From: Zhao Zhili <zhilizhao at tencent.com>
> 
> Move it to transfer_get_formats.
> ---
>  libavutil/hwcontext_vulkan.c | 25 +++++++++++++------------
>  1 file changed, 13 insertions(+), 12 deletions(-)
> 
> diff --git a/libavutil/hwcontext_vulkan.c
> b/libavutil/hwcontext_vulkan.c index 8dd40cb66f..8481427b42 100644
> --- a/libavutil/hwcontext_vulkan.c
> +++ b/libavutil/hwcontext_vulkan.c
> @@ -1664,11 +1664,6 @@ static int
> vulkan_frames_get_constraints(AVHWDeviceContext *ctx, NULL, NULL,
> NULL, NULL, 0, 0) >= 0; }
>  
> -#if CONFIG_CUDA
> -    if (p->dev_is_nvidia)
> -        count++;
> -#endif
> -
>      constraints->valid_sw_formats = av_malloc_array(count + 1,
>                                                      sizeof(enum
> AVPixelFormat)); if (!constraints->valid_sw_formats)
> @@ -1684,10 +1679,6 @@ static int
> vulkan_frames_get_constraints(AVHWDeviceContext *ctx, }
>      }
>  
> -#if CONFIG_CUDA
> -    if (p->dev_is_nvidia)
> -        constraints->valid_sw_formats[count++] = AV_PIX_FMT_CUDA;
> -#endif
>      constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
>  
>      constraints->min_width  = 1;
> @@ -2416,12 +2407,22 @@ static int
> vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum
> AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
>  {
> -    enum AVPixelFormat *fmts = av_malloc_array(2, sizeof(*fmts));
> +    enum AVPixelFormat *fmts;
> +    int n = 2;
> +
> +#if CONFIG_CUDA
> +    n++;
> +#endif
> +    fmts = av_malloc_array(n, sizeof(*fmts));
>      if (!fmts)
>          return AVERROR(ENOMEM);
>  
> -    fmts[0] = hwfc->sw_format;
> -    fmts[1] = AV_PIX_FMT_NONE;
> +    n = 0;
> +    fmts[n++] = hwfc->sw_format;
> +#if CONFIG_CUDA
> +    fmts[n++] = AV_PIX_FMT_CUDA;
> +#endif
> +    fmts[n++] = AV_PIX_FMT_NONE;
>  
>      *formats = fmts;
>      return 0;

LGTM. Thanks!


--phil