[FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle given video parameters

[wm4]

On Mon,  2 Jan 2017 14:26:45 -0700
pkoshevoy at gmail.com wrote:

> From: Pavel Koshevoy <pkoshevoy at gmail.com>
> 
> Evidently CUVID doesn't support decoding 422 or 444 chroma formats,
> and only a limited set of resolutions per codec are supported.
> 
> Given that stream resolution and pixel format are typically known as a
> result of probing it is better to use this info during avcodec_open2
> call and fail early in case the video parameters are not supported,
> rather than failing later during avcodec_send_packet calls.
> 
> This problem surfaced when trying to decode 5120x2700 h246 video on
> Geforce GT 730, or when decoding 422 mpeg2 stream on same GPU --
> avcodec_open2 succeeded but decoding failed.
> ---
>  libavcodec/cuvid.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++-----
>  1 file changed, 53 insertions(+), 5 deletions(-)
> 
> diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
> index 8fc713d..febdd71 100644
> --- a/libavcodec/cuvid.c
> +++ b/libavcodec/cuvid.c
> @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
>      return 0;
>  }
>  
> -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
> +static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
> +                                    const CUVIDPARSERPARAMS *cuparseinfo,
> +                                    cudaVideoChromaFormat probed_chroma_format,
> +                                    int probed_width,
> +                                    int probed_height)
>  {
>      CuvidContext *ctx = avctx->priv_data;
>      CUVIDDECODECREATEINFO cuinfo;
> @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
>      memset(&cuinfo, 0, sizeof(cuinfo));
>  
>      cuinfo.CodecType = cuparseinfo->CodecType;
> -    cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
> +    cuinfo.ChromaFormat = probed_chroma_format;
>      cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
>  
> -    cuinfo.ulWidth = 1280;
> -    cuinfo.ulHeight = 720;
> +    cuinfo.ulWidth = probed_width;
> +    cuinfo.ulHeight = probed_height;
>      cuinfo.ulTargetWidth = cuinfo.ulWidth;
>      cuinfo.ulTargetHeight = cuinfo.ulHeight;
>  
> @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
>      return 0;
>  }
>  
> +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt,
> +                                               cudaVideoChromaFormat *out)
> +{
> +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
> +    if (!(out && desc &&
> +          (desc->nb_components == 1 || desc->nb_components == 3) &&
> +          (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2)))
> +    {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    if (desc->nb_components == 1)
> +    {
> +        *out = cudaVideoChromaFormat_Monochrome;
> +    }
> +    else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR)
> +    {
> +        *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 :
> +                (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 :
> +                cudaVideoChromaFormat_420);
> +    }
> +    else
> +    {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    // unfortunately, 420 is the only one that works:
> +    return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL;
> +}
> +
>  static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>  {
>      CuvidContext *ctx = avctx->priv_data;
> @@ -663,12 +697,23 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>      CUcontext cuda_ctx = NULL;
>      CUcontext dummy;
>      const AVBitStreamFilter *bsf;
> +    cudaVideoChromaFormat probed_chroma_format;
> +    int probed_width;
> +    int probed_height;
>      int ret = 0;
>  
>      enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
>                                         AV_PIX_FMT_NV12,
>                                         AV_PIX_FMT_NONE };
>  
> +    ret = convert_to_cuda_video_chroma_format(avctx->pix_fmt, &probed_chroma_format);
> +    if (ret < 0) {
> +        // pixel format is not supported:
> +        return ret;
> +    }
> +    probed_width = avctx->coded_width ? avctx->coded_width : 1280;
> +    probed_height = avctx->coded_height ? avctx->coded_height : 720;

IMO relying on these fields is a bit of a problem, since they don't
even need to be set (the h264 bitstream can contain them).

It's especially bad for pix_fmt, since my application for example
doesn't set this field at all in many cases.

> +
>      // Accelerated transcoding scenarios with 'ffmpeg' require that the
>      // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the
>      // pix_fmt for non-accelerated transcoding, do not need to be correct
> @@ -824,7 +869,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>      if (ret < 0)
>          goto error;
>  
> -    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo);
> +    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo,
> +                                   probed_chroma_format,
> +                                   probed_width,
> +                                   probed_height);
>      if (ret < 0)
>          goto error;
>