[FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle video parameters

[Pavel Koshevoy]

On Sat, Jan 21, 2017 at 10:27 AM,  <pkoshevoy at gmail.com> wrote:
> From: Pavel Koshevoy <pkoshevoy at gmail.com>
>
> Evidently CUVID doesn't support decoding 422 or 444 chroma formats,
> and only a limited set of resolutions per codec are supported.
>
> Unfortunately CUVID silently drops packets for video of unsupported
> resolution.  However, it will error-out at cuvidCreateDecoder call
> if the indicated video resolution is not supported.
>
> Given that stream resolution and pixel format are typically known as
> a result of probing it is better to use this information during
> avcodec_open2 call to fail immediately, rather than proceeding to
> decode and never receiving any frames from the decoder nor receiving
> any indication of decode failure.
> ---
>  libavcodec/cuvid.c | 62 +++++++++++++++++++++++++++++++++++++++++++++++++-----
>  1 file changed, 57 insertions(+), 5 deletions(-)
>
> diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
> index 8fc713d..f9c29a1 100644
> --- a/libavcodec/cuvid.c
> +++ b/libavcodec/cuvid.c
> @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
>      return 0;
>  }
>
> -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
> +static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
> +                                    const CUVIDPARSERPARAMS *cuparseinfo,
> +                                    cudaVideoChromaFormat probed_chroma_format,
> +                                    int probed_width,
> +                                    int probed_height)
>  {
>      CuvidContext *ctx = avctx->priv_data;
>      CUVIDDECODECREATEINFO cuinfo;
> @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
>      memset(&cuinfo, 0, sizeof(cuinfo));
>
>      cuinfo.CodecType = cuparseinfo->CodecType;
> -    cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
> +    cuinfo.ChromaFormat = probed_chroma_format;
>      cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
>
> -    cuinfo.ulWidth = 1280;
> -    cuinfo.ulHeight = 720;
> +    cuinfo.ulWidth = probed_width;
> +    cuinfo.ulHeight = probed_height;
>      cuinfo.ulTargetWidth = cuinfo.ulWidth;
>      cuinfo.ulTargetHeight = cuinfo.ulHeight;
>
> @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
>      return 0;
>  }
>
> +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt,
> +                                               cudaVideoChromaFormat *out)
> +{
> +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
> +    if (!(out && desc &&
> +          (desc->nb_components == 1 || desc->nb_components == 3) &&
> +          (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2)))
> +    {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    if (desc->nb_components == 1)
> +    {
> +        *out = cudaVideoChromaFormat_Monochrome;
> +    }
> +    else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR)
> +    {
> +        *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 :
> +                (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 :
> +                cudaVideoChromaFormat_420);
> +    }
> +    else
> +    {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    // unfortunately, 420 is the only one that works:
> +    return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL;
> +}
> +
>  static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>  {
>      CuvidContext *ctx = avctx->priv_data;
> @@ -663,12 +697,27 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>      CUcontext cuda_ctx = NULL;
>      CUcontext dummy;
>      const AVBitStreamFilter *bsf;
> +    cudaVideoChromaFormat probed_chroma_format;
> +    int probed_width;
> +    int probed_height;
>      int ret = 0;
>
>      enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
>                                         AV_PIX_FMT_NV12,
>                                         AV_PIX_FMT_NONE };
>
> +    enum AVPixelFormat probed_pix_fmt = (avctx->pix_fmt <= 0 ?
> +                                         AV_PIX_FMT_YUV420P :
> +                                         avctx->pix_fmt);
> +
> +    ret = convert_to_cuda_video_chroma_format(probed_pix_fmt, &probed_chroma_format);
> +    if (ret < 0) {
> +        // pixel format is not supported:
> +        return ret;
> +    }
> +    probed_width = avctx->coded_width ? avctx->coded_width : 1280;
> +    probed_height = avctx->coded_height ? avctx->coded_height : 720;
> +
>      // Accelerated transcoding scenarios with 'ffmpeg' require that the
>      // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the
>      // pix_fmt for non-accelerated transcoding, do not need to be correct
> @@ -824,7 +873,10 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>      if (ret < 0)
>          goto error;
>
> -    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo);
> +    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo,
> +                                   probed_chroma_format,
> +                                   probed_width,
> +                                   probed_height);
>      if (ret < 0)
>          goto error;
>
> --
> 2.9.2
>


Despite wm4 objections I am resubmitting this patch, because I can
find no other reasonable method for rejecting ffmpeg cuvid decoder
when it can't handle a given video stream.  An unreasonable
alternative would be to duplicate the work that cuvid.c does
internally on my application side, and that would be of little benefit
to me and to ffmpeg -- I might as well use the nvenc APIs directly
then.  I still see this patch as an improvement.

Pavel.