[FFmpeg-devel] [PATCH] avcodec/nvdec: Add support for decoding monochrome av1

[Philip Langdale]

The nvidia hardware explicitly supports decoding monochrome content,
presumably for the AVIF alpha channel. Supporting this requires an
adjustment in av1dec and explicit monochrome detection in nvdec.

I'm not sure why the monochrome path in av1dec did what it did - it
seems non-functional - YUV440P doesn't seem a logical pix_fmt for
monochrome and conditioning on chroma sub-sampling doesn't make sense.
So I changed it.

I've tested 8bit content, but I haven't found a way to create a 10bit
sample, so that path is untested for now.

Signed-off-by: Philip Langdale <philipl at overt.org>
---
 libavcodec/av1dec.c  | 23 ++++++++++++++++++-----
 libavcodec/nvdec.c   |  3 +++
 libavcodec/version.h |  2 +-
 3 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/libavcodec/av1dec.c b/libavcodec/av1dec.c
index d7b2ac9d46..f973d93edc 100644
--- a/libavcodec/av1dec.c
+++ b/libavcodec/av1dec.c
@@ -387,11 +387,14 @@ static int get_pixel_format(AVCodecContext *avctx)
                 av_log(avctx, AV_LOG_WARNING, "Unknown AV1 pixel format.\n");
         }
     } else {
-        if (seq->color_config.subsampling_x == 1 &&
-            seq->color_config.subsampling_y == 1)
-            pix_fmt = AV_PIX_FMT_YUV440P;
-        else
-            av_log(avctx, AV_LOG_WARNING, "Unknown AV1 pixel format.\n");
+            if (bit_depth == 8)
+                pix_fmt = AV_PIX_FMT_GRAY8;
+            else if (bit_depth == 10)
+                pix_fmt = AV_PIX_FMT_GRAY10;
+            else if (bit_depth == 12)
+                pix_fmt = AV_PIX_FMT_GRAY12;
+            else
+                av_log(avctx, AV_LOG_WARNING, "Unknown AV1 pixel format.\n");
     }
 
     av_log(avctx, AV_LOG_DEBUG, "AV1 decode get format: %s.\n",
@@ -430,6 +433,16 @@ static int get_pixel_format(AVCodecContext *avctx)
 #endif
 #if CONFIG_AV1_VAAPI_HWACCEL
         *fmtp++ = AV_PIX_FMT_VAAPI;
+#endif
+        break;
+    case AV_PIX_FMT_GRAY8:
+#if CONFIG_AV1_NVDEC_HWACCEL
+        *fmtp++ = AV_PIX_FMT_CUDA;
+#endif
+        break;
+    case AV_PIX_FMT_GRAY10:
+#if CONFIG_AV1_NVDEC_HWACCEL
+        *fmtp++ = AV_PIX_FMT_CUDA;
 #endif
         break;
     }
diff --git a/libavcodec/nvdec.c b/libavcodec/nvdec.c
index dd7dcccafa..d45804994f 100644
--- a/libavcodec/nvdec.c
+++ b/libavcodec/nvdec.c
@@ -84,6 +84,9 @@ static int map_chroma_format(enum AVPixelFormat pix_fmt)
 {
     int shift_h = 0, shift_v = 0;
 
+    if (av_pix_fmt_count_planes(pix_fmt) == 1)
+        return cudaVideoChromaFormat_Monochrome;
+
     av_pix_fmt_get_chroma_sub_sample(pix_fmt, &shift_h, &shift_v);
 
     if (shift_h == 1 && shift_v == 1)
diff --git a/libavcodec/version.h b/libavcodec/version.h
index e4b81da7cb..d1f6fb440a 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -29,7 +29,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR  58
 #define LIBAVCODEC_VERSION_MINOR 114
-#define LIBAVCODEC_VERSION_MICRO 100
+#define LIBAVCODEC_VERSION_MICRO 101
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \
-- 
2.27.0