[FFmpeg-devel] [PATCH v2] avcodec/libaom: Support monochrome encoding with libaom >= 2.0.1

[Philip Langdale]

Monochrome encoding with libaom was buggy for a long time, but this was
finally sorted out in libaom 2.0.1 (2.0.0 is almost there but was still
buggy in realtime mode).

Signed-off-by: Philip Langdale <philipl at overt.org>
---
 Changelog              |  1 +
 libavcodec/libaomenc.c | 42 ++++++++++++++++++++++++++++++++++++++++--
 libavcodec/version.h   |  2 +-
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/Changelog b/Changelog
index 503317dfae..8f5e849f8d 100644
--- a/Changelog
+++ b/Changelog
@@ -51,6 +51,7 @@ version <next>:
 - asubcut filter
 - Microsoft Paint (MSP) version 2 decoder
 - Microsoft Paint (MSP) demuxer
+- AV1 monochrome encoding support via libaom >= 2.0.1
 
 
 version 4.3:
diff --git a/libavcodec/libaomenc.c b/libavcodec/libaomenc.c
index 2b0581b15a..342d0883e4 100644
--- a/libavcodec/libaomenc.c
+++ b/libavcodec/libaomenc.c
@@ -338,6 +338,9 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->pix_fmt);
     enccfg->g_bit_depth = enccfg->g_input_bit_depth = desc->comp[0].depth;
     switch (avctx->pix_fmt) {
+    case AV_PIX_FMT_GRAY8:
+        enccfg->monochrome = 1;
+        /* Fall-through */
     case AV_PIX_FMT_YUV420P:
         enccfg->g_profile = FF_PROFILE_AV1_MAIN;
         *img_fmt = AOM_IMG_FMT_I420;
@@ -351,6 +354,10 @@ static int set_pix_fmt(AVCodecContext *avctx, aom_codec_caps_t codec_caps,
         enccfg->g_profile = FF_PROFILE_AV1_HIGH;
         *img_fmt = AOM_IMG_FMT_I444;
         return 0;
+    case AV_PIX_FMT_GRAY10:
+    case AV_PIX_FMT_GRAY12:
+        enccfg->monochrome = 1;
+        /* Fall-through */
     case AV_PIX_FMT_YUV420P10:
     case AV_PIX_FMT_YUV420P12:
         if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH) {
@@ -1158,6 +1165,15 @@ static const enum AVPixelFormat av1_pix_fmts[] = {
     AV_PIX_FMT_NONE
 };
 
+static const enum AVPixelFormat av1_pix_fmts_with_gray[] = {
+    AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_YUV422P,
+    AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GBRP,
+    AV_PIX_FMT_GRAY8,
+    AV_PIX_FMT_NONE
+};
+
 static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_YUV422P,
@@ -1174,13 +1190,35 @@ static const enum AVPixelFormat av1_pix_fmts_highbd[] = {
     AV_PIX_FMT_NONE
 };
 
+static const enum AVPixelFormat av1_pix_fmts_highbd_with_gray[] = {
+    AV_PIX_FMT_YUV420P,
+    AV_PIX_FMT_YUV422P,
+    AV_PIX_FMT_YUV444P,
+    AV_PIX_FMT_GBRP,
+    AV_PIX_FMT_YUV420P10,
+    AV_PIX_FMT_YUV422P10,
+    AV_PIX_FMT_YUV444P10,
+    AV_PIX_FMT_YUV420P12,
+    AV_PIX_FMT_YUV422P12,
+    AV_PIX_FMT_YUV444P12,
+    AV_PIX_FMT_GBRP10,
+    AV_PIX_FMT_GBRP12,
+    AV_PIX_FMT_GRAY8,
+    AV_PIX_FMT_GRAY10,
+    AV_PIX_FMT_GRAY12,
+    AV_PIX_FMT_NONE
+};
+
 static av_cold void av1_init_static(AVCodec *codec)
 {
+    int supports_monochrome = aom_codec_version() >= 20001;
     aom_codec_caps_t codec_caps = aom_codec_get_caps(aom_codec_av1_cx());
     if (codec_caps & AOM_CODEC_CAP_HIGHBITDEPTH)
-        codec->pix_fmts = av1_pix_fmts_highbd;
+        codec->pix_fmts = supports_monochrome ? av1_pix_fmts_highbd_with_gray :
+                                                av1_pix_fmts_highbd;
     else
-        codec->pix_fmts = av1_pix_fmts;
+        codec->pix_fmts = supports_monochrome ? av1_pix_fmts_with_gray :
+                                                av1_pix_fmts;
 
     if (aom_codec_version_major() < 2)
         codec->capabilities |= AV_CODEC_CAP_EXPERIMENTAL;
diff --git a/libavcodec/version.h b/libavcodec/version.h
index 1c10d105f6..5b92afe60a 100644
--- a/libavcodec/version.h
+++ b/libavcodec/version.h
@@ -29,7 +29,7 @@
 
 #define LIBAVCODEC_VERSION_MAJOR  58
 #define LIBAVCODEC_VERSION_MINOR 115
-#define LIBAVCODEC_VERSION_MICRO 101
+#define LIBAVCODEC_VERSION_MICRO 102
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \
                                                LIBAVCODEC_VERSION_MINOR, \
-- 
2.27.0