[FFmpeg-devel] [RFC PATCH] avcodec/nvenc: add alpha layer encoding support

[Zhao Zhili]

From: Zhao Zhili <zhilizhao at tencent.com>

Signed-off-by: Zhao Zhili <zhilizhao at tencent.com>
---
I need help. A lot of things doesn't work as Nvidia's doc:

1. Global header is broken. NV_ENC_SEQUENCE_PARAM_PAYLOAD doesn't work with
   multi SPS/PPS. It output VPS, SPS_0, SPS_1, PPS_0, with PPS_1 missing.
2. NV_ENC_LOCK_BITSTREAM.alphaLayerSizeInBytes is always 0, although the data is there.
3. The alpha layer bitstream is corrupted. It can be decoded with a lot of errors.

 libavcodec/nvenc.c      | 13 +++++++++++++
 libavcodec/nvenc.h      |  1 +
 libavcodec/nvenc_hevc.c |  2 ++
 3 files changed, 16 insertions(+)

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index f6df7cb6ac..e46cea4302 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -578,6 +578,12 @@ static int nvenc_check_capabilities(AVCodecContext *avctx)
         return AVERROR(ENOSYS);
     }
 
+    if (ctx->alpha_bitrate_ratio > 0 &&
+        nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_ALPHA_LAYER_ENCODING) <= 0) {
+        av_log(avctx, AV_LOG_ERROR, "Alpha layer encoding not supported\n");
+        return AVERROR(ENOSYS);
+    }
+
     ctx->support_dyn_bitrate = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE);
 
     return 0;
@@ -1290,6 +1296,13 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
         hevc->enableConstrainedEncoding = 1;
 #endif
 
+    if (ctx->alpha_bitrate_ratio > 0 && (pixdesc->flags & AV_PIX_FMT_FLAG_ALPHA)) {
+        hevc->enableAlphaLayerEncoding = 1;
+        cc->rcParams.alphaLayerBitrateRatio = ctx->alpha_bitrate_ratio;
+        av_log(avctx, AV_LOG_DEBUG, "Enable alpha layer encoding for pix_fmt %s\n",
+                                    pixdesc->name);
+    }
+
     hevc->disableSPSPPS = (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) ? 1 : 0;
     hevc->repeatSPSPPS  = (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) ? 0 : 1;
     hevc->outputAUD     = ctx->aud;
diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
index 05a7ac48b1..f210d71eeb 100644
--- a/libavcodec/nvenc.h
+++ b/libavcodec/nvenc.h
@@ -240,6 +240,7 @@ typedef struct NvencContext
     int udu_sei;
     int timing_info;
     int highbitdepth;
+    int alpha_bitrate_ratio;
 } NvencContext;
 
 int ff_nvenc_encode_init(AVCodecContext *avctx);
diff --git a/libavcodec/nvenc_hevc.c b/libavcodec/nvenc_hevc.c
index 5ad423444a..67a649fad2 100644
--- a/libavcodec/nvenc_hevc.c
+++ b/libavcodec/nvenc_hevc.c
@@ -177,6 +177,8 @@ static const AVOption options[] = {
                                                             OFFSET(single_slice_intra_refresh), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
     { "constrained-encoding", "Enable constrainedFrame encoding where each slice in the constrained picture is independent of other slices",
                                                             OFFSET(constrained_encoding), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
+    { "alpha-bitrate-ratio", "Specifies the ratio in which bitrate should be split between base and alpha layer, 0 for disable alpha layer encoding",
+                                                            OFFSET(alpha_bitrate_ratio),  AV_OPT_TYPE_INT,  { .i64 = 0 }, 0, INT_MAX, VE },
     { NULL }
 };
 
-- 
2.34.1