[FFmpeg-devel] [PATCH 2/2] avcodec/libx265: add alpha layer encoding support

[Timo Rothenpieler]

---
 libavcodec/libx265.c | 51 ++++++++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/libavcodec/libx265.c b/libavcodec/libx265.c
index 63cc497f83..1bf2068f36 100644
--- a/libavcodec/libx265.c
+++ b/libavcodec/libx265.c
@@ -42,6 +42,14 @@
 #include "atsc_a53.h"
 #include "sei.h"
 
+#if defined(X265_ENABLE_ALPHA)
+#define FF_X265_MAX_LAYERS MAX_LAYERS
+#elif X265_BUILD >= 210
+#define FF_X265_MAX_LAYERS 2
+#else
+#define FF_X265_MAX_LAYERS 1
+#endif
+
 typedef struct ReorderedData {
     int64_t duration;
 
@@ -539,6 +547,15 @@ FF_ENABLE_DEPRECATION_WARNINGS
                                 ctx->dovi.cfg.dv_bl_signal_compatibility_id;
 #endif
 
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    if (desc->flags & AV_PIX_FMT_FLAG_ALPHA) {
+        if (ctx->api->param_parse(ctx->params, "alpha", "1") < 0) {
+            av_log(avctx, AV_LOG_ERROR, "Loaded libx265 does not support alpha layer encoding.\n");
+            return AVERROR(ENOTSUP);
+        }
+    }
+#endif
+
     ctx->encoder = ctx->api->encoder_open(ctx->params);
     if (!ctx->encoder) {
         av_log(avctx, AV_LOG_ERROR, "Cannot open libx265 encoder.\n");
@@ -661,13 +678,10 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
 {
     libx265Context *ctx = avctx->priv_data;
     x265_picture x265pic;
+    x265_picture x265pic_out[FF_X265_MAX_LAYERS] = { 0 };
 #if (X265_BUILD >= 210) && (X265_BUILD < 213)
-    x265_picture x265pic_layers_out[MAX_SCALABLE_LAYERS];
-    x265_picture* x265pic_lyrptr_out[MAX_SCALABLE_LAYERS];
-#else
-    x265_picture x265pic_solo_out = { 0 };
+    x265_picture *x265pic_lyrptr_out[FF_X265_MAX_LAYERS];
 #endif
-    x265_picture* x265pic_out;
     x265_nal *nal;
     x265_sei *sei;
     uint8_t *dst;
@@ -687,7 +701,11 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
         ReorderedData *rd;
         int rd_idx;
 
+#if X265_BUILD >= 210
+        for (i = 0; i < 4; i++) {
+#else
         for (i = 0; i < 3; i++) {
+#endif
            x265pic.planes[i] = pic->data[i];
            x265pic.stride[i] = pic->linesize[i];
         }
@@ -806,14 +824,14 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     }
 
 #if (X265_BUILD >= 210) && (X265_BUILD < 213)
-    for (i = 0; i < MAX_SCALABLE_LAYERS; i++)
-        x265pic_lyrptr_out[i] = &x265pic_layers_out[i];
+    for (i = 0; i < FF_ARRAY_ELEMS(x265pic_out); i++)
+        x265pic_lyrptr_out[i] = &x265pic_out[i];
 
     ret = ctx->api->encoder_encode(ctx->encoder, &nal, &nnal,
                                    pic ? &x265pic : NULL, x265pic_lyrptr_out);
 #else
     ret = ctx->api->encoder_encode(ctx->encoder, &nal, &nnal,
-                                   pic ? &x265pic : NULL, &x265pic_solo_out);
+                                   pic ? &x265pic : NULL, x265pic_out);
 #endif
 
     for (i = 0; i < sei->numPayloads; i++)
@@ -844,12 +862,6 @@ static int libx265_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
             pkt->flags |= AV_PKT_FLAG_KEY;
     }
 
-#if (X265_BUILD >= 210) && (X265_BUILD < 213)
-    x265pic_out = x265pic_lyrptr_out[0];
-#else
-    x265pic_out = &x265pic_solo_out;
-#endif
-
     pkt->pts = x265pic_out->pts;
     pkt->dts = x265pic_out->dts;
 
@@ -907,6 +919,9 @@ static const enum AVPixelFormat x265_csp_eight[] = {
     AV_PIX_FMT_YUVJ444P,
     AV_PIX_FMT_GBRP,
     AV_PIX_FMT_GRAY8,
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    AV_PIX_FMT_YUVA420P,
+#endif
     AV_PIX_FMT_NONE
 };
 
@@ -924,6 +939,10 @@ static const enum AVPixelFormat x265_csp_ten[] = {
     AV_PIX_FMT_GBRP10,
     AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_GRAY10,
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    AV_PIX_FMT_YUVA420P,
+    AV_PIX_FMT_YUVA420P10,
+#endif
     AV_PIX_FMT_NONE
 };
 
@@ -946,6 +965,10 @@ static const enum AVPixelFormat x265_csp_twelve[] = {
     AV_PIX_FMT_GRAY8,
     AV_PIX_FMT_GRAY10,
     AV_PIX_FMT_GRAY12,
+#if X265_BUILD >= 210 && FF_X265_MAX_LAYERS > 1
+    AV_PIX_FMT_YUVA420P,
+    AV_PIX_FMT_YUVA420P10,
+#endif
     AV_PIX_FMT_NONE
 };
 
-- 
2.45.2