[FFmpeg-devel] [PATCH 7/9] avcodec/nvenc: add MV-HEVC encoding support

[Timo Rothenpieler]

From: Diego de Souza <ddesouza at nvidia.com>

Added support for MV-HEVC encoding for stereoscopic videos (2 views
only). Compatible with the framepack filter when using the
AV_STEREO3D_FRAMESEQUENCE format.

Signed-off-by: Diego de Souza <ddesouza at nvidia.com>
---
 libavcodec/nvenc.c      | 49 +++++++++++++++++++++++++++++++++++++++++
 libavcodec/nvenc.h      |  4 ++++
 libavcodec/nvenc_hevc.c |  6 +++++
 3 files changed, 59 insertions(+)

diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index 3403fa8996..a528f1ce93 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -35,6 +35,7 @@
 #include "libavutil/mem.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/mathematics.h"
+#include "libavutil/stereo3d.h"
 #include "atsc_a53.h"
 #include "codec_desc.h"
 #include "encode.h"
@@ -656,6 +657,14 @@ static int nvenc_check_capabilities(AVCodecContext *avctx)
 
     ctx->support_dyn_bitrate = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_DYN_BITRATE_CHANGE);
 
+#ifdef NVENC_HAVE_MVHEVC
+    ret = nvenc_check_cap(avctx, NV_ENC_CAPS_SUPPORT_MVHEVC_ENCODE);
+    if(ctx->multiview && ret <= 0) {
+        av_log(avctx, AV_LOG_WARNING, "Multiview not supported by the device\n");
+        return AVERROR(ENOSYS);
+    }
+#endif
+
     return 0;
 }
 
@@ -1544,6 +1553,11 @@ static av_cold int nvenc_setup_hevc_config(AVCodecContext *avctx)
     }
 #endif
 
+#ifdef NVENC_HAVE_MVHEVC
+    hevc->enableMVHEVC = ctx->multiview;
+    hevc->outputHevc3DReferenceDisplayInfo = ctx->display_sei;
+#endif
+
     return 0;
 }
 
@@ -2469,6 +2483,9 @@ static int nvenc_set_timestamp(AVCodecContext *avctx,
 
     // This can be more than necessary, but we don't know the real reorder delay.
     delay = FFMAX(ctx->encode_config.frameIntervalP - 1, 0);
+#ifdef NVENC_HAVE_MVHEVC
+    delay *= ctx->multiview + 1;
+#endif
     if (ctx->output_frame_num >= delay) {
         pkt->dts = timestamp_queue_dequeue(ctx->timestamp_list);
         ctx->output_frame_num++;
@@ -2875,6 +2892,9 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
     int res, res2;
     int sei_count = 0;
     int i;
+#ifdef NVENC_HAVE_MVHEVC
+    HEVC_3D_REFERENCE_DISPLAY_INFO ref_disp_info = { 0 };
+#endif
 
     NvencContext *ctx = avctx->priv_data;
     NvencDynLoadFunctions *dl_fn = &ctx->nvenc_dload_funcs;
@@ -2952,6 +2972,35 @@ static int nvenc_send_frame(AVCodecContext *avctx, const AVFrame *frame)
     if (res < 0)
         return res;
 
+#ifdef NVENC_HAVE_MVHEVC
+    if (ctx->multiview)
+    {
+        const AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_STEREO3D);
+        if (sd)
+        {
+            const AVStereo3D *stereo = (const AVStereo3D*)sd->data;
+            if (stereo->type == AV_STEREO3D_FRAMESEQUENCE) {
+                ctx->next_view_id = (stereo->view == AV_STEREO3D_VIEW_LEFT) ? 0 : 1;
+            } else {
+                sd = NULL;
+                av_log(avctx, AV_LOG_ERROR, "Stereo format %d not supported! Only AV_STEREO3D_FRAMESEQUENCE is supported!\n", stereo->type);
+            }
+        }
+
+        pic_params.codecPicParams.hevcPicParams.viewId = ctx->next_view_id;
+        if (!sd)
+            ctx->next_view_id ^= 1;
+
+        if (ctx->display_sei)
+        {
+            ref_disp_info.precRefDisplayWidth = 31;
+            ref_disp_info.leftViewId[0] = 0;
+            ref_disp_info.rightViewId[0] = 1;
+            pic_params.codecPicParams.hevcPicParams.p3DReferenceDisplayInfo = &ref_disp_info;
+        }
+    }
+#endif
+
     nv_status = p_nvenc->nvEncEncodePicture(ctx->nvencoder, &pic_params);
 
     for (i = 0; i < sei_count; i++)
diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
index 6f7f8d4e7f..77aef3f188 100644
--- a/libavcodec/nvenc.h
+++ b/libavcodec/nvenc.h
@@ -99,6 +99,7 @@ typedef void ID3D11Device;
 #define NVENC_HAVE_422_SUPPORT
 #define NVENC_HAVE_AV1_UHQ_TUNING
 #define NVENC_HAVE_H264_AND_AV1_TEMPORAL_FILTER
+#define NVENC_HAVE_MVHEVC
 #endif
 
 typedef struct NvencSurface
@@ -245,6 +246,7 @@ typedef struct NvencContext
     void *nvencoder;
 
     uint32_t frame_idx_counter;
+    uint32_t next_view_id;
 
     int preset;
     int profile;
@@ -291,6 +293,7 @@ typedef struct NvencContext
     int single_slice_intra_refresh;
     int constrained_encoding;
     int udu_sei;
+    int display_sei;
     int timing_info;
     int highbitdepth;
     int max_slice_size;
@@ -299,6 +302,7 @@ typedef struct NvencContext
     int lookahead_level;
     int unidir_b;
     int split_encode_mode;
+    int multiview;
 } NvencContext;
 
 int ff_nvenc_encode_init(AVCodecContext *avctx);
diff --git a/libavcodec/nvenc_hevc.c b/libavcodec/nvenc_hevc.c
index 5696e14dd4..f821eaee50 100644
--- a/libavcodec/nvenc_hevc.c
+++ b/libavcodec/nvenc_hevc.c
@@ -194,6 +194,12 @@ static const AVOption options[] = {
                                                             OFFSET(extra_sei),    AV_OPT_TYPE_BOOL,  { .i64 = 1 }, 0, 1, VE },
     { "udu_sei",      "Pass on user data unregistered SEI if available",
                                                             OFFSET(udu_sei),      AV_OPT_TYPE_BOOL,  { .i64 = 0 }, 0, 1, VE },
+#ifdef NVENC_HAVE_MVHEVC
+    { "display_sei",  "Pass on 3D reference display information SEI message for MV-HEVC",
+                                                            OFFSET(display_sei),  AV_OPT_TYPE_BOOL,  { .i64 = 0 }, 0, 1, VE },
+    { "mvhevc",       "Set to 1 to enable MV-HEVC. This feature currently supports only 2 views",
+                                                            OFFSET(multiview),    AV_OPT_TYPE_BOOL,  { .i64 = 0 }, 0, 1, VE },
+#endif
     { "intra-refresh","Use Periodic Intra Refresh instead of IDR frames",
                                                             OFFSET(intra_refresh),AV_OPT_TYPE_BOOL,  { .i64 = 0 }, 0, 1, VE },
     { "single-slice-intra-refresh", "Use single slice intra refresh",
-- 
2.45.2