[FFmpeg-devel] [PATCH v5 6/9] avcodec/vaapi_encode: extract a get_recon_format function to base layer

[tong1.wu at intel.com]

From: Tong Wu <tong1.wu at intel.com>

Get constraints and set recon frame format can be shared with other HW
encoder such as D3D12. Extract this part as a new function to base
layer.

Signed-off-by: Tong Wu <tong1.wu at intel.com>
---
 libavcodec/hw_base_encode.c | 58 +++++++++++++++++++++++++++++++++++++
 libavcodec/hw_base_encode.h |  2 ++
 libavcodec/vaapi_encode.c   | 51 ++------------------------------
 3 files changed, 63 insertions(+), 48 deletions(-)

diff --git a/libavcodec/hw_base_encode.c b/libavcodec/hw_base_encode.c
index bb9fe70239..7497e0397e 100644
--- a/libavcodec/hw_base_encode.c
+++ b/libavcodec/hw_base_encode.c
@@ -836,6 +836,64 @@ int ff_hw_base_init_gop_structure(AVCodecContext *avctx, uint32_t ref_l0, uint32
     return 0;
 }
 
+int ff_hw_base_get_recon_format(AVCodecContext *avctx, const void *hwconfig, enum AVPixelFormat *fmt)
+{
+    HWBaseEncodeContext *ctx = avctx->priv_data;
+    AVHWFramesConstraints *constraints = NULL;
+    enum AVPixelFormat recon_format;
+    int err, i;
+
+    constraints = av_hwdevice_get_hwframe_constraints(ctx->device_ref,
+                                                      hwconfig);
+    if (!constraints) {
+        err = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    // Probably we can use the input surface format as the surface format
+    // of the reconstructed frames.  If not, we just pick the first (only?)
+    // format in the valid list and hope that it all works.
+    recon_format = AV_PIX_FMT_NONE;
+    if (constraints->valid_sw_formats) {
+        for (i = 0; constraints->valid_sw_formats[i] != AV_PIX_FMT_NONE; i++) {
+            if (ctx->input_frames->sw_format ==
+                constraints->valid_sw_formats[i]) {
+                recon_format = ctx->input_frames->sw_format;
+                break;
+            }
+        }
+        if (recon_format == AV_PIX_FMT_NONE) {
+            // No match.  Just use the first in the supported list and
+            // hope for the best.
+            recon_format = constraints->valid_sw_formats[0];
+        }
+    } else {
+        // No idea what to use; copy input format.
+        recon_format = ctx->input_frames->sw_format;
+    }
+    av_log(avctx, AV_LOG_DEBUG, "Using %s as format of "
+           "reconstructed frames.\n", av_get_pix_fmt_name(recon_format));
+
+    if (ctx->surface_width  < constraints->min_width  ||
+        ctx->surface_height < constraints->min_height ||
+        ctx->surface_width  > constraints->max_width ||
+        ctx->surface_height > constraints->max_height) {
+        av_log(avctx, AV_LOG_ERROR, "Hardware does not support encoding at "
+               "size %dx%d (constraints: width %d-%d height %d-%d).\n",
+               ctx->surface_width, ctx->surface_height,
+               constraints->min_width,  constraints->max_width,
+               constraints->min_height, constraints->max_height);
+        err = AVERROR(EINVAL);
+        goto fail;
+    }
+
+    *fmt = recon_format;
+    err = 0;
+fail:
+    av_hwframe_constraints_free(&constraints);
+    return err;
+}
+
 int ff_hw_base_encode_init(AVCodecContext *avctx)
 {
     HWBaseEncodeContext *ctx = avctx->priv_data;
diff --git a/libavcodec/hw_base_encode.h b/libavcodec/hw_base_encode.h
index ca72b7e118..e0133d65f0 100644
--- a/libavcodec/hw_base_encode.h
+++ b/libavcodec/hw_base_encode.h
@@ -279,6 +279,8 @@ int ff_hw_base_rc_mode_configure(AVCodecContext *avctx, const HWBaseEncodeRCMode
 int ff_hw_base_init_gop_structure(AVCodecContext *avctx, uint32_t ref_l0, uint32_t ref_l1,
                                   int flags, int prediction_pre_only);
 
+int ff_hw_base_get_recon_format(AVCodecContext *avctx, const void *hwconfig, enum AVPixelFormat *fmt);
+
 int ff_hw_base_encode_init(AVCodecContext *avctx);
 
 int ff_hw_base_encode_close(AVCodecContext *avctx);
diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index 0bce3ce105..84a81559e1 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1898,9 +1898,8 @@ static av_cold int vaapi_encode_create_recon_frames(AVCodecContext *avctx)
     HWBaseEncodeContext *base_ctx = avctx->priv_data;
     VAAPIEncodeContext       *ctx = avctx->priv_data;
     AVVAAPIHWConfig *hwconfig = NULL;
-    AVHWFramesConstraints *constraints = NULL;
     enum AVPixelFormat recon_format;
-    int err, i;
+    int err;
 
     hwconfig = av_hwdevice_hwconfig_alloc(base_ctx->device_ref);
     if (!hwconfig) {
@@ -1909,52 +1908,9 @@ static av_cold int vaapi_encode_create_recon_frames(AVCodecContext *avctx)
     }
     hwconfig->config_id = ctx->va_config;
 
-    constraints = av_hwdevice_get_hwframe_constraints(base_ctx->device_ref,
-                                                      hwconfig);
-    if (!constraints) {
-        err = AVERROR(ENOMEM);
-        goto fail;
-    }
-
-    // Probably we can use the input surface format as the surface format
-    // of the reconstructed frames.  If not, we just pick the first (only?)
-    // format in the valid list and hope that it all works.
-    recon_format = AV_PIX_FMT_NONE;
-    if (constraints->valid_sw_formats) {
-        for (i = 0; constraints->valid_sw_formats[i] != AV_PIX_FMT_NONE; i++) {
-            if (base_ctx->input_frames->sw_format ==
-                constraints->valid_sw_formats[i]) {
-                recon_format = base_ctx->input_frames->sw_format;
-                break;
-            }
-        }
-        if (recon_format == AV_PIX_FMT_NONE) {
-            // No match.  Just use the first in the supported list and
-            // hope for the best.
-            recon_format = constraints->valid_sw_formats[0];
-        }
-    } else {
-        // No idea what to use; copy input format.
-        recon_format = base_ctx->input_frames->sw_format;
-    }
-    av_log(avctx, AV_LOG_DEBUG, "Using %s as format of "
-           "reconstructed frames.\n", av_get_pix_fmt_name(recon_format));
-
-    if (base_ctx->surface_width  < constraints->min_width  ||
-        base_ctx->surface_height < constraints->min_height ||
-        base_ctx->surface_width  > constraints->max_width ||
-        base_ctx->surface_height > constraints->max_height) {
-        av_log(avctx, AV_LOG_ERROR, "Hardware does not support encoding at "
-               "size %dx%d (constraints: width %d-%d height %d-%d).\n",
-               base_ctx->surface_width, base_ctx->surface_height,
-               constraints->min_width,  constraints->max_width,
-               constraints->min_height, constraints->max_height);
-        err = AVERROR(EINVAL);
+    err = ff_hw_base_get_recon_format(avctx, (const void*)hwconfig, &recon_format);
+    if (err < 0)
         goto fail;
-    }
-
-    av_freep(&hwconfig);
-    av_hwframe_constraints_free(&constraints);
 
     base_ctx->recon_frames_ref = av_hwframe_ctx_alloc(base_ctx->device_ref);
     if (!base_ctx->recon_frames_ref) {
@@ -1978,7 +1934,6 @@ static av_cold int vaapi_encode_create_recon_frames(AVCodecContext *avctx)
     err = 0;
   fail:
     av_freep(&hwconfig);
-    av_hwframe_constraints_free(&constraints);
     return err;
 }
 
-- 
2.41.0.windows.1