[FFmpeg-devel] [PATCH v2 1/3] vaapi_encode_h265: Ensure that tile sizes conform to profile constraints

[Mark Thompson]

While libavcodec doesn't care about these constraints, the HM reference
decoder does enforce them and other decoders may run into problems.
---
 libavcodec/vaapi_encode_h265.c | 35 +++++++++++++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 316030914c..279d0ff1f7 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -1105,7 +1105,7 @@ static av_cold int vaapi_encode_h265_configure(AVCodecContext *avctx)
 {
     VAAPIEncodeContext      *ctx = avctx->priv_data;
     VAAPIEncodeH265Context *priv = avctx->priv_data;
-    int err;
+    int err, i;
 
     err = ff_cbs_init(&priv->cbc, AV_CODEC_ID_HEVC, avctx);
     if (err < 0)
@@ -1143,6 +1143,39 @@ static av_cold int vaapi_encode_h265_configure(AVCodecContext *avctx)
 
     ctx->roi_quant_range = 51 + 6 * (ctx->profile->depth - 8);
 
+    // Ensure that tile sizes conform to profile constraints.
+    if (ctx->tile_cols > 1 || ctx->tile_rows > 1) {
+        if (ctx->surface_width < 256 ||
+            ctx->surface_height < 64) {
+            // Special case to give a more helpful error message when
+            // tiles are not usable at all.
+            av_log(avctx, AV_LOG_ERROR, "The codec does not support "
+                   "tiles at this resolution (%dx%d).\n",
+                   avctx->width, avctx->height);
+            return AVERROR(EINVAL);
+        }
+        for (i = 0; i < ctx->tile_cols; i++) {
+            int width = ctx->tile_col_width[i] * ctx->slice_block_width;
+            if (width < 256) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "Tile column %d with width %d CTUs (%d pixels) "
+                       "is too thin - consider using fewer tile columns.\n",
+                       i, ctx->tile_col_width[i], width);
+                return AVERROR(EINVAL);
+            }
+        }
+        for (i = 0; i < ctx->tile_rows; i++) {
+            int height = ctx->tile_row_height[i] * ctx->slice_block_height;
+            if (height < 64) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "Tile row %d with height %d CTUs (%d pixels) "
+                       "is too short - consider using fewer tile rows.\n",
+                       i, ctx->tile_row_height[i], height);
+                return AVERROR(EINVAL);
+            }
+        }
+    }
+
     return 0;
 }
 
-- 
2.28.0