[FFmpeg-devel] [PATCH v3 19/41] vaapi_encode_h264: Properly set constraint flags

[Mark Thompson]

constraint_set1_flag should be set for constrained baseline and main
profiles, because the stream conforms to main profile.

constraint_set3_flag should be set for high profile when the stream
is intra-only.

constraint_set4_flag should always be set for main and high profiles
because interlaced encoding is not supported.

constraint_set5_flag should be set for main and high profiles when
B-frames are not used.

Also fix the setting of max_num_ref_frames - use the gop_size value
to check for intra-only rather than the constraint flag (which is not
necessarily set).
---
 libavcodec/vaapi_encode_h264.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c
index b233a91d04..bba6848f03 100644
--- a/libavcodec/vaapi_encode_h264.c
+++ b/libavcodec/vaapi_encode_h264.c
@@ -305,10 +305,19 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
     sps->nal_unit_header.nal_unit_type = H264_NAL_SPS;
 
     sps->profile_idc = avctx->profile & 0xff;
-    sps->constraint_set1_flag =
-        !!(avctx->profile & FF_PROFILE_H264_CONSTRAINED);
-    sps->constraint_set3_flag =
-        !!(avctx->profile & FF_PROFILE_H264_INTRA);
+
+    if (avctx->profile == FF_PROFILE_H264_CONSTRAINED_BASELINE ||
+        avctx->profile == FF_PROFILE_H264_MAIN)
+        sps->constraint_set1_flag = 1;
+
+    if (avctx->profile == FF_PROFILE_H264_HIGH)
+        sps->constraint_set3_flag = ctx->gop_size == 1;
+
+    if (avctx->profile == FF_PROFILE_H264_MAIN ||
+        avctx->profile == FF_PROFILE_H264_HIGH) {
+        sps->constraint_set4_flag = 1;
+        sps->constraint_set5_flag = ctx->b_per_p == 0;
+    }
 
     sps->level_idc = avctx->level;
 
@@ -321,8 +330,7 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
         av_clip(av_log2(ctx->b_per_p + 1) - 2, 0, 12);
 
     sps->max_num_ref_frames =
-        (avctx->profile & FF_PROFILE_H264_INTRA) ? 0 :
-        1 + (ctx->b_per_p > 0);
+        ctx->gop_size == 1 ? 0 : 1 + (ctx->b_per_p > 0);
 
     sps->pic_width_in_mbs_minus1        = priv->mb_width  - 1;
     sps->pic_height_in_map_units_minus1 = priv->mb_height - 1;
-- 
2.18.0