[FFmpeg-devel] [PATCH] lavc/vaapi_encode_h265: fix the hevc vaapi encoder output shaking issue.

[Jun Zhao]

From 5db0b2cd0930ed88d853114f13b69a80a44d9c4c Mon Sep 17 00:00:00 2001
From: Jun Zhao <jun.zhao at intel.com>
Date: Fri, 11 Nov 2016 10:04:53 +0800
Subject: [PATCH] lavc/vaapi_encode_h265: fix the hevc vaapi encoder output
 shaking issue.

the max_num_reorder_pics/max_dec_pic_buffering_minus in vps/sps are incorrect.

merge libav commit 314b421dd8910ebed7a8d419a0dbea239a00044e
        vaapi_encode: Decide on GOP setup before initialising sequence parameters

merge libav commit b8cac1e83066aa87e8402c146c81b77a11b5eec3
        vaapi_h265: Fix buffering parameters

Merged-by: Wang, Yi A <yi.a.wang at intel.com>
Merged-by: Jun Zhao <jun.zhao at intel.com>
---
 libavcodec/vaapi_encode.c      | 13 ++++++-------
 libavcodec/vaapi_encode_h265.c |  8 ++++----
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index 21a8656..233f749 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1181,6 +1181,12 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
     ctx->decode_delay = 1;
     ctx->output_order = - ctx->output_delay - 1;
 
+    // Currently we never generate I frames, only IDR.
+    ctx->i_per_idr = 0;
+    ctx->p_per_i = ((avctx->gop_size + avctx->max_b_frames) /
+                    (avctx->max_b_frames + 1));
+    ctx->b_per_p = avctx->max_b_frames;
+
     if (ctx->codec->sequence_params_size > 0) {
         ctx->codec_sequence_params =
             av_mallocz(ctx->codec->sequence_params_size);
@@ -1206,13 +1212,6 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx,
             goto fail;
         }
     }
-
-    // All I are IDR for now.
-    ctx->i_per_idr = 0;
-    ctx->p_per_i = ((avctx->gop_size + avctx->max_b_frames) /
-                    (avctx->max_b_frames + 1));
-    ctx->b_per_p = avctx->max_b_frames;
-
     // This should be configurable somehow.  (Needs testing on a machine
     // where it actually overlaps properly, though.)
     ctx->issue_mode = ISSUE_MODE_MAXIMISE_THROUGHPUT;
diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c
index 45f6f6d..33de041 100644
--- a/libavcodec/vaapi_encode_h265.c
+++ b/libavcodec/vaapi_encode_h265.c
@@ -897,12 +897,12 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
 
         mseq->log2_max_pic_order_cnt_lsb_minus4 = 8;
         mseq->vps_sub_layer_ordering_info_present_flag = 0;
-        mseq->vps_max_dec_pic_buffering_minus1[0] = 1;
-        mseq->vps_max_num_reorder_pics[0]         = ctx->b_per_p;
+        mseq->vps_max_dec_pic_buffering_minus1[0] = (avctx->max_b_frames > 0) + 1;
+        mseq->vps_max_num_reorder_pics[0]         = (avctx->max_b_frames > 0);
         mseq->vps_max_latency_increase_plus1[0]   = 0;
         mseq->sps_sub_layer_ordering_info_present_flag = 0;
-        mseq->sps_max_dec_pic_buffering_minus1[0] = 1;
-        mseq->sps_max_num_reorder_pics[0]         = ctx->b_per_p;
+        mseq->sps_max_dec_pic_buffering_minus1[0] = (avctx->max_b_frames > 0) + 1;
+        mseq->sps_max_num_reorder_pics[0]         = (avctx->max_b_frames > 0);
         mseq->sps_max_latency_increase_plus1[0]   = 0;
 
         mseq->vps_timing_info_present_flag = 1;
-- 
1.8.3.1