[FFmpeg-devel] [PATCH] libavfilter/vf_codecview: enable qp visualization for H264 and VP9

[Yongle Lin]

Add qp visualization in codecview filter which supports H264 and VP9 codecs. Add options for luma/chroma qp and AC/DC qp as well. There is a old way to visualize it but it's deprecated since version 58.
example command line to visualize qp:
./ffmpeg -export_side_data +venc_params -i input.mp4 -vf codecview=qp=true output.mp4
---
 doc/filters.texi           |  6 ++++
 libavfilter/vf_codecview.c | 69 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 74 insertions(+), 1 deletion(-)

diff --git a/doc/filters.texi b/doc/filters.texi
index 84567dec16..f4a57e993f 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -7285,6 +7285,12 @@ backward predicted MVs of B-frames
 @item qp
 Display quantization parameters using the chroma planes.
 
+ at item chroma_qp
+Display chroma quantization parameters (default luma qp) using the chroma planes. Should use with qp option. (e.g. codecview=qp=true:chroma_qp=true)
+
+ at item dc_qp
+Display DC quantization parameters (default AC qp) using the chroma planes. Should use with qp option. (e.g. codecview=qp=true:dc_qp=true)
+
 @item mv_type, mvt
 Set motion vectors type to visualize. Includes MVs from all frames unless specified by @var{frame_type} option.
 
diff --git a/libavfilter/vf_codecview.c b/libavfilter/vf_codecview.c
index 331bfba777..f585dfe28e 100644
--- a/libavfilter/vf_codecview.c
+++ b/libavfilter/vf_codecview.c
@@ -34,6 +34,7 @@
 #include "libavutil/opt.h"
 #include "avfilter.h"
 #include "internal.h"
+#include "libavutil/video_enc_params.h"
 
 #define MV_P_FOR  (1<<0)
 #define MV_B_FOR  (1<<1)
@@ -51,6 +52,8 @@ typedef struct CodecViewContext {
     unsigned mv_type;
     int hsub, vsub;
     int qp;
+    int chroma_qp;
+    int dc_qp;
 } CodecViewContext;
 
 #define OFFSET(x) offsetof(CodecViewContext, x)
@@ -63,6 +66,8 @@ static const AVOption codecview_options[] = {
         CONST("bf", "forward predicted MVs of B-frames",  MV_B_FOR,  "mv"),
         CONST("bb", "backward predicted MVs of B-frames", MV_B_BACK, "mv"),
     { "qp", NULL, OFFSET(qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
+    { "chroma_qp", NULL, OFFSET(chroma_qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
+    { "dc_qp", NULL, OFFSET(dc_qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
     { "mv_type", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
     { "mvt",     "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
         CONST("fp", "forward predicted MVs",  MV_TYPE_FOR,  "mv_type"),
@@ -212,6 +217,52 @@ static void draw_arrow(uint8_t *buf, int sx, int sy, int ex,
     draw_line(buf, sx, sy, ex, ey, w, h, stride, color);
 }
 
+static int qp_color_calculate(int qp, enum AVVideoEncParamsType type) {
+    return type == AV_VIDEO_ENC_PARAMS_H264 ? qp * 128 / 31 : qp;
+}
+
+static void get_block_color(AVVideoEncParams *par, AVVideoBlockParams *b, CodecViewContext *s, enum AVColorRange color_range, int *cu, int *cv)
+{
+    const int plane_qp_cu_index = s->chroma_qp ? 1 : 0;
+    const int plane_qp_cv_index = s->chroma_qp ? 2 : 0;
+    const int ac_dc_index = s->dc_qp ? 0 : 1;
+    *cu = qp_color_calculate(par->qp + par->delta_qp[plane_qp_cu_index][ac_dc_index] + b->delta_qp, par->type);
+    *cv = qp_color_calculate(par->qp + par->delta_qp[plane_qp_cv_index][ac_dc_index] + b->delta_qp, par->type);
+    if (color_range == AVCOL_RANGE_MPEG) {
+        // map jpeg color range(0-255) to mpeg color range(16-235)
+        *cu = av_rescale(*cu, 73, 85) + 16;
+        *cv = av_rescale(*cv, 73, 85) + 16;
+    }
+}
+
+static void color_block(AVFrame *frame, CodecViewContext *s, const int src_x, const int src_y, const int b_w, const int b_h, const int cu, const int cv)
+{
+    const int w = AV_CEIL_RSHIFT(frame->width,  s->hsub);
+    const int h = AV_CEIL_RSHIFT(frame->height, s->vsub);
+    const int lzu = frame->linesize[1];
+    const int lzv = frame->linesize[2];
+
+    const int plane_src_x = src_x >> s->hsub;
+    const int plane_src_y = src_y >> s->vsub;
+    const int plane_b_w = b_w >> s->hsub;
+    const int plane_b_h = b_h >> s->vsub;
+    uint8_t *pu = frame->data[1] + plane_src_y * lzu;
+    uint8_t *pv = frame->data[2] + plane_src_y * lzv;
+
+    for (int y = plane_src_y; y < plane_src_y + plane_b_h; y++) {
+        for (int x = plane_src_x; x < plane_src_x + plane_b_w; x++) {
+            if (x >= w)
+                break;
+            pu[x] = cu;
+            pv[x] = cv;
+        }
+        if (y >= h)
+            break;
+        pu += lzu;
+        pv += lzv;
+    }
+}
+
 static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
 {
     AVFilterContext *ctx = inlink->dst;
@@ -240,8 +291,24 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
                 pv += lzv;
             }
         }
-    }
 
+        AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_VIDEO_ENC_PARAMS);
+        if (sd) {
+            AVVideoEncParams *par = (AVVideoEncParams*)sd->data;
+
+            if (par->nb_blocks) {
+                for (int i = 0; i < par->nb_blocks; i++) {
+                    AVVideoBlockParams *b = av_video_enc_params_block(par, i);
+                    int cu, cv;
+                    get_block_color(par, b, s, frame->color_range, &cu, &cv);
+                    color_block(frame, s, b->src_x, b->src_y, b->w, b->h, cu, cv);
+                }
+            } else {
+                const c = qp_color_calculate(par->qp, par->type);
+                color_block(frame, s, 0, 0, frame->width, frame->height, c, c);
+            }
+        }
+    }
     if (s->mv || s->mv_type) {
         AVFrameSideData *sd = av_frame_get_side_data(frame, AV_FRAME_DATA_MOTION_VECTORS);
         if (sd) {
-- 
2.27.0.111.gc72c7da667-goog