[FFmpeg-devel] [PATCH] libavfilter/vf_signalstats.c: add new hsl(Hue, Saturation, Lightness) for filter signalstats v3(fix fate case)
liuyizhuo
yizhuo.liu753 at gmail.com
Thu Mar 9 15:35:17 EET 2023
From: "yizhuo.liu753 at gmail.com" <yizhuo.liu753 at gmail.com>
---
libavfilter/vf_signalstats.c | 227 +++++++++++++++++-
.../fate/filter-metadata-signalstats-yuv420p | 2 +-
.../filter-metadata-signalstats-yuv420p10 | 2 +-
3 files changed, 228 insertions(+), 3 deletions(-)
diff --git a/libavfilter/vf_signalstats.c b/libavfilter/vf_signalstats.c
index b4d1029296..183420f398 100644
--- a/libavfilter/vf_signalstats.c
+++ b/libavfilter/vf_signalstats.c
@@ -2,6 +2,8 @@
* Copyright (c) 2010 Mark Heath mjpeg0 @ silicontrip dot org
* Copyright (c) 2014 Clément Bœsch
* Copyright (c) 2014 Dave Rice @dericed
+ * Copyright (c) 2022 Wang Wei <wangwei1237 at gmail.com>
+ * Copyright (c) 2022 Liu yizhuo <yizhuo.liu753 at gmail.com>
*
* This file is part of FFmpeg.
*
@@ -18,6 +20,11 @@
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+* @file
+ * Add Calculate the HSL color space information.
+ * HSL is a color space that is more perceptually uniform than RGB :
+ * https://en.wikipedia.org/wiki/HSL_and_HSV
*/
#include "libavutil/intreadwrite.h"
@@ -25,6 +32,7 @@
#include "libavutil/pixdesc.h"
#include "filters.h"
#include "internal.h"
+#include "libswscale/swscale.h"
enum FilterMode {
FILTER_NONE = -1,
@@ -36,6 +44,8 @@ enum FilterMode {
typedef struct SignalstatsContext {
const AVClass *class;
+ int lumah; // height of luma plane
+ int lumaw; // width of luma plane
int chromah; // height of chroma plane
int chromaw; // width of chroma plane
int hsub; // horizontal subsampling
@@ -56,6 +66,11 @@ typedef struct SignalstatsContext {
AVFrame *frame_sat;
AVFrame *frame_hue;
+ AVFrame *frame_rgb;
+
+ int *hsl_h;
+ int *hsl_s;
+ int *hsl_l;
} SignalstatsContext;
typedef struct ThreadData {
@@ -65,9 +80,21 @@ typedef struct ThreadData {
typedef struct ThreadDataHueSatMetrics {
const AVFrame *src;
- AVFrame *dst_sat, *dst_hue;
+ AVFrame *dst_sat, *dst_hue, *dst_h, *dst_s, *dst_l;
} ThreadDataHueSatMetrics;
+typedef struct ThreadDataHSLMetrics {
+ const AVFrame *src;
+ int *dst_h, *dst_s, *dst_l;
+} ThreadDataHSLMetrics;
+
+typedef float num;
+
+static const float EPSILON = 1e-9;
+
+/** @brief Equal of A and B */
+#define EQ(A,B) ((fabs((A) - (B)) < EPSILON) ? 1 : 0)
+
#define OFFSET(x) offsetof(SignalstatsContext, x)
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
@@ -110,11 +137,15 @@ static av_cold void uninit(AVFilterContext *ctx)
av_frame_free(&s->frame_prev);
av_frame_free(&s->frame_sat);
av_frame_free(&s->frame_hue);
+ av_frame_free(&s->frame_rgb);
av_freep(&s->jobs_rets);
av_freep(&s->histy);
av_freep(&s->histu);
av_freep(&s->histv);
av_freep(&s->histsat);
+ av_freep(&s->hsl_h);
+ av_freep(&s->hsl_s);
+ av_freep(&s->hsl_l);
}
// TODO: add more
@@ -151,6 +182,23 @@ static AVFrame *alloc_frame(enum AVPixelFormat pixfmt, int w, int h)
return frame;
}
+static int config_input(AVFilterLink *inlink)
+{
+ // Video input data avilable
+ AVFilterContext *ctx = inlink->dst;
+ SignalstatsContext *s = ctx->priv;
+
+ // free previous buffers in case they are allocated already
+ av_frame_free(&s->frame_rgb);
+ s->frame_rgb = alloc_frame(AV_PIX_FMT_RGB24, inlink->w, inlink->h);
+
+ if (!s->frame_rgb) {
+ return AVERROR(ENOMEM);
+ }
+
+ return 0;
+}
+
static int config_output(AVFilterLink *outlink)
{
AVFilterContext *ctx = outlink->src;
@@ -172,12 +220,22 @@ static int config_output(AVFilterLink *outlink)
outlink->w = inlink->w;
outlink->h = inlink->h;
+ s->lumaw = inlink->w;
+ s->lumah = inlink->h;
+
s->chromaw = AV_CEIL_RSHIFT(inlink->w, s->hsub);
s->chromah = AV_CEIL_RSHIFT(inlink->h, s->vsub);
s->fs = inlink->w * inlink->h;
s->cfs = s->chromaw * s->chromah;
+ s->hsl_h = av_malloc_array(s->lumah, sizeof(*s->hsl_h));
+ s->hsl_s = av_malloc_array(s->lumah, sizeof(*s->hsl_s));
+ s->hsl_l = av_malloc_array(s->lumah, sizeof(*s->hsl_l));
+ if (!s->hsl_h || !s->hsl_s || !s->hsl_l) {
+ return AVERROR(ENOMEM);
+ }
+
s->nb_jobs = FFMAX(1, FFMIN(inlink->h, ff_filter_get_nb_threads(ctx)));
s->jobs_rets = av_malloc_array(s->nb_jobs, sizeof(*s->jobs_rets));
if (!s->jobs_rets)
@@ -455,6 +513,110 @@ static const struct {
{NULL}
};
+static void YUV2RGB(const AVFrame* src, enum AVPixelFormat dstFormat, AVFrame* dst)
+{
+ int width = src->width;
+ int height = src->height;
+
+ struct SwsContext* conversion = NULL;
+ conversion = sws_getContext(width,
+ height,
+ (enum AVPixelFormat)src->format,
+ width,
+ height,
+ dstFormat,
+ SWS_FAST_BILINEAR,
+ NULL,
+ NULL,
+ NULL);
+ sws_scale(conversion, (const uint8_t * const *)src->data, src->linesize, 0, height, dst->data, dst->linesize);
+ sws_freeContext(conversion);
+}
+
+/**
+ * @brief Convert an sRGB color to Hue-Saturation-Lightness (HSL)
+ *
+ * @param H, S, L pointers to hold the result
+ * @param R, G, B the input sRGB values scaled in [0,1]
+ *
+ * This routine transforms from sRGB to the double hexcone HSL color space
+ * The sRGB values are assumed to be between 0 and 1. The outputs are
+ * H = hexagonal hue angle (0 <= H < 360),
+ * S = { C/(2L) if L <= 1/2 (0 <= S <= 1),
+ * { C/(2 - 2L) if L > 1/2
+ * L = (max(R',G',B') + min(R',G',B'))/2 (0 <= L <= 1),
+ * where C = max(R',G',B') - min(R',G',B').
+ *
+ * Wikipedia: http://en.wikipedia.org/wiki/HSL_and_HSV
+ */
+static void RGB2HSL(num *H, num *S, num *L, num R, num G, num B)
+{
+ num Max = FFMAX3(R, G, B);
+ num Min = FFMIN3(R, G, B);
+ num C = Max - Min;
+
+ *L = (Max + Min) / 2;
+
+ if (C > 0) {
+ if (EQ(Max, R)) {
+ *H = (G - B) / C;
+
+ if (G < B) {
+ *H += 6;
+ }
+ } else if (EQ(Max, G)) {
+ *H = 2 + (B - R) / C;
+ } else {
+ *H = 4 + (R - G) / C;
+ }
+
+ *H *= 60;
+ *S = (*L <= 0.5) ? (C/(2*(*L))) : (C/(2 - 2*(*L)));
+ } else {
+ *H = *S = 0;
+ }
+}
+
+static int compute_hsl(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
+{
+ int i, j;
+ num H, S, L;
+ ThreadDataHSLMetrics *td = arg;
+ const SignalstatsContext *s = ctx->priv;
+ const AVFrame *src = td->src;
+ int *dst_h = td->dst_h;
+ int *dst_s = td->dst_s;
+ int *dst_l = td->dst_l;
+
+ const int slice_start = (s->lumah * jobnr ) / nb_jobs;
+ const int slice_end = (s->lumah * (jobnr+1)) / nb_jobs;
+
+ const int lsz_src = src->linesize[0];
+ const uint8_t *p_src = src->data[0] + slice_start * lsz_src;
+
+ for (j = slice_start; j < slice_end; j++) {
+ int line_h = 0, line_s = 0, line_l = 0;
+ for (i = 0; i < s->lumaw; i++) {
+ const uint8_t rgbr = p_src[3 * i];
+ const uint8_t rgbg = p_src[3 * i + 1];
+ const uint8_t rgbb = p_src[3 * i + 2];
+
+ RGB2HSL(&H, &S, &L, 1.0 * rgbr / 255, 1.0 * rgbg / 255, 1.0 * rgbb / 255);
+ line_h += (uint16_t)(H + 0.5);
+ line_s += (uint8_t)(S * 100 + 0.5);
+ line_l += (uint8_t)(L * 100 + 0.5);
+ }
+
+ dst_h[j] = line_h / s->lumaw;
+ dst_s[j] = line_s / s->lumaw;
+ dst_l[j] = line_l / s->lumaw;
+
+ p_src += lsz_src;
+ }
+
+ return 0;
+}
+
static int compute_sat_hue_metrics8(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
{
int i, j;
@@ -562,6 +724,7 @@ static int filter_frame8(AVFilterLink *link, AVFrame *in)
int accy, accu, accv;
int accsat, acchue = 0;
int medhue, maxhue;
+ int avgh = 0, avgs = 0, avgl = 0;
int toty = 0, totu = 0, totv = 0, totsat=0;
int tothue = 0;
int dify = 0, difu = 0, difv = 0;
@@ -582,6 +745,16 @@ static int filter_frame8(AVFilterLink *link, AVFrame *in)
.dst_hue = hue,
};
+ ThreadDataHSLMetrics td_hsl = {
+ .dst_h = s->hsl_h,
+ .dst_s = s->hsl_s,
+ .dst_l = s->hsl_l,
+ };
+
+ int sumh = 0;
+ int sums = 0;
+ int suml = 0;
+
if (!s->frame_prev)
s->frame_prev = av_frame_clone(in);
@@ -604,6 +777,22 @@ static int filter_frame8(AVFilterLink *link, AVFrame *in)
ff_filter_execute(ctx, compute_sat_hue_metrics8, &td_huesat,
NULL, FFMIN(s->chromah, ff_filter_get_nb_threads(ctx)));
+ // Calculate HSL information.
+ YUV2RGB(in, AV_PIX_FMT_RGB24, s->frame_rgb);
+ td_hsl.src = s->frame_rgb;
+ ff_filter_execute(ctx, compute_hsl, &td_hsl,
+ NULL, FFMIN(link->h, ff_filter_get_nb_threads(ctx)));
+
+ for (j = 0; j < s->lumah; j++) {
+ sumh += s->hsl_h[j];
+ sums += s->hsl_s[j];
+ suml += s->hsl_l[j];
+ }
+
+ avgh = sumh / s->lumah;
+ avgs = sums / s->lumah;
+ avgl = suml / s->lumah;
+
// Calculate luma histogram and difference with previous frame or field.
memset(s->histy, 0, s->maxsize * sizeof(*s->histy));
for (j = 0; j < link->h; j++) {
@@ -746,6 +935,10 @@ static int filter_frame8(AVFilterLink *link, AVFrame *in)
SET_META("HUEMED", "%d", medhue);
SET_META("HUEAVG", "%g", 1.0 * tothue / s->cfs);
+ SET_META("HAVG", "%d", avgh);
+ SET_META("SAVG", "%d", avgs);
+ SET_META("LAVG", "%d", avgl);
+
SET_META("YDIF", "%g", 1.0 * dify / s->fs);
SET_META("UDIF", "%g", 1.0 * difu / s->cfs);
SET_META("VDIF", "%g", 1.0 * difv / s->cfs);
@@ -793,6 +986,7 @@ static int filter_frame16(AVFilterLink *link, AVFrame *in)
int accy, accu, accv;
int accsat, acchue = 0;
int medhue, maxhue;
+ int avgh = 0, avgs = 0, avgl = 0;
int64_t toty = 0, totu = 0, totv = 0, totsat=0;
int64_t tothue = 0;
int64_t dify = 0, difu = 0, difv = 0;
@@ -813,6 +1007,16 @@ static int filter_frame16(AVFilterLink *link, AVFrame *in)
.dst_hue = hue,
};
+ ThreadDataHSLMetrics td_hsl = {
+ .dst_h = s->hsl_h,
+ .dst_s = s->hsl_s,
+ .dst_l = s->hsl_l,
+ };
+
+ int sumh = 0;
+ int sums = 0;
+ int suml = 0;
+
if (!s->frame_prev)
s->frame_prev = av_frame_clone(in);
@@ -835,6 +1039,22 @@ static int filter_frame16(AVFilterLink *link, AVFrame *in)
ff_filter_execute(ctx, compute_sat_hue_metrics16, &td_huesat,
NULL, FFMIN(s->chromah, ff_filter_get_nb_threads(ctx)));
+ // Calculate HSL information.
+ YUV2RGB(in, AV_PIX_FMT_RGB24, s->frame_rgb);
+ td_hsl.src = s->frame_rgb;
+ ff_filter_execute(ctx, compute_hsl, &td_hsl,
+ NULL, FFMIN(link->h, ff_filter_get_nb_threads(ctx)));
+
+ for (j = 0; j < s->lumah; j++) {
+ sumh += s->hsl_h[j];
+ sums += s->hsl_s[j];
+ suml += s->hsl_l[j];
+ }
+
+ avgh = sumh / s->lumah;
+ avgs = sums / s->lumah;
+ avgl = suml / s->lumah;
+
// Calculate luma histogram and difference with previous frame or field.
memset(s->histy, 0, s->maxsize * sizeof(*s->histy));
for (j = 0; j < link->h; j++) {
@@ -976,6 +1196,10 @@ static int filter_frame16(AVFilterLink *link, AVFrame *in)
SET_META("UDIF", "%g", 1.0 * difu / s->cfs);
SET_META("VDIF", "%g", 1.0 * difv / s->cfs);
+ SET_META("HAVG", "%d", avgh);
+ SET_META("SAVG", "%d", avgs);
+ SET_META("LAVG", "%d", avgl);
+
SET_META("YBITDEPTH", "%d", compute_bit_depth(masky));
SET_META("UBITDEPTH", "%d", compute_bit_depth(masku));
SET_META("VBITDEPTH", "%d", compute_bit_depth(maskv));
@@ -1009,6 +1233,7 @@ static const AVFilterPad signalstats_inputs[] = {
{
.name = "default",
.type = AVMEDIA_TYPE_VIDEO,
+ .config_props = config_input,
.filter_frame = filter_frame,
},
};
diff --git a/tests/ref/fate/filter-metadata-signalstats-yuv420p b/tests/ref/fate/filter-metadata-signalstats-yuv420p
index a42acc8f2b..e098409102 100644
--- a/tests/ref/fate/filter-metadata-signalstats-yuv420p
+++ b/tests/ref/fate/filter-metadata-signalstats-yuv420p
@@ -1 +1 @@
-pts=0|tag:lavfi.signalstats.UBITDEPTH=1|tag:lavfi.signalstats.YMIN=235|tag:lavfi.signalstats.YLOW=235|tag:lavfi.signalstats.YAVG=235|tag:lavfi.signalstats.YHIGH=235|tag:lavfi.signalstats.YMAX=235|tag:lavfi.signalstats.UMIN=128|tag:lavfi.signalstats.ULOW=128|tag:lavfi.signalstats.UAVG=128|tag:lavfi.signalstats.UHIGH=128|tag:lavfi.signalstats.UMAX=128|tag:lavfi.signalstats.VMIN=128|tag:lavfi.signalstats.VLOW=128|tag:lavfi.signalstats.VAVG=128|tag:lavfi.signalstats.VHIGH=128|tag:lavfi.signalstats.VMAX=128|tag:lavfi.signalstats.SATMIN=0|tag:lavfi.signalstats.SATLOW=0|tag:lavfi.signalstats.SATAVG=0|tag:lavfi.signalstats.SATHIGH=0|tag:lavfi.signalstats.SATMAX=0|tag:lavfi.signalstats.HUEMED=180|tag:lavfi.signalstats.HUEAVG=180|tag:lavfi.signalstats.YDIF=0|tag:lavfi.signalstats.UDIF=0|tag:lavfi.signalstats.VDIF=0|tag:lavfi.signalstats.YBITDEPTH=6|tag:lavfi.signalstats.VBITDEPTH=1
+pts=0|tag:lavfi.signalstats.UBITDEPTH=1|tag:lavfi.signalstats.YMIN=235|tag:lavfi.signalstats.YLOW=235|tag:lavfi.signalstats.YAVG=235|tag:lavfi.signalstats.YHIGH=235|tag:lavfi.signalstats.YMAX=235|tag:lavfi.signalstats.UMIN=128|tag:lavfi.signalstats.ULOW=128|tag:lavfi.signalstats.UAVG=128|tag:lavfi.signalstats.UHIGH=128|tag:lavfi.signalstats.UMAX=128|tag:lavfi.signalstats.VMIN=128|tag:lavfi.signalstats.VLOW=128|tag:lavfi.signalstats.VAVG=128|tag:lavfi.signalstats.VHIGH=128|tag:lavfi.signalstats.VMAX=128|tag:lavfi.signalstats.SATMIN=0|tag:lavfi.signalstats.SATLOW=0|tag:lavfi.signalstats.SATAVG=0|tag:lavfi.signalstats.SATHIGH=0|tag:lavfi.signalstats.SATMAX=0|tag:lavfi.signalstats.HUEMED=180|tag:lavfi.signalstats.HUEAVG=180|tag:lavfi.signalstats.HAVG=0|tag:lavfi.signalstats.SAVG=0|tag:lavfi.signalstats.LAVG=100|tag:lavfi.signalstats.YDIF=0|tag:lavfi.signalstats.UDIF=0|tag:lavfi.signalstats.VDIF=0|tag:lavfi.signalstats.YBITDEPTH=6|tag:lavfi.signalstats.VBITDEPTH=1
\ No newline at end of file
diff --git a/tests/ref/fate/filter-metadata-signalstats-yuv420p10 b/tests/ref/fate/filter-metadata-signalstats-yuv420p10
index 1e5afd0820..7245a61fff 100644
--- a/tests/ref/fate/filter-metadata-signalstats-yuv420p10
+++ b/tests/ref/fate/filter-metadata-signalstats-yuv420p10
@@ -1 +1 @@
-pts=0|tag:lavfi.signalstats.UBITDEPTH=2|tag:lavfi.signalstats.YMIN=943|tag:lavfi.signalstats.YLOW=943|tag:lavfi.signalstats.YAVG=943|tag:lavfi.signalstats.YHIGH=943|tag:lavfi.signalstats.YMAX=943|tag:lavfi.signalstats.UMIN=514|tag:lavfi.signalstats.ULOW=514|tag:lavfi.signalstats.UAVG=514|tag:lavfi.signalstats.UHIGH=514|tag:lavfi.signalstats.UMAX=514|tag:lavfi.signalstats.VMIN=514|tag:lavfi.signalstats.VLOW=514|tag:lavfi.signalstats.VAVG=514|tag:lavfi.signalstats.VHIGH=514|tag:lavfi.signalstats.VMAX=514|tag:lavfi.signalstats.SATMIN=2|tag:lavfi.signalstats.SATLOW=2|tag:lavfi.signalstats.SATAVG=2|tag:lavfi.signalstats.SATHIGH=2|tag:lavfi.signalstats.SATMAX=2|tag:lavfi.signalstats.HUEMED=225|tag:lavfi.signalstats.HUEAVG=225|tag:lavfi.signalstats.YDIF=0|tag:lavfi.signalstats.UDIF=0|tag:lavfi.signalstats.VDIF=0|tag:lavfi.signalstats.YBITDEPTH=8|tag:lavfi.signalstats.VBITDEPTH=2
+pts=0|tag:lavfi.signalstats.UBITDEPTH=2|tag:lavfi.signalstats.YMIN=943|tag:lavfi.signalstats.YLOW=943|tag:lavfi.signalstats.YAVG=943|tag:lavfi.signalstats.YHIGH=943|tag:lavfi.signalstats.YMAX=943|tag:lavfi.signalstats.UMIN=514|tag:lavfi.signalstats.ULOW=514|tag:lavfi.signalstats.UAVG=514|tag:lavfi.signalstats.UHIGH=514|tag:lavfi.signalstats.UMAX=514|tag:lavfi.signalstats.VMIN=514|tag:lavfi.signalstats.VLOW=514|tag:lavfi.signalstats.VAVG=514|tag:lavfi.signalstats.VHIGH=514|tag:lavfi.signalstats.VMAX=514|tag:lavfi.signalstats.SATMIN=2|tag:lavfi.signalstats.SATLOW=2|tag:lavfi.signalstats.SATAVG=2|tag:lavfi.signalstats.SATHIGH=2|tag:lavfi.signalstats.SATMAX=2|tag:lavfi.signalstats.HUEMED=225|tag:lavfi.signalstats.HUEAVG=225|tag:lavfi.signalstats.YDIF=0|tag:lavfi.signalstats.UDIF=0|tag:lavfi.signalstats.VDIF=0|tag:lavfi.signalstats.HAVG=300|tag:lavfi.signalstats.SAVG=100|tag:lavfi.signalstats.LAVG=100|tag:lavfi.signalstats.YBITDEPTH=8|tag:lavfi.signalstats.VBITDEPTH=2
\ No newline at end of file
--
2.24.3 (Apple Git-128)
More information about the ffmpeg-devel
mailing list