[FFmpeg-devel] [PATCH v2 15/32] avfilter/palettegen: change cut score from ∑e² to max e²

[Clément Bœsch]

This is following the results from personal research¹.

¹: https://github.com/ubitux/research/tree/main/color-quantization#results
---
 libavfilter/vf_palettegen.c        | 3 ++-
 tests/ref/fate/filter-palettegen-1 | 2 +-
 tests/ref/fate/filter-palettegen-2 | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/libavfilter/vf_palettegen.c b/libavfilter/vf_palettegen.c
index 7ecb1211ba..00b5f88c49 100644
--- a/libavfilter/vf_palettegen.c
+++ b/libavfilter/vf_palettegen.c
@@ -171,7 +171,8 @@ static void compute_box_stats(PaletteGenContext *s, struct range_box *box)
     if (er2[0] >= er2[1] && er2[0] >= er2[2]) box->major_axis = 0;
     if (er2[1] >= er2[0] && er2[1] >= er2[2]) box->major_axis = 1; // prefer green again
 
-    box->cut_score = er2[0] + er2[1] + er2[2];
+    /* The box that has the axis with the biggest error amongst all boxes will but cut down */
+    box->cut_score = FFMAX3(er2[0], er2[1], er2[2]);
 }
 
 /**
diff --git a/tests/ref/fate/filter-palettegen-1 b/tests/ref/fate/filter-palettegen-1
index 278d831846..57be338b42 100644
--- a/tests/ref/fate/filter-palettegen-1
+++ b/tests/ref/fate/filter-palettegen-1
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 16x16
 #sar 0: 1/1
-0,          0,          0,        1,     1024, 0x394ee723
+0,          0,          0,        1,     1024, 0x21c6e6c4
diff --git a/tests/ref/fate/filter-palettegen-2 b/tests/ref/fate/filter-palettegen-2
index e9bc635c81..bcdf54af95 100644
--- a/tests/ref/fate/filter-palettegen-2
+++ b/tests/ref/fate/filter-palettegen-2
@@ -3,4 +3,4 @@
 #codec_id 0: rawvideo
 #dimensions 0: 16x16
 #sar 0: 1/1
-0,          0,          0,        1,     1024, 0xc54d773d
+0,          0,          0,        1,     1024, 0x630d76b1
-- 
2.39.0