[FFmpeg-devel] [PATCH 2/2 v2] checkasm/flacdsp: sanitize lpc arguments

[James Almer]

Fixes signed integer overflows as reported by ubsan.

Signed-off-by: James Almer <jamrial at gmail.com>
---
Now allowing negative values.

 tests/checkasm/flacdsp.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tests/checkasm/flacdsp.c b/tests/checkasm/flacdsp.c
index 6561b4ed20..d694c1093b 100644
--- a/tests/checkasm/flacdsp.c
+++ b/tests/checkasm/flacdsp.c
@@ -21,6 +21,7 @@
 #include <string.h>
 #include "checkasm.h"
 #include "libavcodec/flacdsp.h"
+#include "libavcodec/mathops.h"
 #include "libavutil/common.h"
 #include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
@@ -54,9 +55,10 @@ static void check_decorrelate(uint8_t **ref_dst, uint8_t **ref_src, uint8_t **ne
     bench_new(new_dst, (int32_t **)new_src, channels, BUF_SIZE / sizeof(int32_t), 8);
 }
 
-static void check_lpc(int pred_order)
+static void check_lpc(int pred_order, int bps)
 {
     int qlevel = rnd() % 16;
+    int coeff_prec = (rnd() % 15) + 1;
     LOCAL_ALIGNED_16(int32_t, coeffs, [32]);
     LOCAL_ALIGNED_16(int32_t, dst,  [BUF_SIZE]);
     LOCAL_ALIGNED_16(int32_t, dst0, [BUF_SIZE]);
@@ -64,10 +66,13 @@ static void check_lpc(int pred_order)
 
     declare_func(void, int32_t *, const int[32], int, int, int);
 
+    if (bps <= 16)
+        coeff_prec = av_clip(coeff_prec, 0, 32 - bps - av_log2(pred_order));
+
     for (int i = 0; i < 32; i++)
-        coeffs[i] = rnd();
+        coeffs[i] = sign_extend(rnd(), coeff_prec);
     for (int i = 0; i < BUF_SIZE; i++)
-        dst[i] = rnd();
+        dst[i] = sign_extend(rnd(), bps);
 
     memcpy(dst0, dst, BUF_SIZE * sizeof (int32_t));
     memcpy(dst1, dst, BUF_SIZE * sizeof (int32_t));
@@ -116,10 +121,10 @@ void checkasm_check_flacdsp(void)
 
     for (i = 0; i < FF_ARRAY_ELEMS(pred_orders); i++)
         if (check_func(h.lpc16, "flac_lpc_16_%d", pred_orders[i]))
-            check_lpc(pred_orders[i]);
+            check_lpc(pred_orders[i], 16);
     for (i = 0; i < FF_ARRAY_ELEMS(pred_orders); i++)
         if (check_func(h.lpc32, "flac_lpc_32_%d", pred_orders[i]))
-            check_lpc(pred_orders[i]);
+            check_lpc(pred_orders[i], 32);
 
     report("lpc");
 }
-- 
2.45.0