[FFmpeg-devel] [PATCH 2/2] checkasm/flacdsp: sanitize lpc arguments

[James Almer]

Fixes signed integer overflows as reported by ubsan.

Signed-off-by: James Almer <jamrial at gmail.com>
---
 tests/checkasm/flacdsp.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/checkasm/flacdsp.c b/tests/checkasm/flacdsp.c
index 6561b4ed20..bf25cea39c 100644
--- a/tests/checkasm/flacdsp.c
+++ b/tests/checkasm/flacdsp.c
@@ -54,9 +54,10 @@ static void check_decorrelate(uint8_t **ref_dst, uint8_t **ref_src, uint8_t **ne
     bench_new(new_dst, (int32_t **)new_src, channels, BUF_SIZE / sizeof(int32_t), 8);
 }
 
-static void check_lpc(int pred_order)
+static void check_lpc(int pred_order, int bps)
 {
     int qlevel = rnd() % 16;
+    int coeff_prec = rnd() % 16;
     LOCAL_ALIGNED_16(int32_t, coeffs, [32]);
     LOCAL_ALIGNED_16(int32_t, dst,  [BUF_SIZE]);
     LOCAL_ALIGNED_16(int32_t, dst0, [BUF_SIZE]);
@@ -64,11 +65,13 @@ static void check_lpc(int pred_order)
 
     declare_func(void, int32_t *, const int[32], int, int, int);
 
+    if (bps <= 16)
+        coeff_prec = av_clip(coeff_prec, 0, 32 - bps - av_log2(pred_order));
+
     for (int i = 0; i < 32; i++)
-        coeffs[i] = rnd();
+        coeffs[i] = av_mod_uintp2(rnd(), coeff_prec);
     for (int i = 0; i < BUF_SIZE; i++)
-        dst[i] = rnd();
-
+        dst[i] = rnd() & ((1LL << bps) - 1);
     memcpy(dst0, dst, BUF_SIZE * sizeof (int32_t));
     memcpy(dst1, dst, BUF_SIZE * sizeof (int32_t));
     call_ref(dst0, coeffs, pred_order, qlevel, BUF_SIZE);
@@ -116,10 +119,10 @@ void checkasm_check_flacdsp(void)
 
     for (i = 0; i < FF_ARRAY_ELEMS(pred_orders); i++)
         if (check_func(h.lpc16, "flac_lpc_16_%d", pred_orders[i]))
-            check_lpc(pred_orders[i]);
+            check_lpc(pred_orders[i], 16);
     for (i = 0; i < FF_ARRAY_ELEMS(pred_orders); i++)
         if (check_func(h.lpc32, "flac_lpc_32_%d", pred_orders[i]))
-            check_lpc(pred_orders[i]);
+            check_lpc(pred_orders[i], 32);
 
     report("lpc");
 }
-- 
2.45.0