[FFmpeg-devel] [PATCH] avcodec/aacps (fixed point): Fix multiple signed integer overflows

[Michael Niedermayer]

Fixes: runtime error: signed integer overflow: 1421978265 - -1810326882 cannot be represented in type 'int'
Fixes: 2527/clusterfuzz-testcase-minimized-5260915396050944

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavcodec/aacps.c | 25 ++++++++-----------------
 1 file changed, 8 insertions(+), 17 deletions(-)

diff --git a/libavcodec/aacps.c b/libavcodec/aacps.c
index 473da7bd43..5758b919a1 100644
--- a/libavcodec/aacps.c
+++ b/libavcodec/aacps.c
@@ -697,26 +697,17 @@ static void decorrelation(PSContext *ps, INTFLOAT (*out)[32][2], const INTFLOAT
     for (i = 0; i < NR_PAR_BANDS[is34]; i++) {
         for (n = n0; n < nL; n++) {
             int decayed_peak;
-            int denom;
-
             decayed_peak = (int)(((int64_t)peak_decay_factor * \
                                            peak_decay_nrg[i] + 0x40000000) >> 31);
             peak_decay_nrg[i] = FFMAX(decayed_peak, power[i][n]);
-            power_smooth[i] += (power[i][n] - power_smooth[i] + 2) >> 2;
-            peak_decay_diff_smooth[i] += (peak_decay_nrg[i] - power[i][n] - \
-                                          peak_decay_diff_smooth[i] + 2) >> 2;
-            denom = peak_decay_diff_smooth[i] + (peak_decay_diff_smooth[i] >> 1);
-            if (denom > power_smooth[i]) {
-              int p = power_smooth[i];
-              while (denom < 0x40000000) {
-                denom <<= 1;
-                p <<= 1;
-              }
-              transient_gain[i][n] = p / (denom >> 16);
-            }
-            else {
-              transient_gain[i][n] = 1 << 16;
-            }
+            power_smooth[i] += (power[i][n] + 2LL - power_smooth[i]) >> 2;
+            peak_decay_diff_smooth[i] += (peak_decay_nrg[i] + 2LL - power[i][n] - \
+                                          peak_decay_diff_smooth[i]) >> 2;
+
+            if (peak_decay_diff_smooth[i]) {
+                transient_gain[i][n] = FFMIN(power_smooth[i]*43691LL / peak_decay_diff_smooth[i], 1<<16);
+            } else
+                transient_gain[i][n] = 1 << 16;
         }
     }
 #else
-- 
2.13.0