[FFmpeg-devel] [PATCH] avcodec/celp_math: avoid overflow in shift

[Michael Niedermayer]

by making gain unsigned we have 1 bit more available
alternatively we can clip twice as in the g729 reference

Fixes: left shift of 23404 by 17 places cannot be represented in type 'int'
Fixes: 61728/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_ACELP_KELVIN_fuzzer-6280412547383296

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavcodec/celp_math.h      | 2 +-
 libavcodec/g729postfilter.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/libavcodec/celp_math.h b/libavcodec/celp_math.h
index 18888a425d..99a0470719 100644
--- a/libavcodec/celp_math.h
+++ b/libavcodec/celp_math.h
@@ -78,7 +78,7 @@ int64_t ff_dot_product(const int16_t *a, const int16_t *b, int length);
  *
  * @return value << offset, if offset>=0; value >> -offset - otherwise
  */
-static inline int bidir_sal(int value, int offset)
+static inline unsigned bidir_sal(unsigned value, int offset)
 {
     if(offset < 0) return value >> -offset;
     else           return value <<  offset;
diff --git a/libavcodec/g729postfilter.c b/libavcodec/g729postfilter.c
index 26e937f0ba..382db92432 100644
--- a/libavcodec/g729postfilter.c
+++ b/libavcodec/g729postfilter.c
@@ -581,7 +581,7 @@ void ff_g729_postfilter(AudioDSPContext *adsp, int16_t* ht_prev_data, int* voici
 int16_t ff_g729_adaptive_gain_control(int gain_before, int gain_after, int16_t *speech,
                                    int subframe_size, int16_t gain_prev)
 {
-    int gain; // (3.12)
+    unsigned gain; // (3.12)
     int n;
     int exp_before, exp_after;
 
@@ -603,7 +603,7 @@ int16_t ff_g729_adaptive_gain_control(int gain_before, int gain_after, int16_t *
             gain = ((gain_before - gain_after) << 14) / gain_after + 0x4000;
             gain = bidir_sal(gain, exp_after - exp_before);
         }
-        gain = av_clip_int16(gain);
+        gain = FFMIN(gain, 32767);
         gain = (gain * G729_AGC_FAC1 + 0x4000) >> 15; // gain * (1-0.9875)
     } else
         gain = 0;
-- 
2.17.1