[FFmpeg-devel] [PATCH] avcodec/aacenc_quantization: Fix undefined behavior and instead detect and print an error

[Michael Niedermayer]

This is a hotfix and not a real fix of the underlaying bug
The underlaying bug is ATM not fully understood

iam not sure if we should apply this or not

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavcodec/aacenc_quantization.h |   13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libavcodec/aacenc_quantization.h b/libavcodec/aacenc_quantization.h
index 4250407..d367be0 100644
--- a/libavcodec/aacenc_quantization.h
+++ b/libavcodec/aacenc_quantization.h
@@ -141,8 +141,17 @@ static av_always_inline float quantize_and_encode_band_cost_template(
             if (BT_ESC) {
                 for (j = 0; j < 2; j++) {
                     if (ff_aac_codebook_vectors[cb-1][curidx*2+j] == 64.0f) {
-                        int coef = av_clip_uintp2(quant(fabsf(in[i+j]), Q, ROUNDING), 13);
-                        int len = av_log2(coef);
+                        float a = fabsf(in[i+j]) * Q;
+                        double f = sqrtf(a * sqrtf(a)) + ROUNDING;
+                        int coef, len;
+
+                        if (f > INT_MAX || f < 16) {
+                            av_log(NULL, AV_LOG_ERROR, "f %f is out of range this is a internal error\n", f);
+                            f = INT_MAX;
+                        }
+
+                        coef = av_clip_uintp2(f, 13);
+                        len = av_log2(coef);
 
                         put_bits(pb, len - 4 + 1, (1 << (len - 4 + 1)) - 2);
                         put_sbits(pb, len, coef);
-- 
1.7.9.5