[FFmpeg-cvslog] swscale: prevent overflow in coefficient calculation.

[Ronald S. Bultje]

ffmpeg | branch: master | Ronald S. Bultje <rsbultje at gmail.com> | Wed Oct 19 19:56:56 2011 -0700| [8305041e137f4f2a49669dd588bf6ccfbbac2b58] | committer: Ronald S. Bultje

swscale: prevent overflow in coefficient calculation.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=8305041e137f4f2a49669dd588bf6ccfbbac2b58
---

 libswscale/utils.c |   21 +++++++++++----------
 1 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/libswscale/utils.c b/libswscale/utils.c
index 8e5daf9..ea5a1ab 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -271,19 +271,20 @@ static int initFilter(int16_t **outFilter, int16_t **filterPos, int *outFilterSi
                 floatd= d * (1.0/(1<<30));
 
                 if (flags & SWS_BICUBIC) {
-#define SQRT_INT64_MAX 0xb504f333
                     int64_t B= (param[0] != SWS_PARAM_DEFAULT ? param[0] :   0) * (1<<24);
                     int64_t C= (param[1] != SWS_PARAM_DEFAULT ? param[1] : 0.6) * (1<<24);
-                    int64_t dd  = d > SQRT_INT64_MAX ? ((d  >> 1) * d) >> 29 : (d  * d) >> 30;
-                    int64_t ddd = d > SQRT_INT64_MAX || dd > SQRT_INT64_MAX ?
-                                                       ((dd >> 2) * d) >> 28 : (dd * d) >> 30;
-
-                    if      (d < 1LL<<30)
-                        coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30);
-                    else if (d < 1LL<<31)
-                        coeff = (-B-6*C)*ddd + (6*B+30*C)*dd + (-12*B-48*C)*d + (8*B+24*C)*(1<<30);
-                    else
-                        coeff=0.0;
+
+                    if (d >= 1LL<<31) {
+                        coeff = 0.0;
+                    } else {
+                        int64_t dd  = (d  * d) >> 30;
+                        int64_t ddd = (dd * d) >> 30;
+
+                        if (d < 1LL<<30)
+                            coeff = (12*(1<<24)-9*B-6*C)*ddd + (-18*(1<<24)+12*B+6*C)*dd + (6*(1<<24)-2*B)*(1<<30);
+                        else
+                            coeff = (-B-6*C)*ddd + (6*B+30*C)*dd + (-12*B-48*C)*d + (8*B+24*C)*(1<<30);
+                    }
                     coeff *= fone>>(30+24);
                 }
 /*                else if (flags & SWS_X) {