[FFmpeg-devel] [PATCH 2/2] swscale/output: Bias 16bps output calculations to improve non overflowing range for GBRP16/GBRPF32

[Michael Niedermayer]

Fixes: integer overflow
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libswscale/output.c       | 25 +++++++++++--------------
 libswscale/x86/output.asm | 16 +++++++++++++++-
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/libswscale/output.c b/libswscale/output.c
index df4647adde..5c85bff971 100644
--- a/libswscale/output.c
+++ b/libswscale/output.c
@@ -2372,18 +2372,15 @@ yuv2gbrp16_full_X_c(SwsContext *c, const int16_t *lumFilter,
 
         Y -= c->yuv2rgb_y_offset;
         Y *= c->yuv2rgb_y_coeff;
-        Y += 1 << 13;
+        Y += (1 << 13) - (1 << 29);
         R = V * c->yuv2rgb_v2r_coeff;
         G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff;
         B =                            U * c->yuv2rgb_u2b_coeff;
 
-        R = av_clip_uintp2(Y + R, 30);
-        G = av_clip_uintp2(Y + G, 30);
-        B = av_clip_uintp2(Y + B, 30);
+        dest16[2][i] = av_clip_uintp2(((Y + R) >> 14) + (1<<15), 16);
+        dest16[0][i] = av_clip_uintp2(((Y + G) >> 14) + (1<<15), 16);
+        dest16[1][i] = av_clip_uintp2(((Y + B) >> 14) + (1<<15), 16);
 
-        dest16[0][i] = G >> 14;
-        dest16[1][i] = B >> 14;
-        dest16[2][i] = R >> 14;
         if (hasAlpha)
             dest16[3][i] = av_clip_uintp2(A, 30) >> 14;
     }
@@ -2448,18 +2445,18 @@ yuv2gbrpf32_full_X_c(SwsContext *c, const int16_t *lumFilter,
 
         Y -= c->yuv2rgb_y_offset;
         Y *= c->yuv2rgb_y_coeff;
-        Y += 1 << 13;
+        Y += (1 << 13) - (1 << 29);
         R = V * c->yuv2rgb_v2r_coeff;
         G = V * c->yuv2rgb_v2g_coeff + U * c->yuv2rgb_u2g_coeff;
         B =                            U * c->yuv2rgb_u2b_coeff;
 
-        R = av_clip_uintp2(Y + R, 30);
-        G = av_clip_uintp2(Y + G, 30);
-        B = av_clip_uintp2(Y + B, 30);
+        R = av_clip_uintp2(((Y + R) >> 14) + (1<<15), 16);
+        G = av_clip_uintp2(((Y + G) >> 14) + (1<<15), 16);
+        B = av_clip_uintp2(((Y + B) >> 14) + (1<<15), 16);
 
-        dest32[0][i] = av_float2int(float_mult * (float)(G >> 14));
-        dest32[1][i] = av_float2int(float_mult * (float)(B >> 14));
-        dest32[2][i] = av_float2int(float_mult * (float)(R >> 14));
+        dest32[0][i] = av_float2int(float_mult * (float)G);
+        dest32[1][i] = av_float2int(float_mult * (float)B);
+        dest32[2][i] = av_float2int(float_mult * (float)R);
         if (hasAlpha)
             dest32[3][i] = av_float2int(float_mult * (float)(av_clip_uintp2(A, 30) >> 14));
     }
diff --git a/libswscale/x86/output.asm b/libswscale/x86/output.asm
index 84e94baaf6..f943a27534 100644
--- a/libswscale/x86/output.asm
+++ b/libswscale/x86/output.asm
@@ -44,11 +44,13 @@ pd_yuv2gbrp_y_start:       times 8 dd  (1 << 9)
 pd_yuv2gbrp_uv_start:      times 8 dd  ((1 << 9) - (128 << 19))
 pd_yuv2gbrp_a_start:       times 8 dd  (1 << 18)
 pd_yuv2gbrp16_offset:      times 8 dd  0x10000  ;(1 << 16)
-pd_yuv2gbrp16_round13:     times 8 dd  0x02000  ;(1 << 13)
+pd_yuv2gbrp16_round13:     times 8 dd  0xE0002000  ;(1 << 13) - (1 << 29)
 pd_yuv2gbrp16_a_offset:    times 8 dd  0x20002000
 pd_yuv2gbrp16_upper30:     times 8 dd  0x3FFFFFFF ;(1<<30) - 1
 pd_yuv2gbrp16_upper27:     times 8 dd  0x07FFFFFF ;(1<<27) - 1
+pd_yuv2gbrp16_upper16:     times 8 dd  0x0000FFFF ;(1<<16) - 1
 pd_yuv2gbrp16_upperC:      times 8 dd  0xC0000000
+pd_yuv2gbrp_debias:        times 8 dd  0x00008000 ;(1 << 29 - 14)
 pb_pack_shuffle8:       db  0,  4,  8, 12, \
                            -1, -1, -1, -1, \
                            -1, -1, -1, -1, \
@@ -883,14 +885,26 @@ cglobal yuv2%1_full_X, 12, 14, 16, ptr, lumFilter, lumSrcx, lumFilterSize, chrFi
         paddd G, Y
         paddd B, Y
 
+%if  DEPTH < 16
         CLIPP2 R, 30
         CLIPP2 G, 30
         CLIPP2 B, 30
+%endif
 
         psrad R, RGB_SHIFT
         psrad G, RGB_SHIFT
         psrad B, RGB_SHIFT
 
+%if  DEPTH >= 16
+        paddd R, [pd_yuv2gbrp_debias]
+        paddd G, [pd_yuv2gbrp_debias]
+        paddd B, [pd_yuv2gbrp_debias]
+
+        CLIPP2 R, 16
+        CLIPP2 G, 16
+        CLIPP2 B, 16
+%endif
+
 %if FLOAT
         cvtdq2ps R, R
         cvtdq2ps G, G
-- 
2.17.1