[FFmpeg-devel] [PATCH 3/3] avcodec/mpegvideo_enc: use 64bit multiplication in dct_quantize_trellis_c and dct_quantize_c

[Marton Balint]

Fixes corruption with:

ffmpeg -t 1 -filter_complex "sine=f=21,showwaves=scale=cbrt:mode=line:colors=white:draw=full" -c:v mpeg2video -non_linear_quant 1 -qmin 1 -qmax 1 -cpuflags 0 out.mpg

or

ffmpeg -t 1 -filter_complex "sine=f=21,showwaves=scale=cbrt:mode=line:colors=white:draw=full" -c:v mpeg2video -non_linear_quant 1 -qmin 1 -qmax 1 -trellis 1 out.mpg

Signed-off-by: Marton Balint <cus at passwd.hu>
---
 libavcodec/mpegvideo_enc.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index 7001d5a566..d844961a5d 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -3984,9 +3984,9 @@ static int dct_quantize_trellis_c(MpegEncContext *s,
 
     for(i=63; i>=start_i; i--) {
         const int j = scantable[i];
-        int level = block[j] * qmat[j];
+        int64_t level = (int64_t)block[j] * qmat[j];
 
-        if(((unsigned)(level+threshold1))>threshold2){
+        if(((uint64_t)(level+threshold1))>threshold2){
             last_non_zero = i;
             break;
         }
@@ -3994,11 +3994,11 @@ static int dct_quantize_trellis_c(MpegEncContext *s,
 
     for(i=start_i; i<=last_non_zero; i++) {
         const int j = scantable[i];
-        int level = block[j] * qmat[j];
+        int64_t level = (int64_t)block[j] * qmat[j];
 
 //        if(   bias+level >= (1<<(QMAT_SHIFT - 3))
 //           || bias-level >= (1<<(QMAT_SHIFT - 3))){
-        if(((unsigned)(level+threshold1))>threshold2){
+        if(((uint64_t)(level+threshold1))>threshold2){
             if(level>0){
                 level= (bias + level)>>QMAT_SHIFT;
                 coeff[0][i]= level;
@@ -4587,7 +4587,7 @@ static int dct_quantize_c(MpegEncContext *s,
                           int16_t *block, int n,
                           int qscale, int *overflow)
 {
-    int i, j, level, last_non_zero, q, start_i;
+    int i, last_non_zero, q, start_i;
     const int *qmat;
     const uint8_t *scantable;
     int bias;
@@ -4627,10 +4627,10 @@ static int dct_quantize_c(MpegEncContext *s,
     threshold1= (1<<QMAT_SHIFT) - bias - 1;
     threshold2= (threshold1<<1);
     for(i=63;i>=start_i;i--) {
-        j = scantable[i];
-        level = block[j] * qmat[j];
+        const int j = scantable[i];
+        int64_t level = (int64_t)block[j] * qmat[j];
 
-        if(((unsigned)(level+threshold1))>threshold2){
+        if(((uint64_t)(level+threshold1))>threshold2){
             last_non_zero = i;
             break;
         }else{
@@ -4638,12 +4638,12 @@ static int dct_quantize_c(MpegEncContext *s,
         }
     }
     for(i=start_i; i<=last_non_zero; i++) {
-        j = scantable[i];
-        level = block[j] * qmat[j];
+        const int j = scantable[i];
+        int64_t level = (int64_t)block[j] * qmat[j];
 
 //        if(   bias+level >= (1<<QMAT_SHIFT)
 //           || bias-level >= (1<<QMAT_SHIFT)){
-        if(((unsigned)(level+threshold1))>threshold2){
+        if(((uint64_t)(level+threshold1))>threshold2){
             if(level>0){
                 level= (bias + level)>>QMAT_SHIFT;
                 block[j]= level;
-- 
2.43.0