[MPlayer-cvslog] r36295 - trunk/libmpcodecs/vf_mcdeint.c

[cehoyos]

Author: cehoyos
Date: Sun Jun  2 01:07:27 2013
New Revision: 36295

Log:
mcdeint: avoid uninited data read

Do not read padding or out-of-buffer values when computing the output
value for a pixel close to the image buffer edge.

This avoids non visible artifacts which affected the output checksum.

Patch by Stefano Sabatini, stefasab gmail

Modified:
   trunk/libmpcodecs/vf_mcdeint.c

Modified: trunk/libmpcodecs/vf_mcdeint.c
==============================================================================
--- trunk/libmpcodecs/vf_mcdeint.c	Sat Jun  1 06:05:02 2013	(r36294)
+++ trunk/libmpcodecs/vf_mcdeint.c	Sun Jun  2 01:07:27 2013	(r36295)
@@ -115,27 +115,49 @@ static void filter(struct vf_priv_s *p, 
         for(y=0; y<h; y++){
             if((y ^ p->parity) & 1){
                 for(x=0; x<w; x++){
-                    if((x-2)+(y-1)*w>=0 && (x+2)+(y+1)*w<w*h){ //FIXME either alloc larger images or optimize this
+                    if(y>0 && y<h-1){
+                        int is_edge= x<3 || x>w-4;
                         uint8_t *filp= &p->frame_dec->data[i][x + y*fils];
                         uint8_t *srcp= &src[i][x + y*srcs];
                         int diff0= filp[-fils] - srcp[-srcs];
                         int diff1= filp[+fils] - srcp[+srcs];
-                        int spatial_score= ABS(srcp[-srcs-1] - srcp[+srcs-1])
-                                          +ABS(srcp[-srcs  ] - srcp[+srcs  ])
-                                          +ABS(srcp[-srcs+1] - srcp[+srcs+1]) - 1;
                         int temp= filp[0];
 
+#define DELTA(j) av_clip(j, -x, w-1-x)
+
+#define GET_SCORE_EDGE(j)\
+   ABS(srcp[-srcs+DELTA(-1+(j))] - srcp[+srcs+DELTA(-1-(j))])+\
+   ABS(srcp[-srcs+DELTA(j)     ] - srcp[+srcs+DELTA(  -(j))])+\
+   ABS(srcp[-srcs+DELTA(1+(j)) ] - srcp[+srcs+DELTA( 1-(j))])
+
+#define GET_SCORE(j)\
+   ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])+\
+   ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])+\
+   ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)])
+
+#define CHECK_EDGE(j)\
+    {   int score= GET_SCORE_EDGE(j);\
+        if(score < spatial_score){\
+            spatial_score= score;\
+            diff0= filp[-fils+DELTA(j)]    - srcp[-srcs+DELTA(j)];\
+            diff1= filp[+fils+DELTA(-(j))] - srcp[+srcs+DELTA(-(j))];\
+
 #define CHECK(j)\
-    {   int score= ABS(srcp[-srcs-1+j] - srcp[+srcs-1-j])\
-                 + ABS(srcp[-srcs  +j] - srcp[+srcs  -j])\
-                 + ABS(srcp[-srcs+1+j] - srcp[+srcs+1-j]);\
+    {   int score= GET_SCORE(j);\
         if(score < spatial_score){\
             spatial_score= score;\
-            diff0= filp[-fils+j] - srcp[-srcs+j];\
-            diff1= filp[+fils-j] - srcp[+srcs-j];
+            diff0= filp[-fils+(j)] - srcp[-srcs+(j)];\
+            diff1= filp[+fils-(j)] - srcp[+srcs-(j)];\
 
-                        CHECK(-1) CHECK(-2) }} }}
-                        CHECK( 1) CHECK( 2) }} }}
+                        if (is_edge) {
+                            int spatial_score= GET_SCORE_EDGE(0)-1;
+                            CHECK_EDGE(-1) CHECK_EDGE(-2) }} }}
+                            CHECK_EDGE( 1) CHECK_EDGE( 2) }} }}
+                        } else {
+                            int spatial_score= GET_SCORE(0)-1;
+                            CHECK(-1) CHECK(-2) }} }}
+                            CHECK( 1) CHECK( 2) }} }}
+                        }
 #if 0
                         if((diff0 ^ diff1) > 0){
                             int mindiff= ABS(diff0) > ABS(diff1) ? diff1 : diff0;