[FFmpeg-devel] [PATCH] lavfi/mp/mcdeint: avoid uninited data read

[Stefano Sabatini]

Always read in-buffer pixels when computing the value for a pixel close
to the image buffer edge.

This avoids non visible artifacts which affect the output checksum.
---
 libavfilter/libmpcodecs/vf_mcdeint.c |   32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/libavfilter/libmpcodecs/vf_mcdeint.c b/libavfilter/libmpcodecs/vf_mcdeint.c
index b9ffaf2..eb3e08a 100644
--- a/libavfilter/libmpcodecs/vf_mcdeint.c
+++ b/libavfilter/libmpcodecs/vf_mcdeint.c
@@ -115,24 +115,38 @@ static void filter(struct vf_priv_s *p, uint8_t *dst[3], uint8_t *src[3], int ds
         for(y=0; y<h; y++){
             if((y ^ p->parity) & 1){
                 for(x=0; x<w; x++){
-                    if((x-2)+(y-1)*w>=0 && (x+2)+(y+1)*w<w*h){ //FIXME either alloc larger images or optimize this
+                    if(y>0 && y<h-1){
+                        int is_edge= x<3 || x>w-4;
                         uint8_t *filp= &p->frame_dec->data[i][x + y*fils];
                         uint8_t *srcp= &src[i][x + y*srcs];
                         int diff0= filp[-fils] - srcp[-srcs];
                         int diff1= filp[+fils] - srcp[+srcs];
-                        int spatial_score= ABS(srcp[-srcs-1] - srcp[+srcs-1])
-                                          +ABS(srcp[-srcs  ] - srcp[+srcs  ])
-                                          +ABS(srcp[-srcs+1] - srcp[+srcs+1]) - 1;
+
+#define DELTA(j) av_clip(j, -x, w-1-x)
+
+#define GET_SCORE(j) (is_edge ?\
+   ABS(srcp[-srcs+DELTA(-1+(j))] - srcp[+srcs+DELTA(-1-(j))])+\
+   ABS(srcp[-srcs+DELTA(j)     ] - srcp[+srcs+DELTA(  -(j))])+\
+   ABS(srcp[-srcs+DELTA(1+(j)) ] - srcp[+srcs+DELTA( 1-(j))])\
+   :\
+   ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])+\
+   ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])+\
+   ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)]))
+
+                        int spatial_score= GET_SCORE(0)-1;
                         int temp= filp[0];
 
 #define CHECK(j)\
-    {   int score= ABS(srcp[-srcs-1+(j)] - srcp[+srcs-1-(j)])\
-                 + ABS(srcp[-srcs  +(j)] - srcp[+srcs  -(j)])\
-                 + ABS(srcp[-srcs+1+(j)] - srcp[+srcs+1-(j)]);\
+    {   int score= GET_SCORE(j);\
         if(score < spatial_score){\
             spatial_score= score;\
-            diff0= filp[-fils+(j)] - srcp[-srcs+(j)];\
-            diff1= filp[+fils-(j)] - srcp[+srcs-(j)];
+            if(is_edge){\
+                diff0= filp[-fils+DELTA(j)]    - srcp[-srcs+DELTA(j)];\
+                diff1= filp[+fils+DELTA(-(j))] - srcp[+srcs+DELTA(-(j))]; \
+            }else{\
+                diff0= filp[-fils+(j)] - srcp[-srcs+(j)];\
+                diff1= filp[+fils-(j)] - srcp[+srcs-(j)];\
+            }
 
                         CHECK(-1) CHECK(-2) }} }}
                         CHECK( 1) CHECK( 2) }} }}
-- 
1.7.9.5