[FFmpeg-devel] [PATCH] avfilter/vf_ssim: Fix x86 assembly code for SSIM calculation

[Evgeny Pavlov]

This commit fixes bug #10495

The code had several bugs related to post-loop compensation code:
- test assembly instruction performs bitwise AND operation and
generate flags used by jz branch instruction. Wrong test condition
leads to incorrect branching
- Incorrect compensation code for some branches

Signed-off-by: Evgeny Pavlov <lucenticus at gmail.com>
---
 libavfilter/x86/vf_ssim.asm | 25 +++++++++++--------------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/libavfilter/x86/vf_ssim.asm b/libavfilter/x86/vf_ssim.asm
index 78809305de..e3e0c8104b 100644
--- a/libavfilter/x86/vf_ssim.asm
+++ b/libavfilter/x86/vf_ssim.asm
@@ -228,25 +228,22 @@ cglobal ssim_end_line, 3, 3, 7, sum0, sum1, w
 
     ; subpd the ones we added too much
     test              wd, wd
-    jz .end
+    jz               .end
     add               wd, 4
-    test              wd, 3
-    jz .skip3
-    test              wd, 2
-    jz .skip2
-    test              wd, 1
-    jz .skip1
-.skip3:
+    cmp               wd, 1
+    jz               .skip3
+    cmp               wd, 2
+    jz               .skip2
+.skip1:               ; 3 valid => skip 1 invalid
     psrldq            m5, 8
     subpd             m6, m5
-    jmp .end
-.skip2:
-    psrldq            m5, 8
+    jmp              .end
+.skip2:               ; 2 valid => skip 2 invalid
     subpd             m6, m5
+    jmp              .end
+.skip3:               ; 1 valid => skip 3 invalid
+    psrldq            m3, 8
     subpd             m0, m3
-    jmp .end
-.skip1:
-    psrldq            m3, 16
     subpd             m6, m5
 
 .end:
-- 
2.41.0