[FFmpeg-devel] [PATCH] videodsp: fix 1-byte overread in top/bottom READ_NUM_BYTES iterations.

[Ronald S. Bultje]

This can overread (either before start or beyond end) of the buffer in
Nx1 (i.e. height=1) images.

Fixes mozilla bug 1240080.
---
 libavcodec/x86/videodsp.asm | 21 ++++++---------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/libavcodec/x86/videodsp.asm b/libavcodec/x86/videodsp.asm
index 48f5ac0..a807d3b 100644
--- a/libavcodec/x86/videodsp.asm
+++ b/libavcodec/x86/videodsp.asm
@@ -193,14 +193,10 @@ hvar_fn
     mov            valb, [srcq+%2-1]
 %elif (%2-%%off) == 2
     mov            valw, [srcq+%2-2]
-%elifidn %1, body
+%else
     mov            valb, [srcq+%2-1]
-    sal            vald, 16
+    ror            vald, 16
     mov            valw, [srcq+%2-3]
-%elifidn %1, bottom
-    movd mm %+ %%mmx_idx, [srcq+%2-4]
-%else ; top
-    movd mm %+ %%mmx_idx, [srcq+%2-3]
 %endif
 %endif ; (%2-%%off) >= 1
 %endmacro ; READ_NUM_BYTES
@@ -253,18 +249,13 @@ hvar_fn
     mov     [dstq+%2-1], valb
 %elif (%2-%%off) == 2
     mov     [dstq+%2-2], valw
-%elifidn %1, body
-    mov     [dstq+%2-3], valw
-    sar            vald, 16
-    mov     [dstq+%2-1], valb
 %else
-    movd           vald, mm %+ %%mmx_idx
-%ifidn %1, bottom
-    sar            vald, 8
-%endif
     mov     [dstq+%2-3], valw
-    sar            vald, 16
+    ror            vald, 16
     mov     [dstq+%2-1], valb
+%ifnidn %1, body
+    ror            vald, 16
+%endif
 %endif
 %endif ; (%2-%%off) >= 1
 %endmacro ; WRITE_NUM_BYTES
-- 
2.1.2