[FFmpeg-devel] [PATCH] avcodec/vc1: fix out-of-bounds reference pixel replication

[Michael Niedermayer]

On Sun, May 27, 2018 at 10:27:39PM +0200, Jerome Borsboom wrote:
> Out-of-bounds reference pixel replication should take into account the frame
> coding mode of the reference frame(s), not the frame coding mode of the
> current frame.
> 
> Signed-off-by: Jerome Borsboom <jerome.borsboom at carpalis.nl>
> ---
> This should fix the remaining issue with the SA10180.vc1 test file.
> 
>  libavcodec/vc1_mc.c | 659 ++++++++++++++++++++++++++++++----------------------
>  1 file changed, 379 insertions(+), 280 deletions(-)

This causes segfaults

Program received signal SIGSEGV, Segmentation fault.
0x0000000000c36920 in ff_emu_edge_vfix18_sse ()
(gdb) bt
Python Exception <type 'exceptions.ImportError'> No module named gdb.frames: 
#0  0x0000000000c36920 in ff_emu_edge_vfix18_sse ()
#1  0x00000000009e6426 in emulated_edge_mc (dst=<optimized out>, src=<optimized out>, dst_stride=<optimized out>, src_stride=<optimized out>, block_w=<optimized out>, block_h=<optimized out>, src_x=<optimized out>, src_y=<optimized out>, w=<optimized out>, h=<optimized out>, vfix_tbl=<optimized out>, 
    v_extend_var=<optimized out>, hfix_tbl=<optimized out>, h_extend_var=<optimized out>) at libavcodec/x86/videodsp_init.c:195
#2  0x00000000009e6289 in emulated_edge_mc_sse2 (buf=0x1db3541 "", src=0x300 <error: Cannot access memory at address 0x300>, buf_stride=140737352976960, src_stride=768, block_w=<optimized out>, block_h=<optimized out>, src_x=-1, src_y=-2, w=720, h=240) at libavcodec/x86/videodsp_init.c:256
#3  0x0000000000913cee in ff_vc1_mc_1mv (v=0x1d01200, dir=<optimized out>) at libavcodec/vc1_mc.c:323
#4  0x00000000009086d4 in vc1_decode_p_mb_intfi (v=<optimized out>) at libavcodec/vc1_block.c:1758
#5  0x0000000000906516 in vc1_decode_p_blocks (v=<optimized out>) at libavcodec/vc1_block.c:2796
#6  0x000000000091b1c8 in vc1_decode_frame (avctx=0x1c46500, data=0x1c91ec0, got_frame=0x7fffffffd6c4, avpkt=<optimized out>) at libavcodec/vc1dec.c:1042
#7  0x00000000006ec6fb in decode_simple_internal (avctx=0x1c46500, frame=0x1c91ec0) at libavcodec/decode.c:398
#8  0x00000000006ec647 in decode_simple_receive_frame (avctx=0x1c46500, frame=0x1c91ec0) at libavcodec/decode.c:594
#9  0x00000000006ea0b2 in decode_receive_frame_internal (avctx=<optimized out>, frame=<optimized out>) at libavcodec/decode.c:612
#10 0x00000000006e9e7d in avcodec_send_packet (avctx=0x1c46500, avpkt=<optimized out>) at libavcodec/decode.c:674
#11 0x000000000042abda in decode (avctx=0x1c46500, frame=0x1c92600, got_frame=0x7fffffffd954, pkt=0x300) at fftools/ffmpeg.c:2234
#12 0x000000000042a0e1 in decode_video (ist=0x1c46c40, pkt=0x7fffffffd960, got_output=0x7fffffffd954, duration_pts=0x7fffffffd958, eof=0, decode_failed=0x7fffffffd950) at fftools/ffmpeg.c:2378
#13 0x00000000004234bd in process_input_packet (ist=0x1c46c40, pkt=0x7fffffffdcc0, no_eof=0) at fftools/ffmpeg.c:2619
#14 0x0000000000427574 in process_input (file_index=<optimized out>) at fftools/ffmpeg.c:4457
#15 0x000000000042263d in transcode_step () at fftools/ffmpeg.c:4577
#16 0x0000000000421081 in transcode () at fftools/ffmpeg.c:4631
#17 0x00000000004207b5 in main (argc=<optimized out>, argv=<optimized out>) at fftools/ffmpeg.c:4838

rax            0x0	0
rbx            0x1	1
rcx            0x300	768
rdx            0x7ffff7ee4a40	140737352976960
rsi            0x300	768
rdi            0x1db3541	31143233
rbp            0x7fffffffd390	0x7fffffffd390
rsp            0x7fffffffd338	0x7fffffffd338
r8             0x2	2
r9             0x23	35
r10            0x7ffff7ee4a40	140737352976960
r11            0x1db3540	31143232
r12            0x300	768
r13            0x13	19
r14            0x25	37
r15            0x13	19
rip            0xc36920	0xc36920 <ff_emu_edge_vfix18_sse+16>
eflags         0x10202	[ IF RF ]
cs             0x33	51
ss             0x2b	43
ds             0x0	0
es             0x0	0
fs             0x0	0
gs             0x0	0

   0x0000000000c36900 <ff_emu_edge_vfix17_sse+80>:	(bad)  
   0x0000000000c36901 <ff_emu_edge_vfix17_sse+81>:	movd   %mm0,0xd(%rdi)
   0x0000000000c36905 <ff_emu_edge_vfix17_sse+85>:	add    %rsi,%rdi
   0x0000000000c36908 <ff_emu_edge_vfix17_sse+88>:	dec    %rax
   0x0000000000c3690b <ff_emu_edge_vfix17_sse+91>:	jne    0xc368fe <ff_emu_edge_vfix17_sse+78>
   0x0000000000c3690d <ff_emu_edge_vfix17_sse+93>:	repz retq 
   0x0000000000c3690f <ff_emu_edge_vfix17_sse+95>:	nop
   0x0000000000c36910 <ff_emu_edge_vfix18_sse+0>:	mov    0x8(%rsp),%rax
   0x0000000000c36915 <ff_emu_edge_vfix18_sse+5>:	sub    %r9,%rax
   0x0000000000c36918 <ff_emu_edge_vfix18_sse+8>:	sub    %r8,%r9
   0x0000000000c3691b <ff_emu_edge_vfix18_sse+11>:	test   %r8,%r8
   0x0000000000c3691e <ff_emu_edge_vfix18_sse+14>:	je     0xc36936 <ff_emu_edge_vfix18_sse+38>
=> 0x0000000000c36920 <ff_emu_edge_vfix18_sse+16>:	movups (%rdx),%xmm0
   0x0000000000c36923 <ff_emu_edge_vfix18_sse+19>:	movd   0xe(%rdx),%mm0
   0x0000000000c36927 <ff_emu_edge_vfix18_sse+23>:	movups %xmm0,(%rdi)
   0x0000000000c3692a <ff_emu_edge_vfix18_sse+26>:	movd   %mm0,0xe(%rdi)
   0x0000000000c3692e <ff_emu_edge_vfix18_sse+30>:	add    %rsi,%rdi
   0x0000000000c36931 <ff_emu_edge_vfix18_sse+33>:	dec    %r8
   0x0000000000c36934 <ff_emu_edge_vfix18_sse+36>:	jne    0xc36927 <ff_emu_edge_vfix18_sse+23>
   0x0000000000c36936 <ff_emu_edge_vfix18_sse+38>:	movups (%rdx),%xmm0
   0x0000000000c36939 <ff_emu_edge_vfix18_sse+41>:	movd   0xe(%rdx),%mm0
   0x0000000000c3693d <ff_emu_edge_vfix18_sse+45>:	movups %xmm0,(%rdi)

[...]
-- 
Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB

No human being will ever know the Truth, for even if they happen to say it
by chance, they would not even known they had done so. -- Xenophanes
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: not available
URL: <http://ffmpeg.org/pipermail/ffmpeg-devel/attachments/20180529/c8571cdd/attachment.sig>