[FFmpeg-devel] [PATCH] libavcodec/zmbvenc.c: don't allow motion estimation out of range.

[matthew.w.fearnley at gmail.com]

From: Matthew Fearnley <matthew.w.fearnley at gmail.com>

The maximum allowable range for ZMBV motion estimation is [-64..63], since
the dx,dy values are each stored in the upper 7 bits of a signed char.

(Previously, the range was capped in the code to 127, resulting in an
effective range of [-127..126])

Also fix a range error in the zmbv_me() for-loops ('<' instead of '<='),
which made the limit asymmetrical [-N..N-1], and also prevented it from
reaching the blocks touching the bottom/right edges.
The range is now more symmetrical [-N..N], although this requires separate
range caps of 64 and 63 for negative and positive dx,dy.

Practically, this patch fixes graphical glitches e.g. when reencoding the
Commander Keen sample video with me_range 65 or higher:

    ffmpeg -i keen4e_000.avi -c:v zmbv -me_range 65 keen4e_me65.avi

(Glitches are worse with higher me_range values up to 127.)
---
 libavcodec/zmbvenc.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/libavcodec/zmbvenc.c b/libavcodec/zmbvenc.c
index 4d9147657d..300a53b08e 100644
--- a/libavcodec/zmbvenc.c
+++ b/libavcodec/zmbvenc.c
@@ -45,7 +45,7 @@
 typedef struct ZmbvEncContext {
     AVCodecContext *avctx;
 
-    int range;
+    int lrange, urange;
     uint8_t *comp_buf, *work_buf;
     uint8_t pal[768];
     uint32_t pal2[256]; //for quick comparisons
@@ -101,8 +101,8 @@ static int zmbv_me(ZmbvEncContext *c, uint8_t *src, int sstride, uint8_t *prev,
     bh = FFMIN(ZMBV_BLOCK, c->avctx->height - y);
     bv = block_cmp(c, src, sstride, prev, pstride, bw, bh, xored);
     if(!bv) return 0;
-    for(ty = FFMAX(y - c->range, 0); ty < FFMIN(y + c->range, c->avctx->height - bh); ty++){
-        for(tx = FFMAX(x - c->range, 0); tx < FFMIN(x + c->range, c->avctx->width - bw); tx++){
+    for(ty = FFMAX(y - c->lrange, 0); ty <= FFMIN(y + c->urange, c->avctx->height - bh); ty++){
+        for(tx = FFMAX(x - c->lrange, 0); tx <= FFMIN(x + c->urange, c->avctx->width - bw); tx++){
             if(tx == x && ty == y) continue; // we already tested this block
             dx = tx - x;
             dy = ty - y;
@@ -285,9 +285,13 @@ static av_cold int encode_init(AVCodecContext *avctx)
 
     c->curfrm = 0;
     c->keyint = avctx->keyint_min;
-    c->range = 8;
-    if(avctx->me_range > 0)
-        c->range = FFMIN(avctx->me_range, 127);
+
+    // Motion estimation range: maximum distance is -64..63
+    c->lrange = c->urange = 8;
+    if(avctx->me_range > 0){
+        c->lrange = FFMIN(avctx->me_range, 64);
+        c->urange = FFMIN(avctx->me_range, 63);
+    }
 
     if(avctx->compression_level >= 0)
         lvl = avctx->compression_level;
-- 
2.17.1