[FFmpeg-cvslog] indeo3: do not try to output more lines than we can fit

[Kostya Shishkov]

ffmpeg | branch: master | Kostya Shishkov <kostya.shishkov at gmail.com> | Thu Oct 18 11:45:42 2012 +0200| [169514c440a75e8967f4416d1ec2bd0402a68876] | committer: Kostya Shishkov

indeo3: do not try to output more lines than we can fit

Internally chroma planes have multiple of four height while allocated image
planes might be smaller if CODEC_FLAG_EMU_EDGE is set. Thus we should not
output more lines of chroma than frame can accept.

Also the decoder can be safely switched to direct rendering now.

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=169514c440a75e8967f4416d1ec2bd0402a68876
---

 libavcodec/indeo3.c |   20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/libavcodec/indeo3.c b/libavcodec/indeo3.c
index c99cee8..fc0d2bc 100644
--- a/libavcodec/indeo3.c
+++ b/libavcodec/indeo3.c
@@ -978,14 +978,17 @@ static int decode_frame_headers(Indeo3DecodeContext *ctx, AVCodecContext *avctx,
  *  @param[in]  buf_sel      indicates which frame buffer the input data stored in
  *  @param[out] dst          pointer to the buffer receiving converted pixels
  *  @param[in]  dst_pitch    pitch for moving to the next y line
+ *  @param[in]  dst_height   output plane height
  */
-static void output_plane(const Plane *plane, int buf_sel, uint8_t *dst, int dst_pitch)
+static void output_plane(const Plane *plane, int buf_sel, uint8_t *dst,
+                         int dst_pitch, int dst_height)
 {
     int             x,y;
     const uint8_t   *src  = plane->pixels[buf_sel];
     uint32_t        pitch = plane->pitch;
 
-    for (y = 0; y < plane->height; y++) {
+    dst_height = FFMIN(dst_height, plane->height);
+    for (y = 0; y < dst_height; y++) {
         /* convert four pixels at once using SWAR */
         for (x = 0; x < plane->width >> 2; x++) {
             AV_WN32A(dst, (AV_RN32A(src) & 0x7F7F7F7F) << 1);
@@ -1072,9 +1075,15 @@ static int decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         return res;
     }
 
-    output_plane(&ctx->planes[0], ctx->buf_sel, ctx->frame.data[0], ctx->frame.linesize[0]);
-    output_plane(&ctx->planes[1], ctx->buf_sel, ctx->frame.data[1], ctx->frame.linesize[1]);
-    output_plane(&ctx->planes[2], ctx->buf_sel, ctx->frame.data[2], ctx->frame.linesize[2]);
+    output_plane(&ctx->planes[0], ctx->buf_sel,
+                 ctx->frame.data[0], ctx->frame.linesize[0],
+                 avctx->height);
+    output_plane(&ctx->planes[1], ctx->buf_sel,
+                 ctx->frame.data[1], ctx->frame.linesize[1],
+                 (avctx->height + 3) >> 2);
+    output_plane(&ctx->planes[2], ctx->buf_sel,
+                 ctx->frame.data[2], ctx->frame.linesize[2],
+                 (avctx->height + 3) >> 2);
 
     *data_size      = sizeof(AVFrame);
     *(AVFrame*)data = ctx->frame;
@@ -1103,5 +1112,6 @@ AVCodec ff_indeo3_decoder = {
     .init           = decode_init,
     .close          = decode_close,
     .decode         = decode_frame,
+    .capabilities   = CODEC_CAP_DR1,
     .long_name      = NULL_IF_CONFIG_SMALL("Intel Indeo 3"),
 };