[FFmpeg-devel] [PATCH] Add missing overflow checks in av_image_fill_pointers() and av_image_fill_linesizes().

[Michael Niedermayer]

On Fri, Nov 26, 2010 at 04:23:47PM +0100, Stefano Sabatini wrote:
> ---
>  libavcore/imgutils.c |   17 ++++++++++++++++-
>  1 files changed, 16 insertions(+), 1 deletions(-)
> 
> diff --git a/libavcore/imgutils.c b/libavcore/imgutils.c
> index 554639f..a2adaa6 100644
> --- a/libavcore/imgutils.c
> +++ b/libavcore/imgutils.c
> @@ -71,6 +71,8 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt
>          return AVERROR(EINVAL);
>  
>      if (desc->flags & PIX_FMT_BITSTREAM) {
> +        if (width > (INT_MAX -7) / (desc->comp[0].step_minus1+1))
> +            return AVERROR(EINVAL);
>          linesizes[0] = (width * (desc->comp[0].step_minus1+1) + 7) >> 3;
>          return 0;
>      }
> @@ -78,7 +80,10 @@ int av_image_fill_linesizes(int linesizes[4], enum PixelFormat pix_fmt, int widt
>      av_image_fill_max_pixsteps(max_step, max_step_comp, desc);
>      for (i = 0; i < 4; i++) {
>          int s = (max_step_comp[i] == 1 || max_step_comp[i] == 2) ? desc->log2_chroma_w : 0;
> -        linesizes[i] = max_step[i] * (((width + (1 << s) - 1)) >> s);
> +        int shifted_w = ((width + (1 << s) - 1)) >> s;
> +        if (max_step[i] > SIZE_MAX / shifted_w)


SIZE_MAX ?


[...]


> @@ -98,11 +103,17 @@ int av_image_fill_pointers(uint8_t *data[4], enum PixelFormat pix_fmt, int heigh
>          return AVERROR(EINVAL);
>  
>      data[0] = ptr;
> +    if (linesizes[0] > INT_MAX / height)
> +        return AVERROR(EINVAL);
>      size[0] = linesizes[0] * height;
>  
>      if (desc->flags & PIX_FMT_PAL) {
> +        if (size[0] > INT_MAX -3)
> +            return AVERROR(EINVAL);
>          size[0] = (size[0] + 3) & ~3;
>          data[1] = ptr + size[0]; /* palette is stored here as 256 32 bits words */
> +        if (size[0] > INT_MAX - 256 * 4)
> +            return AVERROR(EINVAL);

these 3 can be merged, it doesnt matter if our max image size is INT_MAX or
INT_MAX- 1024

[...]
-- 
Michael     GnuPG fingerprint: 9FF2128B147EF6730BADF133611EC787040B0FAB

I have never wished to cater to the crowd; for what I know they do not
approve, and what they approve I do not know. -- Epicurus
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 198 bytes
Desc: Digital signature
URL: <http://lists.mplayerhq.hu/pipermail/ffmpeg-devel/attachments/20101201/60aebe27/attachment.pgp>