[FFmpeg-devel] [PATCH 2/4] huffyuvdec: avoid large stack use.

[wm4]

On Wed,  3 Sep 2014 00:01:53 +0200
Reimar Döffinger <Reimar.Doeffinger at gmx.de> wrote:

> Signed-off-by: Reimar Döffinger <Reimar.Doeffinger at gmx.de>
> ---
>  libavcodec/huffyuvdec.c | 15 +++++++++------
>  1 file changed, 9 insertions(+), 6 deletions(-)
> 
> diff --git a/libavcodec/huffyuvdec.c b/libavcodec/huffyuvdec.c
> index 1df77e0..d661328 100644
> --- a/libavcodec/huffyuvdec.c
> +++ b/libavcodec/huffyuvdec.c
> @@ -115,9 +115,9 @@ static int read_len_table(uint8_t *dst, GetBitContext *gb, int n)
>  
>  static int generate_joint_tables(HYuvContext *s)
>  {
> -    uint16_t symbols[1 << VLC_BITS];
> -    uint16_t bits[1 << VLC_BITS];
> -    uint8_t len[1 << VLC_BITS];
> +    uint16_t *symbols = av_mallocz(5 << VLC_BITS);
> +    uint16_t *bits = symbols + (1 << VLC_BITS);

Is that an unchecked malloc?

> +    uint8_t *len = (uint8_t *)(bits + (1 << VLC_BITS));
>      int ret;
>  
>      if (s->bitstream_bpp < 24 || s->version > 2) {
> @@ -147,7 +147,7 @@ static int generate_joint_tables(HYuvContext *s)
>              ff_free_vlc(&s->vlc[4 + p]);
>              if ((ret = ff_init_vlc_sparse(&s->vlc[4 + p], VLC_BITS, i, len, 1, 1,
>                                            bits, 2, 2, symbols, 2, 2, 0)) < 0)
> -                return ret;
> +                goto out;
>          }
>      } else {
>          uint8_t (*map)[4] = (uint8_t(*)[4]) s->pix_bgr_map;
> @@ -191,9 +191,12 @@ static int generate_joint_tables(HYuvContext *s)
>          ff_free_vlc(&s->vlc[4]);
>          if ((ret = init_vlc(&s->vlc[4], VLC_BITS, i, len, 1, 1,
>                              bits, 2, 2, 0)) < 0)
> -            return ret;
> +            goto out;
>      }
> -    return 0;
> +    ret = 0;
> +out:
> +    av_freep(&symbols);
> +    return ret;
>  }
>  
>  static int read_huffman_tables(HYuvContext *s, const uint8_t *src, int length)