[FFmpeg-devel] [PATCH] avfilter/all: handle ff_formats_unref correctly

[Ganesh Ajjanagadde]

On Sun, Nov 15, 2015 at 7:21 PM, Ganesh Ajjanagadde
<gajjanagadde at gmail.com> wrote:
> Recent commits 6aaac24d72a7da631173209841a3944fcb4a3309 and
> 3835554bf8ed78539a3492c239f979c0ab03a15f made progress towards cleaning
> up usage of the formats API, and in particular fixed possible NULL pointer
> dereferences.
>
> This commit addresses the issue of possible resource leaks when one call
> fails and the others don't.
>
> Fixes: CID 1338330, 1338329, 1338327, 1338326.
>
> Signed-off-by: Ganesh Ajjanagadde <gajjanagadde at gmail.com>
> ---
>  libavfilter/af_channelmap.c |  7 +++++--
>  libavfilter/vf_alphamerge.c |  7 +++++--
>  libavfilter/vf_overlay.c    | 30 +++++++++++++++++++++---------
>  3 files changed, 31 insertions(+), 13 deletions(-)
>
> diff --git a/libavfilter/af_channelmap.c b/libavfilter/af_channelmap.c
> index 9e95a98..45000d0 100644
> --- a/libavfilter/af_channelmap.c
> +++ b/libavfilter/af_channelmap.c
> @@ -296,8 +296,11 @@ static int channelmap_query_formats(AVFilterContext *ctx)
>          (ret = ff_set_common_formats     (ctx             , ff_planar_sample_fmts()             )) < 0 ||
>          (ret = ff_set_common_samplerates (ctx             , ff_all_samplerates()                )) < 0 ||
>          (ret = ff_channel_layouts_ref    (layouts         , &ctx->inputs[0]->out_channel_layouts)) < 0 ||
> -        (ret = ff_channel_layouts_ref    (channel_layouts , &ctx->outputs[0]->in_channel_layouts)) < 0)
> -        return ret;
> +        (ret = ff_channel_layouts_ref    (channel_layouts , &ctx->outputs[0]->in_channel_layouts)) < 0) {
> +            ff_channel_layouts_unref(&channel_layouts);
> +            ff_channel_layouts_unref(&layouts);
> +            return ret;
> +    }
>
>      return 0;
>  }
> diff --git a/libavfilter/vf_alphamerge.c b/libavfilter/vf_alphamerge.c
> index 8a1ca22..dcb7f6c 100644
> --- a/libavfilter/vf_alphamerge.c
> +++ b/libavfilter/vf_alphamerge.c
> @@ -65,8 +65,11 @@ static int query_formats(AVFilterContext *ctx)
>          return AVERROR(ENOMEM);
>      if ((ret = ff_formats_ref(main_formats , &ctx->inputs[0]->out_formats)) < 0 ||
>          (ret = ff_formats_ref(alpha_formats, &ctx->inputs[1]->out_formats)) < 0 ||
> -        (ret = ff_formats_ref(main_formats , &ctx->outputs[0]->in_formats)) < 0)
> -        return ret;
> +        (ret = ff_formats_ref(main_formats , &ctx->outputs[0]->in_formats)) < 0) {
> +            ff_formats_unref(&main_formats);
> +            ff_formats_unref(&alpha_formats);
> +            return ret;
> +    }
>      return 0;
>  }
>
> diff --git a/libavfilter/vf_overlay.c b/libavfilter/vf_overlay.c
> index 3c61731..4873775 100644
> --- a/libavfilter/vf_overlay.c
> +++ b/libavfilter/vf_overlay.c
> @@ -252,23 +252,31 @@ static int query_formats(AVFilterContext *ctx)
>      switch (s->format) {
>      case OVERLAY_FORMAT_YUV420:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv420)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv420))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +        }
>          break;
>      case OVERLAY_FORMAT_YUV422:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv422)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv422))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +        }
>          break;
>      case OVERLAY_FORMAT_YUV444:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_yuv444)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_yuv444))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +        }
>          break;
>      case OVERLAY_FORMAT_RGB:
>          if (!(main_formats    = ff_make_format_list(main_pix_fmts_rgb)) ||
> -            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb)))
> -            return AVERROR(ENOMEM);
> +            !(overlay_formats = ff_make_format_list(overlay_pix_fmts_rgb))) {
> +                ret = AVERROR(ENOMEM);
> +                goto fail;
> +        }
>          break;
>      default:
>          av_assert0(0);
> @@ -277,9 +285,13 @@ static int query_formats(AVFilterContext *ctx)
>      if ((ret = ff_formats_ref(main_formats   , &ctx->inputs[MAIN]->out_formats   )) < 0 ||
>          (ret = ff_formats_ref(overlay_formats, &ctx->inputs[OVERLAY]->out_formats)) < 0 ||
>          (ret = ff_formats_ref(main_formats   , &ctx->outputs[MAIN]->in_formats   )) < 0)
> -        return ret;
> +            goto fail;
>
>      return 0;
> +fail:
> +    ff_formats_unref(&main_formats);
> +    ff_formats_unref(&overlay_formats);
> +    return ret;
>  }
>
>  static const enum AVPixelFormat alpha_pix_fmts[] = {
> --
> 2.6.2
>

ping - reasonably important to get fixed and been nearly a week