[FFmpeg-devel] [PATCH 2/4] libavutil/hwcontext_qsv: fix a bug when malloc handle_pairs_internal

[Chen, Wenbin]

> This commandline cause core dumped:
> ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \
> -hwaccel_output_format vaapi -i input.264 \
> -vf "hwmap=derive_device=qsv,format=qsv" \
> -c:v h264_qsv output.264
> 
> reason: We use nb_surfaces to assign surface to handle_pairs_internal
> but handle_pairs_internal is alloced with the size of init_pool_size.
> This lead to access to illegal address.
> 
> Now change it to use nb_surfaces to allocate handle_pairs_internal and the
> core dumped error is unseen. Also change D3D11VA to use nb_surfaces
> to align to VAAPI and DXVA2.
> 
> Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> ---
>  libavutil/hwcontext_qsv.c | 13 ++++++-------
>  1 file changed, 6 insertions(+), 7 deletions(-)
> 
> diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c
> index c18747f7eb..5a285fd25b 100644
> --- a/libavutil/hwcontext_qsv.c
> +++ b/libavutil/hwcontext_qsv.c
> @@ -1123,8 +1123,7 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
>      case AV_HWDEVICE_TYPE_VAAPI:
>          {
>              AVVAAPIFramesContext *src_hwctx = src_ctx->hwctx;
> -            s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
> -                                                 sizeof(*s->handle_pairs_internal));
> +            s->handle_pairs_internal = av_calloc(src_hwctx->nb_surfaces,
> sizeof(*s->handle_pairs_internal));
>              if (!s->handle_pairs_internal)
>                  return AVERROR(ENOMEM);
>              s->surfaces_internal = av_calloc(src_hwctx->nb_surfaces,
> @@ -1146,15 +1145,15 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
>      case AV_HWDEVICE_TYPE_D3D11VA:
>          {
>              AVD3D11VAFramesContext *src_hwctx = src_ctx->hwctx;
> -            s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
> +            s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
>                                                   sizeof(*s->handle_pairs_internal));
>              if (!s->handle_pairs_internal)
>                  return AVERROR(ENOMEM);
> -            s->surfaces_internal = av_calloc(src_ctx->initial_pool_size,
> +            s->surfaces_internal = av_calloc(src_ctx->nb_surfaces,
>                                               sizeof(*s->surfaces_internal));
>              if (!s->surfaces_internal)
>                  return AVERROR(ENOMEM);
> -            for (i = 0; i < src_ctx->initial_pool_size; i++) {
> +            for (i = 0; i < src_ctx->nb_surfaces; i++) {
>                  qsv_init_surface(dst_ctx, &s->surfaces_internal[i]);
>                  s->handle_pairs_internal[i].first = (mfxMemId)src_hwctx-
> >texture_infos[i].texture;
>                  if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
> @@ -1164,7 +1163,7 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
>                  }
>                  s->surfaces_internal[i].Data.MemId = (mfxMemId)&s-
> >handle_pairs_internal[i];
>              }
> -            dst_hwctx->nb_surfaces = src_ctx->initial_pool_size;
> +            dst_hwctx->nb_surfaces = src_ctx->nb_surfaces;
>              if (src_hwctx->BindFlags & D3D11_BIND_RENDER_TARGET) {
>                  dst_hwctx->frame_type |=
> MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET;
>              } else {
> @@ -1177,7 +1176,7 @@ static int
> qsv_frames_derive_to(AVHWFramesContext *dst_ctx,
>      case AV_HWDEVICE_TYPE_DXVA2:
>          {
>              AVDXVA2FramesContext *src_hwctx = src_ctx->hwctx;
> -            s->handle_pairs_internal = av_calloc(src_ctx->initial_pool_size,
> +            s->handle_pairs_internal = av_calloc(src_ctx->nb_surfaces,
>                                                   sizeof(*s->handle_pairs_internal));
>              if (!s->handle_pairs_internal)
>                  return AVERROR(ENOMEM);
> --
> 2.25.1

ping