[FFmpeg-devel] [PATCH v3] libavutil/hwcontext_qsv: Align width and heigh when download qsv frame

[Xiang, Haihao]

On Wed, 2022-04-06 at 18:10 +0800, Wenbin Chen wrote:
> The width and height for qsv frame to download need to be
> aligned with 16. Add the alignment operation.
> Now the following command works:
> ffmpeg -hwaccel qsv -f rawvideo -s 1920x1080 -pix_fmt yuv420p -i \
> input.yuv -vf "hwupload=extra_hw_frames=16,format=qsv,hwdownload, \
> format=nv12" -f null -
> 
> Signed-off-by: Wenbin Chen <wenbin.chen at intel.com>
> ---
>  libavutil/hwcontext_qsv.c | 47 ++++++++++++++++++++++++++++++++++-----
>  1 file changed, 42 insertions(+), 5 deletions(-)
> 
> diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c
> index 95f8071abe..66c0e38955 100644
> --- a/libavutil/hwcontext_qsv.c
> +++ b/libavutil/hwcontext_qsv.c
> @@ -91,7 +91,8 @@ typedef struct QSVFramesContext {
>  
>      mfxExtOpaqueSurfaceAlloc opaque_alloc;
>      mfxExtBuffer *ext_buffers[1];
> -    AVFrame realigned_tmp_frame;
> +    AVFrame realigned_upload_frame;
> +    AVFrame realigned_download_frame;
>  } QSVFramesContext;
>  
>  static const struct {
> @@ -303,7 +304,8 @@ static void qsv_frames_uninit(AVHWFramesContext *ctx)
>      av_freep(&s->surface_ptrs);
>      av_freep(&s->surfaces_internal);
>      av_freep(&s->handle_pairs_internal);
> -    av_frame_unref(&s->realigned_tmp_frame);
> +    av_frame_unref(&s->realigned_upload_frame);
> +    av_frame_unref(&s->realigned_download_frame);
>      av_buffer_unref(&s->child_frames_ref);
>  }
>  
> @@ -1058,21 +1060,46 @@ static int qsv_transfer_data_from(AVHWFramesContext
> *ctx, AVFrame *dst,
>      mfxSyncPoint sync = NULL;
>      mfxStatus err;
>      int ret = 0;
> +    /* download to temp frame if the output is not padded as libmfx requires
> */
> +    AVFrame *tmp_frame = &s->realigned_download_frame;
> +    AVFrame *dst_frame;
> +    int realigned = 0;
>  
>      ret = qsv_internal_session_check_init(ctx, 0);
>      if (ret < 0)
>          return ret;
>  
> +    /* According to MSDK spec for mfxframeinfo, "Width must be a multiple of
> 16.
> +     * Height must be a multiple of 16 for progressive frame sequence and a
> +     * multiple of 32 otherwise.", so allign all frames to 16 before
> downloading. */
> +    if (dst->height & 15 || dst->linesize[0] & 15) {
> +        realigned = 1;
> +        if (tmp_frame->format != dst->format ||
> +            tmp_frame->width  != FFALIGN(dst->linesize[0], 16) ||
> +            tmp_frame->height != FFALIGN(dst->height, 16)) {
> +            av_frame_unref(tmp_frame);
> +
> +            tmp_frame->format = dst->format;
> +            tmp_frame->width  = FFALIGN(dst->linesize[0], 16);
> +            tmp_frame->height = FFALIGN(dst->height, 16);
> +            ret = av_frame_get_buffer(tmp_frame, 0);
> +            if (ret < 0)
> +                return ret;
> +        }
> +    }
> +
> +    dst_frame = realigned ? tmp_frame : dst;
> +
>      if (!s->session_download) {
>          if (s->child_frames_ref)
> -            return qsv_transfer_data_child(ctx, dst, src);
> +            return qsv_transfer_data_child(ctx, dst_frame, src);
>  
>          av_log(ctx, AV_LOG_ERROR, "Surface download not possible\n");
>          return AVERROR(ENOSYS);
>      }
>  
>      out.Info = in->Info;
> -    map_frame_to_surface(dst, &out);
> +    map_frame_to_surface(dst_frame, &out);
>  
>      do {
>          err = MFXVideoVPP_RunFrameVPPAsync(s->session_download, in, &out,
> NULL, &sync);
> @@ -1093,6 +1120,16 @@ static int qsv_transfer_data_from(AVHWFramesContext
> *ctx, AVFrame *dst,
>          return AVERROR_UNKNOWN;
>      }
>  
> +    if (realigned) {
> +        tmp_frame->width  = dst->width;
> +        tmp_frame->height = dst->height;
> +        ret = av_frame_copy(dst, tmp_frame);
> +        tmp_frame->width  = FFALIGN(dst->linesize[0], 16);
> +        tmp_frame->height = FFALIGN(dst->height, 16);
> +        if (ret < 0)
> +            return ret;
> +    }
> +
>      return 0;
>  }
>  
> @@ -1108,7 +1145,7 @@ static int qsv_transfer_data_to(AVHWFramesContext *ctx,
> AVFrame *dst,
>      mfxStatus err;
>      int ret = 0;
>      /* make a copy if the input is not padded as libmfx requires */
> -    AVFrame *tmp_frame = &s->realigned_tmp_frame;
> +    AVFrame *tmp_frame = &s->realigned_upload_frame;
>      const AVFrame *src_frame;
>      int realigned = 0;

LGTM and applied, thx

-Haihao