[FFmpeg-devel] [PATCH v3 2/2] fftools/ffplay: add hwaccel decoding support

[Lynne]

Oct 28, 2023, 20:14 by quinkblack at foxmail.com:

> From: Zhao Zhili <zhilizhao at tencent.com>
>
> ---
> v3: shared vulkan instance between libplacebo and hwcontext
>
>  fftools/ffplay.c          |  43 ++++++
>  fftools/ffplay_renderer.c | 316 +++++++++++++++++++++++++++++++++++++-
>  fftools/ffplay_renderer.h |   2 +
>  3 files changed, 355 insertions(+), 6 deletions(-)
>
> diff --git a/fftools/ffplay.c b/fftools/ffplay.c
> index 305d72d8b8..29b37a5e46 100644
> --- a/fftools/ffplay.c
> +++ b/fftools/ffplay.c
> @@ -352,6 +352,7 @@ static int autorotate = 1;
>  static int find_stream_info = 1;
>  static int filter_nbthreads = 0;
>  static int enable_vulkan = 0;
> +static const char *hwaccel = NULL;
>  
>  /* current context */
>  static int is_full_screen;
> @@ -2557,6 +2558,37 @@ static int audio_open(void *opaque, AVChannelLayout *wanted_channel_layout, int
>  return spec.size;
>  }
>  
> +static int create_hwaccel(AVBufferRef **device_ctx)
> +{
> +    enum AVHWDeviceType type;
> +    int ret;
> +    AVBufferRef *vk_dev;
> +
> +    *device_ctx = NULL;
> +
> +    if (!hwaccel)
> +        return 0;
> +
> +    type = av_hwdevice_find_type_by_name(hwaccel);
> +    if (type == AV_HWDEVICE_TYPE_NONE)
> +        return AVERROR(ENOTSUP);
> +
> +    ret = vk_renderer_get_hw_dev(vk_renderer, &vk_dev);
> +    if (ret < 0)
> +        return ret;
> +
> +    ret = av_hwdevice_ctx_create_derived(device_ctx, type, vk_dev, 0);
> +    if (!ret)
> +        return 0;
> +
> +    if (ret != AVERROR(ENOSYS))
> +        return ret;
> +
> +    av_log(NULL, AV_LOG_WARNING, "Derive %s from vulkan not supported.\n", hwaccel);
> +    ret = av_hwdevice_ctx_create(device_ctx, type, NULL, NULL, 0);
> +    return ret;
> +}
> +
>  /* open a given stream. Return 0 if OK */
>  static int stream_component_open(VideoState *is, int stream_index)
>  {
> @@ -2624,6 +2656,12 @@ static int stream_component_open(VideoState *is, int stream_index)
>  
>  av_dict_set(&opts, "flags", "+copy_opaque", AV_DICT_MULTIKEY);
>  
> +    if (avctx->codec_type == AVMEDIA_TYPE_VIDEO) {
> +        ret = create_hwaccel(&avctx->hw_device_ctx);
> +        if (ret < 0)
> +            goto fail;
> +    }
> +
>  if ((ret = avcodec_open2(avctx, codec, &opts)) < 0) {
>  goto fail;
>  }
> @@ -3625,6 +3663,7 @@ static const OptionDef options[] = {
>  "read and decode the streams to fill missing information with heuristics" },
>  { "filter_threads", HAS_ARG | OPT_INT | OPT_EXPERT, { &filter_nbthreads }, "number of filter threads per graph" },
>  { "enable_vulkan", OPT_BOOL, { &enable_vulkan }, "enable vulkan render" },
> +    { "hwaccel", HAS_ARG | OPT_STRING | OPT_EXPERT, { &hwaccel }, "use HW accelerated decoding" },
>  { NULL, },
>  };
>  
> @@ -3739,6 +3778,10 @@ int main(int argc, char **argv)
>  #ifdef SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR
>  SDL_SetHint(SDL_HINT_VIDEO_X11_NET_WM_BYPASS_COMPOSITOR, "0");
>  #endif
> +        if (hwaccel && !enable_vulkan) {
> +            av_log(NULL, AV_LOG_INFO, "Enable vulkan renderer to support hwaccel %s\n", hwaccel);
> +            enable_vulkan = 1;
> +        }
>  if (enable_vulkan) {
>  vk_renderer = vk_get_renderer();
>  if (vk_renderer) {
> diff --git a/fftools/ffplay_renderer.c b/fftools/ffplay_renderer.c
> index 49e5516d5d..497d758ec7 100644
> --- a/fftools/ffplay_renderer.c
> +++ b/fftools/ffplay_renderer.c
> @@ -35,6 +35,8 @@ struct VkRenderer {
>  
>  int (*create)(VkRenderer *renderer, SDL_Window *window);
>  
> +    int (*get_hw_dev)(VkRenderer *renderer, AVBufferRef **dev);
> +
>  int (*display)(VkRenderer *renderer, AVFrame *frame);
>  
>  int (*resize)(VkRenderer *renderer, int width, int height);
> @@ -54,6 +56,13 @@ typedef struct RendererContext {
>  pl_tex tex[4];
>  
>  pl_log vk_log;
> +
> +    AVBufferRef *hw_device_ref;
> +    AVBufferRef *hw_frame_ref;
> +    enum AVPixelFormat *transfer_formats;
> +    AVHWFramesConstraints *constraints;
> +
> +    AVFrame *vk_frame;
>  } RendererContext;
>  
>  static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg) {
> @@ -74,35 +83,46 @@ static void vk_log_cb(void *log_priv, enum pl_log_level level, const char *msg)
>  static int create(VkRenderer *renderer, SDL_Window *window)
>  {
>  int ret = 0;
> -    unsigned ext_num = 0;
> +    unsigned num_ext = 0;
>  const char **ext = NULL;
>  int w, h;
>  struct pl_log_params vk_log_params = {
>  .log_cb = vk_log_cb,
> -            .log_level = PL_LOG_WARN,
> +            .log_level = PL_LOG_DEBUG,
>  .log_priv = renderer,
>  };
>  RendererContext *ctx = (RendererContext *)renderer;
>  
> -    if (!SDL_Vulkan_GetInstanceExtensions(window, &ext_num, NULL)) {
> +    static const char *opt_exts[] = {
> +        VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME,
> +        VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME,
> +        VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME,
> +        VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME,
> +        VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME,
> +        VK_KHR_VIDEO_QUEUE_EXTENSION_NAME,
> +        "VK_MESA_video_decode_av1",
> +    };
> +    const int num_opt_exts = FF_ARRAY_ELEMS(opt_exts);
> +
> +    if (!SDL_Vulkan_GetInstanceExtensions(window, &num_ext, NULL)) {
>  av_log(NULL, AV_LOG_FATAL, "Failed to get vulkan extensions: %s\n", SDL_GetError());
>  return AVERROR_EXTERNAL;
>

That's not a complete list. Look at optional_device_exts.
There are also device features that both we and libplacebo would like
to be enabled. Look into vulkan_device_create_internal(), the list there
is up to date. FFmpeg requires timeline semaphores, and descriptor_buffer
for lavfi filters to work.

Is there no option to let libavutil create the hwdevice for both SDL and libplacebo?
That way, that code path can also be tested.