[FFmpeg-devel] libavcodec: add Vulkan common video decoding code
Anton Khirnov
anton at khirnov.net
Thu May 18 11:54:23 EEST 2023
> commit adb671b921d006255597ac126f85adb05f9d6677
> Author: Lynne <dev at lynne.ee>
> Date: Mon Jan 16 07:23:27 2023 +0100
>
> libavcodec: add Vulkan common video decoding code
>
> diff --git a/libavcodec/Makefile b/libavcodec/Makefile
> index d99f7bd25a..362ea31e3e 100644
> --- a/libavcodec/Makefile
> +++ b/libavcodec/Makefile
> @@ -1289,7 +1289,7 @@ SKIPHEADERS-$(CONFIG_XVMC) += xvmc.h
> SKIPHEADERS-$(CONFIG_VAAPI) += vaapi_decode.h vaapi_hevc.h vaapi_encode.h
> SKIPHEADERS-$(CONFIG_VDPAU) += vdpau.h vdpau_internal.h
> SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX) += videotoolbox.h vt_internal.h
> -SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h
> +SKIPHEADERS-$(CONFIG_VULKAN) += vulkan.h vulkan_video.h vulkan_decode.h
> SKIPHEADERS-$(CONFIG_V4L2_M2M) += v4l2_buffers.h v4l2_context.h v4l2_m2m.h
> SKIPHEADERS-$(CONFIG_ZLIB) += zlib_wrapper.h
>
> diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c
> new file mode 100644
> index 0000000000..d07b9aa3eb
> --- /dev/null
> +++ b/libavcodec/vulkan_decode.c
> @@ -0,0 +1,1182 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#include "vulkan_video.h"
> +#include "vulkan_decode.h"
> +#include "config_components.h"
> +#include "libavutil/hwcontext_internal.h"
what for?
> +static AVFrame *vk_get_dpb_pool(FFVulkanDecodeShared *ctx)
> +{
> + AVFrame *avf = av_frame_alloc();
> + AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->dpb_hwfc_ref->data;
> + if (!avf)
> + return NULL;
> +
> + avf->hw_frames_ctx = av_buffer_ref(ctx->dpb_hwfc_ref);
> + if (!avf->hw_frames_ctx)
> + av_frame_free(&avf);
> + avf->buf[0] = av_buffer_pool_get(dpb_frames->pool);
> + if (!avf->buf[0])
> + av_frame_free(&avf);
> + avf->data[0] = avf->buf[0]->data;
Why is this not av_hwframe_get_buffer()?
> +void ff_vk_decode_free_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vp)
> +{
> + FFVulkanFunctions *vk;
> + VkSemaphoreWaitInfo sem_wait;
> + FFVulkanDecodeShared *ctx;
> +
> + // TODO: investigate why this happens
> + if (!dec || !dec->shared_ref) {
My guess is that this is called from a different thread than the one
whose hwaccel_priv_data you gave to ff_hwaccel_frame_priv_alloc().
You have to attach everything you need to hwaccel_priv_buf itself.
> +/* Since to even get decoder capabilities, we have to initialize quite a lot,
> + * this function does initialization and saves it to hwaccel_priv_data if
> + * available. */
> +static int vulkan_decode_check_init(AVCodecContext *avctx, AVBufferRef *frames_ref,
> + VulkanVideoProfile *profile_data,
> + int *width_align, int *height_align,
> + enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt,
> + int *dpb_dedicate)
> +{
> + VkResult ret;
> + int err, max_level;
> + const struct FFVkCodecMap *vk_codec = &ff_vk_codec_map[avctx->codec_id];
> + AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data;
> + AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
> + AVVulkanDeviceContext *hwctx = device->hwctx;
> + enum AVPixelFormat source_format;
> + enum AVPixelFormat best_format;
> + VkFormat best_vkfmt;
> + int base_profile, cur_profile = avctx->profile;
> +
> + int dedicated_dpb;
> + int layered_dpb;
> +
> + FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data;
> + FFVulkanDecodeShared *ctx = (FFVulkanDecodeShared *)dec->shared_ref->data;
> +
> + FFVulkanExtensions local_extensions = 0x0;
> + FFVulkanExtensions *extensions = ctx ? &ctx->s.extensions : &local_extensions;
> + FFVulkanFunctions local_vk = { 0 };
> + FFVulkanFunctions *vk = ctx ? &ctx->s.vkfn : &local_vk;
> + VkVideoCapabilitiesKHR local_caps = { 0 };
> + VkVideoCapabilitiesKHR *caps = ctx ? &ctx->common.caps : &local_caps;
> + VkVideoDecodeCapabilitiesKHR local_dec_caps = { 0 };
> + VkVideoDecodeCapabilitiesKHR *dec_caps = ctx ? &ctx->dec_caps : &local_dec_caps;
> +
> + VkVideoDecodeH264ProfileInfoKHR local_h264_profile = { 0 };
> + VkVideoDecodeH264ProfileInfoKHR *h264_profile = profile_data ?
> + &profile_data->h264_profile :
> + &local_h264_profile;
> +
> + VkVideoDecodeH264ProfileInfoKHR local_h265_profile = { 0 };
> + VkVideoDecodeH264ProfileInfoKHR *h265_profile = profile_data ?
> + &profile_data->h265_profile :
> + &local_h265_profile;
> +
> + VkVideoDecodeUsageInfoKHR local_usage = { 0 };
> + VkVideoDecodeUsageInfoKHR *usage = profile_data ?
> + &profile_data->usage : &local_usage;
> + VkVideoProfileInfoKHR local_profile = { 0 };
> + VkVideoProfileInfoKHR *profile = profile_data ?
> + &profile_data->profile : &local_profile;
> + VkVideoProfileListInfoKHR local_profile_list = { 0 };
> + VkVideoProfileListInfoKHR *profile_list = profile_data ?
> + &profile_data->profile_list :
> + &local_profile_list;
> +
> + VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
> + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
> + .pNext = profile_list,
> + };
> + VkVideoDecodeH264CapabilitiesKHR h264_caps = {
> + .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_CAPABILITIES_KHR,
> + };
> + VkVideoDecodeH265CapabilitiesKHR h265_caps = {
> + .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
> + };
> + VkVideoFormatPropertiesKHR *ret_info;
> + uint32_t nb_out_fmts = 0;
> +
> + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(avctx->sw_pix_fmt);
> + if (!desc)
> + return AVERROR(EINVAL);
> +
> + if (ctx && ctx->init)
> + return 0;
> +
> + if (!vk_codec->decode_op)
> + return AVERROR(EINVAL);
> +
> + *extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
> + hwctx->nb_enabled_dev_extensions);
> +
> + if (!(*extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
> + av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
> + VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
> + return AVERROR(ENOSYS);
> + } else if (!vk_codec->decode_extension) {
> + av_log(avctx, AV_LOG_ERROR, "Unsupported codec for Vulkan decoding: %s!\n",
> + avcodec_get_name(avctx->codec_id));
> + return AVERROR(ENOSYS);
> + } else if (!(vk_codec->decode_extension & *extensions)) {
> + av_log(avctx, AV_LOG_ERROR, "Device does not support decoding %s!\n",
> + avcodec_get_name(avctx->codec_id));
> + return AVERROR(ENOSYS);
> + }
> +
> + err = ff_vk_load_functions(device, vk, *extensions, 1, 1);
> + if (err < 0)
> + return err;
> +
> +repeat:
> + if (avctx->codec_id == AV_CODEC_ID_H264) {
> + base_profile = FF_PROFILE_H264_CONSTRAINED_BASELINE;
> + dec_caps->pNext = &h264_caps;
> + usage->pNext = h264_profile;
> + h264_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR;
> + h264_profile->stdProfileIdc = cur_profile;
> + h264_profile->pictureLayout = avctx->field_order == AV_FIELD_UNKNOWN ||
> + avctx->field_order == AV_FIELD_PROGRESSIVE ?
> + VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR :
> + VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
> + } else if (avctx->codec_id == AV_CODEC_ID_H265) {
> + base_profile = FF_PROFILE_HEVC_MAIN;
> + dec_caps->pNext = &h265_caps;
> + usage->pNext = h265_profile;
> + h265_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR;
> + h265_profile->stdProfileIdc = cur_profile;
> + }
> +
> + usage->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
> + usage->videoUsageHints = VK_VIDEO_DECODE_USAGE_DEFAULT_KHR;
> +
> + profile->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
> + profile->pNext = usage;
> + profile->videoCodecOperation = vk_codec->decode_op;
> + profile->chromaSubsampling = ff_vk_subsampling_from_av_desc(desc);
> + profile->lumaBitDepth = ff_vk_depth_from_av_depth(desc->comp[0].depth);
> + profile->chromaBitDepth = profile->lumaBitDepth;
> +
> + profile_list->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
> + profile_list->profileCount = 1;
> + profile_list->pProfiles = profile;
> +
> + /* Get the capabilities of the decoder for the given profile */
> + caps->sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
> + caps->pNext = dec_caps;
> + dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
> + /* dec_caps->pNext already filled in */
> +
> + ret = vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
> + caps);
> + if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR &&
> + avctx->flags & AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH &&
> + cur_profile != base_profile) {
> + cur_profile = base_profile;
> + av_log(avctx, AV_LOG_VERBOSE, "%s profile %s not supported, attempting "
> + "again with profile %s\n",
> + avcodec_get_name(avctx->codec_id),
> + avcodec_profile_name(avctx->codec_id, avctx->profile),
> + avcodec_profile_name(avctx->codec_id, base_profile));
> + goto repeat;
This function is long and ugly enough even without backward gotos. What
would Dijkstra say?
> +#endif /* AVCODEC_VULKAN_DECODE_H */
> diff --git a/libavutil/vulkan.c b/libavutil/vulkan.c
> index 0ab90c8f3c..db47956198 100644
> --- a/libavutil/vulkan.c
> +++ b/libavutil/vulkan.c
> @@ -510,8 +510,8 @@ void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e)
> AVVkFrame *vkf = (AVVkFrame *)f->data[0];
> vkfc->unlock_frame(hwfc, vkf);
> e->frame_locked[j] = 0;
> - e->frame_update[j] = 0;
> }
> + e->frame_update[j] = 0;
unrelated?
--
Anton Khirnov
More information about the ffmpeg-devel
mailing list