[FFmpeg-devel] [PATCH 2/3] lavfi: add common Vulkan filtering code
Mark Thompson
sw at jkqxz.net
Mon Apr 2 23:50:20 EEST 2018
On 30/03/18 04:14, Rostislav Pehlivanov wrote:
> This commit adds a common code for use in Vulkan filters. It attempts
> to ease the burden of writing Vulkan image filtering to a minimum,
> which is pretty much a requirement considering how verbose the API is.
>
> It supports both compute and graphic pipelines and manages to abstract
> the API to such a level there's no need to call any Vulkan functions
> inside the init path of the code. Handling shader descriptors is probably
> the bulk of the code, and despite the abstraction, it loses none of the
> features for describing shader IO.
>
> In order to produce linkable shaders, it depends on the libshaderc
> library (and depends on the latest stable version of it). This allows
> for greater performance and flexibility than static built-in shaders
> and also eliminates the cumbersome process of interfacing with glslang
> to compile GLSL to SPIR-V.
>
> It's based off of the common opencl and provides similar interfaces for
> filter pad init and config, with the addition that it also supports
> in-place filtering.
If we're going to have in-place filtering of hardware frames then we need to work out tracking the writability of them properly. E.g. presumably you can write on decoder references frames here while they're still being used?
>
> Signed-off-by: Rostislav Pehlivanov <atomnuker at gmail.com>
> ---
> configure | 3 +
> libavfilter/vulkan.c | 1101 ++++++++++++++++++++++++++++++++++++++++++++++++++
> libavfilter/vulkan.h | 179 ++++++++
> 3 files changed, 1283 insertions(+)
> create mode 100644 libavfilter/vulkan.c
> create mode 100644 libavfilter/vulkan.h
>
> diff --git a/configure b/configure
> index 2213f0452d..3621b5cdeb 100755
> --- a/configure
> +++ b/configure
> @@ -252,6 +252,7 @@ External library support:
> --enable-librsvg enable SVG rasterization via librsvg [no]
> --enable-librubberband enable rubberband needed for rubberband filter [no]
> --enable-librtmp enable RTMP[E] support via librtmp [no]
> + --enable-libshaderc enable GLSL->SPIRV compilation via libshaderc [no]
> --enable-libshine enable fixed-point MP3 encoding via libshine [no]
> --enable-libsmbclient enable Samba protocol via libsmbclient [no]
> --enable-libsnappy enable Snappy compression, needed for hap encoding [no]
> @@ -1702,6 +1703,7 @@ EXTERNAL_LIBRARY_LIST="
> libpulse
> librsvg
> librtmp
> + libshaderc
> libshine
> libsmbclient
> libsnappy
> @@ -6020,6 +6022,7 @@ enabled libpulse && require_pkg_config libpulse libpulse pulse/pulseaud
> enabled librsvg && require_pkg_config librsvg librsvg-2.0 librsvg-2.0/librsvg/rsvg.h rsvg_handle_render_cairo
> enabled librtmp && require_pkg_config librtmp librtmp librtmp/rtmp.h RTMP_Socket
> enabled librubberband && require_pkg_config librubberband "rubberband >= 1.8.1" rubberband/rubberband-c.h rubberband_new -lstdc++ && append librubberband_extralibs "-lstdc++"
> +enabled libshaderc && require libshaderc shaderc/shaderc.h shaderc_compiler_initialize -lshaderc_shared
> enabled libshine && require_pkg_config libshine shine shine/layer3.h shine_encode_buffer
> enabled libsmbclient && { check_pkg_config libsmbclient smbclient libsmbclient.h smbc_init ||
> require libsmbclient libsmbclient.h smbc_init -lsmbclient; }
> diff --git a/libavfilter/vulkan.c b/libavfilter/vulkan.c
> new file mode 100644
> index 0000000000..c2e02f5d0a
> --- /dev/null
> +++ b/libavfilter/vulkan.c
> @@ -0,0 +1,1101 @@
> ...
> +
> +int ff_vk_filter_init(AVFilterContext *avctx)
> +{
> + VulkanFilterContext *s = avctx->priv;
> + const shaderc_env_version opt_ver = shaderc_env_version_vulkan_1_1;
> + const shaderc_optimization_level opt_lvl = shaderc_optimization_level_size;
How much does this affect the output? Is it worth making it confgurable?
> +
> + s->output_format = AV_PIX_FMT_NONE;
> +
> + s->sc_compiler = shaderc_compiler_initialize();
> + if (!s->sc_compiler)
> + return AVERROR_EXTERNAL;
> +
> + s->sc_opts = shaderc_compile_options_initialize();
> + if (!s->sc_compiler)
> + return AVERROR_EXTERNAL;
> +
> + shaderc_compile_options_set_target_env(s->sc_opts,
> + shaderc_target_env_vulkan,
> + opt_ver);
> + shaderc_compile_options_set_optimization_level(s->sc_opts, opt_lvl);
> +
> + return 0;
> +}
> +
> ...
> +
> +static VkSamplerYcbcrModelConversion conv_primaries(enum AVColorPrimaries color_primaries)
> +{
> + switch(color_primaries) {
> + case AVCOL_PRI_BT470BG:
> + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601;
> + case AVCOL_PRI_BT709:
> + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709;
> + case AVCOL_PRI_BT2020:
> + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020;
> + }
> + return VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY;
You probably don't want to use RGB for YUV not in the given list - you can at least make it only a bit wrong, rather than totally wrong.
> +}
> +
> +int ff_vk_init_sampler(AVFilterContext *avctx, AVFrame *input)
> +{
> + VkResult ret;
> + VulkanFilterContext *s = avctx->priv;
> + VkSamplerYcbcrConversionCreateInfo c_info;
> +
> + if (input) {
> + c_info.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO;
> + c_info.format = av_vkfmt_from_pixfmt(s->input_format);
> + c_info.chromaFilter = VK_FILTER_LINEAR;
> + c_info.ycbcrModel = conv_primaries(input->color_primaries);
> + c_info.ycbcrRange = input->color_range == AVCOL_RANGE_JPEG ?
> + VK_SAMPLER_YCBCR_RANGE_ITU_FULL :
> + VK_SAMPLER_YCBCR_RANGE_ITU_NARROW;
> + c_info.xChromaOffset = input->chroma_location == AVCHROMA_LOC_CENTER ?
> + VK_CHROMA_LOCATION_MIDPOINT :
> + VK_CHROMA_LOCATION_COSITED_EVEN;
> + c_info.forceExplicitReconstruction = 0;
> +
> + VkComponentMapping comp_map = {
> + .r = VK_COMPONENT_SWIZZLE_IDENTITY,
> + .g = VK_COMPONENT_SWIZZLE_IDENTITY,
> + .b = VK_COMPONENT_SWIZZLE_IDENTITY,
> + .a = VK_COMPONENT_SWIZZLE_ONE,
> + };
> +
> + c_info.components = comp_map;
> +
> + VK_LOAD_PFN(s->hwctx->inst, vkCreateSamplerYcbcrConversionKHR);
> +
> + s->yuv_sampler.sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO;
> +
> + ret = pfn_vkCreateSamplerYcbcrConversionKHR(s->hwctx->act_dev, &c_info,
> + NULL, &s->yuv_sampler.conversion);
> + if (ret != VK_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Unable to init conversion: %s\n", av_vk_ret2str(ret));
> + return AVERROR_EXTERNAL;
> + }
> + }
> +
> + VkSamplerCreateInfo sampler = {
> + .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO,
> + .pNext = input ? &s->yuv_sampler : NULL,
> + .magFilter = VK_FILTER_LINEAR,
> + .minFilter = VK_FILTER_LINEAR,
> + .mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST,
> + .addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE,
> + .addressModeV = sampler.addressModeU,
> + .addressModeW = sampler.addressModeU,
> + .anisotropyEnable = VK_FALSE,
> + .compareOp = VK_COMPARE_OP_NEVER,
> + .borderColor = VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK,
> + .unnormalizedCoordinates = 1,
> + };
> +
> + ret = vkCreateSampler(s->hwctx->act_dev, &sampler, NULL, &s->sampler);
> + if (ret != VK_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Unable to init sampler: %s\n", av_vk_ret2str(ret));
> + return AVERROR_EXTERNAL;
> + }
> +
> + return 0;
> +}
> +
> ...
> +
Pretty huge amount of boilerplate in this file. I didn't read it carefully, but the top level looks sensible.
> diff --git a/libavfilter/vulkan.h b/libavfilter/vulkan.h
> new file mode 100644
> index 0000000000..6e059731b7
> --- /dev/null
> +++ b/libavfilter/vulkan.h
> @@ -0,0 +1,179 @@
> +/*
> + * Vulkan utilities
> + * Copyright (c) 2018 Rostislav Pehlivanov <atomnuker at gmail.com>
> + *
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +#ifndef AVFILTER_VULKAN_COMMON_H
> +#define AVFILTER_VULKAN_COMMON_H
> +
> +#include "avfilter.h"
> +#include "libavutil/pixdesc.h"
> +#include "libavutil/bprint.h"
> +#include "libavutil/hwcontext.h"
> +#include "libavutil/hwcontext_vulkan.h"
> +
> +#include <shaderc/shaderc.h>
> +
> +/* GLSL management macros */
> +#define INDENT(N) INDENT_##N
> +#define INDENT_0
> +#define INDENT_1 INDENT_0 " "
> +#define INDENT_2 INDENT_1 INDENT_1
> +#define INDENT_3 INDENT_2 INDENT_1
> +#define INDENT_4 INDENT_3 INDENT_1
> +#define INDENT_5 INDENT_4 INDENT_1
> +#define INDENT_6 INDENT_5 INDENT_1
> +#define C(N, S) INDENT(N) #S "\n"
> +#define GLSLC(N, S) av_bprintf(&shd->src, C(N, S))
> +#define GLSLA(...) av_bprintf(&shd->src, __VA_ARGS__)
> +#define GLSLF(N, S, ...) av_bprintf(&shd->src, C(N, S), __VA_ARGS__)
> +#define GLSLD(D) GLSLC(0, ); \
> + av_bprint_append_data(&shd->src, D, strlen(D)); \
> + GLSLC(0, )
This string construction, while nice for short shaders, is going to get unwieldy very quickly for longer ones.
Can you do anything like the CUDA or OpenCL code to embed files, at least for the core part of your shader? (When debugging OpenCL, including the "#line" was incredibly useful to make sense of compiler messages.)
> +
> ...
> +
> +#endif /* AVFILTER_VULKAN_COMMON_H */
>
More information about the ffmpeg-devel
mailing list