[FFmpeg-devel] [PATCH] avfilter: add vf_overlay_videotoolbox
gnattu
gnattuoc at me.com
Wed Feb 21 03:18:06 EET 2024
Overlay filter for VideoToolbox hwframes. Unlike most hardware
overlay filters, this filter does not require the two inputs to
have the same pixel format; instead, it will perform format
conversion automatically with hardware accelerated methods.
Signed-off-by: Gnattu OC <gnattuoc at me.com>
---
Changelog | 1 +
configure | 1 +
libavfilter/Makefile | 3 +
libavfilter/allfilters.c | 1 +
libavfilter/metal/utils.h | 7 +
libavfilter/metal/utils.m | 28 +
.../metal/vf_overlay_videotoolbox.metal | 58 ++
libavfilter/vf_overlay_videotoolbox.m | 504 ++++++++++++++++++
8 files changed, 603 insertions(+)
create mode 100644 libavfilter/metal/vf_overlay_videotoolbox.metal
create mode 100644 libavfilter/vf_overlay_videotoolbox.m
diff --git a/Changelog b/Changelog
index 610ee61dd6..3ecfdab81b 100644
--- a/Changelog
+++ b/Changelog
@@ -27,6 +27,7 @@ version <next>:
- a C11-compliant compiler is now required; note that this requirement
will be bumped to C17 in the near future, so consider updating your
build environment if it lacks C17 support
+- VideoToolbox overlay filter
version 6.1:
- libaribcaption decoder
diff --git a/configure b/configure
index 23066efa32..a7c349d126 100755
--- a/configure
+++ b/configure
@@ -3807,6 +3807,7 @@ overlay_qsv_filter_deps="libmfx"
overlay_qsv_filter_select="qsvvpp"
overlay_vaapi_filter_deps="vaapi VAProcPipelineCaps_blend_flags"
overlay_vulkan_filter_deps="vulkan spirv_compiler"
+overlay_videotoolbox_filter_deps="metal corevideo coreimage videotoolbox"
owdenoise_filter_deps="gpl"
pad_opencl_filter_deps="opencl"
pan_filter_deps="swresample"
diff --git a/libavfilter/Makefile b/libavfilter/Makefile
index f6c1d641d6..330924fadf 100644
--- a/libavfilter/Makefile
+++ b/libavfilter/Makefile
@@ -413,6 +413,9 @@ OBJS-$(CONFIG_OVERLAY_OPENCL_FILTER) += vf_overlay_opencl.o opencl.o \
OBJS-$(CONFIG_OVERLAY_QSV_FILTER) += vf_overlay_qsv.o framesync.o
OBJS-$(CONFIG_OVERLAY_VAAPI_FILTER) += vf_overlay_vaapi.o framesync.o vaapi_vpp.o
OBJS-$(CONFIG_OVERLAY_VULKAN_FILTER) += vf_overlay_vulkan.o vulkan.o vulkan_filter.o
+OBJS-$(CONFIG_OVERLAY_VIDEOTOOLBOX_FILTER) += vf_overlay_videotoolbox.o \
+ metal/vf_overlay_videotoolbox.metallib.o \
+ metal/utils.o
OBJS-$(CONFIG_OWDENOISE_FILTER) += vf_owdenoise.o
OBJS-$(CONFIG_PAD_FILTER) += vf_pad.o
OBJS-$(CONFIG_PAD_OPENCL_FILTER) += vf_pad_opencl.o opencl.o opencl/pad.o
diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c
index 149bf50997..ec9d975ecb 100644
--- a/libavfilter/allfilters.c
+++ b/libavfilter/allfilters.c
@@ -389,6 +389,7 @@ extern const AVFilter ff_vf_overlay_qsv;
extern const AVFilter ff_vf_overlay_vaapi;
extern const AVFilter ff_vf_overlay_vulkan;
extern const AVFilter ff_vf_overlay_cuda;
+extern const AVFilter ff_vf_overlay_videotoolbox;
extern const AVFilter ff_vf_owdenoise;
extern const AVFilter ff_vf_pad;
extern const AVFilter ff_vf_pad_opencl;
diff --git a/libavfilter/metal/utils.h b/libavfilter/metal/utils.h
index 7350d42a35..a2142b6472 100644
--- a/libavfilter/metal/utils.h
+++ b/libavfilter/metal/utils.h
@@ -56,4 +56,11 @@ CVMetalTextureRef ff_metal_texture_from_pixbuf(void *avclass,
MTLPixelFormat format)
API_AVAILABLE(macos(10.11), ios(8.0));
+CVMetalTextureRef ff_metal_texture_from_non_planer_pixbuf(void *avclass,
+ CVMetalTextureCacheRef textureCache,
+ CVPixelBufferRef pixbuf,
+ int plane,
+ MTLPixelFormat format)
+API_AVAILABLE(macos(10.11), ios(8.0));
+
#endif /* AVFILTER_METAL_UTILS_H */
diff --git a/libavfilter/metal/utils.m b/libavfilter/metal/utils.m
index f365d3ceea..b6a4ba16ff 100644
--- a/libavfilter/metal/utils.m
+++ b/libavfilter/metal/utils.m
@@ -74,3 +74,31 @@ CVMetalTextureRef ff_metal_texture_from_pixbuf(void *ctx,
return tex;
}
+
+CVMetalTextureRef ff_metal_texture_from_non_planer_pixbuf(void *ctx,
+ CVMetalTextureCacheRef textureCache,
+ CVPixelBufferRef pixbuf,
+ int plane,
+ MTLPixelFormat format)
+{
+ CVMetalTextureRef tex = NULL;
+ CVReturn ret;
+
+ ret = CVMetalTextureCacheCreateTextureFromImage(
+ NULL,
+ textureCache,
+ pixbuf,
+ NULL,
+ format,
+ CVPixelBufferGetWidth(pixbuf),
+ CVPixelBufferGetHeight(pixbuf),
+ plane,
+ &tex
+ );
+ if (ret != kCVReturnSuccess) {
+ av_log(ctx, AV_LOG_ERROR, "ff_metal_texture_from_non_planer_pixbuf Failed to create CVMetalTexture from image: %d\n", ret);
+ return NULL;
+ }
+
+ return tex;
+}
diff --git a/libavfilter/metal/vf_overlay_videotoolbox.metal b/libavfilter/metal/vf_overlay_videotoolbox.metal
new file mode 100644
index 0000000000..936e57e03e
--- /dev/null
+++ b/libavfilter/metal/vf_overlay_videotoolbox.metal
@@ -0,0 +1,58 @@
+/*
+ * Copyright (C) 2024 Gnattu OC <gnattuoc at me.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <metal_stdlib>
+#include <metal_integer>
+#include <metal_texture>
+
+using namespace metal;
+
+struct mtlBlendParams {
+ uint x_position;
+ uint y_position;
+};
+
+/*
+ * Blend shader for premultiplied alpha textures
+ */
+kernel void blend_shader(
+ texture2d<float, access::read> source [[ texture(0) ]],
+ texture2d<float, access::read> mask [[ texture(1) ]],
+ texture2d<float, access::write> dest [[ texture(2) ]],
+ constant mtlBlendParams& params [[ buffer(3) ]],
+ uint2 gid [[ thread_position_in_grid ]])
+{
+ const auto mask_size = uint2(mask.get_width(),
+ mask.get_height());
+ const auto loc_overlay = uint2(params.x_position, params.y_position);
+ if (gid.x < loc_overlay.x ||
+ gid.y < loc_overlay.y ||
+ gid.x >= mask_size.x + loc_overlay.x ||
+ gid.y >= mask_size.y + loc_overlay.y)
+ {
+ float4 source_color = source.read(gid);
+ dest.write(source_color, gid);
+ } else {
+ float4 source_color = source.read(gid);
+ float4 mask_color = mask.read((gid - loc_overlay));
+ float4 result_color = source_color * (1.0f - mask_color.w) + (mask_color * mask_color.w);
+ dest.write(result_color, gid);
+ }
+}
diff --git a/libavfilter/vf_overlay_videotoolbox.m b/libavfilter/vf_overlay_videotoolbox.m
new file mode 100644
index 0000000000..e100523088
--- /dev/null
+++ b/libavfilter/vf_overlay_videotoolbox.m
@@ -0,0 +1,504 @@
+/*
+ * Copyright (C) 2024 Gnattu OC <gnattuoc at me.com>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <CoreImage/CoreImage.h>
+#include <VideoToolbox/VideoToolbox.h>
+#include "internal.h"
+#include "metal/utils.h"
+#include "framesync.h"
+#include "libavutil/hwcontext.h"
+#include "libavutil/pixdesc.h"
+#include "libavutil/opt.h"
+#include "libavutil/objc.h"
+#include "video.h"
+
+#include <assert.h>
+
+extern char ff_vf_overlay_videotoolbox_metallib_data[];
+extern unsigned int ff_vf_overlay_videotoolbox_metallib_len;
+
+typedef struct API_AVAILABLE(macos(10.11), ios(8.0)) OverlayVideoToolboxContext {
+ AVBufferRef *device_ref;
+ FFFrameSync fs;
+ CVMetalTextureCacheRef textureCache;
+ CVPixelBufferRef inputMainPixelBufferCache;
+ CVPixelBufferRef outputPixelBufferCache;
+ CVPixelBufferRef inputOverlayPixelBufferCache;
+ CIContext *coreImageCtx;
+ VTPixelTransferSessionRef vtSession;
+
+ id<MTLDevice> mtlDevice;
+ id<MTLLibrary> mtlLibrary;
+ id<MTLCommandQueue> mtlQueue;
+ id<MTLComputePipelineState> mtlPipeline;
+ id<MTLFunction> mtlFunction;
+ id<MTLBuffer> mtlParamsBuffer;
+
+ int output_configured;
+ uint x_position;
+ uint y_position;
+ enum AVPixelFormat output_format;
+} OverlayVideoToolboxContext API_AVAILABLE(macos(10.11), ios(8.0));
+
+struct mtlBlendParams {
+ uint x_position;
+ uint y_position;
+};
+
+// Using sizeof(OverlayVideoToolboxContext) without an availability check will error
+// if we're targeting an older OS version, so we need to calculate the size ourselves
+// (we'll statically verify it's correct in overlay_videotoolbox_init behind a check)
+#define OVERLAY_VT_CTX_SIZE (sizeof(FFFrameSync) + sizeof(int) * 1 + sizeof(uint) * 2 + sizeof(void*) * 13 + sizeof(enum AVPixelFormat))
+
+static void call_kernel(AVFilterContext *avctx,
+ id<MTLTexture> dst,
+ id<MTLTexture> main,
+ id<MTLTexture> overlay,
+ uint x_position,
+ uint y_position) API_AVAILABLE(macos(10.11), ios(8.0))
+{
+ OverlayVideoToolboxContext *ctx = avctx->priv;
+ id<MTLCommandBuffer> buffer = ctx->mtlQueue.commandBuffer;
+ id<MTLComputeCommandEncoder> encoder = buffer.computeCommandEncoder;
+
+ struct mtlBlendParams *params = (struct mtlBlendParams *)ctx->mtlParamsBuffer.contents;
+ *params = (struct mtlBlendParams){
+ .x_position = x_position,
+ .y_position = y_position,
+ };
+ [encoder setTexture:main atIndex:0];
+ [encoder setTexture:overlay atIndex:1];
+ [encoder setTexture:dst atIndex:2];
+ [encoder setBuffer:ctx->mtlParamsBuffer offset:0 atIndex:3];
+ ff_metal_compute_encoder_dispatch(ctx->mtlDevice, ctx->mtlPipeline, encoder, dst.width, dst.height);
+ [encoder endEncoding];
+ [buffer commit];
+ [buffer waitUntilCompleted];
+}
+
+static int overlay_vt_blend(FFFrameSync *fs) API_AVAILABLE(macos(10.11), ios(8.0))
+{
+ AVFilterContext *avctx = fs->parent;
+ OverlayVideoToolboxContext *ctx = avctx->priv;
+ AVFilterLink *outlink = avctx->outputs[0];
+ AVFilterLink *inlink = avctx->inputs[0];
+ AVFilterLink *inlink_overlay = avctx->inputs[1];
+ AVFrame *input_main, *input_overlay;
+ AVFrame *output;
+ AVHWFramesContext *frames_ctx = (AVHWFramesContext*)inlink->hw_frames_ctx->data;
+ AVHWFramesContext *frames_ctx_overlay = (AVHWFramesContext*)inlink_overlay->hw_frames_ctx->data;
+ const AVPixFmtDescriptor *in_overlay_desc;
+
+ CIImage *main_image = NULL;
+ CIImage *output_image = NULL;
+ CVMetalTextureRef main, dst, overlay;
+ id<MTLCommandBuffer> mtl_buffer = ctx->mtlQueue.commandBuffer;
+ id<MTLTexture> tex_main, tex_overlay, tex_dst;
+
+ MTLPixelFormat format = MTLPixelFormatBGRA8Unorm;
+ int ret;
+ int i, overlay_planes = 0;
+ in_overlay_desc = av_pix_fmt_desc_get(frames_ctx_overlay->sw_format);
+ // read main and overlay frames from inputs
+ ret = ff_framesync_get_frame(fs, 0, &input_main, 0);
+ if (ret < 0)
+ return ret;
+ ret = ff_framesync_get_frame(fs, 1, &input_overlay, 0);
+ if (ret < 0)
+ return ret;
+ if (!input_main)
+ return AVERROR_BUG;
+ if (!input_overlay)
+ return ff_filter_frame(outlink, input_main);
+
+ output = ff_get_video_buffer(outlink, outlink->w, outlink->h);
+ ret = av_frame_copy_props(output, input_main);
+ if (ret < 0)
+ return ret;
+ [mtl_buffer commit];
+ for (i = 0; i < in_overlay_desc->nb_components; i++)
+ overlay_planes = FFMAX(overlay_planes,
+ in_overlay_desc->comp[i].plane + 1);
+ if (overlay_planes > 1) {
+ if (@available(macOS 10.8, iOS 16.0, *)) {
+ if (!ctx->vtSession) {
+ ret = VTPixelTransferSessionCreate(NULL, &ctx->vtSession);
+ if (ret < 0)
+ return ret;
+ }
+ if (!ctx->inputOverlayPixelBufferCache) {
+ ret = CVPixelBufferCreate(kCFAllocatorDefault,
+ CVPixelBufferGetWidthOfPlane((CVPixelBufferRef)input_overlay->data[3], 0),
+ CVPixelBufferGetHeightOfPlane((CVPixelBufferRef)input_overlay->data[3], 0),
+ kCVPixelFormatType_32BGRA,
+ (__bridge CFDictionaryRef)@{
+ (NSString *)kCVPixelBufferCGImageCompatibilityKey: @(YES),
+ (NSString *)kCVPixelBufferMetalCompatibilityKey: @(YES)
+ },
+ &ctx->inputOverlayPixelBufferCache);
+ if (ret < 0)
+ return ret;
+ }
+ // The YUV formatted overlays will be hwuploaded to kCVPixelFormatType_4444AYpCbCr16, which is not render-able using CoreImage.
+ // As a fallback, use the (much) slower VTPixelTransferSessionTransferImage instead.
+ // This should work on all macOS version provides Metal, but is only available on iOS >=16.
+ ret = VTPixelTransferSessionTransferImage(ctx->vtSession,(CVPixelBufferRef)input_overlay->data[3] ,ctx->inputOverlayPixelBufferCache);
+ if (ret < 0)
+ return ret;
+ overlay = ff_metal_texture_from_non_planer_pixbuf(avctx, ctx->textureCache, ctx->inputOverlayPixelBufferCache, 0, format);
+ } else {
+ av_log(ctx, AV_LOG_ERROR, "VTPixelTransferSessionTransferImage is not available on this OS version\n");
+ av_log(ctx, AV_LOG_ERROR, "Try an overlay with kCVPixelFormatType_32BGRA\n");
+ return AVERROR(ENOSYS);
+ }
+ } else {
+ overlay = ff_metal_texture_from_non_planer_pixbuf(avctx, ctx->textureCache, (CVPixelBufferRef)input_overlay->data[3], 0, format);
+ }
+ main_image = CFBridgingRetain([CIImage imageWithCVPixelBuffer: (CVPixelBufferRef)input_main->data[3]]);
+ if (!ctx->inputMainPixelBufferCache) {
+ ret = CVPixelBufferCreate(kCFAllocatorDefault,
+ CVPixelBufferGetWidthOfPlane((CVPixelBufferRef)input_main->data[3], 0),
+ CVPixelBufferGetHeightOfPlane((CVPixelBufferRef)input_main->data[3], 0),
+ kCVPixelFormatType_32BGRA,
+ (__bridge CFDictionaryRef)@{
+ (NSString *)kCVPixelBufferCGImageCompatibilityKey: @(YES),
+ (NSString *)kCVPixelBufferMetalCompatibilityKey: @(YES)
+ },
+ &ctx->inputMainPixelBufferCache);
+ if (ret < 0)
+ return ret;
+ }
+ if (!ctx->outputPixelBufferCache) {
+ ret = CVPixelBufferCreate(kCFAllocatorDefault,
+ CVPixelBufferGetWidthOfPlane((CVPixelBufferRef)input_main->data[3], 0),
+ CVPixelBufferGetHeightOfPlane((CVPixelBufferRef)input_main->data[3], 0),
+ kCVPixelFormatType_32BGRA,
+ (__bridge CFDictionaryRef)@{
+ (NSString *)kCVPixelBufferCGImageCompatibilityKey: @(YES),
+ (NSString *)kCVPixelBufferMetalCompatibilityKey: @(YES)
+ },
+ &ctx->outputPixelBufferCache);
+ if (ret < 0)
+ return ret;
+ }
+ [(__bridge CIContext*)ctx->coreImageCtx render: (__bridge CIImage*)main_image toCVPixelBuffer: ctx->inputMainPixelBufferCache];
+ [mtl_buffer waitUntilCompleted];
+ main = ff_metal_texture_from_non_planer_pixbuf(avctx, ctx->textureCache, ctx->inputMainPixelBufferCache, 0, format);
+ dst = ff_metal_texture_from_non_planer_pixbuf(avctx, ctx->textureCache, ctx->outputPixelBufferCache, 0, format);
+ tex_main = CVMetalTextureGetTexture(main);
+ tex_overlay = CVMetalTextureGetTexture(overlay);
+ tex_dst = CVMetalTextureGetTexture(dst);
+ call_kernel(avctx, tex_dst, tex_main, tex_overlay, ctx->x_position, ctx->y_position);
+ output_image = CFBridgingRetain([CIImage imageWithCVPixelBuffer: ctx->outputPixelBufferCache]);
+ [(__bridge CIContext*)ctx->coreImageCtx render: (__bridge CIImage*)output_image toCVPixelBuffer: (CVPixelBufferRef)output->data[3]];
+ [mtl_buffer waitUntilCompleted];
+ CFRelease(main);
+ CFRelease(overlay);
+ CFRelease(dst);
+ CFRelease(main_image);
+ CFRelease(output_image);
+ CVBufferPropagateAttachments((CVPixelBufferRef)input_main->data[3], (CVPixelBufferRef)output->data[3]);
+
+ return ff_filter_frame(outlink, output);
+}
+
+static av_cold void do_uninit(AVFilterContext *avctx) API_AVAILABLE(macos(10.11), ios(8.0))
+{
+ OverlayVideoToolboxContext *ctx = avctx->priv;
+ if(ctx->coreImageCtx) {
+ CFRelease(ctx->coreImageCtx);
+ ctx->coreImageCtx = NULL;
+ }
+ if (ctx->output_configured) {
+ av_buffer_unref(&ctx->device_ref);
+ }
+
+ ff_objc_release(&ctx->mtlParamsBuffer);
+ ff_objc_release(&ctx->mtlFunction);
+ ff_objc_release(&ctx->mtlPipeline);
+ ff_objc_release(&ctx->mtlQueue);
+ ff_objc_release(&ctx->mtlLibrary);
+ ff_objc_release(&ctx->mtlDevice);
+
+ if (ctx->textureCache) {
+ CFRelease(ctx->textureCache);
+ ctx->textureCache = NULL;
+ }
+ if (ctx->inputMainPixelBufferCache) {
+ CFRelease(ctx->inputMainPixelBufferCache);
+ ctx->inputMainPixelBufferCache = NULL;
+ }
+ if (ctx->inputOverlayPixelBufferCache) {
+ CFRelease(ctx->inputOverlayPixelBufferCache);
+ ctx->inputOverlayPixelBufferCache = NULL;
+ }
+ if (ctx->outputPixelBufferCache) {
+ CFRelease(ctx->outputPixelBufferCache);
+ ctx->outputPixelBufferCache = NULL;
+ }
+ if(ctx->vtSession) {
+ VTPixelTransferSessionInvalidate(ctx->vtSession);
+ CFRelease(ctx->vtSession);
+ ctx->vtSession = NULL;
+ }
+ ff_framesync_uninit(&ctx->fs);
+}
+
+static av_cold void overlay_videotoolbox_uninit(AVFilterContext *ctx)
+{
+ if (@available(macOS 10.11, iOS 8.0, *)) {
+ do_uninit(ctx);
+ }
+}
+
+static av_cold int do_init(AVFilterContext *ctx) API_AVAILABLE(macos(10.11), ios(8.0))
+{
+ OverlayVideoToolboxContext *s = ctx->priv;
+ NSError *err = nil;
+ CVReturn ret;
+ dispatch_data_t libData;
+
+ s->mtlDevice = MTLCreateSystemDefaultDevice();
+ if (!s->mtlDevice) {
+ av_log(ctx, AV_LOG_ERROR, "Unable to find Metal device\n");
+ goto fail;
+ }
+
+ av_log(ctx, AV_LOG_INFO, "Using Metal device: %s\n", s->mtlDevice.name.UTF8String);
+
+ libData = dispatch_data_create(
+ ff_vf_overlay_videotoolbox_metallib_data,
+ ff_vf_overlay_videotoolbox_metallib_len,
+ nil,
+ nil);
+
+ s->mtlLibrary = [s->mtlDevice newLibraryWithData:libData error:&err];
+ dispatch_release(libData);
+ libData = nil;
+ s->mtlFunction = [s->mtlLibrary newFunctionWithName:@"blend_shader"];
+ if (!s->mtlFunction) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create Metal function!\n");
+ goto fail;
+ }
+
+ s->mtlQueue = s->mtlDevice.newCommandQueue;
+ if (!s->mtlQueue) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create Metal command queue!\n");
+ goto fail;
+ }
+
+ s->mtlPipeline = [s->mtlDevice
+ newComputePipelineStateWithFunction:s->mtlFunction
+ error:&err];
+ if (err) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create Metal compute pipeline: %s\n", err.description.UTF8String);
+ goto fail;
+ }
+
+ s->mtlParamsBuffer = [s->mtlDevice
+ newBufferWithLength:sizeof(struct mtlBlendParams)
+ options:MTLResourceStorageModeShared];
+ if (!s->mtlParamsBuffer) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create Metal buffer for parameters\n");
+ goto fail;
+ }
+
+ ret = CVMetalTextureCacheCreate(
+ NULL,
+ NULL,
+ s->mtlDevice,
+ NULL,
+ &s->textureCache
+ );
+ if (ret != kCVReturnSuccess) {
+ av_log(ctx, AV_LOG_ERROR, "Failed to create CVMetalTextureCache: %d\n", ret);
+ goto fail;
+ }
+
+ s->coreImageCtx = CFBridgingRetain([CIContext contextWithMTLCommandQueue: s->mtlQueue]);
+ s->fs.on_event = &overlay_vt_blend;
+ s->output_format = AV_PIX_FMT_NONE;
+ av_log(ctx, AV_LOG_INFO, "do_init!\n");
+
+ return 0;
+fail:
+ overlay_videotoolbox_uninit(ctx);
+ return AVERROR_EXTERNAL;
+}
+
+static av_cold int overlay_videotoolbox_init(AVFilterContext *ctx)
+{
+ if (@available(macOS 10.11, iOS 8.0, *)) {
+ // Ensure we calculated OVERLAY_VT_CTX_SIZE correctly
+ static_assert(OVERLAY_VT_CTX_SIZE == sizeof(OverlayVideoToolboxContext), "Incorrect OVERLAY_VT_CTX_SIZE value!");
+ return do_init(ctx);
+ } else {
+ av_log(ctx, AV_LOG_ERROR, "Metal is not available on this OS version\n");
+ return AVERROR(ENOSYS);
+ }
+}
+
+static int do_config_input(AVFilterLink *inlink) API_AVAILABLE(macos(10.11), ios(8.0))
+{
+ AVFilterContext *avctx = inlink->dst;
+ OverlayVideoToolboxContext *ctx = avctx->priv;
+ AVBufferRef *input_ref;
+ AVHWFramesContext *input_frames;
+
+ if (!inlink->hw_frames_ctx) {
+ av_log(avctx, AV_LOG_ERROR, "A hardware frames reference is "
+ "required to associate the processing device.\n");
+ return AVERROR(EINVAL);
+ }
+ input_ref = av_buffer_ref(inlink->hw_frames_ctx);
+ input_frames = (AVHWFramesContext*)input_ref->data;
+ av_assert0(input_frames);
+ ctx->device_ref = av_buffer_ref(input_frames->device_ref);
+
+ if (!ctx->device_ref) {
+ av_log(ctx, AV_LOG_ERROR, "A device reference create "
+ "failed.\n");
+ return AVERROR(ENOMEM);
+ }
+ if (ctx->output_format == AV_PIX_FMT_NONE)
+ ctx->output_format = input_frames->sw_format;
+ ctx->output_configured = 1;
+
+ return 0;
+}
+
+static int config_input(AVFilterLink *inlink)
+{
+ AVFilterContext *ctx = inlink->dst;
+ if (@available(macOS 10.13, iOS 9.0, *)) {
+ return do_config_input(inlink);
+ } else {
+ av_log(ctx, AV_LOG_ERROR, "Metal is not available on this OS version\n");
+ return AVERROR(ENOSYS);
+ }
+}
+
+static int do_config_output(AVFilterLink *link) API_AVAILABLE(macos(10.11), ios(8.0))
+{
+ AVHWFramesContext *output_frames;
+ AVFilterContext *avctx = link->src;
+ OverlayVideoToolboxContext *ctx = avctx->priv;
+ int ret = 0;
+
+ av_log(avctx, AV_LOG_INFO, "do_config_output!\n");
+ link->hw_frames_ctx = av_hwframe_ctx_alloc(ctx->device_ref);
+ if (!link->hw_frames_ctx) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to create HW frame context "
+ "for output.\n");
+ ret = AVERROR(ENOMEM);
+ return ret;
+ }
+
+ output_frames = (AVHWFramesContext*)link->hw_frames_ctx->data;
+
+ output_frames->format = AV_PIX_FMT_VIDEOTOOLBOX;
+ output_frames->sw_format = ctx->output_format;
+ output_frames->width = avctx->inputs[0]->w;
+ output_frames->height = avctx->inputs[0]->h;
+
+ ret = ff_filter_init_hw_frames(avctx, link, 10);
+ if (ret < 0)
+ return ret;
+
+ ret = av_hwframe_ctx_init(link->hw_frames_ctx);
+ if (ret < 0) {
+ av_log(avctx, AV_LOG_ERROR, "Failed to initialise VideoToolbox frame "
+ "context for output: %d\n", ret);
+ return ret;
+ }
+
+ ret = ff_framesync_init_dualinput(&ctx->fs, avctx);
+ if (ret < 0)
+ return ret;
+
+ ret = ff_framesync_configure(&ctx->fs);
+ return ret;
+}
+
+static int config_output(AVFilterLink *link)
+{
+ AVFilterContext *ctx = link->src;
+ if (@available(macOS 10.13, iOS 9.0, *)) {
+ return do_config_output(link);
+ } else {
+ av_log(ctx, AV_LOG_ERROR, "Metal is not available on this OS version\n");
+ return AVERROR(ENOSYS);
+ }
+}
+
+static int overlay_videotoolbox_activate(AVFilterContext *avctx) {
+ OverlayVideoToolboxContext *ctx = avctx->priv;
+ return ff_framesync_activate(&ctx->fs);
+}
+
+#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
+#define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, INT_MIN, INT_MAX, FLAGS, unit }
+#define OFFSET(x) offsetof(OverlayVideoToolboxContext, x)
+
+static const AVOption overlay_videotoolbox_options[] = {
+ { "x", "Overlay x position",
+ OFFSET(x_position), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+ { "y", "Overlay y position",
+ OFFSET(y_position), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, .flags = FLAGS },
+ { NULL },
+};
+
+AVFILTER_DEFINE_CLASS(overlay_videotoolbox);
+
+static const AVFilterPad overlay_videotoolbox_inputs[] = {
+ {
+ .name = "main",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = config_input,
+ },
+ {
+ .name = "overlay",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = config_input,
+ },
+};
+
+static const AVFilterPad overlay_videotoolbox_outputs[] = {
+ {
+ .name = "default",
+ .type = AVMEDIA_TYPE_VIDEO,
+ .config_props = config_output,
+ },
+};
+
+const AVFilter ff_vf_overlay_videotoolbox = {
+ .name = "overlay_videotoolbox",
+ .description = NULL_IF_CONFIG_SMALL("Overlay filter for VideoToolbox frames using Metal compute"),
+ .priv_size = OVERLAY_VT_CTX_SIZE,
+ .priv_class = &overlay_videotoolbox_class,
+ .init = overlay_videotoolbox_init,
+ .uninit = overlay_videotoolbox_uninit,
+ .activate = overlay_videotoolbox_activate,
+ FILTER_SINGLE_PIXFMT(AV_PIX_FMT_VIDEOTOOLBOX),
+ FILTER_INPUTS(overlay_videotoolbox_inputs),
+ FILTER_OUTPUTS(overlay_videotoolbox_outputs),
+ .flags = AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL,
+ .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
+};
--
2.39.3 (Apple Git-145)
More information about the ffmpeg-devel
mailing list