[FFmpeg-devel] [PATCH] avutil/hwcontext_videotoolbox: implement hwupload to convert AVFrame to CVPixelBuffer
Aman Karmani
ffmpeg at tmm1.net
Mon Jul 5 21:37:36 EEST 2021
On Tue, Jun 29, 2021 at 3:08 PM Aman Karmani <ffmpeg at tmm1.net> wrote:
> From: Aman Karmani <aman at tmm1.net>
>
> Teach AV_HWDEVICE_TYPE_VIDEOTOOLBOX to be able to create AVFrames of type
> AV_PIX_FMT_VIDEOTOOLBOX. This can be used to hwupload a regular AVFrame
> into its CVPixelBuffer equivalent.
>
> ffmpeg -init_hw_device videotoolbox -f lavfi -i color=black:640x480
> -vf hwupload -c:v h264_videotoolbox -f null -y /dev/null
Will merge this in a few days if no one objects.
Do I need to bump avutil patch or minor?
Aman
>
> Signed-off-by: Aman Karmani <aman at tmm1.net>
> ---
> libavutil/hwcontext_videotoolbox.c | 418 ++++++++++++++++++++++++++++-
> libavutil/hwcontext_videotoolbox.h | 7 +-
> 2 files changed, 420 insertions(+), 5 deletions(-)
>
> diff --git a/libavutil/hwcontext_videotoolbox.c
> b/libavutil/hwcontext_videotoolbox.c
> index bded9873fe..58095a1fc9 100644
> --- a/libavutil/hwcontext_videotoolbox.c
> +++ b/libavutil/hwcontext_videotoolbox.c
> @@ -24,6 +24,7 @@
> #include <VideoToolbox/VideoToolbox.h>
>
> #include "buffer.h"
> +#include "buffer_internal.h"
> #include "common.h"
> #include "hwcontext.h"
> #include "hwcontext_internal.h"
> @@ -32,6 +33,10 @@
> #include "pixfmt.h"
> #include "pixdesc.h"
>
> +typedef struct VTFramesContext {
> + CVPixelBufferPoolRef pool;
> +} VTFramesContext;
> +
> static const struct {
> uint32_t cv_fmt;
> bool full_range;
> @@ -50,6 +55,39 @@ static const struct {
> #endif
> };
>
> +static const enum AVPixelFormat supported_formats[] = {
> + AV_PIX_FMT_NV12,
> + AV_PIX_FMT_YUV420P,
> + AV_PIX_FMT_UYVY422,
> + AV_PIX_FMT_P010,
> + AV_PIX_FMT_BGRA,
> +};
> +
> +static int vt_frames_get_constraints(AVHWDeviceContext *ctx,
> + const void *hwconfig,
> + AVHWFramesConstraints *constraints)
> +{
> + int i;
> +
> + constraints->valid_sw_formats =
> av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1,
> +
> sizeof(*constraints->valid_sw_formats));
> + if (!constraints->valid_sw_formats)
> + return AVERROR(ENOMEM);
> +
> + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
> + constraints->valid_sw_formats[i] = supported_formats[i];
> + constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] =
> AV_PIX_FMT_NONE;
> +
> + constraints->valid_hw_formats = av_malloc_array(2,
> sizeof(*constraints->valid_hw_formats));
> + if (!constraints->valid_hw_formats)
> + return AVERROR(ENOMEM);
> +
> + constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX;
> + constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
> +
> + return 0;
> +}
> +
> enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
> {
> int i;
> @@ -75,11 +113,134 @@ uint32_t
> av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, boo
> return 0;
> }
>
> +static int vt_pool_alloc(AVHWFramesContext *ctx)
> +{
> + VTFramesContext *fctx = ctx->internal->priv;
> + CVReturn err;
> + CFNumberRef w, h, pixfmt;
> + uint32_t cv_pixfmt;
> + CFMutableDictionaryRef attributes, iosurface_properties;
> +
> + attributes = CFDictionaryCreateMutable(
> + NULL,
> + 2,
> + &kCFTypeDictionaryKeyCallBacks,
> + &kCFTypeDictionaryValueCallBacks);
> +
> + cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format);
> + pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt);
> + CFDictionarySetValue(
> + attributes,
> + kCVPixelBufferPixelFormatTypeKey,
> + pixfmt);
> + CFRelease(pixfmt);
> +
> + iosurface_properties = CFDictionaryCreateMutable(
> + NULL,
> + 0,
> + &kCFTypeDictionaryKeyCallBacks,
> + &kCFTypeDictionaryValueCallBacks);
> + CFDictionarySetValue(attributes,
> kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties);
> + CFRelease(iosurface_properties);
> +
> + w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width);
> + h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height);
> + CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w);
> + CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h);
> + CFRelease(w);
> + CFRelease(h);
> +
> + err = CVPixelBufferPoolCreate(
> + NULL,
> + NULL,
> + attributes,
> + &fctx->pool);
> + CFRelease(attributes);
> +
> + if (err == kCVReturnSuccess)
> + return 0;
> +
> + av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n",
> err);
> + return AVERROR_EXTERNAL;
> +}
> +
> +static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size)
> +{
> + return NULL;
> +}
> +
> +static void vt_frames_uninit(AVHWFramesContext *ctx)
> +{
> + VTFramesContext *fctx = ctx->internal->priv;
> + if (fctx->pool) {
> + CVPixelBufferPoolRelease(fctx->pool);
> + fctx->pool = NULL;
> + }
> +}
> +
> +static int vt_frames_init(AVHWFramesContext *ctx)
> +{
> + int i, ret;
> +
> + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) {
> + if (ctx->sw_format == supported_formats[i])
> + break;
> + }
> + if (i == FF_ARRAY_ELEMS(supported_formats)) {
> + av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n",
> + av_get_pix_fmt_name(ctx->sw_format));
> + return AVERROR(ENOSYS);
> + }
> +
> + // create a dummy pool so av_hwframe_get_buffer doesn't EINVAL
> + if (!ctx->pool) {
> + ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx,
> vt_dummy_pool_alloc, NULL);
> + if (!ctx->internal->pool_internal)
> + return AVERROR(ENOMEM);
> + }
> +
> + ret = vt_pool_alloc(ctx);
> + if (ret < 0)
> + return ret;
> +
> + return 0;
> +}
> +
> +static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
> +{
> + CVPixelBufferRelease((CVPixelBufferRef)data);
> +}
> +
> static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame)
> {
> - frame->buf[0] = av_buffer_pool_get(ctx->pool);
> - if (!frame->buf[0])
> - return AVERROR(ENOMEM);
> + VTFramesContext *fctx = ctx->internal->priv;
> +
> + if (ctx->pool && ctx->pool->size != 0) {
> + frame->buf[0] = av_buffer_pool_get(ctx->pool);
> + if (!frame->buf[0])
> + return AVERROR(ENOMEM);
> + } else {
> + CVPixelBufferRef pixbuf;
> + AVBufferRef *buf = NULL;
> + CVReturn err;
> +
> + err = CVPixelBufferPoolCreatePixelBuffer(
> + NULL,
> + fctx->pool,
> + &pixbuf
> + );
> + if (err != kCVReturnSuccess) {
> + av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from
> pool: %d\n", err);
> + return AVERROR_EXTERNAL;
> + }
> +
> + buf = av_buffer_create((uint8_t *)pixbuf, 1,
> videotoolbox_buffer_release, NULL, 0);
> + if (!buf) {
> + CVPixelBufferRelease(pixbuf);
> + return AVERROR(ENOMEM);
> + }
> + frame->buf[0] = buf;
> + }
>
> frame->data[3] = frame->buf[0]->data;
> frame->format = AV_PIX_FMT_VIDEOTOOLBOX;
> @@ -111,6 +272,248 @@ static void vt_unmap(AVHWFramesContext *ctx,
> HWMapDescriptor *hwmap)
> CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv);
> }
>
> +static int vt_pixbuf_set_par(AVHWFramesContext *hwfc,
> + CVPixelBufferRef pixbuf, const AVFrame *src)
> +{
> + CFMutableDictionaryRef par = NULL;
> + CFNumberRef num = NULL, den = NULL;
> + AVRational avpar = src->sample_aspect_ratio;
> +
> + if (avpar.num == 0)
> + return 0;
> +
> + av_reduce(&avpar.num, &avpar.den,
> + avpar.num, avpar.den,
> + 0xFFFFFFFF);
> +
> + num = CFNumberCreate(kCFAllocatorDefault,
> + kCFNumberIntType,
> + &avpar.num);
> +
> + den = CFNumberCreate(kCFAllocatorDefault,
> + kCFNumberIntType,
> + &avpar.den);
> +
> + par = CFDictionaryCreateMutable(kCFAllocatorDefault,
> + 2,
> + &kCFCopyStringDictionaryKeyCallBacks,
> + &kCFTypeDictionaryValueCallBacks);
> +
> + if (!par || !num || !den) {
> + if (par) CFRelease(par);
> + if (num) CFRelease(num);
> + if (den) CFRelease(den);
> + return AVERROR(ENOMEM);
> + }
> +
> + CFDictionarySetValue(
> + par,
> + kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
> + num);
> + CFDictionarySetValue(
> + par,
> + kCVImageBufferPixelAspectRatioVerticalSpacingKey,
> + den);
> +
> + CVBufferSetAttachment(
> + pixbuf,
> + kCVImageBufferPixelAspectRatioKey,
> + par,
> + kCVAttachmentMode_ShouldPropagate
> + );
> +
> + CFRelease(par);
> + CFRelease(num);
> + CFRelease(den);
> +
> + return 0;
> +}
> +
> +static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc,
> + CVPixelBufferRef pixbuf, const AVFrame
> *src)
> +{
> + CFStringRef loc = NULL;
> +
> + switch (src->chroma_location) {
> + case AVCHROMA_LOC_LEFT:
> + loc = kCVImageBufferChromaLocation_Left;
> + break;
> + case AVCHROMA_LOC_CENTER:
> + loc = kCVImageBufferChromaLocation_Center;
> + break;
> + case AVCHROMA_LOC_TOP:
> + loc = kCVImageBufferChromaLocation_Top;
> + break;
> + case AVCHROMA_LOC_BOTTOM:
> + loc = kCVImageBufferChromaLocation_Bottom;
> + break;
> + case AVCHROMA_LOC_TOPLEFT:
> + loc = kCVImageBufferChromaLocation_TopLeft;
> + break;
> + case AVCHROMA_LOC_BOTTOMLEFT:
> + loc = kCVImageBufferChromaLocation_BottomLeft;
> + break;
> + }
> +
> + if (loc) {
> + CVBufferSetAttachment(
> + pixbuf,
> + kCVImageBufferChromaLocationTopFieldKey,
> + loc,
> + kCVAttachmentMode_ShouldPropagate);
> + }
> +
> + return 0;
> +}
> +
> +static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc,
> + CVPixelBufferRef pixbuf, const
> AVFrame *src)
> +{
> + CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL;
> + Float32 gamma = 0;
> +
> + switch (src->colorspace) {
> + case AVCOL_SPC_BT2020_CL:
> + case AVCOL_SPC_BT2020_NCL:
> + if (__builtin_available(macOS 10.11, *))
> + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020;
> + else
> + colormatrix = CFSTR("ITU_R_2020");
> + break;
> + case AVCOL_SPC_BT470BG:
> + case AVCOL_SPC_SMPTE170M:
> + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
> + break;
> + case AVCOL_SPC_BT709:
> + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
> + break;
> + case AVCOL_SPC_SMPTE240M:
> + colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
> + break;
> + case AVCOL_SPC_UNSPECIFIED:
> + break;
> + default:
> + av_log(hwfc, AV_LOG_WARNING, "Color space %s is not
> supported.\n", av_color_space_name(src->colorspace));
> + }
> +
> + switch (src->color_primaries) {
> + case AVCOL_PRI_BT2020:
> + if (__builtin_available(macOS 10.11, *))
> + colorpri = kCVImageBufferColorPrimaries_ITU_R_2020;
> + else
> + colorpri = CFSTR("ITU_R_2020");
> + break;
> + case AVCOL_PRI_BT709:
> + colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2;
> + break;
> + case AVCOL_PRI_SMPTE170M:
> + colorpri = kCVImageBufferColorPrimaries_SMPTE_C;
> + break;
> + case AVCOL_PRI_BT470BG:
> + colorpri = kCVImageBufferColorPrimaries_EBU_3213;
> + break;
> + case AVCOL_PRI_UNSPECIFIED:
> + break;
> + default:
> + av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not
> supported.\n", av_color_primaries_name(src->color_primaries));
> + }
> +
> + switch (src->color_trc) {
> + case AVCOL_TRC_SMPTE2084:
> + if (__builtin_available(macOS 10.13, *))
> + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
> + else
> + colortrc = CFSTR("SMPTE_ST_2084_PQ");
> + break;
> + case AVCOL_TRC_BT2020_10:
> + case AVCOL_TRC_BT2020_12:
> + if (__builtin_available(macOS 10.11, *))
> + colortrc = kCVImageBufferTransferFunction_ITU_R_2020;
> + else
> + colortrc = CFSTR("ITU_R_2020");
> + break;
> + case AVCOL_TRC_BT709:
> + colortrc = kCVImageBufferTransferFunction_ITU_R_709_2;
> + break;
> + case AVCOL_TRC_SMPTE240M:
> + colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
> + break;
> + case AVCOL_TRC_SMPTE428:
> + if (__builtin_available(macOS 10.12, *))
> + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1;
> + else
> + colortrc = CFSTR("SMPTE_ST_428_1");
> + break;
> + case AVCOL_TRC_ARIB_STD_B67:
> + if (__builtin_available(macOS 10.13, *))
> + colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
> + else
> + colortrc = CFSTR("ITU_R_2100_HLG");
> + break;
> + case AVCOL_TRC_GAMMA22:
> + gamma = 2.2;
> + colortrc = kCVImageBufferTransferFunction_UseGamma;
> + break;
> + case AVCOL_TRC_GAMMA28:
> + gamma = 2.8;
> + colortrc = kCVImageBufferTransferFunction_UseGamma;
> + break;
> + case AVCOL_TRC_UNSPECIFIED:
> + break;
> + default:
> + av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not
> supported.\n", av_color_transfer_name(src->color_trc));
> + }
> +
> + if (colormatrix) {
> + CVBufferSetAttachment(
> + pixbuf,
> + kCVImageBufferYCbCrMatrixKey,
> + colormatrix,
> + kCVAttachmentMode_ShouldPropagate);
> + }
> + if (colorpri) {
> + CVBufferSetAttachment(
> + pixbuf,
> + kCVImageBufferColorPrimariesKey,
> + colorpri,
> + kCVAttachmentMode_ShouldPropagate);
> + }
> + if (colortrc) {
> + CVBufferSetAttachment(
> + pixbuf,
> + kCVImageBufferTransferFunctionKey,
> + colortrc,
> + kCVAttachmentMode_ShouldPropagate);
> + }
> + if (gamma != 0) {
> + CFNumberRef gamma_level = CFNumberCreate(NULL,
> kCFNumberFloat32Type, &gamma);
> + CVBufferSetAttachment(
> + pixbuf,
> + kCVImageBufferGammaLevelKey,
> + gamma_level,
> + kCVAttachmentMode_ShouldPropagate);
> + CFRelease(gamma_level);
> + }
> +
> + return 0;
> +}
> +
> +static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc,
> + CVPixelBufferRef pixbuf, const
> AVFrame *src)
> +{
> + int ret;
> + ret = vt_pixbuf_set_par(hwfc, pixbuf, src);
> + if (ret < 0)
> + return ret;
> + ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src);
> + if (ret < 0)
> + return ret;
> + ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src);
> + if (ret < 0)
> + return ret;
> + return 0;
> +}
> +
> static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const
> AVFrame *src,
> int flags)
> {
> @@ -223,6 +626,10 @@ static int vt_transfer_data_to(AVHWFramesContext
> *hwfc,
> if (err)
> goto fail;
>
> + err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3],
> src);
> + if (err)
> + goto fail;
> +
> err = 0;
> fail:
> av_frame_free(&map);
> @@ -244,8 +651,13 @@ const HWContextType ff_hwcontext_type_videotoolbox = {
> .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
> .name = "videotoolbox",
>
> + .frames_priv_size = sizeof(VTFramesContext),
> +
> .device_create = vt_device_create,
> + .frames_init = vt_frames_init,
> .frames_get_buffer = vt_get_buffer,
> + .frames_get_constraints = vt_frames_get_constraints,
> + .frames_uninit = vt_frames_uninit,
> .transfer_get_formats = vt_transfer_get_formats,
> .transfer_data_to = vt_transfer_data_to,
> .transfer_data_from = vt_transfer_data_from,
> diff --git a/libavutil/hwcontext_videotoolbox.h
> b/libavutil/hwcontext_videotoolbox.h
> index 5074d79e68..62cde07c51 100644
> --- a/libavutil/hwcontext_videotoolbox.h
> +++ b/libavutil/hwcontext_videotoolbox.h
> @@ -29,11 +29,14 @@
> * @file
> * An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX.
> *
> - * This API currently does not support frame allocation, as the raw
> VideoToolbox
> - * API does allocation, and FFmpeg itself never has the need to allocate
> frames.
> + * This API supports frame allocation using a native CVPixelBufferPool
> + * instead of an AVBufferPool.
> *
> * If the API user sets a custom pool, AVHWFramesContext.pool must return
> * AVBufferRefs whose data pointer is a CVImageBufferRef or
> CVPixelBufferRef.
> + * Note that the underlying CVPixelBuffer could be retained by OS
> frameworks
> + * depending on application usage, so it is preferable to let CoreVideo
> manage
> + * the pool using the default implementation.
> *
> * Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are
> always
> * NULL.
> --
> 2.29.2
>
>
More information about the ffmpeg-devel
mailing list