[FFmpeg-devel] [PATCH] lavc/videotoolbox: Consistent fallback to external software encoding/decoding.

[Thilo Borgmann]

Hi,

trying to handle software fallback more consistently for videotoolbox and
probably other hardware accelerations.

Addresses ticket #5352 where software fallback is demanded which has been
removed on purpose before. With this patch the user can configure the desired
behaviour.

-Thilo
-------------- next part --------------
From b7d6e9f40171e59c703d8a59c8097eb9f37c2e14 Mon Sep 17 00:00:00 2001
From: Thilo Borgmann <thilo.borgmann at mail.de>
Date: Sun, 20 Mar 2016 16:40:59 +0100
Subject: [PATCH] lavc/videotoolbox: Consistent fallback to external software
 encoding/decoding.

    Fallback to external software encoding or decoding only on user demand
    by specifying codec flag ext_sw_fallback. Fail otherwise.

    Fixes ticket #5352.
---
 configure                    |  2 +-
 libavcodec/avcodec.h         |  4 ++++
 libavcodec/options_table.h   |  1 +
 libavcodec/videotoolbox.c    |  8 +++++++-
 libavcodec/videotoolboxenc.c | 26 +++++---------------------
 5 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/configure b/configure
index 8a1ac38..a09f8e5 100755
--- a/configure
+++ b/configure
@@ -2673,7 +2673,7 @@ pcm_mulaw_at_encoder_deps="audiotoolbox"
 pcm_mulaw_at_encoder_select="audio_frame_queue"
 chromaprint_muxer_deps="chromaprint"
 h264_videotoolbox_encoder_deps="videotoolbox_encoder pthreads"
-h264_videotoolbox_encoder_select="bzlib zlib iconv"
+#h264_videotoolbox_encoder_select="bzlib zlib iconv"
 libcelt_decoder_deps="libcelt"
 libdcadec_decoder_deps="libdcadec"
 libfaac_encoder_deps="libfaac"
diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
index 637984b..c46400a 100644
--- a/libavcodec/avcodec.h
+++ b/libavcodec/avcodec.h
@@ -744,6 +744,10 @@ typedef struct RcOverride{
  */
 #define AV_CODEC_FLAG_QPEL            (1 <<  4)
 /**
+ * Allow external software fallback for external hardware accellerators
+ */
+#define AV_CODEC_FLAG_EXT_SW_FALLBACK (1 <<  5)
+/**
  * Use internal 2pass ratecontrol in first pass mode.
  */
 #define AV_CODEC_FLAG_PASS1           (1 <<  9)
diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h
index d29996f..04a1e4e 100644
--- a/libavcodec/options_table.h
+++ b/libavcodec/options_table.h
@@ -52,6 +52,7 @@ static const AVOption avcodec_options[] = {
 {"unaligned", "allow decoders to produce unaligned output", 0, AV_OPT_TYPE_CONST, { .i64 = AV_CODEC_FLAG_UNALIGNED }, INT_MIN, INT_MAX, V | D, "flags" },
 {"mv4", "use four motion vectors per macroblock (MPEG-4)", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_4MV }, INT_MIN, INT_MAX, V|E, "flags"},
 {"qpel", "use 1/4-pel motion compensation", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_QPEL }, INT_MIN, INT_MAX, V|E, "flags"},
+{"ext_sw_fallback", "allow external software fallback for external hardware accellerators", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_EXT_SW_FALLBACK }, INT_MIN, INT_MAX, V|E|D, "flags"},
 {"loop", "use loop filter", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_LOOP_FILTER }, INT_MIN, INT_MAX, V|E, "flags"},
 {"qscale", "use fixed qscale", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG_QSCALE }, INT_MIN, INT_MAX, 0, "flags"},
 #if FF_API_GMC
diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
index 2f4d531..8554f02 100644
--- a/libavcodec/videotoolbox.c
+++ b/libavcodec/videotoolbox.c
@@ -32,6 +32,10 @@
 #include "h264.h"
 #include "mpegvideo.h"
 
+#ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
+#  define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
+#endif
+
 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
 #  define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
 #endif
@@ -412,7 +416,9 @@ static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType codec
                                                                    &kCFTypeDictionaryValueCallBacks);
 
     CFDictionarySetValue(config_info,
-                         kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
+                         (avctx->flags & AV_CODEC_FLAG_EXT_SW_FALLBACK) ?
+                            kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder :
+                            kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
                          kCFBooleanTrue);
 
     if (avctx->extradata_size) {
diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index 3ed1f64..68d46ce 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -492,8 +492,11 @@ static av_cold int vtenc_init(AVCodecContext *avctx)
     if (!enc_info) return AVERROR(ENOMEM);
 
 #if !TARGET_OS_IPHONE
-    CFDictionarySetValue(enc_info, kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder, kCFBooleanTrue);
-    CFDictionarySetValue(enc_info, kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,  kCFBooleanTrue);
+    CFDictionarySetValue(enc_info,
+                         (avctx->flags & AV_CODEC_FLAG_EXT_SW_FALLBACK) ?
+                            kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder :
+                            kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
+                         kCFBooleanTrue);
 #endif
 
     status = VTCompressionSessionCreate(
@@ -509,25 +512,6 @@ static av_cold int vtenc_init(AVCodecContext *avctx)
         &vtctx->session
     );
 
-#if !TARGET_OS_IPHONE
-    if (status != 0 || !vtctx->session) {
-        CFDictionaryRemoveValue(enc_info, kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder);
-
-        status = VTCompressionSessionCreate(
-            kCFAllocatorDefault,
-            avctx->width,
-            avctx->height,
-            codec_type,
-            enc_info,
-            NULL,
-            kCFAllocatorDefault,
-            vtenc_output_callback,
-            avctx,
-            &vtctx->session
-        );
-    }
-#endif
-
     CFRelease(enc_info);
 
     if (status || !vtctx->session) {
-- 
1.8.3.2