[FFmpeg-devel] [PATCH] hwcontext: Return a sensible error when deriving an unsupported device type

[Mark Thompson]

This previously returned ENOMEM because the failure was detected during
av_hwdevice_ctx_alloc(), which is not helpful.  Check earlier in order to
return the correct ENOSYS instead.
---
Before:

$ ./ffmpeg_g -init_hw_device vaapi=d0:/dev/dri/renderD128 -init_hw_device dxva2=d1 at d0
...
Device creation failed: -12.
Failed to set value 'dxva2=d1 at d0' for option 'init_hw_device': Cannot allocate memory
Error parsing global options: Cannot allocate memory

After:

$ ./ffmpeg_g -init_hw_device vaapi=d0:/dev/dri/renderD128 -init_hw_device dxva2=d1 at d0
Device creation failed: -38.
Failed to set value 'dxva2=d1 at d0' for option 'init_hw_device': Function not implemented
Error parsing global options: Function not implemented


  libavutil/hwcontext.c | 24 ++++++++++++++++--------
  1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c
index d13d0f7c9b..4a307369c4 100644
--- a/libavutil/hwcontext.c
+++ b/libavutil/hwcontext.c
@@ -139,19 +139,22 @@ static void hwdevice_ctx_free(void *opaque, uint8_t *data)
      av_freep(&ctx);
  }

+static const HWContextType *hwcontext_find_type(enum AVHWDeviceType type)
+{
+    for (int i = 0; hw_table[i]; i++) {
+        if (hw_table[i]->type == type)
+            return hw_table[i];
+    }
+    return NULL;
+}
+
  AVBufferRef *av_hwdevice_ctx_alloc(enum AVHWDeviceType type)
  {
      AVHWDeviceContext *ctx;
      AVBufferRef *buf;
-    const HWContextType *hw_type = NULL;
-    int i;
+    const HWContextType *hw_type;

-    for (i = 0; hw_table[i]; i++) {
-        if (hw_table[i]->type == type) {
-            hw_type = hw_table[i];
-            break;
-        }
-    }
+    hw_type = hwcontext_find_type(type);
      if (!hw_type)
          return NULL;

@@ -652,6 +655,11 @@ int av_hwdevice_ctx_create_derived_opts(AVBufferRef **dst_ref_ptr,
      AVHWDeviceContext *dst_ctx, *tmp_ctx;
      int ret = 0;

+    if (!hwcontext_find_type(type)) {
+        // The type we want to derive is not supported in this build.
+        return AVERROR(ENOSYS);
+    }
+
      tmp_ref = src_ref;
      while (tmp_ref) {
          tmp_ctx = (AVHWDeviceContext*)tmp_ref->data;
-- 
2.29.2