[FFmpeg-devel] [PATCH 2/2] avutil/hwcontext_vaapi: return ENOSYS for unsupported operation

[Zhao Zhili]

From: Zhao Zhili <zhilizhao at tencent.com>

av_hwframe_transfer_data try with src_ctx first. If the operation
failed with AVERROR(ENOSYS), it will try again with dst_ctx. Return
AVERROR(EINVAL) makes the second step being skipped.
---
 libavutil/hwcontext_vaapi.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c
index 558fed94c6..bb28bcf588 100644
--- a/libavutil/hwcontext_vaapi.c
+++ b/libavutil/hwcontext_vaapi.c
@@ -217,7 +217,7 @@ static int vaapi_get_image_format(AVHWDeviceContext *hwdev,
             return 0;
         }
     }
-    return AVERROR(EINVAL);
+    return AVERROR(ENOSYS);
 }
 
 static int vaapi_frames_get_constraints(AVHWDeviceContext *hwdev,
@@ -805,19 +805,19 @@ static int vaapi_map_frame(AVHWFramesContext *hwfc,
 
     if (!ctx->derive_works && (flags & AV_HWFRAME_MAP_DIRECT)) {
         // Requested direct mapping but it is not possible.
-        return AVERROR(EINVAL);
+        return AVERROR(ENOSYS);
     }
     if (dst->format == AV_PIX_FMT_NONE)
         dst->format = hwfc->sw_format;
     if (dst->format != hwfc->sw_format && (flags & AV_HWFRAME_MAP_DIRECT)) {
         // Requested direct mapping but the formats do not match.
-        return AVERROR(EINVAL);
+        return AVERROR(ENOSYS);
     }
 
     err = vaapi_get_image_format(hwfc->device_ctx, dst->format, &image_format);
     if (err < 0) {
         // Requested format is not a valid output format.
-        return AVERROR(EINVAL);
+        return err;
     }
 
     map = av_malloc(sizeof(*map));
@@ -992,7 +992,7 @@ static int vaapi_map_to_memory(AVHWFramesContext *hwfc, AVFrame *dst,
     if (dst->format != AV_PIX_FMT_NONE) {
         err = vaapi_get_image_format(hwfc->device_ctx, dst->format, NULL);
         if (err < 0)
-            return AVERROR(ENOSYS);
+            return err;
     }
 
     err = vaapi_map_frame(hwfc, dst, src, flags);
-- 
2.25.1