[FFmpeg-devel] [PATCH] vulkan: Fix win/i386 calling convention and pointer/int conversion error

[Martin Storsjö]

This fixes the following two errors when compiling with a modern
version of Clang for Windows/i386:

src/libavutil/hwcontext_vulkan.c:738:32: error: incompatible function pointer types initializing 'PFN_vkDebugUtilsMessengerCallbackEXT' (aka 'unsigned int (*)(enum VkDebugUtilsMessageSeverityFlagBitsEXT, unsigned int, const struct VkDebugUtilsMessengerCallbackDataEXT *, void *) __attribute__((stdcall))') with an expression of type 'VkBool32 (VkDebugUtilsMessageSeverityFlagBitsEXT, VkDebugUtilsMessageTypeFlagsEXT, const VkDebugUtilsMessengerCallbackDataEXT *, void *)' (aka 'unsigned int (enum VkDebugUtilsMessageSeverityFlagBitsEXT, unsigned int, const struct VkDebugUtilsMessengerCallbackDataEXT *, void *)') [-Wincompatible-function-pointer-types]
            .pfnUserCallback = vk_dbg_callback,
                               ^~~~~~~~~~~~~~~
src/libavutil/hwcontext_vulkan.c:1152:15: error: incompatible pointer to integer conversion assigning to 'VkCommandPool' (aka 'unsigned long long') from 'void *' [-Wint-conversion]
    cmd->pool = NULL;
              ^ ~~~~
2 errors generated.

The calling convention mismatch is an issue only on Windows/i386,
while the latter is a hard error by default in modern Clang whenever
pointers are smaller than unsigned long long.

Signed-off-by: Martin Storsjö <martin at martin.st>
---
 libavutil/hwcontext_vulkan.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index 2a9b5f4aac..2ae469283d 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -401,10 +401,10 @@ static const char *vk_ret2str(VkResult res)
 #undef CASE
 }
 
-static VkBool32 vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
-                                VkDebugUtilsMessageTypeFlagsEXT messageType,
-                                const VkDebugUtilsMessengerCallbackDataEXT *data,
-                                void *priv)
+static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
+                                           VkDebugUtilsMessageTypeFlagsEXT messageType,
+                                           const VkDebugUtilsMessengerCallbackDataEXT *data,
+                                           void *priv)
 {
     int l;
     AVHWDeviceContext *ctx = priv;
@@ -1149,7 +1149,7 @@ static void free_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd)
 
     av_freep(&cmd->queues);
     av_freep(&cmd->bufs);
-    cmd->pool = NULL;
+    cmd->pool = 0;
 }
 
 static VkCommandBuffer get_buf_exec_ctx(AVHWFramesContext *hwfc, VulkanExecCtx *cmd)
-- 
2.34.1