[vulkan] Dynamically load Vulkan loader library. Avoid Validation Layer crash on exit.

Makefile

@@ -335,9 +335,6 @@ endif
 ifneq ($(TEST_OPENCL), )
 OPENCL_LD_FLAGS ?= -lOpenCL
 endif
-ifneq ($(TEST_VULKAN), )
-VULKAN_LD_FLAGS ?= -lvulkan
-endif
 HOST_OS=linux
 endif
 
@@ -349,10 +346,6 @@ endif
 ifneq ($(TEST_OPENCL), )
 OPENCL_LD_FLAGS ?= -framework OpenCL
 endif
-ifneq ($(TEST_VULKAN), )
-# The Vulkan loader is distributed as a dylib on OSX (not a framework)
-VULKAN_LD_FLAGS ?= -lvulkan
-endif
 ifneq ($(TEST_METAL), )
 METAL_LD_FLAGS ?= -framework Metal -framework Foundation
 endif

README_vulkan.md

@@ -57,9 +57,9 @@ For Vulkan device drivers, consult the appropriate hardware vendor for your devi
 
 ## Linux 
 
-On Ubuntu Linux v22.04, the vulkan runtime is distributed in the `vulkan-tools` package. For earlier versions of Ubuntu (e.g. v20.x or v18.x) the contents of the `vulkan-tools` package was distributed as `vulkan-utils` so use that package instead.
+The Vulkan SDK packages are now being maintained by LunarG.  These include the Vulkan Loader library, as well as the Vulkan Tools packages. Instructions for installing these can be found on their [Getting Started Guide](https://vulkan.lunarg.com/doc/view/latest/linux/getting_started_ubuntu.html).
 
-Proprietary drivers can be installed via 'apt' using PPA's for each vendor. Examples for AMD and NVIDIA are provided below.
+Once the SDK has been installed, you need to install the appropriate driver for your device.  Proprietary drivers can be installed via 'apt' using PPA's for each vendor. Examples for AMD and NVIDIA are provided below.
 
 For AMD on Ubuntu v22.04:
 ```

cmake/HalideGeneratorHelpers.cmake

@@ -712,11 +712,6 @@ function(_Halide_add_targets_to_runtime TARGET)
 endfunction()
 
 function(_Halide_target_link_gpu_libs TARGET VISIBILITY)
-    if ("${ARGN}" MATCHES "vulkan")
-        find_package(Vulkan REQUIRED)
-        target_link_libraries(${TARGET} ${VISIBILITY} Vulkan::Vulkan)
-    endif ()
-
     if ("${ARGN}" MATCHES "metal")
         find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
         find_library(METAL_LIBRARY Metal REQUIRED)

src/runtime/mini_vulkan.h

@@ -2639,18 +2639,16 @@ typedef void(VKAPI_PTR *PFN_vkCmdNextSubpass)(VkCommandBuffer commandBuffer, VkS
 typedef void(VKAPI_PTR *PFN_vkCmdEndRenderPass)(VkCommandBuffer commandBuffer);
 typedef void(VKAPI_PTR *PFN_vkCmdExecuteCommands)(VkCommandBuffer commandBuffer, uint32_t commandBufferCount, const VkCommandBuffer *pCommandBuffers);
 
-// This appears to be exported by the loader
+#ifndef VK_NO_PROTOTYPES
 VKAPI_ATTR VkResult VKAPI_CALL vkCreateInstance(
     const VkInstanceCreateInfo *pCreateInfo,
     const VkAllocationCallbacks *pAllocator,
     VkInstance *pInstance);
 
-// Same as above ... these two methods are the only prototypes we depend upon
 VKAPI_ATTR PFN_vkVoidFunction VKAPI_CALL vkGetInstanceProcAddr(
     VkInstance instance,
     const char *pName);
 
-#ifndef VK_NO_PROTOTYPES
 VKAPI_ATTR void VKAPI_CALL vkDestroyInstance(
     VkInstance instance,
     const VkAllocationCallbacks *pAllocator);

src/runtime/vulkan.cpp

@@ -210,47 +210,36 @@ WEAK int halide_vulkan_device_release(void *user_context) {
     debug(user_context)
         << "halide_vulkan_device_release (user_context: " << user_context << ")\n";
 
-    VulkanMemoryAllocator *allocator;
-    VkInstance instance;
-    VkDevice device;
-    VkCommandPool command_pool;
-    VkPhysicalDevice physical_device;
-    VkQueue queue;
-    uint32_t _throwaway;
-
+    VulkanMemoryAllocator *allocator = nullptr;
+    VkInstance instance = nullptr;
+    VkDevice device = nullptr;
+    VkCommandPool command_pool = VkInvalidCommandPool;
+    VkPhysicalDevice physical_device = nullptr;
+    VkQueue queue = nullptr;
+    uint32_t queue_family_index = 0;
+
+    int destroy_status = halide_error_code_success;
     int acquire_status = halide_vulkan_acquire_context(user_context,
                                                        reinterpret_cast<halide_vulkan_memory_allocator **>(&allocator),
-                                                       &instance, &device, &physical_device, &command_pool, &queue, &_throwaway, false);
+                                                       &instance, &device, &physical_device, &command_pool, &queue, &queue_family_index, false);
 
-    if ((acquire_status == halide_error_code_success) && (instance != nullptr)) {
-        vkQueueWaitIdle(queue);
-        if (command_pool == cached_command_pool) {
-            cached_command_pool = 0;
-        }
-        if (reinterpret_cast<halide_vulkan_memory_allocator *>(allocator) == cached_allocator) {
+    if (acquire_status == halide_error_code_success) {
+        // Destroy the context if we created it
+        if ((instance == cached_instance) && (device == cached_device)) {
+            destroy_status = vk_destroy_context(user_context, allocator, instance, device, physical_device, command_pool, queue);
+            cached_command_pool = VkInvalidCommandPool;
             cached_allocator = nullptr;
-        }
-
-        vk_destroy_command_pool(user_context, allocator, command_pool);
-        vk_destroy_shader_modules(user_context, allocator);
-        vk_destroy_memory_allocator(user_context, allocator);
-
-        if (device == cached_device) {
             cached_device = nullptr;
             cached_physical_device = nullptr;
             cached_queue = nullptr;
             cached_queue_family_index = 0;
-        }
-        vkDestroyDevice(device, nullptr);
-
-        if (instance == cached_instance) {
             cached_instance = nullptr;
         }
-        vkDestroyInstance(instance, nullptr);
+
         halide_vulkan_release_context(user_context, instance, device, queue);
     }
 
-    return halide_error_code_success;
+    return destroy_status;
 }
 
 WEAK int halide_vulkan_device_malloc(void *user_context, halide_buffer_t *buf) {
@@ -1409,7 +1398,18 @@ WEAK __attribute__((constructor)) void register_vulkan_allocation_pool() {
 }
 
 WEAK __attribute__((destructor)) void halide_vulkan_cleanup() {
-    halide_vulkan_device_release(nullptr);
+    // FIXME: When the VK_LAYER_KHRONOS_validation is intercepting calls to the API, it will
+    //        cause a segfault if it's invoked inside the destructor since it uses it's own global
+    //        state to track object usage which is no longer valid once this call is invoked.
+    //        Calling this destructor with the validator hooks in place will cause an uncaught
+    //        exception for an uninitialized mutex lock. We can avoid the crash on exit by \
+    //        bypassing the device release call and leak (!!!!)
+    // ISSUE: https://github.com/halide/Halide/issues/8290
+    const char *layer_names = vk_get_layer_names_internal(nullptr);
+    bool has_validation_layer = strstr(layer_names, "VK_LAYER_KHRONOS_validation");
+    if (!has_validation_layer) {
+        halide_vulkan_device_release(nullptr);
+    }
 }
 
 // --------------------------------------------------------------------------

src/runtime/vulkan_context.h

@@ -24,7 +24,7 @@ halide_vulkan_memory_allocator *WEAK cached_allocator = nullptr;
 // Cached instance related handles for device resources
 VkInstance WEAK cached_instance = nullptr;
 VkDevice WEAK cached_device = nullptr;
-VkCommandPool WEAK cached_command_pool = 0;
+VkCommandPool WEAK cached_command_pool = VkInvalidCommandPool;
 VkQueue WEAK cached_queue = nullptr;
 VkPhysicalDevice WEAK cached_physical_device = nullptr;
 uint32_t WEAK cached_queue_family_index = 0;
@@ -42,7 +42,7 @@ class VulkanContext {
     VulkanMemoryAllocator *allocator = nullptr;
     VkInstance instance = nullptr;
     VkDevice device = nullptr;
-    VkCommandPool command_pool = 0;
+    VkCommandPool command_pool = VkInvalidCommandPool;
     VkPhysicalDevice physical_device = nullptr;
     VkQueue queue = nullptr;
     uint32_t queue_family_index = 0;  // used for operations requiring queue family
@@ -88,27 +88,52 @@ int vk_find_compute_capability(void *user_context, int *major, int *minor) {
     VkPhysicalDevice physical_device = nullptr;
     uint32_t queue_family_index = 0;
 
+    if (!lib_vulkan) {
+        // If the Vulkan loader can't be found, we want to return compute
+        // capability of (0, 0) ... this is not considered an error. So we
+        // should be very careful about looking for Vulkan without tripping
+        // any errors in the rest of this runtime.
+        void *sym = halide_vulkan_get_symbol(user_context, "vkCreateInstance");
+        if (!sym) {
+            *major = *minor = 0;
+            return halide_error_code_success;
+        }
+    }
+
+    if (vkCreateInstance == nullptr) {
+        vk_load_vulkan_loader_functions(user_context);
+        if (vkCreateInstance == nullptr) {
+            debug(user_context) << "  no valid vulkan loader library was found ...\n";
+            *major = *minor = 0;
+            return halide_error_code_success;
+        }
+    }
+
     StringTable requested_layers;
     vk_get_requested_layers(user_context, requested_layers);
 
+    // Attempt to find a suitable instance that can support the requested layers
     const VkAllocationCallbacks *alloc_callbacks = halide_vulkan_get_allocation_callbacks(user_context);
     int status = vk_create_instance(user_context, requested_layers, &instance, alloc_callbacks);
     if (status != halide_error_code_success) {
         debug(user_context) << "  no valid vulkan runtime was found ...\n";
-        *major = 0;
-        *minor = 0;
+        *major = *minor = 0;
         return 0;
     }
 
     if (vkCreateDevice == nullptr) {
-        vk_load_vulkan_functions(instance);
+        vk_load_vulkan_functions(user_context, instance);
+        if (vkCreateDevice == nullptr) {
+            debug(user_context) << "  no valid vulkan runtime was found ...\n";
+            *major = *minor = 0;
+            return halide_error_code_success;
+        }
     }
 
     status = vk_select_device_for_context(user_context, &instance, &device, &physical_device, &queue_family_index);
     if (status != halide_error_code_success) {
         debug(user_context) << "  no valid vulkan device was found ...\n";
-        *major = 0;
-        *minor = 0;
+        *major = *minor = 0;
         return 0;
     }
 
@@ -425,6 +450,14 @@ int vk_create_context(void *user_context, VulkanMemoryAllocator **allocator,
 
     debug(user_context) << " vk_create_context (user_context: " << user_context << ")\n";
 
+    if (vkCreateInstance == nullptr) {
+        vk_load_vulkan_loader_functions(user_context);
+        if (vkCreateInstance == nullptr) {
+            error(user_context) << "Vulkan: Failed to resolve loader library methods to create instance!\n";
+            return halide_error_code_symbol_not_found;
+        }
+    }
+
     StringTable requested_layers;
     uint32_t requested_layer_count = vk_get_requested_layers(user_context, requested_layers);
     debug(user_context) << "  requested " << requested_layer_count << " layers for instance!\n";
@@ -440,7 +473,11 @@ int vk_create_context(void *user_context, VulkanMemoryAllocator **allocator,
     }
 
     if (vkCreateDevice == nullptr) {
-        vk_load_vulkan_functions(*instance);
+        vk_load_vulkan_functions(user_context, *instance);
+        if (vkCreateDevice == nullptr) {
+            error(user_context) << "Vulkan: Failed to resolve API library methods to create device!\n";
+            return halide_error_code_symbol_not_found;
+        }
     }
 
     error_code = vk_select_device_for_context(user_context, instance, device, physical_device, queue_family_index);
@@ -470,6 +507,32 @@ int vk_create_context(void *user_context, VulkanMemoryAllocator **allocator,
     return halide_error_code_success;
 }
 
+// Destroys the context and all associated resources (used by halide_vulkan_device_release)
+// NOTE: This should be called inside an acquire_context/release_context scope
+int vk_destroy_context(void *user_context, VulkanMemoryAllocator *allocator,
+                       VkInstance instance, VkDevice device, VkPhysicalDevice physical_device,
+                       VkCommandPool command_pool, VkQueue queue) {
+
+    debug(user_context)
+        << "vk_destroy_context (user_context: " << user_context << ")\n";
+
+    if (device != nullptr) {
+        vkDeviceWaitIdle(device);
+    }
+    if ((command_pool != VkInvalidCommandPool) && (allocator != nullptr)) {
+        vk_destroy_command_pool(user_context, allocator, command_pool);
+        vk_destroy_shader_modules(user_context, allocator);
+        vk_destroy_memory_allocator(user_context, allocator);
+    }
+    if (device != nullptr) {
+        vkDestroyDevice(device, nullptr);
+    }
+    if (instance != nullptr) {
+        vkDestroyInstance(instance, nullptr);
+    }
+    return halide_error_code_success;
+}
+
 // --------------------------------------------------------------------------
 
 }  // namespace

src/runtime/vulkan_functions.h

@@ -1,6 +1,8 @@
-// NOTE: vkCreateInstance is already defined in the mini_vulkan header
+// NOTE: vkCreateInstance and vkGetInstanceProcAddr are defined in the loader library and will be resolved seperately
+// The rest of these are resolved via vkGetInstanceProcAddr which the loader exports and maps to the driver implementation
 VULKAN_FN(vkDestroyInstance)
 VULKAN_FN(vkCreateDevice)
+VULKAN_FN(vkDeviceWaitIdle)
 VULKAN_FN(vkDestroyDevice)
 VULKAN_FN(vkGetDeviceQueue)
 VULKAN_FN(vkCreateBuffer)

src/runtime/vulkan_interface.h

@@ -37,25 +37,70 @@ namespace Vulkan {
 
 // --------------------------------------------------------------------------
 
-// Halide device interface struct for runtime specific funtion table
+// Halide device interface struct for runtime specific function table
 extern WEAK halide_device_interface_t vulkan_device_interface;
 
 // --------------------------------------------------------------------------
 
+// The default implementation of halide_vulkan_get_symbol attempts to load
+// the Vulkan loader shared library/DLL, and then get the symbol from it.
+WEAK void *lib_vulkan = nullptr;
+
+extern "C" WEAK void *halide_vulkan_get_symbol(void *user_context, const char *name) {
+    // Only try to load the library if the library isn't already
+    // loaded, or we can't load the symbol from the process already.
+    void *symbol = halide_get_library_symbol(lib_vulkan, name);
+    if (symbol) {
+        return symbol;
+    }
+
+    const char *lib_names[] = {
+#ifdef WINDOWS
+        "vulkan-1.dll",
+#else
+        "libvulkan.so.1",
+        "libvulkan.1.dylib",
+#endif
+    };
+    for (auto &lib_name : lib_names) {
+        lib_vulkan = halide_load_library(lib_name);
+        if (lib_vulkan) {
+            debug(user_context) << "    Loaded Vulkan loader library: " << lib_name << "\n";
+            break;
+        } else {
+            debug(user_context) << "    Missing Vulkan loader library: " << lib_name << "\n";
+        }
+    }
+
+    return halide_get_library_symbol(lib_vulkan, name);
+}
+
+// Declare all the function pointers for the Vulkan API methods that will be resolved dynamically
 // clang-format off
 #define VULKAN_FN(fn) WEAK PFN_##fn fn; 
+VULKAN_FN(vkCreateInstance)
+VULKAN_FN(vkGetInstanceProcAddr)
 #include "vulkan_functions.h"
 #undef VULKAN_FN
 // clang-format on
 
-void WEAK vk_load_vulkan_functions(VkInstance instance) {
+// Get the function pointers from the Vulkan loader to create an Instance (to find all available driver implementations)
+void WEAK vk_load_vulkan_loader_functions(void *user_context) {
+    debug(user_context) << "    vk_load_vulkan_loader_functions (user_context: " << user_context << ")\n";
+#define VULKAN_FN(fn) fn = (PFN_##fn)halide_vulkan_get_symbol(user_context, #fn);
+    VULKAN_FN(vkCreateInstance)
+    VULKAN_FN(vkGetInstanceProcAddr)
+#undef VULKAN_FN
+}
+
+// Get the function pointers from the Vulkan instance for the resolved driver API methods.
+void WEAK vk_load_vulkan_functions(void *user_context, VkInstance instance) {
+    debug(user_context) << "    vk_load_vulkan_functions (user_context: " << user_context << ")\n";
 #define VULKAN_FN(fn) fn = (PFN_##fn)vkGetInstanceProcAddr(instance, #fn);
 #include "vulkan_functions.h"
 #undef VULKAN_FN
 }
 
-// --
-
 // --------------------------------------------------------------------------
 
 }  // namespace Vulkan

src/runtime/vulkan_internal.h

@@ -58,6 +58,15 @@ int vk_create_context(
     VkCommandPool *command_pool,
     VkQueue *queue, uint32_t *queue_family_index);
 
+int vk_destroy_context(
+    void *user_context,
+    VulkanMemoryAllocator *allocator,
+    VkInstance instance,
+    VkDevice device,
+    VkPhysicalDevice physical_device,
+    VkCommandPool command_pool,
+    VkQueue queue);
+
 int vk_find_compute_capability(void *user_context, int *major, int *minor);
 
 int vk_create_instance(void *user_context, const StringTable &requested_layers, VkInstance *instance, const VkAllocationCallbacks *alloc_callbacks);
@@ -92,6 +101,9 @@ bool vk_validate_required_extension_support(void *user_context,
 int vk_create_command_pool(void *user_context, VulkanMemoryAllocator *allocator, uint32_t queue_index, VkCommandPool *command_pool);
 int vk_destroy_command_pool(void *user_context, VulkanMemoryAllocator *allocator, VkCommandPool command_pool);
 
+// Command pools are uint64_t and zero may be valid, so use this as a sentinel for invalid
+const VkCommandPool VkInvalidCommandPool(uint64_t(-1));
+
 // -- Command Buffer
 int vk_create_command_buffer(void *user_context, VulkanMemoryAllocator *allocator, VkCommandPool pool, VkCommandBuffer *command_buffer);
 int vk_destroy_command_buffer(void *user_context, VulkanMemoryAllocator *allocator, VkCommandPool command_pool, VkCommandBuffer command_buffer);