GS/Vulkan: Simplify loader using DynamicLibrary

Backport of 8e3284d8c6

common/CocoaTools.h

@@ -17,6 +17,8 @@ namespace CocoaTools
 	void AddThemeChangeHandler(void* ctx, void(handler)(void* ctx));
 	/// Remove a handler previously added using AddThemeChangeHandler with the given context
 	void RemoveThemeChangeHandler(void* ctx);
+	/// Returns the bundle path.
+	std::optional<std::string> GetBundlePath();
 	/// Get the bundle path to the actual application without any translocation fun
 	std::optional<std::string> GetNonTranslocatedBundlePath();
 	/// Move the given file to the trash, and return the path to its new location

common/CocoaTools.mm

@@ -133,6 +133,17 @@ bool Common::PlaySoundAsync(const char* path)
 
 // MARK: - Updater
 
+std::optional<std::string> CocoaTools::GetBundlePath()
+{
+  std::optional<std::string> ret;
+  @autoreleasepool {
+    NSURL* url = [NSURL fileURLWithPath:[[NSBundle mainBundle] bundlePath]];
+    if (url)
+      ret = std::string([url fileSystemRepresentation]);
+  }
+  return ret;
+}
+
 std::optional<std::string> CocoaTools::GetNonTranslocatedBundlePath()
 {
 	// See https://objective-see.com/blog/blog_0x15.html

common/DynamicLibrary.cpp

@@ -5,7 +5,9 @@
 #include "common/Assertions.h"
 #include "common/Console.h"
 #include "common/Error.h"
+#include "common/FileSystem.h"
 #include "common/SmallString.h"
+#include "common/Path.h"
 #include "common/StringUtil.h"
 
 #include <cstring>
@@ -15,6 +17,9 @@
 #include "common/RedtapeWindows.h"
 #else
 #include <dlfcn.h>
+#ifdef __APPLE__
+#include "common/CocoaTools.h"
+#endif
 #endif
 
 DynamicLibrary::DynamicLibrary() = default;
@@ -91,6 +96,27 @@ bool DynamicLibrary::Open(const char* filename, Error* error)
 	m_handle = dlopen(filename, RTLD_NOW);
 	if (!m_handle)
 	{
+#ifdef __APPLE__
+		// On MacOS, try searching in Frameworks.
+		if (!Path::IsAbsolute(filename))
+		{
+			std::optional<std::string> bundle_path = CocoaTools::GetBundlePath();
+			if (bundle_path.has_value())
+			{
+				std::string frameworks_path = fmt::format("{}/Contents/Frameworks/{}", bundle_path.value(), filename);
+				if (FileSystem::FileExists(frameworks_path.c_str()))
+				{
+					m_handle = dlopen(frameworks_path.c_str(), RTLD_NOW);
+					if (m_handle)
+					{
+						Error::Clear(error);
+						return true;
+					}
+				}
+			}
+		}
+#endif
+
 		const char* err = dlerror();
 		Error::SetStringFmt(error, "Loading {} failed: {}", filename, err ? err : "<UNKNOWN>");
 		return false;

common/Error.cpp

@@ -30,6 +30,12 @@ void Error::Clear()
 	m_description = {};
 }
 
+void Error::Clear(Error* errptr)
+{
+	if (errptr)
+		errptr->Clear();
+}
+
 void Error::SetErrno(int err)
 {
 	SetErrno(std::string_view(), err);

common/Error.h

@@ -66,6 +66,7 @@ public:
 #endif
 
 	// helpers for setting
+	static void Clear(Error* errptr);
 	static void SetErrno(Error* errptr, int err);
 	static void SetErrno(Error* errptr, std::string_view prefix, int err);
 	static void SetSocket(Error* errptr, int err);

pcsx2/GS/Renderers/Vulkan/GSDeviceVK.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
+// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
 // SPDX-License-Identifier: LGPL-3.0+
 
 #include "GS/GS.h"
@@ -14,6 +14,7 @@
 
 #include "common/Console.h"
 #include "common/BitUtils.h"
+#include "common/Error.h"
 #include "common/HostSys.h"
 #include "common/Path.h"
 #include "common/ScopedGuard.h"
@@ -373,8 +374,7 @@ bool GSDeviceVK::SelectDeviceExtensions(ExtensionList* extension_list, bool enab
 	m_optional_extensions.vk_ext_calibrated_timestamps =
 		SupportsExtension(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME, false);
 	m_optional_extensions.vk_ext_rasterization_order_attachment_access =
-		SupportsExtension(VK_EXT_RASTERIZATION_ORDER_ATTACHMENT_ACCESS_EXTENSION_NAME, false) ||
-		SupportsExtension(VK_ARM_RASTERIZATION_ORDER_ATTACHMENT_ACCESS_EXTENSION_NAME, false);
+		SupportsExtension(VK_EXT_RASTERIZATION_ORDER_ATTACHMENT_ACCESS_EXTENSION_NAME, false);
 	m_optional_extensions.vk_ext_line_rasterization = SupportsExtension(VK_EXT_LINE_RASTERIZATION_EXTENSION_NAME,
 		require_line_rasterization);
 	m_optional_extensions.vk_khr_driver_properties = SupportsExtension(VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME, false);
@@ -2029,9 +2029,8 @@ void GSDeviceVK::GetAdaptersAndFullscreenModes(
 	}
 	else
 	{
-		if (Vulkan::LoadVulkanLibrary())
+		if (Vulkan::LoadVulkanLibrary(nullptr))
 		{
-			ScopedGuard lib_guard([]() { Vulkan::UnloadVulkanLibrary(); });
 			const VkInstance instance = CreateVulkanInstance(WindowInfo(), false, false);
 			if (instance != VK_NULL_HANDLE)
 			{
@@ -2040,6 +2039,8 @@ void GSDeviceVK::GetAdaptersAndFullscreenModes(
 
 				vkDestroyInstance(instance, nullptr);
 			}
+
+			Vulkan::UnloadVulkanLibrary();
 		}
 	}
 }
@@ -2497,9 +2498,11 @@ bool GSDeviceVK::CreateDeviceAndSwapChain()
 	bool enable_debug_utils = GSConfig.UseDebugDevice;
 	bool enable_validation_layer = GSConfig.UseDebugDevice;
 
-	if (!Vulkan::LoadVulkanLibrary())
+	Error error;
+	if (!Vulkan::LoadVulkanLibrary(&error))
 	{
-		Host::ReportErrorAsync("Error", "Failed to load Vulkan library. Does your GPU and/or driver support Vulkan?");
+		Error::AddPrefix(&error, "Failed to load Vulkan library. Does your GPU and/or driver support Vulkan?\nThe error was:\n");
+		Host::ReportErrorAsync("Error", error.GetDescription());
 		return false;
 	}
 

pcsx2/GS/Renderers/Vulkan/VKLoader.cpp

@@ -1,9 +1,12 @@
-// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
+// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
 // SPDX-License-Identifier: LGPL-3.0+
 
 #include "GS/Renderers/Vulkan/VKLoader.h"
 
 #include "common/Assertions.h"
+#include "common/Console.h"
+#include "common/DynamicLibrary.h"
+#include "common/Error.h"
 
 #include <cstdarg>
 #include <cstdio>
@@ -11,14 +14,6 @@
 #include <cstring>
 #include <string>
 
-#ifndef _WIN32
-#include <dlfcn.h>
-#endif
-
-#ifdef __APPLE__
-#include <mach-o/dyld.h>
-#endif
-
 extern "C" {
 
 #define VULKAN_MODULE_ENTRY_POINT(name, required) PFN_##name name;
@@ -41,134 +36,51 @@ void Vulkan::ResetVulkanLibraryFunctionPointers()
 #undef VULKAN_MODULE_ENTRY_POINT
 }
 
-#if defined(_WIN32)
-
-static HMODULE s_vulkan_module;
+static DynamicLibrary s_vulkan_library;
 
 bool Vulkan::IsVulkanLibraryLoaded()
 {
-	return s_vulkan_module != NULL;
+	return s_vulkan_library.IsOpen();
 }
 
-bool Vulkan::LoadVulkanLibrary()
+bool Vulkan::LoadVulkanLibrary(Error* error)
 {
-	pxAssertRel(!s_vulkan_module, "Vulkan module is not loaded.");
+	pxAssertRel(!s_vulkan_library.IsOpen(), "Vulkan module is not loaded.");
 
-	s_vulkan_module = LoadLibraryA("vulkan-1.dll");
-	if (!s_vulkan_module)
-	{
-		std::fprintf(stderr, "Failed to load vulkan-1.dll\n");
-		return false;
-	}
-
-	bool required_functions_missing = false;
-	auto LoadFunction = [&required_functions_missing](FARPROC* func_ptr, const char* name, bool is_required) {
-		*func_ptr = GetProcAddress(s_vulkan_module, name);
-		if (!(*func_ptr) && is_required)
-		{
-			std::fprintf(stderr, "Vulkan: Failed to load required module function %s\n", name);
-			required_functions_missing = true;
-		}
-	};
-
-#define VULKAN_MODULE_ENTRY_POINT(name, required) LoadFunction(reinterpret_cast<FARPROC*>(&name), #name, required);
-#include "VKEntryPoints.inl"
-#undef VULKAN_MODULE_ENTRY_POINT
-
-	if (required_functions_missing)
-	{
-		ResetVulkanLibraryFunctionPointers();
-		FreeLibrary(s_vulkan_module);
-		s_vulkan_module = nullptr;
-		return false;
-	}
-
-	return true;
-}
-
-void Vulkan::UnloadVulkanLibrary()
-{
-	ResetVulkanLibraryFunctionPointers();
-	if (s_vulkan_module)
-		FreeLibrary(s_vulkan_module);
-	s_vulkan_module = nullptr;
-}
-
-#else
-
-static void* s_vulkan_module;
-
-bool Vulkan::IsVulkanLibraryLoaded()
-{
-	return s_vulkan_module != nullptr;
-}
-
-bool Vulkan::LoadVulkanLibrary()
-{
-	pxAssertRel(!s_vulkan_module, "Vulkan module is not loaded.");
-
-#if defined(__APPLE__)
+#ifdef __APPLE__
 	// Check if a path to a specific Vulkan library has been specified.
 	char* libvulkan_env = getenv("LIBVULKAN_PATH");
 	if (libvulkan_env)
-		s_vulkan_module = dlopen(libvulkan_env, RTLD_NOW);
-	if (!s_vulkan_module)
+		s_vulkan_library.Open(libvulkan_env, error);
+	if (!s_vulkan_library.IsOpen() &&
+		!s_vulkan_library.Open(DynamicLibrary::GetVersionedFilename("MoltenVK").c_str(), error))
 	{
-		unsigned path_size = 0;
-		_NSGetExecutablePath(nullptr, &path_size);
-		std::string path;
-		path.resize(path_size);
-		if (_NSGetExecutablePath(path.data(), &path_size) == 0)
-		{
-			path[path_size] = 0;
-
-			size_t pos = path.rfind('/');
-			if (pos != std::string::npos)
-			{
-				path.erase(pos);
-				path += "/../Frameworks/libMoltenVK.dylib";
-				s_vulkan_module = dlopen(path.c_str(), RTLD_NOW);
+		return false;
 	}
-		}
-	}
-	if (!s_vulkan_module)
-		s_vulkan_module = dlopen("libvulkan.dylib", RTLD_NOW);
 #else
-	// Names of libraries to search. Desktop should use libvulkan.so.1 or libvulkan.so.
-	static const char* search_lib_names[] = {"libvulkan.so.1", "libvulkan.so"};
-	for (size_t i = 0; i < sizeof(search_lib_names) / sizeof(search_lib_names[0]); i++)
+	// try versioned first, then unversioned.
+	if (!s_vulkan_library.Open(DynamicLibrary::GetVersionedFilename("vulkan", 1).c_str(), error) &&
+		!s_vulkan_library.Open(DynamicLibrary::GetVersionedFilename("vulkan").c_str(), error))
 	{
-		s_vulkan_module = dlopen(search_lib_names[i], RTLD_NOW);
-		if (s_vulkan_module)
-			break;
+		return false;
 	}
 #endif
 
-	if (!s_vulkan_module)
-	{
-		std::fprintf(stderr, "Failed to load or locate libvulkan.so\n");
-		return false;
-	}
-
 	bool required_functions_missing = false;
-	auto LoadFunction = [&required_functions_missing](void** func_ptr, const char* name, bool is_required) {
-		*func_ptr = dlsym(s_vulkan_module, name);
-		if (!(*func_ptr) && is_required)
-		{
-			std::fprintf(stderr, "Vulkan: Failed to load required module function %s\n", name);
-			required_functions_missing = true;
+#define VULKAN_MODULE_ENTRY_POINT(name, required) \
+	if (!s_vulkan_library.GetSymbol(#name, &name)) \
+	{ \
+		ERROR_LOG("Vulkan: Failed to load required module function {}", #name); \
+		required_functions_missing = true; \
 	}
-	};
 
-#define VULKAN_MODULE_ENTRY_POINT(name, required) LoadFunction(reinterpret_cast<void**>(&name), #name, required);
 #include "VKEntryPoints.inl"
 #undef VULKAN_MODULE_ENTRY_POINT
 
 	if (required_functions_missing)
 	{
 		ResetVulkanLibraryFunctionPointers();
-		dlclose(s_vulkan_module);
-		s_vulkan_module = nullptr;
+		s_vulkan_library.Close();
 		return false;
 	}
 
@@ -178,13 +90,9 @@ bool Vulkan::LoadVulkanLibrary()
 void Vulkan::UnloadVulkanLibrary()
 {
 	ResetVulkanLibraryFunctionPointers();
-	if (s_vulkan_module)
-		dlclose(s_vulkan_module);
-	s_vulkan_module = nullptr;
+	s_vulkan_library.Close();
 }
 
-#endif
-
 bool Vulkan::LoadVulkanInstanceFunctions(VkInstance instance)
 {
 	bool required_functions_missing = false;

pcsx2/GS/Renderers/Vulkan/VKLoader.h

@@ -1,8 +1,10 @@
-// SPDX-FileCopyrightText: 2002-2023 PCSX2 Dev Team
+// SPDX-FileCopyrightText: 2002-2024 PCSX2 Dev Team
 // SPDX-License-Identifier: LGPL-3.0+
 
 #pragma once
 
+class Error;
+
 #define VK_NO_PROTOTYPES
 
 #ifdef _WIN32
@@ -86,7 +88,7 @@
 namespace Vulkan
 {
 	bool IsVulkanLibraryLoaded();
-	bool LoadVulkanLibrary();
+	bool LoadVulkanLibrary(Error* error);
 	bool LoadVulkanInstanceFunctions(VkInstance instance);
 	bool LoadVulkanDeviceFunctions(VkDevice device);
 	void UnloadVulkanLibrary();