rsx: Make X11 optional on Linux

This makes it possible to build rpcs3 on a pure Wayland system, without
the Xlib installed.

rpcs3/CMakeLists.txt

@@ -81,9 +81,15 @@ if (WIN32)
 endif()
 
 if(UNIX AND NOT APPLE)
-	find_package(X11 REQUIRED)
+	find_package(X11)
-	target_include_directories(rpcs3 PUBLIC ${X11_INCLUDE_DIR})
+	if(X11_FOUND)
-	target_link_libraries(rpcs3 ${X11_LIBRARIES})
+		target_include_directories(rpcs3 PUBLIC ${X11_INCLUDE_DIR})
+		target_link_libraries(rpcs3 ${X11_LIBRARIES})
+		target_compile_definitions(rpcs3 INTERFACE -DHAVE_X11)
+	# This has been checked in 3rdparty/CMakeLists.txt already.
+	elseif(NOT WAYLAND_FOUND)
+		message(FATAL_ERROR "RPCS3 requires either X11 or Wayland (or both) for Vulkan.")
+	endif()
 endif()
 
 if(UNIX)

rpcs3/Emu/RSX/GL/OpenGL.cpp

@@ -34,7 +34,7 @@ void gl::init()
 #ifdef __unix__
 	glewExperimental = true;
 	glewInit();
-#ifndef __APPLE__
+#ifdef HAVE_X11
 	glxewInit();
 #endif
 #endif
@@ -44,7 +44,7 @@ void gl::set_swapinterval(int interval)
 {
 #ifdef _WIN32
 	wglSwapIntervalEXT(interval);
-#elif !defined(__APPLE__)
+#elif defined(HAVE_X11)
 	if (glXSwapIntervalEXT)
 	{
 		if (auto window = glXGetCurrentDrawable())

rpcs3/Emu/RSX/GL/OpenGL.h

@@ -22,11 +22,13 @@ typedef BOOL (WINAPI* PFNWGLSWAPINTERVALEXTPROC) (int interval);
 #include <OpenGL/glu.h>
 
 #else
-#include <GL/glxew.h>
 #include <GL/gl.h>
+#ifdef HAVE_X11
+#include <GL/glxew.h>
 #include <GL/glx.h>
 #include <GL/glxext.h>
 #endif
+#endif
 
 #ifndef GL_TEXTURE_BUFFER_BINDING
 //During spec release, this enum was removed during upgrade from ARB equivalent

rpcs3/Emu/RSX/GSRender.h

@@ -8,7 +8,7 @@
 #include <windows.h>
 #elif defined(__APPLE__)
 // nothing
-#else
+#elif defined(HAVE_X11)
 // Cannot include Xlib.h before Qt5
 // and we don't need all of Xlib anyway
 using Display = struct _XDisplay;
@@ -57,9 +57,12 @@ using display_handle_t = HWND;
 using display_handle_t = void*; // NSView
 #else
 using display_handle_t = std::variant<
+#if defined(HAVE_X11) && defined(VK_USE_PLATFORM_WAYLAND_KHR)
+	std::pair<Display*, Window>, std::pair<wl_display*, wl_surface*>
+#elif defined(HAVE_X11)
 	std::pair<Display*, Window>
-#ifdef VK_USE_PLATFORM_WAYLAND_KHR
+#elif defined(VK_USE_PLATFORM_WAYLAND_KHR)
-	, std::pair<wl_display*, wl_surface*>
+	std::pair<wl_display*, wl_surface*>
 #endif
 >;
 #endif

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -366,7 +366,7 @@ VKGSRender::VKGSRender() : GSRender()
 
 	display_handle_t display = m_frame->handle();
 
-#if !defined(_WIN32) && !defined(__APPLE__)
+#ifdef HAVE_X11
 	std::visit([this](auto&& p) {
 		using T = std::decay_t<decltype(p)>;
 		if constexpr (std::is_same_v<T, std::pair<Display*, Window>>)
@@ -653,7 +653,7 @@ VKGSRender::~VKGSRender()
 	m_swapchain->destroy();
 	m_thread_context.close();
 
-#if !defined(_WIN32) && !defined(__APPLE__) && defined(HAVE_VULKAN)
+#if defined(HAVE_X11) && defined(HAVE_VULKAN)
 	if (m_display_handle)
 		XCloseDisplay(m_display_handle);
 #endif

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -416,7 +416,7 @@ private:
 	//Vertex layout
 	rsx::vertex_input_layout m_vertex_layout;
 
-#if !defined(_WIN32) && !defined(__APPLE__) && defined(HAVE_VULKAN)
+#if defined(HAVE_X11) && defined(HAVE_VULKAN)
 	Display *m_display_handle = nullptr;
 #endif
 

rpcs3/Emu/RSX/VK/VKHelpers.h

@@ -10,7 +10,7 @@
 #include <variant>
 #include <stack>
 
-#if !defined(_WIN32) && !defined(__APPLE__)
+#ifdef HAVE_X11
 #include <X11/Xutil.h>
 #endif
 
@@ -2029,7 +2029,7 @@ public:
 		{
 			fmt::throw_exception("Native macOS swapchain is not implemented yet!");
 		}
-#else
+#elif defined(HAVE_X11)
 
 	class swapchain_X11 : public native_swapchain_base
 	{
@@ -2136,6 +2136,37 @@ public:
 			src.first = false;
 			return VK_SUCCESS;
 		}
+#else
+
+	class swapchain_Wayland : public native_swapchain_base
+	{
+
+	public:
+		swapchain_Wayland(physical_device &gpu, uint32_t _present_queue, uint32_t _graphics_queue, VkFormat format = VK_FORMAT_B8G8R8A8_UNORM)
+		: native_swapchain_base(gpu, _present_queue, _graphics_queue, format)
+		{}
+
+		~swapchain_Wayland(){}
+
+		bool init() override
+		{
+			fmt::throw_exception("Native Wayland swapchain is not implemented yet!");
+		}
+
+		void create(display_handle_t& window_handle) override
+		{
+			fmt::throw_exception("Native Wayland swapchain is not implemented yet!");
+		}
+
+		void destroy(bool full=true) override
+		{
+			fmt::throw_exception("Native Wayland swapchain is not implemented yet!");
+		}
+
+		VkResult present(VkSemaphore /*semaphore*/, u32 index) override
+		{
+			fmt::throw_exception("Native Wayland swapchain is not implemented yet!");
+		}
 #endif
 
 		VkResult acquire_next_swapchain_image(VkSemaphore /*semaphore*/, u64 /*timeout*/, u32* result) override
@@ -2539,11 +2570,13 @@ public:
 				extensions.push_back(VK_MVK_MACOS_SURFACE_EXTENSION_NAME);
 #else
 				bool found_surface_ext = false;
+#ifdef HAVE_X11
 				if (support.is_supported(VK_KHR_XLIB_SURFACE_EXTENSION_NAME))
 				{
 					extensions.push_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
 					found_surface_ext = true;
 				}
+#endif
 #ifdef VK_USE_PLATFORM_WAYLAND_KHR
 				if (support.is_supported(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME))
 				{
@@ -2642,12 +2675,17 @@ public:
 
 			CHECK_RESULT(vkCreateMacOSSurfaceMVK(m_instance, &createInfo, NULL, &m_surface));
 #else
+#ifdef HAVE_X11
 			using swapchain_NATIVE = swapchain_X11;
+#else
+			using swapchain_NATIVE = swapchain_Wayland;
+#endif
 
 			std::visit([&](auto&& p)
 			{
 				using T = std::decay_t<decltype(p)>;
 
+#ifdef HAVE_X11
 				if constexpr (std::is_same_v<T, std::pair<Display*, Window>>)
 				{
 					VkXlibSurfaceCreateInfoKHR createInfo = {};
@@ -2656,8 +2694,10 @@ public:
 					createInfo.window                     = p.second;
 					CHECK_RESULT(vkCreateXlibSurfaceKHR(this->m_instance, &createInfo, nullptr, &m_surface));
 				}
+				else
+#endif
 #ifdef VK_USE_PLATFORM_WAYLAND_KHR
-				else if constexpr (std::is_same_v<T, std::pair<wl_display*, wl_surface*>>)
+				if constexpr (std::is_same_v<T, std::pair<wl_display*, wl_surface*>>)
 				{
 					VkWaylandSurfaceCreateInfoKHR createInfo = {};
 					createInfo.sType                         = VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR;
@@ -2667,10 +2707,10 @@ public:
 					force_wm_reporting_off = true;
 				}
 				else
+#endif
 				{
 					static_assert(std::conditional_t<true, std::false_type, T>::value, "Unhandled window_handle type in std::variant");
 				}
-#endif
 			}, window_handle);
 #endif
 

rpcs3/Emu/RSX/VK/VulkanAPI.h

@@ -4,7 +4,7 @@
 #define VK_USE_PLATFORM_WIN32_KHR
 #elif defined(__APPLE__)
 #define VK_USE_PLATFORM_MACOS_MVK
-#else
+#elif HAVE_X11
 #define VK_USE_PLATFORM_XLIB_KHR
 #endif
 

rpcs3/rpcs3qt/gs_frame.cpp

@@ -26,8 +26,10 @@
 #include <QGuiApplication>
 #include <qpa/qplatformnativeinterface.h>
 #endif
+#ifdef HAVE_X11
 #include <X11/Xlib.h>
 #endif
+#endif
 
 constexpr auto qstr = QString::fromStdString;
 
@@ -235,7 +237,11 @@ display_handle_t gs_frame::handle() const
 	else
 	{
 #endif
+#ifdef HAVE_X11
 		return std::make_pair(XOpenDisplay(0), static_cast<ulong>(this->winId()));
+#else
+		fmt::throw_exception("Vulkan X11 support disabled at compile-time.");
+#endif
 #ifdef VK_USE_PLATFORM_WAYLAND_KHR
 	}
 #endif