rsx/qt: Implement native window hooks for win32 windows to allow communication between WndProc thread and rsx::thread

- This communication is important in communicating window events. Helps properly synchronize swapchain management on vulkan and stops nvidia crashing
- Do not block the message queue lest the driver detect window as not responding

rpcs3/Emu/RSX/D3D12/D3D12GSRender.cpp

@@ -301,6 +301,12 @@ void D3D12GSRender::on_exit()
 	return GSRender::on_exit();
 }
 
+void D3D12GSRender::do_local_task()
+{
+	//TODO
+	m_frame->clear_wm_events();
+}
+
 bool D3D12GSRender::do_method(u32 cmd, u32 arg)
 {
 	switch (cmd)

rpcs3/Emu/RSX/D3D12/D3D12GSRender.h

@@ -173,6 +173,7 @@ private:
 protected:
 	virtual void on_init_thread() override;
 	virtual void on_exit() override;
+	virtual void do_local_task() override;
 	virtual bool do_method(u32 cmd, u32 arg) override;
 	virtual void end() override;
 	virtual void flip(int buffer) override;

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -1231,6 +1231,8 @@ void GLGSRender::on_notify_memory_unmapped(u32 address_base, u32 size)
 
 void GLGSRender::do_local_task()
 {
+	m_frame->clear_wm_events();
+
 	std::lock_guard<std::mutex> lock(queue_guard);
 
 	work_queue.remove_if([](work_item &q) { return q.received; });

rpcs3/Emu/RSX/GSRender.h

@@ -18,6 +18,15 @@ struct RSXDebuggerProgram
 	}
 };
 
+enum wm_event
+{
+	none, //nothing
+	geometry_change_notice, //about to start resizing and/or moving the window
+	geometry_change_in_progress, //window being resized and/or moved
+	window_resized, //window was resized
+	window_moved //window moved without resize
+};
+
 using RSXDebuggerPrograms = std::vector<RSXDebuggerProgram>;
 
 using draw_context_t = std::shared_ptr<void>;
@@ -45,6 +54,35 @@ public:
 protected:
 	virtual void delete_context(void* ctx) = 0;
 	virtual void* make_context() = 0;
+
+	//window manager event management
+	wm_event m_raised_event;
+	std::atomic_bool wm_event_raised = {};
+
+public:
+	//synchronize native window access
+	std::mutex wm_event_lock;
+
+	virtual wm_event get_default_wm_event() const = 0;
+
+	void clear_wm_events()
+	{
+		m_raised_event = wm_event::none;
+		wm_event_raised.store(false);
+	}
+
+	wm_event get_wm_event()
+	{
+		if (wm_event_raised.load(std::memory_order_consume))
+		{
+			auto result = m_raised_event;
+			m_raised_event = wm_event::none;
+			wm_event_raised.store(false);
+			return result;
+		}
+
+		return get_default_wm_event();
+	}
 };
 
 class GSRender : public rsx::thread

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -1584,13 +1584,27 @@ void VKGSRender::present(frame_context_t *ctx)
 	verify(HERE), ctx->present_image != UINT32_MAX;
 	VkSwapchainKHR swap_chain = (VkSwapchainKHR)(*m_swap_chain);
 
+	if (!present_surface_dirty_flag)
+	{
 		VkPresentInfoKHR present = {};
 		present.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
 		present.pNext = nullptr;
 		present.swapchainCount = 1;
 		present.pSwapchains = &swap_chain;
 		present.pImageIndices = &ctx->present_image;
-	CHECK_RESULT(m_swap_chain->queuePresentKHR(m_swap_chain->get_present_queue(), &present));
+
+		switch (VkResult error = m_swap_chain->queuePresentKHR(m_swap_chain->get_present_queue(), &present))
+		{
+		case VK_SUCCESS:
+		case VK_SUBOPTIMAL_KHR:
+			break;
+		case VK_ERROR_OUT_OF_DATE_KHR:
+			present_surface_dirty_flag = true;
+			break;
+		default:
+			vk::die_with_error(HERE, error);
+		}
+	}
 
 	//Presentation image released; reset value
 	ctx->present_image = UINT32_MAX;
@@ -1698,6 +1712,75 @@ void VKGSRender::do_local_task()
 		if (!cb->pending)
 			m_last_flushable_cb = -1;
 	}
+
+#ifdef _WIN32
+
+	switch (m_frame->get_wm_event())
+	{
+	case wm_event::none:
+		break;
+	case wm_event::geometry_change_notice:
+	{
+		//Stall until finish notification is received. Also, raise surface dirty flag
+		u32 timeout = 1000;
+		bool handled = false;
+
+		while (timeout)
+		{
+			switch (m_frame->get_wm_event())
+			{
+			default:
+				break;
+			case wm_event::window_resized:
+				handled = true;
+				present_surface_dirty_flag = true;
+				break;
+			case wm_event::window_moved:
+				handled = true;
+				break;
+			case wm_event::geometry_change_in_progress:
+				timeout += 10; //extend timeout to wait for user to finish resizing
+				break;
+			}
+
+			if (handled)
+				break;
+			else
+			{
+				//wait for window manager event
+				std::this_thread::sleep_for(10ms);
+				timeout -= 10;
+			}
+		}
+
+		if (!timeout)
+		{
+			LOG_ERROR(RSX, "wm event handler timed out");
+		}
+
+		break;
+	}
+	case wm_event::window_resized:
+	{
+		LOG_ERROR(RSX, "wm_event::window_resized received without corresponding wm_event::geometry_change_notice!");
+		std::this_thread::sleep_for(100ms);
+		break;
+	}
+	}
+
+#else
+
+	const auto frame_width = m_frame->client_width();
+	const auto frame_height = m_frame->client_height();
+
+	if (m_client_height != frame_height ||
+		m_client_width != frame_width)
+	{
+		if (!!frame_width && !!frame_height)
+			present_surface_dirty_flag = true;
+	}
+
+#endif
 }
 
 bool VKGSRender::do_method(u32 cmd, u32 arg)
@@ -2336,6 +2419,8 @@ void VKGSRender::reinitialize_swapchain()
 
 	m_current_command_buffer->reset();
 	open_command_buffer();
+
+	present_surface_dirty_flag = false;
 }
 
 void VKGSRender::flip(int buffer)
@@ -2357,18 +2442,6 @@ void VKGSRender::flip(int buffer)
 		return;
 	}
 
-	bool resize_screen = false;
-
-	const auto frame_width = m_frame->client_width();
-	const auto frame_height = m_frame->client_height();
-
-	if (m_client_height != frame_height ||
-		m_client_width != frame_width)
-	{
-		if (!!frame_width && !!frame_height)
-			resize_screen = true;
-	}
-
 	std::chrono::time_point<steady_clock> flip_start = steady_clock::now();
 
 	close_render_pass();
@@ -2399,8 +2472,12 @@ void VKGSRender::flip(int buffer)
 		process_swap_request(m_current_frame, true);
 	}
 
-	if (!resize_screen)
+	if (present_surface_dirty_flag)
 	{
+		//Recreate swapchain and continue as usual
+		reinitialize_swapchain();
+	}
+
 	u32 buffer_width = display_buffers[buffer].width;
 	u32 buffer_height = display_buffers[buffer].height;
 	u32 buffer_pitch = display_buffers[buffer].pitch;
@@ -2409,7 +2486,7 @@ void VKGSRender::flip(int buffer)
 
 	coordi aspect_ratio;
 
-		sizei csize = { frame_width, frame_height };
+	sizei csize = { (s32)m_client_width, (s32)m_client_height };
 	sizei new_size = csize;
 
 	if (!g_cfg.video.stretch_to_display_area)
@@ -2467,6 +2544,7 @@ void VKGSRender::flip(int buffer)
 		}
 		case VK_ERROR_OUT_OF_DATE_KHR:
 			LOG_ERROR(RSX, "vkAcquireNextImageKHR failed with VK_ERROR_OUT_OF_DATE_KHR. Flip request ignored until surface is recreated.");
+			present_surface_dirty_flag = true;
 			reinitialize_swapchain();
 			return;
 		default:
@@ -2556,13 +2634,6 @@ void VKGSRender::flip(int buffer)
 	}
 
 	queue_swap_request();
-	}
-	else
-	{
-		//Recreate the swapchain to resize it
-		//NOTE: Nvidia driver does not invalidate the swapchain immediately on window size changes. The event will fire after a while and an OUT_OF_DATE error will be returned in subsequent rendering commands
-		reinitialize_swapchain();
-	}
 
 	std::chrono::time_point<steady_clock> flip_end = steady_clock::now();
 	m_flip_time = std::chrono::duration_cast<std::chrono::microseconds>(flip_end - flip_start).count();

rpcs3/Emu/RSX/VK/VKGSRender.h

@@ -159,6 +159,8 @@ private:
 
 	std::unique_ptr<vk::framebuffer_holder> m_draw_fbo;
 
+	bool present_surface_dirty_flag = false;
+
 	u64 m_last_heap_sync_time = 0;
 	vk::vk_data_heap m_attrib_ring_info;
 	vk::vk_data_heap m_uniform_buffer_ring_info;

rpcs3/rpcs3qt/gs_frame.cpp

@@ -12,6 +12,10 @@
 
 #include "rpcs3_version.h"
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 constexpr auto qstr = QString::fromStdString;
 
 gs_frame::gs_frame(const QString& title, int w, int h, QIcon appIcon, bool disableMouse)
@@ -238,3 +242,88 @@ bool gs_frame::event(QEvent* ev)
 	}
 	return QWindow::event(ev);
 }
+
+bool gs_frame::nativeEvent(const QByteArray &eventType, void *message, long *result)
+{
+#ifdef _WIN32
+	//Wait for consumer to clear notification pending flag
+	while (wm_event_raised.load(std::memory_order_consume));
+
+	{
+		std::lock_guard<std::mutex> lock(wm_event_lock);
+		MSG* msg = static_cast<MSG*>(message);
+		switch (msg->message)
+		{
+		case WM_WINDOWPOSCHANGING:
+		{
+			const auto flags = reinterpret_cast<LPWINDOWPOS>(msg->lParam)->flags & SWP_NOSIZE;
+			if (m_in_sizing_event || flags != 0)
+				break;
+
+			//About to resize
+			m_in_sizing_event = true;
+			m_interactive_resize = false;
+			m_raised_event = wm_event::geometry_change_notice;
+			wm_event_raised.store(true);
+			break;
+		}
+		case WM_WINDOWPOSCHANGED:
+		{
+			const auto flags = reinterpret_cast<LPWINDOWPOS>(msg->lParam)->flags & (SWP_NOSIZE | SWP_NOMOVE);
+			if (!m_in_sizing_event || m_user_interaction_active || flags == (SWP_NOSIZE | SWP_NOMOVE))
+				break;
+
+			if (flags & SWP_NOSIZE)
+				m_raised_event = wm_event::window_moved;
+			else
+				m_raised_event = wm_event::window_resized;
+
+			//Just finished resizing using maximize or SWP
+			m_in_sizing_event = false;
+			wm_event_raised.store(true);
+			break;
+		}
+		case WM_ENTERSIZEMOVE:
+			m_user_interaction_active = true;
+			break;
+		case WM_EXITSIZEMOVE:
+			m_user_interaction_active = false;
+			if (m_in_sizing_event && !m_user_interaction_active)
+			{
+				//Just finished resizing using manual interaction. The corresponding WM_SIZE is consumed before this event fires
+				m_raised_event = m_interactive_resize ? wm_event::window_resized : wm_event::window_moved;
+				m_in_sizing_event = false;
+				wm_event_raised.store(true);
+			}
+			break;
+		case WM_SIZE:
+		{
+			if (m_user_interaction_active)
+			{
+				//Interaction is a resize not a move
+				m_interactive_resize = true;
+			}
+			else if (m_in_sizing_event)
+			{
+				//Any other unexpected resize mode will give an unconsumed WM_SIZE event
+				m_raised_event = wm_event::window_resized;
+				m_in_sizing_event = false;
+				wm_event_raised.store(true);
+			}
+			break;
+		}
+		}
+	}
+
+	//Do not enter DefWndProc until the consumer has consumed the message
+	while (wm_event_raised.load(std::memory_order_consume));
+#endif
+
+	//Let the default handler deal with this. Only the notification is required
+	return false;
+}
+
+wm_event gs_frame::get_default_wm_event() const
+{
+	return (m_user_interaction_active) ? wm_event::geometry_change_in_progress : wm_event::none;
+}

rpcs3/rpcs3qt/gs_frame.h

@@ -14,8 +14,14 @@ class gs_frame : public QWindow, public GSFrameBase
 	bool m_show_fps;
 	bool m_disable_mouse;
 
+	bool m_in_sizing_event = false; //a signal that the window is about to be resized was received
+	bool m_user_interaction_active = false; //a signal indicating the window is being manually moved/resized was received
+	bool m_interactive_resize = false; //resize signal received while dragging window
+
 public:
 	gs_frame(const QString& title, int w, int h, QIcon appIcon, bool disableMouse);
+
+	wm_event get_default_wm_event() const override;
 protected:
 	virtual void paintEvent(QPaintEvent *event);
 
@@ -38,6 +44,8 @@ protected:
 	int client_width() override;
 	int client_height() override;
 
+	bool nativeEvent(const QByteArray &eventType, void *message, long *result) override;
+
 	bool event(QEvent* ev) override;
 private Q_SLOTS:
 	void HandleCursor(QWindow::Visibility visibility);