From 9793e546460d0e252e80e8e69e6438d4df3fdd14 Mon Sep 17 00:00:00 2001
From: Flyinghead <raphael.jean@gmail.com>
Date: Sat, 12 Oct 2024 17:31:58 +0200
Subject: [PATCH] avoid deadlock when sh4 cpu is restarted while emu is being
 stopped

The sh4 cpu is stopped/restarted during a soft reset, and dynarec code
reset (arm64, arm, x86). If the emulator is stopped concurrently, the
call may hang. Use a mutex-protected method to restart the cpu only if
the emu is still running.
---
 core/emulator.cpp            | 21 +++++++++++++++++----
 core/emulator.h              |  5 +++++
 core/rec-ARM/rec_arm.cpp     |  5 +++--
 core/rec-ARM64/rec_arm64.cpp |  5 +++--
 core/rec-x86/rec_x86.cpp     |  5 +++--
 5 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/core/emulator.cpp b/core/emulator.cpp
index 1dd05e8d0..0176cc3ea 100644
--- a/core/emulator.cpp
+++ b/core/emulator.cpp
@@ -688,7 +688,8 @@ void Emulator::runInternal()
 			{
 				nvmem::saveFiles();
 				dc_reset(false);
-				sh4_cpu.Start();
+				if (!restartCpu())
+					resetRequested = false;
 			}
 		} while (resetRequested);
 	}
@@ -750,9 +751,12 @@ void Emulator::stop()
 	// Avoid race condition with GGPO restarting the sh4 for a new frame
 	if (config::GGPOEnable)
 		NetworkHandshake::term();
-	// must be updated after GGPO is stopped since it may run some rollback frames
-	state = Loaded;
-	sh4_cpu.Stop();
+	{
+		const std::lock_guard<std::mutex> _(mutex);
+		// must be updated after GGPO is stopped since it may run some rollback frames
+		state = Loaded;
+		sh4_cpu.Stop();
+	}
 	if (config::ThreadedRendering)
 	{
 		rend_cancel_emu_wait();
@@ -1039,4 +1043,13 @@ void Emulator::vblank()
 		sh4_cpu.Stop();
 }
 
+bool Emulator::restartCpu()
+{
+	const std::lock_guard<std::mutex> _(mutex);
+	if (state != Running)
+		return false;
+	sh4_cpu.Start();
+	return true;
+}
+
 Emulator emu;
diff --git a/core/emulator.h b/core/emulator.h
index eb2a698be..354a491db 100644
--- a/core/emulator.h
+++ b/core/emulator.h
@@ -165,6 +165,11 @@ public:
 	 * Called internally on vblank.
 	 */
 	void vblank();
+	/**
+	 * Restart the cpu iff the emu is still running
+	 * Returns true if the cpu was started
+	 */
+	bool restartCpu();
 
 private:
 	bool checkStatus(bool wait = false);
diff --git a/core/rec-ARM/rec_arm.cpp b/core/rec-ARM/rec_arm.cpp
index 15a477db0..ac5c48aa5 100644
--- a/core/rec-ARM/rec_arm.cpp
+++ b/core/rec-ARM/rec_arm.cpp
@@ -50,6 +50,7 @@ using namespace vixl::aarch32;
 #include "cfg/option.h"
 #include "arm_unwind.h"
 #include "oslib/virtmem.h"
+#include "emulator.h"
 
 //#define CANONICALTEST
 
@@ -428,8 +429,8 @@ public:
 			generate_mainloop();
 
 			::mainloop(context);
-			if (restarting)
-				p_sh4rcb->cntx.CpuRunning = 1;
+			if (restarting && !emu.restartCpu())
+				restarting = false;
 		} while (restarting);
 	}
 
diff --git a/core/rec-ARM64/rec_arm64.cpp b/core/rec-ARM64/rec_arm64.cpp
index 4051b5979..485a21c9a 100644
--- a/core/rec-ARM64/rec_arm64.cpp
+++ b/core/rec-ARM64/rec_arm64.cpp
@@ -43,6 +43,7 @@ using namespace vixl::aarch64;
 #include "arm64_regalloc.h"
 #include "hw/mem/addrspace.h"
 #include "oslib/virtmem.h"
+#include "emulator.h"
 
 #undef do_sqw_nommu
 
@@ -2213,8 +2214,8 @@ public:
 				generate_mainloop();
 
 				::mainloop(v_cntx);
-				if (restarting)
-					p_sh4rcb->cntx.CpuRunning = 1;
+				if (restarting && !emu.restartCpu())
+					restarting = false;
 			} while (restarting);
 		} catch (const SH4ThrownException&) {
 			ERROR_LOG(DYNAREC, "SH4ThrownException in mainloop");
diff --git a/core/rec-x86/rec_x86.cpp b/core/rec-x86/rec_x86.cpp
index f78e811bc..a8bc5fc09 100644
--- a/core/rec-x86/rec_x86.cpp
+++ b/core/rec-x86/rec_x86.cpp
@@ -27,6 +27,7 @@
 #include "hw/sh4/sh4_mem.h"
 #include "hw/mem/addrspace.h"
 #include "oslib/unwind_info.h"
+#include "emulator.h"
 
 static void (*mainloop)();
 static void (*ngen_FailedToFindBlock_)();
@@ -900,8 +901,8 @@ public:
 				generate_mainloop();
 
 				::mainloop();
-				if (restarting)
-					p_sh4rcb->cntx.CpuRunning = 1;
+				if (restarting && !emu.restartCpu())
+					restarting = false;
 			} while (restarting);
 		} catch (const SH4ThrownException& e) {
 			ERROR_LOG(DYNAREC, "SH4ThrownException in mainloop %x pc %x", e.expEvn, e.epc);