Handle reentrant calls in waterbox (#3007)

Fixes #2585

src/BizHawk.Client.EmuHawk/Program.cs

@@ -130,7 +130,7 @@ namespace BizHawk.Client.EmuHawk
 				new ExceptionBox(e.Message).ShowDialog();
 			}
 
-			var configPath = cliFlags.cmdConfigFile ?? Path.Combine(PathUtils.ExeDirectoryPath, "config.json");
+			var configPath = cliFlags.cmdConfigFile ?? Path.Combine(PathUtils.ExeDirectoryPath, "config.ini");
 
 			Config initialConfig;
 			try

waterbox/emulibc/emulibc.c

@@ -10,6 +10,7 @@ struct __AddressRange {
 struct __WbxSysLayout {
 	struct __AddressRange elf;
 	struct __AddressRange main_thread;
+	struct __AddressRange alt_thread;
 	struct __AddressRange sbrk;
 	struct __AddressRange sealed;
 	struct __AddressRange invis;

waterbox/waterboxhost/src/cinterface.rs

@@ -37,6 +37,7 @@ impl MemoryLayoutTemplate {
 			a
 		};
 		res.main_thread = add_area(1 << 20);
+		res.alt_thread = add_area(1 << 20);
 		res.sbrk = add_area(self.sbrk_size);
 		res.sealed = add_area(self.sealed_size);
 		res.invis = add_area(self.invis_size);

waterbox/waterboxhost/src/context/interop.s

@@ -6,47 +6,13 @@ struc Context
 	.thread_area resq 1
 	.host_rsp resq 1
 	.guest_rsp resq 1
+	.host_rsp_alt resq 1
+	.guest_rsp_alt resq 1
 	.dispatch_syscall resq 1
 	.host_ptr resq 1
 	.extcall_slots resq 64
 endstruc
 
-times 0-($-$$) int3 ; CALL_GUEST_IMPL_ADDR
-; sets up guest stack and calls a function
-; r11 - guest entry point
-; r10 - address of context structure
-; regular arg registers are 0..6 args passed through to guest
-call_guest_impl:
-	; save host TIB data
-	mov rax, [gs:0x08]
-	push rax
-	mov rax, [gs:0x10]
-	push rax
-
-	; set guest TIB data
-	xor rax, rax
-	mov [gs:0x10], rax
-	sub rax, 1
-	mov [gs:0x08], rax
-
-	mov [gs:0x18], r10
-	mov [r10 + Context.host_rsp], rsp
-	mov rsp, [r10 + Context.guest_rsp]
-	call r11 ; stack hygiene note - this host address is saved on the guest stack
-	mov r10, [gs:0x18]
-	mov [r10 + Context.guest_rsp], rsp ; restore stack so next call using same Context will work
-	mov rsp, [r10 + Context.host_rsp]
-	mov r11, 0
-	mov [gs:0x18], r11
-
-	; restore host TIB data
-	pop r10
-	mov [gs:0x10], r10
-	pop r10
-	mov [gs:0x08], r10
-
-	ret
-
 times 0x80-($-$$) int3
 ; called by guest when it wishes to make a syscall
 ; must be loaded at fixed address, as that address is burned into guest executables
@@ -99,7 +65,74 @@ call_guest_simple:
 	mov r10, rsi
 	jmp call_guest_impl
 
-times 0x200-($-$$) int3 ; EXTCALL_THUNK_ADDR
+times 0x200-($-$$) int3 ; CALL_GUEST_IMPL_ADDR
+; sets up guest stack and calls a function
+; r11 - guest entry point
+; r10 - address of context structure
+; regular arg registers are 0..6 args passed through to guest
+call_guest_impl:
+	; check if we need to swap stacks for a reentrant call
+	mov rax, [r10 + Context.host_rsp]
+	test rax, rax
+	je do_tib
+	mov rax, [r10 + Context.host_rsp_alt]
+	test rax, rax
+	je do_swap
+	int3 ; both stacks exhausted
+
+do_swap:
+	mov rax, [r10 + Context.host_rsp]
+	mov [r10 + Context.host_rsp_alt], rax
+	mov rax, [r10 + Context.guest_rsp]
+	xchg rax, [r10 + Context.guest_rsp_alt]
+	mov [r10 + Context.guest_rsp], rax
+
+do_tib:
+	; save host TIB data
+	mov rax, [gs:0x08]
+	push rax
+	mov rax, [gs:0x10]
+	push rax
+
+	; set guest TIB data
+	xor rax, rax
+	mov [gs:0x10], rax
+	sub rax, 1
+	mov [gs:0x08], rax
+
+	mov [gs:0x18], r10
+	mov [r10 + Context.host_rsp], rsp
+	mov rsp, [r10 + Context.guest_rsp]
+	call r11 ; stack hygiene note - this host address is saved on the guest stack
+	mov r10, [gs:0x18]
+	mov [r10 + Context.guest_rsp], rsp ; restore stack so next call using same Context will work
+	mov rsp, [r10 + Context.host_rsp]
+	mov r11, 0
+	mov [r10 + Context.host_rsp], r11 ; zero out host_rsp so we'll know this callstack is no longer in use
+	mov [gs:0x18], r11
+
+	; check to see if we need to swap back stacks
+	mov r11, [r10 + Context.host_rsp_alt]
+	test r11, r11
+	je do_restore_tib
+
+	mov [r10 + Context.host_rsp], r11
+	mov r11, 0
+	mov [r10 + Context.host_rsp_alt], r11
+	mov r11, [r10 + Context.guest_rsp_alt]
+	xchg r11, [r10 + Context.guest_rsp]
+	mov [r10 + Context.guest_rsp_alt], r11
+
+do_restore_tib:
+	; restore host TIB data
+	pop r10
+	mov [gs:0x10], r10
+	pop r10
+	mov [gs:0x08], r10
+
+	ret
+
+times 0x300-($-$$) int3 ; EXTCALL_THUNK_ADDR
 ; individual thunks to each of 64 call slots
 ; should be in fixed locations for memory hygiene in the core, since they may be stored there for some time
 %macro guest_extcall_thunk 1
@@ -150,7 +183,7 @@ guest_extcall_impl:
 	ret
 guest_extcall_impl_end:
 
-times 0x700-($-$$) int3 ; RUNTIME_TABLE_ADDR
+times 0x800-($-$$) int3 ; RUNTIME_TABLE_ADDR
 runtime_function_table:
 	; https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-runtime_function
 	dd RVA(guest_syscall)

waterbox/waterboxhost/src/context/mod.rs

@@ -9,10 +9,10 @@ pub mod thunks;
 // manually match these up with interop.s
 const ORG: usize = 0x35f00000000;
 
-const CALL_GUEST_IMPL_ADDR: usize = ORG;
 const CALL_GUEST_SIMPLE_ADDR: usize = ORG + 0x100;
-const EXTCALL_THUNK_ADDR: usize = ORG + 0x200;
+const CALL_GUEST_IMPL_ADDR: usize = ORG + 0x200;
-const RUNTIME_TABLE_ADDR: usize = ORG + 0x700;
+const EXTCALL_THUNK_ADDR: usize = ORG + 0x300;
+const RUNTIME_TABLE_ADDR: usize = ORG + 0x800;
 
 pub const CALLBACK_SLOTS: usize = 64;
 /// Retrieves a function pointer suitable for sending to the guest that will cause
@@ -81,6 +81,10 @@ pub struct Context {
 	/// Sets the guest's starting rsp, and used internally to track the guest's most recent rsp when transitioned to extcall or syscall
 	/// can be changed by the host to return to a different guest thread
 	pub guest_rsp: usize,
+	/// Like `host_rsp`, but for the inactive first call when a second reentrant call is made to waterbox code.
+	pub host_rsp_alt: usize,
+	/// Like `guest_rsp`, but for the inactive first call when a second reentrant call is made to waterbox code.
+	pub guest_rsp_alt: usize,
 	/// syscall service function
 	pub dispatch_syscall: SyscallCallback,
 	/// This will be passed as the final parameter to dispatch_syscall, and is not otherwise used by the context tracking code
@@ -90,11 +94,13 @@ pub struct Context {
 }
 impl Context {
 	/// Returns a suitably initialized context.  It's almost ready to use, but host_ptr must be set before each usage
-	pub fn new(initial_guest_rsp: usize, dispatch_syscall: SyscallCallback) -> Context {
+	pub fn new(initial_guest_rsp: usize, initial_guest_rsp_alt: usize, dispatch_syscall: SyscallCallback) -> Context {
 		Context {
 			thread_area: 0,
 			host_rsp: 0,
 			guest_rsp: initial_guest_rsp,
+			host_rsp_alt: 0,
+			guest_rsp_alt: initial_guest_rsp_alt,
 			dispatch_syscall,
 			host_ptr: 0,
 			extcall_slots: [None; 64]

waterbox/waterboxhost/src/elf.rs

@@ -171,6 +171,10 @@ impl ElfLoader {
 		b.mprotect(AddressRange { start: layout.main_thread.start, size: PAGESIZE * 4 }, Protection::None)?;
 		b.mark_invisible(layout.main_thread)?;
 
+		b.mmap_fixed(layout.alt_thread, Protection::RWStack, true)?;
+		b.mprotect(AddressRange { start: layout.alt_thread.start, size: PAGESIZE * 4 }, Protection::None)?;
+		b.mark_invisible(layout.alt_thread)?;
+
 		Ok(ElfLoader {
 			sections,
 			exports,

waterbox/waterboxhost/src/host.rs

@@ -42,7 +42,7 @@ impl WaterboxHost {
 			active: false,
 			sealed: false,
 			image_file,
-			context: Context::new(layout.main_thread.end(), syscall),
+			context: Context::new(layout.main_thread.end(), layout.alt_thread.end(), syscall),
 			thunks,
 			threads: GuestThreadSet::new(),
 		});

waterbox/waterboxhost/src/lib.rs

@@ -91,8 +91,11 @@ fn align_up(p: usize) -> usize {
 #[repr(C)]
 #[derive(Copy, Clone)]
 pub struct WbxSysLayout {
+	// Keep this all in sync with the C code!
+
 	pub elf: AddressRange,
 	pub main_thread: AddressRange,
+	pub alt_thread: AddressRange,
 	pub sbrk: AddressRange,
 	pub sealed: AddressRange,
 	pub invis: AddressRange,