try a new technique to just unstick the frontend when the n64 emulation thread dies

BizHawk.Common/Win32Hacks.cs

@@ -400,9 +400,9 @@ namespace BizHawk.Common
 		/// <summary>
 		/// functionally the same as WaitOne, but does not message pump
 		/// </summary>
-		public static void HackyPinvokeWaitOne(WaitHandle handle)
+		public static void HackyPinvokeWaitOne(WaitHandle handle, uint timeout = 0xFFFFFFFF)
 		{
-			NativeMethods.WaitForSingleObject(handle.SafeWaitHandle, 0xFFFFFFFF);
+			NativeMethods.WaitForSingleObject(handle.SafeWaitHandle, timeout);
 		}
 
 		/// <summary>

BizHawk.Emulation.Cores/Consoles/Nintendo/N64/NativeApi/mupen64plusCoreApi.cs

@@ -733,6 +733,9 @@ namespace BizHawk.Emulation.Cores.Nintendo.N64.NativeApi
 
 		public void frame_advance()
 		{
+			if (!emulator_running)
+				return;
+
 			event_frameend = false;
 			m64pCoreDoCommandPtr(m64p_command.M64CMD_ADVANCE_FRAME, 0, IntPtr.Zero);
 
@@ -751,7 +754,7 @@ namespace BizHawk.Emulation.Cores.Nintendo.N64.NativeApi
 
 			for(;;)
 			{
-				BizHawk.Common.Win32ThreadHacks.HackyPinvokeWaitOne(m64pEvent);
+				BizHawk.Common.Win32ThreadHacks.HackyPinvokeWaitOne(m64pEvent, 200);
 				if (event_frameend)
 					break;
 				if (event_breakpoint)
@@ -771,7 +774,13 @@ namespace BizHawk.Emulation.Cores.Nintendo.N64.NativeApi
 
 					event_breakpoint = false;
 					Resume();
+					continue;
 				}
+				//no event.. must be a timeout
+				//check if the core crashed and bail if it did
+				//otherwise wait longer (could be inside slow emulation or lua logic)
+				if (!emulator_running)
+					break;
 			}
 		}
 

libmupen64plus/mupen64plus-core/src/api/frontend.c

@@ -211,7 +211,15 @@ EXPORT m64p_error CALL CoreDoCommand(m64p_command Command, int ParamInt, void *P
             /* print out plugin-related warning messages */
             plugin_check();
             /* the main_run() function will not return until the player has quit the game */
+						__try
+						{
             rval = main_run((void (*)(void))ParamPtr);
+						}
+						__except(EXCEPTION_EXECUTE_HANDLER)
+						{
+							printf("exception swallowed because mupen is crashy when being fuzzed\n");
+							rval = M64ERR_INTERNAL;
+						}
             return rval;
         case M64CMD_STOP:
             if (!g_EmulatorRunning)