libsnes: recreate the emu cothread at the end of init, to avoid pointer poison that was breaking xor state consistency. Big todo: all init functionality really needs to happen from the main thread, as there are many syscalls that reenter managed in that code, and reentering managed from a cothread stack is a Bad Thing

BizHawk.Emulation.Cores/Consoles/Nintendo/SNES/LibsnesApi.cs

@@ -349,9 +349,14 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 
 		public void Seal()
 		{
-			_core.SetBuffer(0, null, 0);
-			_core.SetBuffer(1, null, 0);
-			_core.SetBuffer(2, null, 0);
+			/* Cothreads can very easily acquire "pointer poison"; because their stack and even registers
+			 * are part of state, any poisoned pointer that's used even temporarily might be persisted longer
+			 * than needed.  Most of the libsnes core cothreads handle internal matters only and aren't very
+			 * vulnerable to pointer poison, but the main boss cothread is used heavily during init, when
+			 * many syscalls happen and many kinds of poison can end up on the stack.  so here, we call
+			 * _core.DllInit() again, which recreates that cothread, zeroing out all of the memory first,
+			 * as well as zeroing out the comm struct. */
+			_core.DllInit();
 			_exe.Seal();
 			_sealed = true;
 		}

waterbox/libco/amd64.c

@@ -18,6 +18,25 @@ extern "C" {
 static long long co_active_buffer[64];
 static cothread_t co_active_handle = 0;
 
+static void* smalloc(size_t size)
+{
+  char* ret = malloc(size + 16);
+  if (ret)
+  {
+    *(size_t*)ret = size;
+    return ret + 16;
+  }
+  return NULL;
+}
+
+static void sfree(void* ptr)
+{
+  char* original = (char*)ptr - 16;
+  size_t size = *(size_t*)original + 16;
+  memset(original, 0, size);
+  free(original);
+}
+
 extern void co_swap(cothread_t, cothread_t);
 
 static void crash() {
@@ -39,7 +58,7 @@ cothread_t co_create(unsigned int size, void (*entrypoint)(void)) {
   size += 512;  /* allocate additional space for storage */
   size &= ~15;  /* align stack to 16-byte boundary */
 
-  if(handle = (cothread_t)malloc(size)) {
+  if(handle = (cothread_t)smalloc(size)) {
     long long *p = (long long*)((char*)handle + size);  /* seek to top of stack */
     *--p = (long long)crash;                            /* crash if entrypoint returns */
     *--p = (long long)entrypoint;                       /* start of function */
@@ -50,7 +69,7 @@ cothread_t co_create(unsigned int size, void (*entrypoint)(void)) {
 }
 
 void co_delete(cothread_t handle) {
-  free(handle);
+  sfree(handle);
 }
 
 void co_switch(cothread_t handle) {

waterbox/libsnes/bsnes/target-libsnes/libsnes_pwrap.cpp

@@ -613,10 +613,16 @@ EXPORT void* DllInit()
 	T(privbuf, 232);
 	#undef T
 
-	memset(&comm,0,sizeof(comm));
+	memset(&comm, 0, sizeof(comm));
 
 	//make a coroutine thread to run the emulation in. we'll switch back to this cothread when communicating with the frontend
 	co_control = co_active();
+	if (co_emu)
+	{
+		// if this was called again, that's OK; delete the old emuthread
+		co_delete(co_emu);
+		co_emu = nullptr;
+	}
 	co_emu = co_create(65536 * sizeof(void*), new_emuthread);
 
 	return &comm;