Rewrite WaterboxHost in rust. (#2190)

This replaces the old managed one.  The only direct effect of this is to fix some hard to reproduce crashes in bsnes.

In the long run, we'll use this new code to help build more waterbox features.

src/BizHawk.BizInvoke/BizInvoker.cs

@@ -558,7 +558,7 @@ namespace BizHawk.BizInvoke
 				var strlenbytes = il.DeclareLocal(typeof(int), false);
 				il.Emit(OpCodes.Ldloc, encoding);
 				il.Emit(OpCodes.Ldarg, (short)idx);
-				il.Emit(OpCodes.Call, typeof(Encoding).GetMethod("GetByteCount", new[] { typeof(string) }));
+				il.EmitCall(OpCodes.Callvirt, typeof(Encoding).GetMethod("GetByteCount", new[] { typeof(string) }), Type.EmptyTypes);
 				il.Emit(OpCodes.Stloc, strlenbytes);
 
 				var strval = il.DeclareLocal(typeof(string), true); // pin!
@@ -590,7 +590,7 @@ namespace BizHawk.BizInvoke
 				// bytelength
 				il.Emit(OpCodes.Ldloc, strlenbytes);
 				// call
-				il.Emit(OpCodes.Call, typeof(Encoding).GetMethod("GetBytes", new[] { typeof(char*), typeof(int), typeof(byte*), typeof(int) }));
+				il.EmitCall(OpCodes.Callvirt, typeof(Encoding).GetMethod("GetBytes", new[] { typeof(char*), typeof(int), typeof(byte*), typeof(int) }), Type.EmptyTypes);
 				// unused ret
 				il.Emit(OpCodes.Pop);
 

src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs

@@ -252,7 +252,7 @@ namespace BizHawk.Client.Common
 			_nextStateIndex = reader.ReadInt32();
 		}
 
-		private class SaveStateStream : Stream
+		private class SaveStateStream : Stream, ISpanStream
 		{
 			/// <summary>
 			/// 
@@ -296,6 +296,7 @@ namespace BizHawk.Client.Common
 			public override int Read(byte[] buffer, int offset, int count) => throw new IOException();
 			public override long Seek(long offset, SeekOrigin origin) => throw new IOException();
 			public override void SetLength(long value) => throw new IOException();
+			public int Read(Span<byte> buffer) => throw new IOException();
 
 			public override void Write(byte[] buffer, int offset, int count)
 			{
@@ -324,6 +325,40 @@ namespace BizHawk.Client.Common
 				}
 			}
 
+			public void Write(ReadOnlySpan<byte> buffer)
+			{
+				long requestedSize = _position + buffer.Length;
+				while (requestedSize > _notifySize)
+					_notifySize = _notifySizeReached();
+				long n = buffer.Length;
+				if (n > 0)
+				{
+					var start = (_position + _offset) & _mask;
+					var end = (start + n) & _mask;
+					if (end < start)
+					{
+						long m = _buffer.LongLength - start;
+
+						// Array.Copy(buffer, offset, _buffer, start, m);
+						buffer.Slice(0, (int)m).CopyTo(new Span<byte>(_buffer, (int)start, (int)m));
+
+						// offset += (int)m;
+						buffer = buffer.Slice((int)m);
+
+						n -= m;
+						_position += m;
+						start = 0;
+					}
+					if (n > 0)
+					{
+						// Array.Copy(buffer, offset, _buffer, start, n);
+						buffer.CopyTo(new Span<byte>(_buffer, (int)start, (int)n));
+
+						_position += n;
+					}
+				}
+			}
+
 			public override void WriteByte(byte value)
 			{
 				long requestedSize = _position + 1;
@@ -333,7 +368,7 @@ namespace BizHawk.Client.Common
 			}
 		}
 
-		private class LoadStateStream : Stream
+		private class LoadStateStream : Stream, ISpanStream
 		{
 			public LoadStateStream(byte[] buffer, long offset, long size, long mask)
 			{
@@ -387,6 +422,38 @@ namespace BizHawk.Client.Common
 				return ret;
 			}
 
+			public unsafe int Read(Span<byte> buffer)
+			{
+				long n = Math.Min(_size - _position, buffer.Length);
+				int ret = (int)n;
+				if (n > 0)
+				{
+					var start = (_position + _offset) & _mask;
+					var end = (start + n) & _mask;
+					if (end < start)
+					{
+						long m = _buffer.LongLength - start;
+
+						// Array.Copy(_buffer, start, buffer, offset, m);
+						new ReadOnlySpan<byte>(_buffer, (int)start, (int)m).CopyTo(buffer);
+
+						// offset += (int)m;
+						buffer = buffer.Slice((int)m);
+	
+						n -= m;
+						_position += m;
+						start = 0;
+					}
+					if (n > 0)
+					{
+						// Array.Copy(_buffer, start, buffer, offset, n);
+						new ReadOnlySpan<byte>(_buffer, (int)start, (int)n).CopyTo(buffer);
+						_position += n;
+					}
+				}
+				return ret;
+			}
+
 			public override int ReadByte()
 			{
 				return _position < _size
@@ -397,6 +464,8 @@ namespace BizHawk.Client.Common
 			public override long Seek(long offset, SeekOrigin origin) => throw new IOException();
 			public override void SetLength(long value) => throw new IOException();
 			public override void Write(byte[] buffer, int offset, int count) => throw new IOException();
+
+			public void Write(ReadOnlySpan<byte> buffer) => throw new IOException();
 		}
 	}
 }

src/BizHawk.Common/BizHawk.Common.csproj

@@ -7,6 +7,7 @@
 	</PropertyGroup>
 	<ItemGroup>
 		<PackageReference Include="System.ComponentModel.Annotations" Version="4.7.0" />
+		<PackageReference Include="System.Memory" Version="4.5.4" />
 		<PackageReference Include="System.Reflection.Emit.Lightweight" Version="4.7.0" PrivateAssets="all" />
 		<PackageReference Include="Menees.Analyzers" Version="2.0.4" Condition=" '$(MachineRunAnalyzersDuringBuild)' != '' " />
 		<PackageReference Include="Microsoft.CodeAnalysis.FxCopAnalyzers" Version="3.0.0" Condition=" '$(MachineRunAnalyzersDuringBuild)' != '' " />

src/BizHawk.Common/SpanStream.cs

@@ -0,0 +1,63 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace BizHawk.Common
+{
+	/// <summary>
+	/// TODO: Switch to dotnet core and remove this junkus
+	/// </summary>
+	public interface ISpanStream
+	{
+		void Write (ReadOnlySpan<byte> buffer);
+		int Read (Span<byte> buffer);
+	}
+	public class SpanStream
+	{
+		/// <summary>
+		/// Returns a stream in spanstream mode, or creates a wrapper that provides that functionality
+		/// </summary>
+		/// <param name="s"></param>
+		/// <returns></returns>
+		public static ISpanStream GetOrBuild(Stream s)
+		{
+			return s as ISpanStream
+				?? new SpanStreamAdapter(s);
+		}
+		private class SpanStreamAdapter : ISpanStream
+		{
+			public SpanStreamAdapter(Stream stream)
+			{
+				_stream = stream;
+			}
+			private byte[] _buffer = new byte[0];
+			private readonly Stream _stream;
+			public unsafe int Read(Span<byte> buffer)
+			{
+				if (buffer.Length > _buffer.Length)
+				{
+					_buffer = new byte[buffer.Length];
+				}
+				var n = _stream.Read(_buffer, 0, buffer.Length);
+				fixed(byte* p = buffer)
+				{
+					Marshal.Copy(_buffer, 0, (IntPtr)p, n);
+				}
+				return n;
+			}
+
+			public unsafe void Write(ReadOnlySpan<byte> buffer)
+			{
+				if (buffer.Length > _buffer.Length)
+				{
+					_buffer = new byte[buffer.Length];
+				}
+				fixed(byte* p = buffer)
+				{
+					Marshal.Copy((IntPtr)p, _buffer, 0, buffer.Length);
+				}
+				_stream.Write(_buffer, 0, buffer.Length);
+			}
+		}
+	}
+}

src/BizHawk.Emulation.Cores/BizHawk.Emulation.Cores.csproj

@@ -6,6 +6,7 @@
 	</PropertyGroup>
 	<ItemGroup>
 		<PackageReference Include="ELFSharp" Version="2.10.0" />
+		<PackageReference Include="System.Memory" Version="4.5.4" />
 		<Reference Include="FlatBuffers.Core" HintPath="$(ProjectDir)../../References/FlatBuffers.Core.dll" Private="true" />
 		<Reference Include="Virtu" HintPath="$(ProjectDir)../../References/Virtu.dll" Private="true" />
 		<ProjectReference Include="$(ProjectDir)../BizHawk.BizInvoke/BizHawk.BizInvoke.csproj" />

src/BizHawk.Emulation.Cores/Consoles/NEC/PCE/TurboNyma.cs

@@ -29,11 +29,13 @@ namespace BizHawk.Emulation.Cores.Consoles.NEC.PCE
 			NymaSettings settings, NymaSyncSettings syncSettings, bool deterministic)
 			: base(comm, "PCE", "PC Engine Controller", settings, syncSettings)
 		{
-			var firmwares = new Dictionary<string, (string, string)>
-			{
-				{ "FIRMWARE:syscard3.pce", ("PCECD", "Bios") },
-				{ "FIRMWARE:gecard.pce", ("PCECD", "GE-Bios") },
-			};
+			var ids = discs.Select(d => new DiscIdentifier(d).DetectDiscType())
+				.ToList();
+			var firmwares = new Dictionary<string, (string, string)>();
+			if (ids.Contains(DiscType.TurboCD))
+				firmwares.Add("FIRMWARE:syscard3.pce", ("PCECD", "Bios"));
+			if (ids.Contains(DiscType.TurboGECD))
+				firmwares.Add("FIRMWARE:gecard.pce", ("PCECD", "GE-Bios"));
 			_turboNyma = DoInit<LibTurboNyma>(game, null, discs, "turbo.wbx", null, deterministic, firmwares);
 		}
 

src/BizHawk.Emulation.Cores/Consoles/Nintendo/Gameboy/Sameboy.cs

@@ -168,7 +168,7 @@ namespace BizHawk.Emulation.Cores.Consoles.Nintendo.Gameboy
 
 		public new byte[] CloneSaveRam()
 		{
-			_exe.AddTransientFile(null, "save.ram");
+			_exe.AddTransientFile(new byte[0], "save.ram");
 			_core.GetSaveRam();
 			return _exe.RemoveTransientFile("save.ram");
 		}

src/BizHawk.Emulation.Cores/Waterbox/EmuLibc.cs

@@ -1,43 +0,0 @@
-using System;
-using System.Runtime.InteropServices;
-using BizHawk.BizInvoke;
-using BizHawk.Common;
-
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	/// <summary>
-	/// implementation for special functions defined in emulibc.h
-	/// </summary>
-	internal class EmuLibc
-	{
-		private readonly WaterboxHost _parent;
-		public EmuLibc(WaterboxHost parent)
-		{
-			_parent = parent;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__walloc_sealed")]
-		public IntPtr AllocSealed(UIntPtr size)
-		{
-			return Z.US(_parent._sealedheap.Allocate((ulong)size, 16));
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__walloc_invisible")]
-		public IntPtr AllocInvisible(UIntPtr size)
-		{
-			return Z.US(_parent._invisibleheap.Allocate((ulong)size, 16));
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__walloc_plain")]
-		public IntPtr AllocPlain(UIntPtr size)
-		{
-			return Z.US(_parent._plainheap.Allocate((ulong)size, 16));
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__w_debug_puts")]
-		public void DebugPuts(IntPtr s)
-		{
-			Console.WriteLine(Mershul.PtrToStringUtf8(s));
-		}
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/NymaCore.cs

@@ -59,22 +59,37 @@ namespace BizHawk.Emulation.Cores.Waterbox
 				var filesToRemove = new List<string>();
 				if (firmwares != null)
 				{
-					_exe.MissingFileCallback = s =>
+					foreach (var kvp in firmwares)
 					{
-						if (firmwares.TryGetValue(s, out var tt))
+						var s = kvp.Key;
+						var tt = kvp.Value;
+						var data = CoreComm.CoreFileProvider.GetFirmware(tt.SystemID, tt.FirmwareID, false,
+							"Firmware files are usually required and may stop your game from loading");
+						if (data != null)
 						{
-							var data = CoreComm.CoreFileProvider.GetFirmware(tt.SystemID, tt.FirmwareID, false,
-								"Firmware files are usually required and may stop your game from loading");
-							if (data != null)
-							{
-								_exe.AddReadonlyFile(data, s);
-								filesToRemove.Add(s);
-								return true;
-							}
+							_exe.AddReadonlyFile(data, kvp.Key);
+							filesToRemove.Add(s);
 						}
-						return false;
-					};
+					}
 				}
+				// if (firmwares != null)
+				// {
+				// 	_exe.MissingFileCallback = s =>
+				// 	{
+				// 		if (firmwares.TryGetValue(s, out var tt))
+				// 		{
+				// 			var data = CoreComm.CoreFileProvider.GetFirmware(tt.SystemID, tt.FirmwareID, false,
+				// 				"Firmware files are usually required and may stop your game from loading");
+				// 			if (data != null)
+				// 			{
+				// 				_exe.AddReadonlyFile(data, s);
+				// 				filesToRemove.Add(s);
+				// 				return true;
+				// 			}
+				// 		}
+				// 		return false;
+				// 	};
+				// }
 				if (discs?.Length > 0)
 				{
 					_disks = discs;
@@ -104,14 +119,14 @@ namespace BizHawk.Emulation.Cores.Waterbox
 
 					_exe.RemoveReadonlyFile(fn);
 				}
-				if (firmwares != null)
-				{
+				// if (firmwares != null)
+				// {
 					foreach (var s in filesToRemove)
 					{
 						_exe.RemoveReadonlyFile(s);
 					}
-					_exe.MissingFileCallback = null;
-				}
+				// 	_exe.MissingFileCallback = null;
+				// }
 
 				var info = *_nyma.GetSystemInfo();
 				_videoBuffer = new int[Math.Max(info.MaxWidth * info.MaxHeight, info.LcmWidth * info.LcmHeight)];

src/BizHawk.Emulation.Cores/Waterbox/Swappable.cs

@@ -1,169 +0,0 @@
-using BizHawk.Common;
-using BizHawk.BizInvoke;
-using BizHawk.Common.BufferExtensions;
-using System;
-using System.Collections.Concurrent;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	/// <summary>
-	/// represents an object that can be swapped in and out of memory to compete with other objects in the same memory
-	/// not suited for general purpose stuff
-	/// </summary>
-	public abstract class Swappable : IMonitor, IDisposable
-	{
-		/// <summary>
-		/// start address
-		/// </summary>
-		private uint _lockkey;
-
-		/// <summary>
-		/// the the relevant lockinfo for this core
-		/// </summary>
-		private LockInfo _currentLockInfo;
-
-		/// <summary>
-		/// everything to swap in for context switches
-		/// </summary>
-		private List<MemoryBlock> _memoryBlocks = new List<MemoryBlock>();
-
-		/// <summary>
-		/// an informative name for each memory block:  used for debugging purposes
-		/// </summary>
-		private List<string> _memoryBlockNames = new List<string>();
-
-		protected void AddMemoryBlock(MemoryBlock block, string name)
-		{
-			_memoryBlocks.Add(block);
-			_memoryBlockNames.Add(name);
-		}
-
-		protected void PurgeMemoryBlocks()
-		{
-			_memoryBlocks = null;
-		}
-
-		protected void Initialize(ulong startAddress)
-		{
-			// any Swappables in the same 4G range are assumed to conflict
-			var lockkey = (uint)(startAddress >> 32);
-
-			_lockkey = lockkey;
-			if (lockkey == 0)
-				throw new NullReferenceException();
-			_currentLockInfo = LockInfos.GetOrAdd(_lockkey, new LockInfo { Sync = new object() });
-		}
-
-		private class LockInfo
-		{
-			public object Sync;
-			private WeakReference LoadedRef = new WeakReference(null);
-#if DEBUG
-			/// <summary>
-			/// recursive lock count
-			/// </summary>
-			public int LockCount;
-#endif
-			public Swappable Loaded
-			{
-				// if somehow an object died without being disposed,
-				// the MemoryBlock finalizer will have unloaded the memory
-				// and so we can treat it as if no Swappable was attached
-				get => (Swappable)LoadedRef.Target;
-				set => LoadedRef.Target = value;
-			}
-		}
-
-		private static readonly ConcurrentDictionary<uint, LockInfo> LockInfos = new ConcurrentDictionary<uint, LockInfo>();
-
-		/// <summary>
-		/// acquire lock and swap this into memory
-		/// </summary>
-		public void Enter()
-		{
-			Monitor.Enter(_currentLockInfo.Sync);
-#if DEBUG
-			if (_currentLockInfo.LockCount++ != 0 && _currentLockInfo.Loaded != this)
-				throw new InvalidOperationException("Woops!");
-#endif
-			if (_currentLockInfo.Loaded != this)
-			{
-				_currentLockInfo.Loaded?.DeactivateInternal();
-				_currentLockInfo.Loaded = null;
-				ActivateInternal();
-				_currentLockInfo.Loaded = this;
-			}
-		}
-
-		/// <summary>
-		/// release lock
-		/// </summary>
-		public void Exit()
-		{
-#if DEBUG
-			// when debugging, if we're releasing the lock then deactivate
-			if (_currentLockInfo.LockCount-- == 1)
-			{
-				if (_currentLockInfo.Loaded != this)
-					throw new InvalidOperationException("Woops!");
-				DeactivateInternal();
-				_currentLockInfo.Loaded = null;
-			}
-#endif
-			Monitor.Exit(_currentLockInfo.Sync);
-		}
-
-		private void DeactivateInternal()
-		{
-			foreach (var m in _memoryBlocks)
-				m.Deactivate();
-		}
-
-		private void ActivateInternal()
-		{
-			foreach (var m in _memoryBlocks)
-				m.Activate();
-		}
-		
-		public void PrintDebuggingInfo()
-		{
-			using (this.EnterExit())
-			{
-				foreach (var a in _memoryBlocks.Zip(_memoryBlockNames, (m, s) => new { m, s }))
-				{
-					Console.WriteLine($"{a.m.FullHash().BytesToHexString()}: {a.s}");
-				}
-			}
-		}
-
-		private bool _disposed = false;
-
-		protected virtual void Dispose(bool disposing)
-		{
-			if (!_disposed)
-			{
-				if (disposing)
-				{
-					lock (_currentLockInfo.Sync)
-					{
-						if (_currentLockInfo.Loaded == this)
-						{
-							DeactivateInternal();
-							_currentLockInfo.Loaded = null;
-						}
-						_currentLockInfo = null;
-					}
-				}
-				_disposed = true;
-			}
-		}
-
-		public void Dispose()
-		{
-			Dispose(true);
-		}
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/Syscalls/Errno.cs

@@ -1,140 +0,0 @@
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	partial class Syscalls
-	{
-		internal const int EPERM = 1;
-		internal const int ENOENT = 2;
-		internal const int ESRCH = 3;
-		internal const int EINTR = 4;
-		internal const int EIO = 5;
-		internal const int ENXIO = 6;
-		internal const int E2BIG = 7;
-		internal const int ENOEXEC = 8;
-		internal const int EBADF = 9;
-		internal const int ECHILD = 10;
-		internal const int EAGAIN = 11;
-		internal const int ENOMEM = 12;
-		internal const int EACCES = 13;
-		internal const int EFAULT = 14;
-		internal const int ENOTBLK = 15;
-		internal const int EBUSY = 16;
-		internal const int EEXIST = 17;
-		internal const int EXDEV = 18;
-		internal const int ENODEV = 19;
-		internal const int ENOTDIR = 20;
-		internal const int EISDIR = 21;
-		internal const int EINVAL = 22;
-		internal const int ENFILE = 23;
-		internal const int EMFILE = 24;
-		internal const int ENOTTY = 25;
-		internal const int ETXTBSY = 26;
-		internal const int EFBIG = 27;
-		internal const int ENOSPC = 28;
-		internal const int ESPIPE = 29;
-		internal const int EROFS = 30;
-		internal const int EMLINK = 31;
-		internal const int EPIPE = 32;
-		internal const int EDOM = 33;
-		internal const int ERANGE = 34;
-		internal const int EDEADLK = 35;
-		internal const int ENAMETOOLONG = 36;
-		internal const int ENOLCK = 37;
-		internal const int ENOSYS = 38;
-		internal const int ENOTEMPTY = 39;
-		internal const int ELOOP = 40;
-		internal const int EWOULDBLOCK = EAGAIN;
-		internal const int ENOMSG = 42;
-		internal const int EIDRM = 43;
-		internal const int ECHRNG = 44;
-		internal const int EL2NSYNC = 45;
-		internal const int EL3HLT = 46;
-		internal const int EL3RST = 47;
-		internal const int ELNRNG = 48;
-		internal const int EUNATCH = 49;
-		internal const int ENOCSI = 50;
-		internal const int EL2HLT = 51;
-		internal const int EBADE = 52;
-		internal const int EBADR = 53;
-		internal const int EXFULL = 54;
-		internal const int ENOANO = 55;
-		internal const int EBADRQC = 56;
-		internal const int EBADSLT = 57;
-		internal const int EDEADLOCK = EDEADLK;
-		internal const int EBFONT = 59;
-		internal const int ENOSTR = 60;
-		internal const int ENODATA = 61;
-		internal const int ETIME = 62;
-		internal const int ENOSR = 63;
-		internal const int ENONET = 64;
-		internal const int ENOPKG = 65;
-		internal const int EREMOTE = 66;
-		internal const int ENOLINK = 67;
-		internal const int EADV = 68;
-		internal const int ESRMNT = 69;
-		internal const int ECOMM = 70;
-		internal const int EPROTO = 71;
-		internal const int EMULTIHOP = 72;
-		internal const int EDOTDOT = 73;
-		internal const int EBADMSG = 74;
-		internal const int EOVERFLOW = 75;
-		internal const int ENOTUNIQ = 76;
-		internal const int EBADFD = 77;
-		internal const int EREMCHG = 78;
-		internal const int ELIBACC = 79;
-		internal const int ELIBBAD = 80;
-		internal const int ELIBSCN = 81;
-		internal const int ELIBMAX = 82;
-		internal const int ELIBEXEC = 83;
-		internal const int EILSEQ = 84;
-		internal const int ERESTART = 85;
-		internal const int ESTRPIPE = 86;
-		internal const int EUSERS = 87;
-		internal const int ENOTSOCK = 88;
-		internal const int EDESTADDRREQ = 89;
-		internal const int EMSGSIZE = 90;
-		internal const int EPROTOTYPE = 91;
-		internal const int ENOPROTOOPT = 92;
-		internal const int EPROTONOSUPPORT = 93;
-		internal const int ESOCKTNOSUPPORT = 94;
-		internal const int EOPNOTSUPP = 95;
-		internal const int ENOTSUP = EOPNOTSUPP;
-		internal const int EPFNOSUPPORT = 96;
-		internal const int EAFNOSUPPORT = 97;
-		internal const int EADDRINUSE = 98;
-		internal const int EADDRNOTAVAIL = 99;
-		internal const int ENETDOWN = 100;
-		internal const int ENETUNREACH = 101;
-		internal const int ENETRESET = 102;
-		internal const int ECONNABORTED = 103;
-		internal const int ECONNRESET = 104;
-		internal const int ENOBUFS = 105;
-		internal const int EISCONN = 106;
-		internal const int ENOTCONN = 107;
-		internal const int ESHUTDOWN = 108;
-		internal const int ETOOMANYREFS = 109;
-		internal const int ETIMEDOUT = 110;
-		internal const int ECONNREFUSED = 111;
-		internal const int EHOSTDOWN = 112;
-		internal const int EHOSTUNREACH = 113;
-		internal const int EALREADY = 114;
-		internal const int EINPROGRESS = 115;
-		internal const int ESTALE = 116;
-		internal const int EUCLEAN = 117;
-		internal const int ENOTNAM = 118;
-		internal const int ENAVAIL = 119;
-		internal const int EISNAM = 120;
-		internal const int EREMOTEIO = 121;
-		internal const int EDQUOT = 122;
-		internal const int ENOMEDIUM = 123;
-		internal const int EMEDIUMTYPE = 124;
-		internal const int ECANCELED = 125;
-		internal const int ENOKEY = 126;
-		internal const int EKEYEXPIRED = 127;
-		internal const int EKEYREVOKED = 128;
-		internal const int EKEYREJECTED = 129;
-		internal const int EOWNERDEAD = 130;
-		internal const int ENOTRECOVERABLE = 131;
-		internal const int ERFKILL = 132;
-		internal const int EHWPOISON = 133;
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/Syscalls/MMan.cs

@@ -1,202 +0,0 @@
-using System;
-using System.Runtime.InteropServices;
-using BizHawk.BizInvoke;
-
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	partial class Syscalls
-	{
-		internal const long MAP_FAILED = -1;
-
-		internal const ulong MAP_SHARED = 0x01;
-		internal const ulong MAP_PRIVATE = 0x02;
-		internal const ulong MAP_SHARED_VALIDATE = 0x03;
-		internal const ulong MAP_TYPE = 0x0f;
-		internal const ulong MAP_FIXED = 0x10;
-		internal const ulong MAP_ANON = 0x20;
-		internal const ulong MAP_ANONYMOUS = MAP_ANON;
-		internal const ulong MAP_NORESERVE = 0x4000;
-		internal const ulong MAP_GROWSDOWN = 0x0100;
-		internal const ulong MAP_DENYWRITE = 0x0800;
-		internal const ulong MAP_EXECUTABLE = 0x1000;
-		internal const ulong MAP_LOCKED = 0x2000;
-		internal const ulong MAP_POPULATE = 0x8000;
-		internal const ulong MAP_NONBLOCK = 0x10000;
-		internal const ulong MAP_STACK = 0x20000;
-		internal const ulong MAP_HUGETLB = 0x40000;
-		internal const ulong MAP_SYNC = 0x80000;
-		internal const ulong MAP_FIXED_NOREPLACE = 0x100000;
-		internal const ulong MAP_FILE = 0;
-
-		internal const ulong MAP_HUGE_SHIFT = 26;
-		internal const ulong MAP_HUGE_MASK = 0x3f;
-		internal const ulong MAP_HUGE_64KB = (16 << 26);
-		internal const ulong MAP_HUGE_512KB = (19 << 26);
-		internal const ulong MAP_HUGE_1MB = (20 << 26);
-		internal const ulong MAP_HUGE_2MB = (21 << 26);
-		internal const ulong MAP_HUGE_8MB = (23 << 26);
-		internal const ulong MAP_HUGE_16MB = (24 << 26);
-		internal const ulong MAP_HUGE_32MB = (25 << 26);
-		internal const ulong MAP_HUGE_256MB = (28 << 26);
-		internal const ulong MAP_HUGE_512MB = (29 << 26);
-		internal const ulong MAP_HUGE_1GB = (30 << 26);
-		internal const ulong MAP_HUGE_2GB = (31 << 26);
-		internal const ulong MAP_HUGE_16GB = (34U << 26);
-
-		internal const ulong PROT_NONE = 0;
-		internal const ulong PROT_READ = 1;
-		internal const ulong PROT_WRITE = 2;
-		internal const ulong PROT_EXEC = 4;
-		internal const ulong PROT_GROWSDOWN = 0x01000000;
-		internal const ulong PROT_GROWSUP = 0x02000000;
-
-		internal const ulong MS_ASYNC = 1;
-		internal const ulong MS_INVALIDATE = 2;
-		internal const ulong MS_SYNC = 4;
-
-		internal const ulong MCL_CURRENT = 1;
-		internal const ulong MCL_FUTURE = 2;
-		internal const ulong MCL_ONFAULT = 4;
-
-		internal const ulong POSIX_MADV_NORMAL = 0;
-		internal const ulong POSIX_MADV_RANDOM = 1;
-		internal const ulong POSIX_MADV_SEQUENTIAL = 2;
-		internal const ulong POSIX_MADV_WILLNEED = 3;
-		internal const ulong POSIX_MADV_DONTNEED = 4;
-
-		internal const ulong MADV_NORMAL = 0;
-		internal const ulong MADV_RANDOM = 1;
-		internal const ulong MADV_SEQUENTIAL = 2;
-		internal const ulong MADV_WILLNEED = 3;
-		internal const ulong MADV_DONTNEED = 4;
-		internal const ulong MADV_FREE = 8;
-		internal const ulong MADV_REMOVE = 9;
-		internal const ulong MADV_DONTFORK = 10;
-		internal const ulong MADV_DOFORK = 11;
-		internal const ulong MADV_MERGEABLE = 12;
-		internal const ulong MADV_UNMERGEABLE = 13;
-		internal const ulong MADV_HUGEPAGE = 14;
-		internal const ulong MADV_NOHUGEPAGE = 15;
-		internal const ulong MADV_DONTDUMP = 16;
-		internal const ulong MADV_DODUMP = 17;
-		internal const ulong MADV_WIPEONFORK = 18;
-		internal const ulong MADV_KEEPONFORK = 19;
-		internal const ulong MADV_COLD = 20;
-		internal const ulong MADV_PAGEOUT = 21;
-		internal const ulong MADV_HWPOISON = 100;
-		internal const ulong MADV_SOFT_OFFLINE = 101;
-
-		internal const ulong MREMAP_MAYMOVE = 1;
-		internal const ulong MREMAP_FIXED = 2;
-
-		internal const ulong MLOCK_ONFAULT = 0x01;
-
-		internal const ulong MFD_CLOEXEC = 0x0001U;
-		internal const ulong MFD_ALLOW_SEALING = 0x0002U;
-		internal const ulong MFD_HUGETLB = 0x0004U;
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[9]")]
-		public IntPtr MMap(IntPtr address, UIntPtr size, ulong prot, ulong flags, int fd, IntPtr offs)
-		{
-			if (address != IntPtr.Zero)
-			{
-				// waterbox cores generally don't know about hardcoded addresses
-				// we could support this, so long as the address is in our heap's range
-				return Z.SS(MAP_FAILED);
-			}
-			MemoryBlock.Protection mprot;
-			switch (prot)
-			{
-				case PROT_NONE:
-					mprot = MemoryBlock.Protection.None;
-					break;
-				default:
-				case PROT_WRITE | PROT_EXEC: // W^X
-				case PROT_READ | PROT_WRITE | PROT_EXEC: // W^X
-				case PROT_EXEC: // exec only????
-				case PROT_WRITE:
-					return Z.SS(MAP_FAILED); // write only????
-				case PROT_READ | PROT_WRITE:
-					mprot = (flags & MAP_STACK) != 0 ? MemoryBlock.Protection.RW_Stack : MemoryBlock.Protection.RW;
-					break;
-				case PROT_READ:
-					mprot = MemoryBlock.Protection.R;
-					break;
-				case PROT_READ | PROT_EXEC:
-					mprot = MemoryBlock.Protection.RX;
-					break;
-			}
-			if ((flags & MAP_ANONYMOUS) == 0)
-			{
-				// anonymous + private is easy
-				// anonymous by itself is hard
-				// nothing needs either right now
-				return Z.SS(MAP_FAILED);
-			}
-			if ((flags & 0xf00) != 0)
-			{
-				// various unsupported flags
-				return Z.SS(MAP_FAILED);
-			}
-
-			var ret = _parent._mmapheap.Map((ulong)size, mprot);
-			return ret == 0 ? Z.SS(MAP_FAILED) : Z.US(ret);
-		}
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[25]")]
-		public IntPtr MRemap(UIntPtr oldAddress, UIntPtr oldSize,
-			UIntPtr newSize, ulong flags)
-		{
-			if ((flags & MREMAP_FIXED) != 0)
-			{
-				// just like mmap.
-				// waterbox cores generally don't know about hardcoded addresses
-				// we could support this, so long as the address is in our heap's range	
-				return Z.SS(MAP_FAILED);
-			}
-			var ret = _parent._mmapheap.Remap((ulong)oldAddress, (ulong)oldSize, (ulong)newSize,
-				(flags & MREMAP_MAYMOVE) != 0);
-			return ret == 0 ? Z.SS(MAP_FAILED) : Z.US(ret);
-		}
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[11]")]
-		public long MUnmap(UIntPtr address, UIntPtr size)
-		{
-			return _parent._mmapheap.Unmap((ulong)address, (ulong)size) ? 0 : MAP_FAILED;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[10]")]
-		public long MProtect(UIntPtr address, UIntPtr size, ulong prot)
-		{
-			MemoryBlock.Protection mprot;
-			switch (prot)
-			{
-				case PROT_NONE:
-					mprot = MemoryBlock.Protection.None;
-					break;
-				default:
-				case PROT_WRITE | PROT_EXEC: // W^X
-				case PROT_READ | PROT_WRITE | PROT_EXEC: // W^X
-				case PROT_EXEC: // exec only????
-				case PROT_WRITE:
-					return MAP_FAILED; // write only????
-				case PROT_READ | PROT_WRITE:
-					mprot = MemoryBlock.Protection.RW;
-					break;
-				case PROT_READ:
-					mprot = MemoryBlock.Protection.R;
-					break;
-				case PROT_READ | PROT_EXEC:
-					mprot = MemoryBlock.Protection.RX;
-					break;
-			}
-			return _parent._mmapheap.Protect((ulong)address, (ulong)size, mprot) ? 0 : MAP_FAILED;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[28]")]
-		public long MAdvise(UIntPtr address, UIntPtr size, ulong advice)
-		{
-			// malloc will call this with DONTNEED in a few situations.  Ignore for now.
-			// TODO: This could be implemented along with munmap returning pages to non-dirty
-			return 0;
-		}
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/Syscalls/NotImplementedSyscalls.cs

@@ -1,422 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Runtime.InteropServices;
-using System.Text.RegularExpressions;
-using BizHawk.BizInvoke;
-using BizHawk.Common;
-
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	/// <summary>
-	/// Provides useful traps for any syscalls that are not implemented by libc
-	/// </summary>
-	internal class NotImplementedSyscalls : IImportResolver
-	{
-		private static readonly Dictionary<int, string> SyscallNames = new Dictionary<int, string>()
-		{
-			{ 0, "read" },
-			{ 1, "write" },
-			{ 2, "open" },
-			{ 3, "close" },
-			{ 4, "stat" },
-			{ 5, "fstat" },
-			{ 6, "lstat" },
-			{ 7, "poll" },
-			{ 8, "lseek" },
-			{ 9, "mmap" },
-			{ 10, "mprotect" },
-			{ 11, "munmap" },
-			{ 12, "brk" },
-			{ 13, "rt_sigaction" },
-			{ 14, "rt_sigprocmask" },
-			{ 15, "rt_sigreturn" },
-			{ 16, "ioctl" },
-			{ 17, "pread64" },
-			{ 18, "pwrite64" },
-			{ 19, "readv" },
-			{ 20, "writev" },
-			{ 21, "access" },
-			{ 22, "pipe" },
-			{ 23, "select" },
-			{ 24, "sched_yield" },
-			{ 25, "mremap" },
-			{ 26, "msync" },
-			{ 27, "mincore" },
-			{ 28, "madvise" },
-			{ 29, "shmget" },
-			{ 30, "shmat" },
-			{ 31, "shmctl" },
-			{ 32, "dup" },
-			{ 33, "dup2" },
-			{ 34, "pause" },
-			{ 35, "nanosleep" },
-			{ 36, "getitimer" },
-			{ 37, "alarm" },
-			{ 38, "setitimer" },
-			{ 39, "getpid" },
-			{ 40, "sendfile" },
-			{ 41, "socket" },
-			{ 42, "connect" },
-			{ 43, "accept" },
-			{ 44, "sendto" },
-			{ 45, "recvfrom" },
-			{ 46, "sendmsg" },
-			{ 47, "recvmsg" },
-			{ 48, "shutdown" },
-			{ 49, "bind" },
-			{ 50, "listen" },
-			{ 51, "getsockname" },
-			{ 52, "getpeername" },
-			{ 53, "socketpair" },
-			{ 54, "setsockopt" },
-			{ 55, "getsockopt" },
-			{ 56, "clone" },
-			{ 57, "fork" },
-			{ 58, "vfork" },
-			{ 59, "execve" },
-			{ 60, "exit" },
-			{ 61, "wait4" },
-			{ 62, "kill" },
-			{ 63, "uname" },
-			{ 64, "semget" },
-			{ 65, "semop" },
-			{ 66, "semctl" },
-			{ 67, "shmdt" },
-			{ 68, "msgget" },
-			{ 69, "msgsnd" },
-			{ 70, "msgrcv" },
-			{ 71, "msgctl" },
-			{ 72, "fcntl" },
-			{ 73, "flock" },
-			{ 74, "fsync" },
-			{ 75, "fdatasync" },
-			{ 76, "truncate" },
-			{ 77, "ftruncate" },
-			{ 78, "getdents" },
-			{ 79, "getcwd" },
-			{ 80, "chdir" },
-			{ 81, "fchdir" },
-			{ 82, "rename" },
-			{ 83, "mkdir" },
-			{ 84, "rmdir" },
-			{ 85, "creat" },
-			{ 86, "link" },
-			{ 87, "unlink" },
-			{ 88, "symlink" },
-			{ 89, "readlink" },
-			{ 90, "chmod" },
-			{ 91, "fchmod" },
-			{ 92, "chown" },
-			{ 93, "fchown" },
-			{ 94, "lchown" },
-			{ 95, "umask" },
-			{ 96, "gettimeofday" },
-			{ 97, "getrlimit" },
-			{ 98, "getrusage" },
-			{ 99, "sysinfo" },
-			{ 100, "times" },
-			{ 101, "ptrace" },
-			{ 102, "getuid" },
-			{ 103, "syslog" },
-			{ 104, "getgid" },
-			{ 105, "setuid" },
-			{ 106, "setgid" },
-			{ 107, "geteuid" },
-			{ 108, "getegid" },
-			{ 109, "setpgid" },
-			{ 110, "getppid" },
-			{ 111, "getpgrp" },
-			{ 112, "setsid" },
-			{ 113, "setreuid" },
-			{ 114, "setregid" },
-			{ 115, "getgroups" },
-			{ 116, "setgroups" },
-			{ 117, "setresuid" },
-			{ 118, "getresuid" },
-			{ 119, "setresgid" },
-			{ 120, "getresgid" },
-			{ 121, "getpgid" },
-			{ 122, "setfsuid" },
-			{ 123, "setfsgid" },
-			{ 124, "getsid" },
-			{ 125, "capget" },
-			{ 126, "capset" },
-			{ 127, "rt_sigpending" },
-			{ 128, "rt_sigtimedwait" },
-			{ 129, "rt_sigqueueinfo" },
-			{ 130, "rt_sigsuspend" },
-			{ 131, "sigaltstack" },
-			{ 132, "utime" },
-			{ 133, "mknod" },
-			{ 134, "uselib" },
-			{ 135, "personality" },
-			{ 136, "ustat" },
-			{ 137, "statfs" },
-			{ 138, "fstatfs" },
-			{ 139, "sysfs" },
-			{ 140, "getpriority" },
-			{ 141, "setpriority" },
-			{ 142, "sched_setparam" },
-			{ 143, "sched_getparam" },
-			{ 144, "sched_setscheduler" },
-			{ 145, "sched_getscheduler" },
-			{ 146, "sched_get_priority_max" },
-			{ 147, "sched_get_priority_min" },
-			{ 148, "sched_rr_get_interval" },
-			{ 149, "mlock" },
-			{ 150, "munlock" },
-			{ 151, "mlockall" },
-			{ 152, "munlockall" },
-			{ 153, "vhangup" },
-			{ 154, "modify_ldt" },
-			{ 155, "pivot_root" },
-			{ 156, "_sysctl" },
-			{ 157, "prctl" },
-			{ 158, "arch_prctl" },
-			{ 159, "adjtimex" },
-			{ 160, "setrlimit" },
-			{ 161, "chroot" },
-			{ 162, "sync" },
-			{ 163, "acct" },
-			{ 164, "settimeofday" },
-			{ 165, "mount" },
-			{ 166, "umount2" },
-			{ 167, "swapon" },
-			{ 168, "swapoff" },
-			{ 169, "reboot" },
-			{ 170, "sethostname" },
-			{ 171, "setdomainname" },
-			{ 172, "iopl" },
-			{ 173, "ioperm" },
-			{ 174, "create_module" },
-			{ 175, "init_module" },
-			{ 176, "delete_module" },
-			{ 177, "get_kernel_syms" },
-			{ 178, "query_module" },
-			{ 179, "quotactl" },
-			{ 180, "nfsservctl" },
-			{ 181, "getpmsg" },
-			{ 182, "putpmsg" },
-			{ 183, "afs_syscall" },
-			{ 184, "tuxcall" },
-			{ 185, "security" },
-			{ 186, "gettid" },
-			{ 187, "readahead" },
-			{ 188, "setxattr" },
-			{ 189, "lsetxattr" },
-			{ 190, "fsetxattr" },
-			{ 191, "getxattr" },
-			{ 192, "lgetxattr" },
-			{ 193, "fgetxattr" },
-			{ 194, "listxattr" },
-			{ 195, "llistxattr" },
-			{ 196, "flistxattr" },
-			{ 197, "removexattr" },
-			{ 198, "lremovexattr" },
-			{ 199, "fremovexattr" },
-			{ 200, "tkill" },
-			{ 201, "time" },
-			{ 202, "futex" },
-			{ 203, "sched_setaffinity" },
-			{ 204, "sched_getaffinity" },
-			{ 205, "set_thread_area" },
-			{ 206, "io_setup" },
-			{ 207, "io_destroy" },
-			{ 208, "io_getevents" },
-			{ 209, "io_submit" },
-			{ 210, "io_cancel" },
-			{ 211, "get_thread_area" },
-			{ 212, "lookup_dcookie" },
-			{ 213, "epoll_create" },
-			{ 214, "epoll_ctl_old" },
-			{ 215, "epoll_wait_old" },
-			{ 216, "remap_file_pages" },
-			{ 217, "getdents64" },
-			{ 218, "set_tid_address" },
-			{ 219, "restart_syscall" },
-			{ 220, "semtimedop" },
-			{ 221, "fadvise64" },
-			{ 222, "timer_create" },
-			{ 223, "timer_settime" },
-			{ 224, "timer_gettime" },
-			{ 225, "timer_getoverrun" },
-			{ 226, "timer_delete" },
-			{ 227, "clock_settime" },
-			{ 228, "clock_gettime" },
-			{ 229, "clock_getres" },
-			{ 230, "clock_nanosleep" },
-			{ 231, "exit_group" },
-			{ 232, "epoll_wait" },
-			{ 233, "epoll_ctl" },
-			{ 234, "tgkill" },
-			{ 235, "utimes" },
-			{ 236, "vserver" },
-			{ 237, "mbind" },
-			{ 238, "set_mempolicy" },
-			{ 239, "get_mempolicy" },
-			{ 240, "mq_open" },
-			{ 241, "mq_unlink" },
-			{ 242, "mq_timedsend" },
-			{ 243, "mq_timedreceive" },
-			{ 244, "mq_notify" },
-			{ 245, "mq_getsetattr" },
-			{ 246, "kexec_load" },
-			{ 247, "waitid" },
-			{ 248, "add_key" },
-			{ 249, "request_key" },
-			{ 250, "keyctl" },
-			{ 251, "ioprio_set" },
-			{ 252, "ioprio_get" },
-			{ 253, "inotify_init" },
-			{ 254, "inotify_add_watch" },
-			{ 255, "inotify_rm_watch" },
-			{ 256, "migrate_pages" },
-			{ 257, "openat" },
-			{ 258, "mkdirat" },
-			{ 259, "mknodat" },
-			{ 260, "fchownat" },
-			{ 261, "futimesat" },
-			{ 262, "newfstatat" },
-			{ 263, "unlinkat" },
-			{ 264, "renameat" },
-			{ 265, "linkat" },
-			{ 266, "symlinkat" },
-			{ 267, "readlinkat" },
-			{ 268, "fchmodat" },
-			{ 269, "faccessat" },
-			{ 270, "pselect6" },
-			{ 271, "ppoll" },
-			{ 272, "unshare" },
-			{ 273, "set_robust_list" },
-			{ 274, "get_robust_list" },
-			{ 275, "splice" },
-			{ 276, "tee" },
-			{ 277, "sync_file_range" },
-			{ 278, "vmsplice" },
-			{ 279, "move_pages" },
-			{ 280, "utimensat" },
-			{ 281, "epoll_pwait" },
-			{ 282, "signalfd" },
-			{ 283, "timerfd_create" },
-			{ 284, "eventfd" },
-			{ 285, "fallocate" },
-			{ 286, "timerfd_settime" },
-			{ 287, "timerfd_gettime" },
-			{ 288, "accept4" },
-			{ 289, "signalfd4" },
-			{ 290, "eventfd2" },
-			{ 291, "epoll_create1" },
-			{ 292, "dup3" },
-			{ 293, "pipe2" },
-			{ 294, "inotify_init1" },
-			{ 295, "preadv" },
-			{ 296, "pwritev" },
-			{ 297, "rt_tgsigqueueinfo" },
-			{ 298, "perf_event_open" },
-			{ 299, "recvmmsg" },
-			{ 300, "fanotify_init" },
-			{ 301, "fanotify_mark" },
-			{ 302, "prlimit64" },
-			{ 303, "name_to_handle_at" },
-			{ 304, "open_by_handle_at" },
-			{ 305, "clock_adjtime" },
-			{ 306, "syncfs" },
-			{ 307, "sendmmsg" },
-			{ 308, "setns" },
-			{ 309, "getcpu" },
-			{ 310, "process_vm_readv" },
-			{ 311, "process_vm_writev" },
-			{ 312, "kcmp" },
-			{ 313, "finit_module" },
-			{ 314, "sched_setattr" },
-			{ 315, "sched_getattr" },
-			{ 316, "renameat2" },
-			{ 317, "seccomp" },
-			{ 318, "getrandom" },
-			{ 319, "memfd_create" },
-			{ 320, "kexec_file_load" },
-			{ 321, "bpf" },
-			{ 322, "execveat" },
-			{ 323, "userfaultfd" },
-			{ 324, "membarrier" },
-			{ 325, "mlock2" },
-			{ 326, "copy_file_range" },
-			{ 327, "preadv2" },
-			{ 328, "pwritev2" },
-			{ 329, "pkey_mprotect" },
-			{ 330, "pkey_alloc" },
-			{ 331, "pkey_free" },
-			{ 332, "statx" },
-			{ 333, "io_pgetevents" },
-			{ 334, "rseq" },
-			{ 424, "pidfd_send_signal" },
-			{ 425, "io_uring_setup" },
-			{ 426, "io_uring_enter" },
-			{ 427, "io_uring_register" },
-			{ 428, "open_tree" },
-			{ 429, "move_mount" },
-			{ 430, "fsopen" },
-			{ 431, "fsconfig" },
-			{ 432, "fsmount" },
-			{ 433, "fspick" },
-			{ 434, "pidfd_open" },
-			{ 435, "clone3" },
-		};
-
-		private class Trap
-		{
-			private readonly int _index;
-			private readonly string _message;
-			private readonly IImportResolver _resolver;
-			public Trap(int index)
-			{
-				_index = index;
-				_resolver = BizExvoker.GetExvoker(this, CallingConventionAdapters.Waterbox);
-				if (!SyscallNames.TryGetValue(_index, out var name))
-					name = "???";
-				_message = $"Trapped on unimplemented syscall {name} (#{_index})";
-			}
-			[BizExport(CallingConvention.Cdecl, EntryPoint="@@")]
-			public void RunTrap()
-			{
-				System.Diagnostics.Debug.WriteLine(_message);
-				// TODO: this unwind never works, so debugbrk instead
-				System.Diagnostics.Debugger.Break();
-				throw new InvalidOperationException(_message);
-			}
-			public IntPtr FunctionPointer => _resolver.GetProcAddrOrThrow("@@");
-		}
-		private readonly List<Trap> _traps;
-		private NotImplementedSyscalls()
-		{
-			_traps = Enumerable.Range(0, 512)
-				.Select(i => new Trap(i))
-				.ToList();
-		}
-
-		private static readonly Regex ExportRegex = new Regex("__wsyscalltab\\[(\\d+)\\]");
-
-		public IntPtr GetProcAddrOrZero(string entryPoint)
-		{
-			var m = ExportRegex.Match(entryPoint);
-			if (m.Success)
-			{
-				return _traps[int.Parse(m.Groups[1].Value)].FunctionPointer;
-			}
-			return IntPtr.Zero;
-		}
-
-		public IntPtr GetProcAddrOrThrow(string entryPoint)
-		{
-			var m = ExportRegex.Match(entryPoint);
-			if (m.Success)
-			{
-				return _traps[int.Parse(m.Groups[1].Value)].FunctionPointer;
-			}
-			throw new InvalidOperationException($"{entryPoint} was not of the format __wsyscalltab[#]");
-		}
-
-		public static NotImplementedSyscalls Instance { get; } = new NotImplementedSyscalls();
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/Syscalls/Stat.cs

@@ -1,120 +0,0 @@
-using System;
-using System.Runtime.InteropServices;
-using BizHawk.BizInvoke;
-
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	unsafe partial class Syscalls
-	{
-		internal const uint S_IFMT = 61440;
-
-		internal const uint S_IFDIR = 16384;
-		internal const uint S_IFCHR = 8192;
-		internal const uint S_IFBLK = 24576;
-		internal const uint S_IFREG = 32768;
-		internal const uint S_IFIFO = 4096;
-		internal const uint S_IFLNK = 40960;
-		internal const uint S_IFSOCK = 49152;
-
-		internal const uint S_ISUID = 2048;
-		internal const uint S_ISGID = 1024;
-		internal const uint S_ISVTX = 512;
-		internal const uint S_IRUSR = 0400;
-		internal const uint S_IWUSR = 256;
-		internal const uint S_IXUSR = 64;
-		internal const uint S_IRWXU = 448;
-		internal const uint S_IRGRP = 32;
-		internal const uint S_IWGRP = 16;
-		internal const uint S_IXGRP = 8;
-		internal const uint S_IRWXG = 56;
-		internal const uint S_IROTH = 4;
-		internal const uint S_IWOTH = 2;
-		internal const uint S_IXOTH = 1;
-		internal const uint S_IRWXO = 7;
-
-
-		[StructLayout(LayoutKind.Sequential)]
-		public struct KStat
-		{
-			public ulong st_dev;
-			public ulong st_ino;
-			public ulong st_nlink;
-
-			public uint st_mode;
-			public uint st_uid;
-			public uint st_gid;
-			public uint __pad0;
-			public ulong st_rdev;
-			public long st_size;
-			public long st_blksize;
-			public long st_blocks;
-
-			public long st_atime_sec;
-			public long st_atime_nsec;
-			public long st_mtime_sec;
-			public long st_mtime_nsec;
-			public long st_ctime_sec;
-			public long st_ctime_nsec;
-			public long __unused0;
-			public long __unused1;
-			public long __unused2;
-		}
-
-		private void StatInternal(KStat* s, IFileObject o)
-		{
-			s->st_dev = 1;
-			s->st_ino = 1;
-			s->st_nlink = 0;
-
-			uint flags = 0;
-			if (o.Stream.CanRead)
-				flags |= S_IRUSR | S_IRGRP | S_IROTH;
-			if (o.Stream.CanWrite)
-				flags |= S_IWUSR | S_IWGRP | S_IWOTH;
-			if (o.Stream.CanSeek)
-				flags |= S_IFREG;
-			else
-				flags |= S_IFIFO;
-			s->st_mode = flags;
-			s->st_uid = 0;
-			s->st_gid = 0;
-			s->__pad0 = 0;
-			s->st_rdev = 0;
-			if (o.Stream.CanSeek)
-				s->st_size = o.Stream.Length;
-			else
-				s->st_size = 0;
-			s->st_blksize = 4096;
-			s->st_blocks = (s->st_size + 511) / 512;
-
-			s->st_atime_sec = 1262304000000;
-			s->st_atime_nsec = 1000000000 / 2;
-			s->st_mtime_sec = 1262304000000;
-			s->st_mtime_nsec = 1000000000 / 2;
-			s->st_ctime_sec = 1262304000000;
-			s->st_ctime_nsec = 1000000000 / 2;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[4]")]
-		public long Stat(string path, KStat* statbuf)
-		{
-			if (!_availableFiles.TryGetValue(path, out var o))
-				return -ENOENT;
-
-			StatInternal(statbuf, o);
-			return 0;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[5]")]
-		public long Fstat(int fd, KStat* statbuf)
-		{
-			if (fd < 0 || fd >= _openFiles.Count)
-				return -EBADF;
-			var o = _openFiles[fd];
-			if (o == null)
-				return -EBADF;
-			StatInternal(statbuf, o);
-			return 0;
-		}
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/Syscalls/Syscalls.cs

@@ -1,498 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Runtime.InteropServices;
-using BizHawk.BizInvoke;
-using BizHawk.Common;
-using BizHawk.Emulation.Common;
-
-namespace BizHawk.Emulation.Cores.Waterbox
-{
-	/// <summary>
-	/// syscall emulation layer
-	/// </summary>
-	internal partial class Syscalls : IBinaryStateable
-	{
-		/// <summary>
-		/// Can be set by the frontend and will be called if the core attempts to open a missing file.
-		/// The callee may add additional files to the waterbox during the callback and return `true` to indicate
-		/// that the right file was added and the scan should be rerun.  The callee may return `false` to indicate
-		/// that the file should be reported as missing.  Do not call other things during this callback.
-		/// Can be called at any time by the core, so you may want to remove your callback entirely after init
-		/// if it was for firmware only.
-		/// </summary>
-		public Func<string, bool> MissingFileCallback { get; set; }
-		public interface IFileObject : IBinaryStateable
-		{
-			bool Open(FileAccess access);
-			bool Close();
-			Stream Stream { get; }
-			string Name { get; }
-		}
-
-		private class SpecialFile : IFileObject
-		{
-			// stdin, stdout, stderr
-			public string Name { get; }
-			public Stream Stream { get; }
-			public bool Close() => false;
-			public bool Open(FileAccess access) => false;
-
-			public void SaveStateBinary(BinaryWriter writer) { }
-			public void LoadStateBinary(BinaryReader reader) { }
-
-			public SpecialFile(Stream stream, string name)
-			{
-				Stream = stream;
-				Name = name;
-			}
-		}
-
-		private class ReadonlyFirmware : IFileObject
-		{
-			private readonly byte[] _data;
-			private readonly byte[] _hash;
-
-			public string Name { get; }
-			public Stream Stream { get; private set; }
-			public bool Close()
-			{
-				if (Stream == null)
-					return false;
-				Stream = null;
-				return true;
-			}
-
-			public bool Open(FileAccess access)
-			{
-				if (Stream != null || access != FileAccess.Read)
-					return false;
-				Stream = new MemoryStream(_data, false);
-				return true;
-			}
-
-			public void LoadStateBinary(BinaryReader br)
-			{
-				if (!br.ReadBytes(_hash.Length).SequenceEqual(_hash))
-					throw new InvalidOperationException("Savestate internal firmware mismatch");
-				var pos = br.ReadInt64();
-				if (pos == -1)
-				{
-					Stream = null;
-				}
-				else
-				{
-					if (Stream == null)
-						Open(FileAccess.Read);
-					Stream.Position = pos;
-				}
-			}
-
-			public void SaveStateBinary(BinaryWriter bw)
-			{
-				bw.Write(_hash);
-				bw.Write(Stream != null ? Stream.Position : -1);
-			}
-
-			public ReadonlyFirmware(byte[] data, string name)
-			{
-				_data = data;
-				_hash = WaterboxUtils.Hash(data);
-				Name = name;
-			}
-		}
-
-		private class TransientFile : IFileObject
-		{
-			private bool _inUse = false;
-			public string Name { get; }
-			public Stream Stream { get; }
-			public bool Close()
-			{
-				if (_inUse)
-				{
-					_inUse = false;
-					return true;
-				}
-				else
-				{
-					return false;
-				}
-			}
-
-			public bool Open(FileAccess access)
-			{
-				if (_inUse)
-				{
-					return false;
-				}
-				else
-				{
-					// TODO: if access != RW, the resultant handle lets you do those all anyway
-					_inUse = true;
-					Stream.Position = 0;
-					return true;
-				}
-			}
-
-			public void LoadStateBinary(BinaryReader br)
-			{
-				throw new InvalidOperationException("Internal savestate error!");
-			}
-
-			public void SaveStateBinary(BinaryWriter bw)
-			{
-				throw new InvalidOperationException("Transient files cannot be savestated!");
-			}
-
-			public TransientFile(byte[] data, string name)
-			{
-				Stream = new MemoryStream();
-				Name = name;
-				if (data != null)
-				{
-					Stream.Write(data, 0, data.Length);
-					Stream.Position = 0;
-				}
-			}
-
-			public byte[] GetContents()
-			{
-				if (_inUse)
-					throw new InvalidOperationException();
-				return ((MemoryStream)Stream).ToArray();
-			}
-		}
-
-		private readonly List<IFileObject> _openFiles = new List<IFileObject>();
-		private readonly Dictionary<string, IFileObject> _availableFiles = new Dictionary<string, IFileObject>();
-
-		private readonly WaterboxHost _parent;
-		public Syscalls(WaterboxHost parent)
-		{
-			_parent = parent;
-			var stdin = new SpecialFile(Stream.Null, "___stdin");
-			var stdout = new SpecialFile(Console.OpenStandardOutput(), "___stdout");
-			var stderr = new SpecialFile(Console.OpenStandardError(), "___stderr");
-
-			_openFiles = new List<IFileObject>
-			{
-				stdin,
-				stdout,
-				stderr
-			};
-			_availableFiles = new Dictionary<string, IFileObject>
-			{
-				[stdin.Name] = stdin,
-				[stdout.Name] = stdout,
-				[stderr.Name] = stderr
-			};
-		}
-
-		private Stream StreamForFd(int fd)
-		{
-			if (fd >= 0 && fd < _openFiles.Count)
-				return _openFiles[fd].Stream;
-			else
-				return null;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[12]")]
-		public UIntPtr Brk(UIntPtr _p)
-		{
-			var heap = _parent._heap;
-
-			var start = heap.Memory.Start;
-			var currentEnd = start + heap.Used;
-			var maxEnd = heap.Memory.EndExclusive;
-
-			var p = (ulong)_p;
-
-			if (p < start || p > maxEnd)
-			{
-				// failure: return current break
-				return Z.UU(currentEnd);
-			}
-			else if (p > currentEnd)
-			{
-				// increase size of heap
-				heap.Allocate(p - currentEnd, 1);
-				return Z.UU(p);
-			}
-			else if (p < currentEnd)
-			{
-				throw new InvalidOperationException("We don't support shrinking heaps");
-			}
-			else
-			{
-				// no change
-				return Z.UU(currentEnd);
-			}
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[16]")]
-		public int IoCtl(int fd, ulong req)
-		{
-			return 0; // sure it worked, honest
-		}
-
-		public struct Iovec
-		{
-			public IntPtr Base;
-			public ulong Length;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[0]")]
-		public long Read(int fd, IntPtr buff, ulong count)
-		{
-			var s = StreamForFd(fd);
-			if (s == null || !s.CanRead)
-				return -1;
-			var tmp = new byte[count];
-			var ret = s.Read(tmp, 0, (int)count);
-			Marshal.Copy(tmp, 0, buff, ret);
-			return ret;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[1]")]
-		public long Write(int fd, IntPtr buff, ulong count)
-		{
-			var s = StreamForFd(fd);
-			if (s == null || !s.CanWrite)
-				return -1;
-			var tmp = new byte[count];
-			Marshal.Copy(buff, tmp, 0, (int)count);
-			s.Write(tmp, 0, tmp.Length);
-			return (long)count;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[19]")]
-		public unsafe long Readv(int fd, Iovec* iov, int iovcnt)
-		{
-			long ret = 0;
-			for (int i = 0; i < iovcnt; i++)
-			{
-				var len = Read(fd, iov[i].Base, iov[i].Length);
-				if (len < 0)
-					return len;
-				ret += len;
-				if (len != (long)iov[i].Length)
-					break;
-			}
-			return ret;
-		}
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[20]")]
-		public unsafe long Writev(int fd, Iovec* iov, int iovcnt)
-		{
-			long ret = 0;
-			for (int i = 0; i < iovcnt; i++)
-			{
-				if (iov[i].Base != IntPtr.Zero)
-					ret += Write(fd, iov[i].Base, iov[i].Length);
-			}
-			return ret;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[2]")]
-		public long Open(string path, int flags, int mode)
-		{
-			if (!_availableFiles.TryGetValue(path, out var o))
-			{
-				var retry = MissingFileCallback?.Invoke(path) == true;
-				if (retry)
-				{
-					if (!_availableFiles.TryGetValue(path, out o))
-						return -ENOENT;
-				}
-				else
-				{
-					return -ENOENT;
-				}
-			}
-			if (_openFiles.Contains(o))
-				return -EACCES;
-			FileAccess access;
-			switch (flags & 3)
-			{
-				case 0:
-					access = FileAccess.Read;
-					break;
-				case 1:
-					access = FileAccess.Write;
-					break;
-				case 2:
-					access = FileAccess.ReadWrite;
-					break;
-				default:
-					return -EINVAL;
-			}
-			if (!o.Open(access))
-				return -EACCES;
-			int fd;
-			for (fd = 0; fd < _openFiles.Count; fd++)
-				if (_openFiles[fd] == null)
-					break;
-			if (fd == _openFiles.Count)
-				_openFiles.Add(o);
-			else
-				_openFiles[fd] = o;
-			return fd;
-		}
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[3]")]
-		public long Close(int fd)
-		{
-			if (fd < 0 || fd >= _openFiles.Count)
-				return -EBADF;
-			var o = _openFiles[fd];
-			if (o == null)
-				return -EBADF;
-			if (!o.Close())
-				return -EIO;
-			_openFiles[fd] = null;
-			return 0;
-		}
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[8]")]
-		public long Seek(int fd, long offset, int type)
-		{
-			var s = StreamForFd(fd);
-			if (s == null)
-				return -EBADF;
-			if (!s.CanSeek)
-				return -ESPIPE;
-			SeekOrigin o;
-			switch (type)
-			{
-				case 0:
-					o = SeekOrigin.Begin;
-					break;
-				case 1:
-					o = SeekOrigin.Current;
-					break;
-				case 2:
-					o = SeekOrigin.End;
-					break;
-				default:
-					return -EINVAL;
-			}
-			try
-			{
-				return s.Seek(offset, o);
-			}
-			catch (IOException)
-			{
-				// usually means bad offset, since we already filtered out on CanSeek
-				return -EINVAL;
-			}
-		}
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[77]")]
-		public long FTruncate(int fd, long length)
-		{
-			var s = StreamForFd(fd);
-			if (s == null)
-				return -EBADF;
-			if (!s.CanWrite)
-				return -ESPIPE;
-			if (!s.CanSeek)
-				return -ESPIPE;
-			s.SetLength(length);
-			return 0;
-		}
-
-		// TODO: Remove this entirely once everything is compiled against the new libc
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[205]")]
-		public long SetThreadArea(IntPtr uinfo)
-		{
-			return 38; // ENOSYS
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[218]")]
-		public long SetTidAddress(IntPtr address)
-		{
-			// pretend we succeeded
-			return 8675309; // arbitrary thread id
-		}
-
-		[StructLayout(LayoutKind.Sequential)]
-		public struct TimeSpec
-		{
-			public long Seconds;
-			public long NanoSeconds;
-		}
-
-		[BizExport(CallingConvention.Cdecl, EntryPoint = "__wsyscalltab[228]")]
-		public unsafe int SysClockGetTime(int which, TimeSpec* time)
-		{
-			time->Seconds = 1495889068;
-			time->NanoSeconds = 0;
-			return 0;
-		}
-
-		public void SaveStateBinary(BinaryWriter bw)
-		{
-			bw.Write(_availableFiles.Count);
-			foreach (var f in _availableFiles.Values.OrderBy(f => f.Name))
-			{
-				bw.Write(f.Name);
-				f.SaveStateBinary(bw);
-			}
-			bw.Write(_openFiles.Count);
-			foreach (var f in _openFiles)
-			{
-				bw.Write(f != null);
-				if (f != null)
-					bw.Write(f.Name);
-			}
-		}
-
-		public void LoadStateBinary(BinaryReader br)
-		{
-			if (_availableFiles.Count != br.ReadInt32())
-				throw new InvalidOperationException("Internal savestate error:  Filelist change");
-			foreach (var f in _availableFiles.Values.OrderBy(f => f.Name))
-			{
-				if (br.ReadString() != f.Name)
-					throw new InvalidOperationException("Internal savestate error:  Filelist change");
-				f.LoadStateBinary(br);
-			}
-			var c = br.ReadInt32();
-			_openFiles.Clear();
-			for (int i = 0; i < c; i++)
-			{
-				_openFiles.Add(br.ReadBoolean() ? _availableFiles[br.ReadString()] : null);
-			}
-		}
-
-		private T RemoveFileInternal<T>(string name)
-			where T : IFileObject
-		{
-			if (!_availableFiles.TryGetValue(name, out var o))
-				throw new InvalidOperationException("File was never registered!");
-			if (o.GetType() != typeof(T))
-				throw new InvalidOperationException("Object was not a the right kind of file");
-			if (_openFiles.Contains(o))
-				throw new InvalidOperationException("Core never closed the file!");
-			_availableFiles.Remove(name);
-			return (T)o;
-		}
-
-		public void AddReadonlyFile(byte[] data, string name)
-		{
-			_availableFiles.Add(name, new ReadonlyFirmware(data, name));
-		}
-
-		public void RemoveReadonlyFile(string name)
-		{
-			RemoveFileInternal<ReadonlyFirmware>(name);
-		}
-
-		public void AddTransientFile(byte[] data, string name)
-		{
-			_availableFiles.Add(name, new TransientFile(data, name));
-		}
-		public byte[] RemoveTransientFile(string name)
-		{
-			return RemoveFileInternal<TransientFile>(name).GetContents();
-		}
-	}
-}

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHost.cs

@@ -7,6 +7,7 @@ using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
 using System.Threading;
+using static BizHawk.Emulation.Cores.Waterbox.WaterboxHostNative;
 
 namespace BizHawk.Emulation.Cores.Waterbox
 {
@@ -53,11 +54,6 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// </summary>
 		public uint MmapHeapSizeKB { get; set; }
 
-		/// <summary>
-		/// start address in memory
-		/// </summary>
-		public ulong StartAddress { get; set; } = WaterboxHost.CanonicalStart;
-
 		/// <summary>
 		/// Skips the check that the wbx file and other associated dlls match from state save to state load.
 		/// DO NOT SET THIS TO TRUE.  A different executable most likely means different meanings for memory locations,
@@ -72,222 +68,84 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		public bool SkipMemoryConsistencyCheck { get; set; } = false;
 	}
 
-	public class WaterboxHost : Swappable, IImportResolver, IBinaryStateable
+	public unsafe class WaterboxHost : IMonitor, IImportResolver, IBinaryStateable, IDisposable
 	{
+		private IntPtr _nativeHost;
+		private IntPtr _activatedNativeHost;
+		private int _enterCount;
+		private object _keepAliveDelegate;
+
+		private static readonly WaterboxHostNative NativeImpl;
 		static WaterboxHost()
 		{
-			if (OSTailoredCode.IsUnixHost)
-			{
-				WaterboxLibcoLinuxStartup.Setup();
-			}
+			NativeImpl = BizInvoker.GetInvoker<WaterboxHostNative>(
+				new DynamicLibraryImportResolver(OSTailoredCode.IsUnixHost ? "libwaterboxhost.so" : "waterboxhost.dll", eternal: true),
+				CallingConventionAdapters.Native);
+			#if !DEBUG
+			NativeImpl.wbx_set_always_evict_blocks(false);
+			#endif
 		}
 
-		/// <summary>
-		/// usual starting point for the executable
-		/// </summary>
-		public const ulong CanonicalStart = 0x0000036f00000000;
-
-		/// <summary>
-		/// the next place where we can put a module or heap
-		/// </summary>
-		private ulong _nextStart = CanonicalStart;
-
-		/// <summary>
-		/// increment _nextStart after adding a module
-		/// </summary>
-		private void ComputeNextStart(ulong size)
+		private ReadCallback Reader(Stream s)
 		{
-			_nextStart += size;
-			// align to 1MB, then increment 16MB
-			_nextStart = ((_nextStart - 1) | 0xfffff) + 0x1000001;
+			var ret = MakeCallbackForReader(s);
+			_keepAliveDelegate = s;
+			return ret;
 		}
-
-		/// <summary>
-		/// standard malloc() heap
-		/// </summary>
-		internal Heap _heap;
-
-		/// <summary>
-		/// sealed heap (writable only during init)
-		/// </summary>
-		internal Heap _sealedheap;
-
-		/// <summary>
-		/// invisible heap (not savestated, use with care)
-		/// </summary>
-		internal Heap _invisibleheap;
-
-		/// <summary>
-		/// extra savestated heap
-		/// </summary>
-		internal Heap _plainheap;
-
-		/// <summary>
-		/// memory map emulation
-		/// </summary>
-		internal MapHeap _mmapheap;
-
-		/// <summary>
-		/// the loaded elf file
-		/// </summary>
-		private ElfLoader _module;
-
-		/// <summary>
-		/// all loaded heaps
-		/// </summary>
-		private readonly List<Heap> _heaps = new List<Heap>();
-
-		/// <summary>
-		/// anything at all that needs to be disposed on finish
-		/// </summary>
-		private readonly List<IDisposable> _disposeList = new List<IDisposable>();
-
-		/// <summary>
-		/// anything at all that needs its state saved and loaded
-		/// </summary>
-		private readonly List<IBinaryStateable> _savestateComponents = new List<IBinaryStateable>();
-
-		private readonly EmuLibc _emu;
-		private readonly Syscalls _syscalls;
-
-		/// <summary>
-		/// the set of functions made available for the elf module
-		/// </summary>
-		private readonly IImportResolver _imports;
-
-		/// <summary>
-		/// timestamp of creation acts as a sort of "object id" in the savestate
-		/// </summary>
-		private readonly long _createstamp = WaterboxUtils.Timestamp();
-
-		private Heap CreateHeapHelper(uint sizeKB, string name, bool saveStated)
+		private WriteCallback Writer(Stream s)
 		{
-			if (sizeKB != 0)
-			{
-				var heap = new Heap(_nextStart, sizeKB * 1024, name);
-				heap.Memory.Activate();
-				ComputeNextStart(sizeKB * 1024);
-				AddMemoryBlock(heap.Memory, name);
-				if (saveStated)
-					_savestateComponents.Add(heap);
-				_disposeList.Add(heap);
-				_heaps.Add(heap);
-				return heap;
-			}
-			else
-			{
-				return null;
-			}
+			var ret = MakeCallbackForWriter(s);
+			_keepAliveDelegate = s;
+			return ret;
 		}
 
 		public WaterboxHost(WaterboxOptions opt)
 		{
-			_nextStart = opt.StartAddress;
-			Initialize(_nextStart);
-			using (this.EnterExit())
+			var nativeOpts = new MemoryLayoutTemplate
 			{
-				_emu = new EmuLibc(this);
-				_syscalls = new Syscalls(this);
+				sbrk_size = Z.UU(opt.SbrkHeapSizeKB * 1024),
+				sealed_size = Z.UU(opt.SealedHeapSizeKB * 1024),
+				invis_size = Z.UU(opt.InvisibleHeapSizeKB * 1024),
+				plain_size = Z.UU(opt.PlainHeapSizeKB * 1024),
+				mmap_size = Z.UU(opt.MmapHeapSizeKB * 1024),
+			};
 
-				_imports = new PatchImportResolver(
-					NotImplementedSyscalls.Instance,
-					BizExvoker.GetExvoker(_emu, CallingConventionAdapters.Waterbox),
-					BizExvoker.GetExvoker(_syscalls, CallingConventionAdapters.Waterbox)
-				);
+			var moduleName = opt.Filename;
 
-				if (true)
-				{
-					var moduleName = opt.Filename;
-
-					var path = Path.Combine(opt.Path, moduleName);
-					var gzpath = path + ".gz";
-					byte[] data;
-					if (File.Exists(gzpath))
-					{
-						using var fs = new FileStream(gzpath, FileMode.Open, FileAccess.Read);
-						data = Util.DecompressGzipFile(fs);
-					}
-					else
-					{
-						data = File.ReadAllBytes(path);
-					}
-
-					_module = new ElfLoader(moduleName, data, _nextStart, opt.SkipCoreConsistencyCheck, opt.SkipMemoryConsistencyCheck);
-
-					ComputeNextStart(_module.Memory.Size);
-					AddMemoryBlock(_module.Memory, moduleName);
-					_savestateComponents.Add(_module);
-					_disposeList.Add(_module);
-				}
-
-				ConnectAllImports();
-
-				// load all heaps
-				_heap = CreateHeapHelper(opt.SbrkHeapSizeKB, "brk-heap", true);
-				_sealedheap = CreateHeapHelper(opt.SealedHeapSizeKB, "sealed-heap", true);
-				_invisibleheap = CreateHeapHelper(opt.InvisibleHeapSizeKB, "invisible-heap", false);
-				_plainheap = CreateHeapHelper(opt.PlainHeapSizeKB, "plain-heap", true);
-
-				if (opt.MmapHeapSizeKB != 0)
-				{
-					_mmapheap = new MapHeap(_nextStart, opt.MmapHeapSizeKB * 1024, "mmap-heap");
-					_mmapheap.Memory.Activate();
-					ComputeNextStart(opt.MmapHeapSizeKB * 1024);
-					AddMemoryBlock(_mmapheap.Memory, "mmap-heap");
-					_savestateComponents.Add(_mmapheap);
-					_disposeList.Add(_mmapheap);
-				}
-
-				// TODO: This debugger stuff doesn't work on nix?
-				System.Diagnostics.Debug.WriteLine($"About to enter unmanaged code for {opt.Filename}");
-				if (OSTailoredCode.IsUnixHost)
-				{
-					if (System.Diagnostics.Debugger.IsAttached)
-						System.Diagnostics.Debugger.Break();
-				}
-				else
-				{
-					if (!System.Diagnostics.Debugger.IsAttached && Win32Imports.IsDebuggerPresent())
-					{
-						// this means that GDB or another unconventional debugger is attached.
-						// if that's the case, and it's observing this core, it probably wants a break
-						System.Diagnostics.Debugger.Break();
-					}
-				}
-				_module.RunNativeInit();
+			var path = Path.Combine(opt.Path, moduleName);
+			var gzpath = path + ".gz";
+			byte[] data;
+			if (File.Exists(gzpath))
+			{
+				using var fs = new FileStream(gzpath, FileMode.Open, FileAccess.Read);
+				data = Util.DecompressGzipFile(fs);
 			}
+			else
+			{
+				data = File.ReadAllBytes(path);
+			}
+
+			var retobj = new ReturnData();
+			NativeImpl.wbx_create_host(nativeOpts, opt.Filename, Reader(new MemoryStream(data, false)), IntPtr.Zero, retobj);
+			_nativeHost = retobj.GetDataOrThrow();
 		}
 
 		public IntPtr GetProcAddrOrZero(string entryPoint)
 		{
-			var addr = _module.GetProcAddrOrZero(entryPoint);
-			if (addr != IntPtr.Zero)
+			using (this.EnterExit())
 			{
-				var exclude = _imports.GetProcAddrOrZero(entryPoint);
-				if (exclude != IntPtr.Zero)
-				{
-					// don't reexport anything that's part of waterbox internals
-					return IntPtr.Zero;
-				}	
+				var retobj = new ReturnData();
+				NativeImpl.wbx_get_proc_addr(_activatedNativeHost, entryPoint, retobj);
+				return retobj.GetDataOrThrow();
 			}
-			return addr;
 		}
 
 		public IntPtr GetProcAddrOrThrow(string entryPoint)
 		{
-			var addr = _module.GetProcAddrOrZero(entryPoint);
+			var addr = GetProcAddrOrZero(entryPoint);
 			if (addr != IntPtr.Zero)
 			{
-				var exclude = _imports.GetProcAddrOrZero(entryPoint);
-				if (exclude != IntPtr.Zero)
-				{
-					// don't reexport anything that's part of waterbox internals
-					throw new InvalidOperationException($"Tried to resolve {entryPoint}, but it should not be exported");
-				}
-				else
-				{
-					return addr;
-				}
+				return addr;
 			}
 			else
 			{
@@ -299,38 +157,13 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		{
 			using (this.EnterExit())
 			{
-				// if libco is used, the jmp_buf for the main cothread can have stack stuff in it.
-				// this isn't a problem, since we only savestate when the core is not running, and
-				// the next time it's run, that buf will be overridden again.
-				// but it breaks xor state verification, so when we seal, nuke it.
-
-				// this could be the responsibility of something else other than the PeRunner; I am not sure yet...
-
-				// TODO: MAKE SURE THIS STILL WORKS
-				IntPtr co_clean;
-				if ((co_clean = _module.GetProcAddrOrZero("co_clean")) != IntPtr.Zero)
-				{
-					Console.WriteLine("Calling co_clean().");
-					CallingConventionAdapters.Waterbox.GetDelegateForFunctionPointer<Action>(co_clean)();
-				}
-
-				_sealedheap.Seal();
-				foreach (var h in _heaps)
-				{
-					if (h != _invisibleheap && h != _sealedheap) // TODO: if we have more non-savestated heaps, refine this hack
-						h.Memory.Seal();
-				}
-				_module.SealImportsAndTakeXorSnapshot();
-				_mmapheap?.Memory.Seal();
+				var retobj = new ReturnData();
+				NativeImpl.wbx_seal(_activatedNativeHost, retobj);
+				retobj.GetDataOrThrow();
 			}
 			Console.WriteLine("WaterboxHost Sealed!");
 		}
 
-		private void ConnectAllImports()
-		{
-			_module.ConnectSyscalls(_imports);
-		}
-
 		/// <summary>
 		/// Adds a file that will appear to the waterbox core's libc.  the file will be read only.
 		/// All savestates must have the same file list, so either leave it up forever or remove it during init!
@@ -338,7 +171,12 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// <param name="name">the filename that the unmanaged core will access the file by</param>
 		public void AddReadonlyFile(byte[] data, string name)
 		{
-			_syscalls.AddReadonlyFile((byte[])data.Clone(), name);
+			using (this.EnterExit())
+			{
+				var retobj = new ReturnData();
+				NativeImpl.wbx_mount_file(_activatedNativeHost, name, Reader(new MemoryStream(data, false)), IntPtr.Zero, false, retobj);
+				retobj.GetDataOrThrow();
+			}
 		}
 
 		/// <summary>
@@ -347,7 +185,12 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// </summary>
 		public void RemoveReadonlyFile(string name)
 		{
-			_syscalls.RemoveReadonlyFile(name);
+			using (this.EnterExit())
+			{
+				var retobj = new ReturnData();
+				NativeImpl.wbx_unmount_file(_activatedNativeHost, name, null, IntPtr.Zero, retobj);
+				retobj.GetDataOrThrow();
+			}
 		}
 
 		/// <summary>
@@ -356,7 +199,12 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// </summary>
 		public void AddTransientFile(byte[] data, string name)
 		{
-			_syscalls.AddTransientFile(data, name); // don't need to clone data, as it's used at init only
+			using (this.EnterExit())
+			{
+				var retobj = new ReturnData();
+				NativeImpl.wbx_mount_file(_activatedNativeHost, name, Reader(new MemoryStream(data, false)), IntPtr.Zero, true, retobj);
+				retobj.GetDataOrThrow();
+			}
 		}
 
 		/// <summary>
@@ -365,108 +213,123 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// <returns>The state of the file when it was removed</returns>
 		public byte[] RemoveTransientFile(string name)
 		{
-			return _syscalls.RemoveTransientFile(name);
+			using (this.EnterExit())
+			{
+				var retobj = new ReturnData();
+				var ms = new MemoryStream();
+				NativeImpl.wbx_unmount_file(_activatedNativeHost, name, Writer(ms), IntPtr.Zero, retobj);
+				retobj.GetDataOrThrow();
+				return ms.ToArray();
+			}
 		}
 
+		// public class MissingFileResult
+		// {
+		// 	public byte[] data;
+		// 	public bool writable;
+		// }
+
 		/// <summary>
 		/// Can be set by the frontend and will be called if the core attempts to open a missing file.
-		/// The callee may add additional files to the waterbox during the callback and return `true` to indicate
-		/// that the right file was added and the scan should be rerun.  The callee may return `false` to indicate
-		/// that the file should be reported as missing.  Do not call other things during this callback.
+		/// The callee returns a result object, either null to indicate that the file should be reported as missing,
+		/// or data and writable status for a file to be just in time mounted.
+		/// Do not call anything on the waterbox things during this callback.
 		/// Can be called at any time by the core, so you may want to remove your callback entirely after init
 		/// if it was for firmware only.
+		/// writable == false is equivalent to AddReadonlyFile, writable == true is equivalent to AddTransientFile
 		/// </summary>
-		public Func<string, bool> MissingFileCallback
-		{
-			get => _syscalls.MissingFileCallback;
-			set => _syscalls.MissingFileCallback = value;
-		}
+		// public Func<string, MissingFileResult> MissingFileCallback
+		// {
+		// 	set
+		// 	{
+		// 		// TODO
+		// 		using (this.EnterExit())
+		// 		{
+		// 			var mfc_o = value == null ? null : new WaterboxHostNative.MissingFileCallback
+		// 			{
+		// 				callback = (_unused, name) =>
+		// 				{
+		// 					var res = value(name);
+		// 				}
+		// 			};
 
-		private const ulong MAGIC = 0x736b776162727477;
-		private const ulong WATERBOXSTATEVERSION = 2;
+		// 			NativeImpl.wbx_set_missing_file_callback(_activatedNativeHost, value == null
+		// 				? null
+		// 				: )
+		// 		}
+		// 	}
+		// 	get => _syscalls.MissingFileCallback;
+		// 	set => _syscalls.MissingFileCallback = value;
+		// }
 
 		public void SaveStateBinary(BinaryWriter bw)
 		{
-			bw.Write(MAGIC);
-			bw.Write(WATERBOXSTATEVERSION);
-			bw.Write(_createstamp);
-			bw.Write(_savestateComponents.Count);
 			using (this.EnterExit())
 			{
-				foreach (var c in _savestateComponents)
-				{
-					c.SaveStateBinary(bw);
-				}
+				var retobj = new ReturnData();
+				NativeImpl.wbx_save_state(_activatedNativeHost, Writer(bw.BaseStream), IntPtr.Zero, retobj);
+				retobj.GetDataOrThrow();
 			}
 		}
 
 		public void LoadStateBinary(BinaryReader br)
 		{
-			if (br.ReadUInt64() != MAGIC)
-				throw new InvalidOperationException("Internal savestate error");
-			if (br.ReadUInt64() != WATERBOXSTATEVERSION)
-				throw new InvalidOperationException("Waterbox savestate version mismatch");
-			var differentCore = br.ReadInt64() != _createstamp; // true if a different core instance created the state
-			if (br.ReadInt32() != _savestateComponents.Count)
-				throw new InvalidOperationException("Internal savestate error");
 			using (this.EnterExit())
 			{
-				foreach (var c in _savestateComponents)
-				{
-					c.LoadStateBinary(br);
-				}
-				if (differentCore)
-				{
-					// if a different runtime instance than this one saved the state,
-					// Exvoker imports need to be reconnected
-					Console.WriteLine($"Restoring {nameof(WaterboxHost)} state from a different core...");
-					ConnectAllImports();
-				}
+				var retobj = new ReturnData();
+				NativeImpl.wbx_load_state(_activatedNativeHost, Reader(br.BaseStream), IntPtr.Zero, retobj);
+				retobj.GetDataOrThrow();
 			}
 		}
 
-		protected override void Dispose(bool disposing)
+		public void Enter()
 		{
-			base.Dispose(disposing);
-			if (disposing)
+			if (_enterCount == 0)
 			{
-				foreach (var d in _disposeList)
-					d.Dispose();
-				_disposeList.Clear();
-				PurgeMemoryBlocks();
-				_module = null;
-				_heap = null;
-				_sealedheap = null;
-				_invisibleheap = null;
-				_plainheap = null;
-				_mmapheap = null;
+				var retobj = new ReturnData();
+				NativeImpl.wbx_activate_host(_nativeHost, retobj);
+				_activatedNativeHost = retobj.GetDataOrThrow();
+			}
+			_enterCount++;
+		}
+
+		public void Exit()
+		{
+			if (_enterCount <= 0)
+			{
+				throw new InvalidOperationException();
+			}
+			else if (_enterCount == 1)
+			{
+				var retobj = new ReturnData();
+				NativeImpl.wbx_deactivate_host(_activatedNativeHost, retobj);
+				retobj.GetDataOrThrow();
+				_activatedNativeHost = IntPtr.Zero;
+			}
+			_enterCount--;
+		}
+
+		public void Dispose()
+		{
+			if (_nativeHost != IntPtr.Zero)
+			{
+				var retobj = new ReturnData();
+				if (_activatedNativeHost != IntPtr.Zero)
+				{
+					NativeImpl.wbx_deactivate_host(_activatedNativeHost, retobj);
+					Console.Error.WriteLine("Warn: Disposed of WaterboxHost which was active");
+					_activatedNativeHost = IntPtr.Zero;
+				}
+				NativeImpl.wbx_destroy_host(_nativeHost, retobj);
+				_enterCount = 0;
+				_nativeHost = IntPtr.Zero;
+				GC.SuppressFinalize(this);
 			}
 		}
-	}
 
-	internal static class WaterboxLibcoLinuxStartup
-	{
-		// TODO: This is a giant mess and get rid of it entirely
-		internal static void Setup()
+		~WaterboxHost()
 		{
-			// Our libco implementation plays around with stuff in the TEB block.  This is bad for a few reasons:
-			// 1. It's windows specific
-			// 2. It's only doing this to satisfy some msvs __stkchk code that should never run
-			// 3. That code is only running because memory allocations in a cothread still send you back
-			// to managed land where things like the .NET jit might run on the costack, oof.
-
-			// We need to stop #3 from happening, probably by making the waterboxhost unmanaged code.  Then if we
-			// still need "GS fiddling" we can have it be part of our syscall layer without having a managed transition
-
-			// Until then, just fake a TEB block for linux -- nothing else uses GS anyway.
-			var ptr = Marshal.AllocHGlobal(0x40);
-			WaterboxUtils.ZeroMemory(ptr, 0x40);
-			Marshal.WriteIntPtr(ptr, 0x30, ptr);
-			if (ArchPrCtl(0x1001 /* SET_GS */, Z.SU((long)ptr)) != 0)
-				throw new InvalidOperationException("ArchPrCtl failed!");
+			Dispose();
 		}
-
-		[DllImport("libc.so.6", EntryPoint = "arch_prctl")]
-		private static extern int ArchPrCtl(int code, UIntPtr addr);
 	}
 }

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHostNative.cs

@@ -0,0 +1,157 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using BizHawk.BizInvoke;
+using BizHawk.Common;
+
+namespace BizHawk.Emulation.Cores.Waterbox
+{
+	public unsafe abstract class WaterboxHostNative
+	{
+		[StructLayout(LayoutKind.Explicit)]
+		public class ReturnData
+		{
+			[FieldOffset(0)]
+			public byte ErrorMessageStart;
+			[FieldOffset(1024)]
+			public IntPtr Data;
+
+			public unsafe IntPtr GetDataOrThrow()
+			{
+				if (ErrorMessageStart != 0)
+				{
+					fixed(byte* p = &ErrorMessageStart)
+						throw new InvalidOperationException(Mershul.PtrToStringUtf8((IntPtr)p));
+				}
+				return Data;
+			}
+		}
+
+		[StructLayout(LayoutKind.Sequential)]
+		public class MemoryLayoutTemplate
+		{
+			/// Memory space to serve brk(2)
+			public UIntPtr sbrk_size;
+			/// Memory space to serve alloc_sealed(3)
+			public UIntPtr sealed_size;
+			/// Memory space to serve alloc_invisible(3)
+			public UIntPtr invis_size;
+			/// Memory space to serve alloc_plain(3)
+			public UIntPtr plain_size;
+			/// Memory space to serve mmap(2) and friends.
+			/// Calls without MAP_FIXED or MREMAP_FIXED will be placed in this area.
+			/// TODO: Are we allowing fixed calls to happen anywhere in the block?
+			public UIntPtr mmap_size;
+		}
+		/// Read bytes into the buffer.  Return number of bytes read on success, or < 0 on failure.
+		/// permitted to read less than the provided buffer size, but must always read at least 1
+		/// byte if EOF is not reached.  If EOF is reached, should return 0.
+		public delegate IntPtr ReadCallback(IntPtr userdata, IntPtr /*byte**/ data, UIntPtr size);
+		/// write bytes.  Return 0 on success, or < 0 on failure.
+		/// Must write all provided bytes in one call or fail, not permitted to write less (unlike reader).
+		public delegate int WriteCallback(IntPtr userdata, IntPtr /*byte**/ data, UIntPtr size);
+		// public delegate UIntPtr /*MissingFileResult*/ FileCallback(IntPtr userdata, UIntPtr /*string*/ name);
+
+		public static WriteCallback MakeCallbackForWriter(Stream s)
+		{
+			var ss = SpanStream.GetOrBuild(s);
+			return (_unused, data, size) =>
+			{
+				try
+				{
+					var count = (int)size;
+					ss.Write(new ReadOnlySpan<byte>((void*)data, (int)size));
+					return 0;
+				}
+				catch
+				{
+					return -1;
+				}
+			};
+		}
+		public static ReadCallback MakeCallbackForReader(Stream s)
+		{
+			var ss = SpanStream.GetOrBuild(s);
+			return (_unused, data, size) =>
+			{
+				try
+				{
+					var count = (int)size;
+					var n = ss.Read(new Span<byte>((void*)data, count));
+					return Z.SS(n);
+				}
+				catch
+				{
+					return Z.SS(-1);
+				}
+			};
+		}
+		// [StructLayout(LayoutKind.Sequential)]
+		// public class MissingFileCallback
+		// {
+		// 	public UIntPtr userdata;
+		// 	public FileCallback callback;
+		// }
+		// [StructLayout(LayoutKind.Sequential)]
+		// public class MissingFileResult : CReader
+		// {
+		// 	public bool writable;
+		// }
+
+		/// Given a guest executable and a memory layout, create a new host environment.  All data will be immediately consumed from the reader,
+		/// which will not be used after this call.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_create_host(MemoryLayoutTemplate layout, string moduleName, ReadCallback wbx, IntPtr userdata, ReturnData /*WaterboxHost*/ ret);
+		/// Tear down a host environment.  May not be called while the environment is active.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_destroy_host(IntPtr /*WaterboxHost*/ obj, ReturnData /*void*/ ret);
+		/// Activate a host environment.  This swaps it into memory and makes it available for use.
+		/// Pointers to inside the environment are only valid while active.  Uses a mutex internally
+		/// so as to not stomp over other host environments in the same 4GiB slice.
+		/// Returns a pointer to the activated object, used to do most other functions.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_activate_host(IntPtr /*WaterboxHost*/ obj, ReturnData /*ActivatedWaterboxHost*/ ret);
+		/// Deactivates a host environment, and releases the mutex.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_deactivate_host(IntPtr /*ActivatedWaterboxHost*/ obj, ReturnData /*void*/ ret);
+		/// Returns the address of an exported function from the guest executable.  This pointer is only valid
+		/// while the host is active.  A missing proc is not an error and simply returns 0.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_get_proc_addr(IntPtr /*ActivatedWaterboxHost*/ obj, string name, ReturnData /*UIntPtr*/ ret);
+		/// Calls the seal operation, which is a one time action that prepares the host to save states.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_seal(IntPtr /*ActivatedWaterboxHost*/ obj, ReturnData /*void*/ ret);
+		/// Mounts a file in the environment.  All data will be immediately consumed from the reader, which will not be used after this call.
+		/// To prevent nondeterminism, adding and removing files is very limited WRT savestates.  If a file is writable, it must never exist
+		/// when save_state is called, and can only be used for transient operations.  If a file is readable, it can appear in savestates,
+		/// but it must exist in every savestate and the exact sequence of add_file calls must be consistent from savestate to savestate.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_mount_file(IntPtr /*ActivatedWaterboxHost*/ obj, string name, ReadCallback reader, IntPtr userdata, bool writable, ReturnData /*void*/ ret);
+		/// Remove a file previously added.  Writer is optional; if provided, the contents of the file at time of removal will be dumped to it.
+		/// It is an error to remove a file which is currently open in the guest.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_unmount_file(IntPtr /*ActivatedWaterboxHost*/ obj, string name, WriteCallback writer, IntPtr userdata, ReturnData /*void*/ ret);
+		/// Set (or clear, with None) a callback to be called whenever the guest tries to load a nonexistant file.
+		/// The callback will be provided with the name of the requested load, and can either return null to signal the waterbox
+		/// to return ENOENT to the guest, or a struct to immediately load that file.  You may not call any wbx methods
+		/// in the callback.  If the MissingFileResult is provided, it will be consumed immediately and will have the same effect
+		/// as wbx_mount_file().  You may free resources associated with the MissingFileResult whenever control next returns to your code.
+		// [BizImport(CallingConvention.Cdecl)]
+		// public abstract void wbx_set_missing_file_callback(IntPtr /*ActivatedWaterboxHost*/ obj, MissingFileCallback mfc_o);
+		/// Save state.  Must not be called before seal.  Must not be called with any writable files mounted.
+		/// Must always be called with the same sequence and contents of readonly files.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_save_state(IntPtr /*ActivatedWaterboxHost*/ obj, WriteCallback writer, IntPtr userdata, ReturnData /*void*/ ret);
+		/// Load state.  Must not be called before seal.  Must not be called with any writable files mounted.
+		/// Must always be called with the same sequence and contents of readonly files that were in the save state.
+		/// Must be called with the same wbx executable and memory layout as in the savestate.
+		/// Errors generally poison the environment; sorry!
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_load_state(IntPtr /*ActivatedWaterboxHost*/ obj, ReadCallback reader, IntPtr userdata, ReturnData /*void*/ ret);
+		/// Control whether the host automatically evicts blocks from memory when they are not active.  For the best performance,
+		/// this should be set to false.  Set to true to help catch dangling pointer issues.  Will be ignored (and forced to true)
+		/// if waterboxhost was built in debug mode.  This is a single global setting.
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_set_always_evict_blocks(bool val);
+	}
+}

waterbox/Notes on Debugging.md

@@ -0,0 +1,87 @@
+# How to Debug Waterbox and Cores
+
+Bring lots of tools, and lots of self loathing.
+
+## Windows
+
+### gdb
+
+* Usually comes from mingw, or something
+* Example script to attach to a running BizHawk instance:
+	```
+	#!/bin/bash
+	PSLINE=$(eval "ps -W | grep EmuHawk")
+
+	if [[ $PSLINE =~ [0-9]+[[:space:]]+[0-9]+[[:space:]]+[0-9]+[[:space:]]+([0-9]+) ]]; then
+		gdb --pid=${BASH_REMATCH[1]}
+	fi
+	```
+* Thinks we're in x86-32 mode because of the PE header on EmuHawk.exe.  We're not.
+	You can put this in `~/.gdbinit` (or maybe add it to the startgdb script?)
+	```
+	set arch i386:x86-64
+	```
+* The waterbox files have DWARF information, which gdb can read.  If you build a waterbox core in debug mode,
+	you even get full source level debugging.  With the new rust based waterboxhost, these symbol files should automatically
+	be registered for you as waterboxhost hits a gdb hook.
+	* The gdb hook is experimental, so be ready with `add-sym foobar.wbx` if needed.
+* Has no way to understand first chance vs second chance exceptions.  Since lazystates was added, the cores now
+	emit lots of benign SIGSEGVs as the waterboxhost discovers what memory space they use.  You can suppress these exceptions:
+	```
+	han SIGSEGV nos nopr
+	```
+	But if the real exception you're trying to break on is a SIGSEGV, this leaves you defenseless.
+* Also understands symbols for waterboxhost.dll, since that was actually built with MINGW.
+* `b rust_panic` to examine rust unwinds before they explode your computer.
+* Breakpoints on symbols in the wbx file just don't work a lot of the time.
+	* This is the single worst part of modern waterbox debugging.  I have no idea what gdb is doing wrong.  It sees the wbx
+		symbols and can print stack information and tell you what function you're in, and print globals, but `b some_emu_core_function`
+		just doesn't get hit.  I think it might have something to do with how we map memory.  This worked in some previous
+		waterbox editions, but I never got to the bottom.
+	* Recompile cores with lots of `__asm__("int3")` in them?  Heh.
+
+### windbg
+
+* `!address`, `!vprot` are useful for examining VirtualQuery information.
+* Can't read any symbols, which makes it mostly useless until you've narrowed things down heavily.
+* Has more useful stack traces than gdb, as gdb usually can't see the stack outside DWARF land.
+* Understands first and second chance exceptions.
+	* `sxd av` will do exactly what we need for lazystate page mapping; break only when our handled decides not to handle it.
+* Can give reasonable information on random WIN32 exceptions with `!analyze`
+
+### OmniSharp
+
+* Great visibility into C# exceptions, pretty worthless otherwise.
+
+### General unwinding hazards
+
+* Within the guest, DWARF unwinding information is available and libunwind is used.  Guest C++ programs can freely use exceptions,
+	and Mednafen does so without issue.
+	* Unwinding through any stack that is not guest content will likely explode.
+* Within the host, rust unwinds usually make some SEH explosion that crashes the application.
+	* None of them were meant to be recoverable anyway.
+* C# exceptions that occur when there is guest code on the stack usually cause a complete termination.
+	* So a callback to managed from within a waterbox core, etc.
+	* I never figured this one out.  Guest execution is fully done on vanilla host stacks, and we should be doing nothing
+		that would make this fail.  But it does.
+	* Since the rust waterboxhost, this now will only happen from emulator core specific callbacks, since the rust waterboxhost
+		does not call out to C# from within a syscall handler, ever.
+* Any unwinding on a libco cothread will likely make me laugh.
+
+## Linux
+
+### gdb
+
+* Only game in town here.
+* Mono seems to use a lot of custom signals for... something.
+	* This script will start gdb, ignore those signals, and start EmuHawk:
+		```
+		#!/bin/bash
+		gdb -iex "han SIG35 nos" -iex "han SIG36 nos" --args mono ./EmuHawk.exe "$@"
+		```
+* Because you're actually in linux, beware function names: `b mmap` can mean the host libc's mmap, the rust implementation of the guest mmap,
+	or the guest libc's mmap.
+* Same general problem with intermittently functional guest breakpoints as Windows.  Heh.
+* Same lazystate SIGSEGV problem (and same solutions) as Windows.
+* Can see some mono symbols, which is sometimes useful.
+* If you're looking for a pure core bug, this might be a better environment to test it on than Windows.  It depends.

waterbox/emulibc/emulibc.c

@@ -1,27 +1,94 @@
 #include "emulibc.h"
+#include <stdio.h>
+#include <sys/mman.h>
 
-#define __WBXSYSCALL __attribute__((section(".wbxsyscall"))) __attribute__((visibility("default")))
+// Keep this in sync with "syscall.h"!!
+struct __AddressRange {
+	unsigned long start;
+	unsigned long size;
+};
+struct __WbxSysLayout {
+	struct __AddressRange elf;
+	struct __AddressRange sbrk;
+	struct __AddressRange sealed;
+	struct __AddressRange invis;
+	struct __AddressRange plain;
+	struct __AddressRange mmap;
+};
+struct __WbxSysSyscall {
+	long ud;
+	void* syscall;
+};
+struct __WbxSysArea {
+	struct __WbxSysLayout layout;
+	struct __WbxSysSyscall syscall;
+};
+extern struct __WbxSysArea __wbxsysarea;
 
-__WBXSYSCALL void *(*__walloc_sealed)(size_t);
-void *alloc_sealed(size_t size)
+void* alloc_helper(size_t size, const struct __AddressRange* range, unsigned long* current, const char* name)
 {
-	return __walloc_sealed(size);
+	if (!*current)
+	{
+		printf("Initializing heap %s at %p:%p\n", name, (void*)range->start, (void*)(range->start + range->size));
+		*current = range->start;
+	}
+
+	unsigned long start = *current;
+	unsigned long end = start + size;
+	end = (end + 15) & ~15ul;
+
+	if (end < start || end > range->start + range->size)
+	{
+		fprintf(stderr, "Failed to satisfy allocation of %lu bytes on %s heap\n", size, name);
+		return NULL;
+	}
+	else
+	{
+		unsigned long pstart = (start + 0xfff) & ~0xffful;
+		unsigned long pend = (end + 0xfff) & ~0xffful;
+		if (pstart < pend)
+		{
+			if (mmap((void*)pstart, pend - pstart, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED | MAP_FIXED_NOREPLACE, -1, 0) == MAP_FAILED)
+			{
+				fprintf(stderr, "VERY STRANGE: mmap() failed to satisfy allocation of %lu bytes on %s heap\n", size, name);
+				return NULL;
+			}
+		}
+		printf("Allocated %lu bytes on %s heap, usage %lu/%lu\n", size, name, end - range->start, range->size);
+		*current = end;
+		return (void*)start;
+	}
 }
 
-__WBXSYSCALL void *(*__walloc_invisible)(size_t);
-void *alloc_invisible(size_t size)
+static unsigned long __sealed_current;
+void* alloc_sealed(size_t size)
 {
-	return __walloc_invisible(size);
+	return alloc_helper(size, &__wbxsysarea.layout.sealed, &__sealed_current, "sealed");
 }
 
-__WBXSYSCALL void *(*__walloc_plain)(size_t);
-void *alloc_plain(size_t size)
+static unsigned long __invisible_current;
+void* alloc_invisible(size_t size)
 {
-	return __walloc_plain(size);
+	return alloc_helper(size, &__wbxsysarea.layout.invis, &__invisible_current, "invisible");
 }
 
-__WBXSYSCALL void (*__w_debug_puts)(const char *);
+static unsigned long __plain_current;
+void* alloc_plain(size_t size)
+{
+	return alloc_helper(size, &__wbxsysarea.layout.plain, &__plain_current, "plain");
+}
+
+// TODO: This existed before we even had stdio support.  Retire?
 void _debug_puts(const char *s)
 {
-	__w_debug_puts(s);
+	fprintf(stderr, "%s\n", s);
+}
+
+ECL_EXPORT void ecl_seal()
+{
+	if (__sealed_current)
+	{
+		if (mprotect((void*)__wbxsysarea.layout.sealed.start, (__sealed_current - __wbxsysarea.layout.sealed.start + 0xfff) & ~0xffful, PROT_READ) != 0)
+			__asm__("int3");
+	}
 }

waterbox/libco/amd64.c

@@ -13,13 +13,14 @@
 #include <sys/mman.h>
 #include <emulibc.h>
 
-// allocations are 16k larger than asked for,
-// and include guard space between the stack and the storage
+// allocations are 16k larger than asked for, which is all used as guard space
+#define GUARD_SIZE 0x4000
 
 typedef struct {
 	// used by coswap.s, has to be at the beginning of the struct
 	uint64_t jmp_buf[32];
 	// points to the lowest address in the stack
+	// NB: because of guard space, this is not valid stack
 	void* stack;
 	// length of the stack that we allocated in bytes
 	uint64_t stack_size;
@@ -30,6 +31,13 @@ static cothread_impl co_host_buffer;
 // what cothread are we in right now
 static cothread_impl* co_active_handle;
 
+static void free_thread(cothread_impl* co)
+{
+	if (munmap(co->stack, co->stack_size) != 0)
+		abort();
+	free(co);
+}
+
 static cothread_impl* alloc_thread(uint64_t size)
 {
 	cothread_impl* co = calloc(1, sizeof(*co));
@@ -37,27 +45,28 @@ static cothread_impl* alloc_thread(uint64_t size)
 		return NULL;
 
 	// align up to 4k
-	size = (size + 4095) & ~4095;
+	size = (size + 4095) & ~4095ul;
+	size += GUARD_SIZE;
 
 	co->stack = mmap(NULL, size,
-		PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_STACK, -1, 0);
+		PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
 	
 	if (co->stack == (void*)(-1))
 	{
 		free(co);
 		return NULL;
 	}
+
+	if (mprotect(co->stack, GUARD_SIZE, PROT_NONE) != 0)
+	{
+		free_thread(co);
+		return NULL;
+	}
+
 	co->stack_size = size;
 	return co;
 }
 
-static void free_thread(cothread_impl* co)
-{
-	if (munmap(co->stack, co->stack_size) != 0)
-		abort();
-	free(co);
-}
-
 extern void co_swap(cothread_impl*, cothread_impl*);
 
 static void crash(void)
@@ -99,39 +108,39 @@ void co_delete(cothread_t handle)
 	free_thread(handle);
 }
 
-static uint64_t hoststart;
-static uint64_t hostend;
+// static uint64_t hoststart;
+// static uint64_t hostend;
 
 void co_switch(cothread_t handle)
 {
 	cothread_impl* co = handle;
 
-	uint64_t start;
-	uint64_t end;
-	if (co_active_handle == &co_host_buffer)
-	{
-		// migrating off of real thread; save stack params
-		__asm__("movq %%gs:0x08, %0": "=r"(end));
-		__asm__("movq %%gs:0x10, %0": "=r"(start));
-		hoststart = start;
-		hostend = end;
-	}
-	if (handle == &co_host_buffer)
-	{
-		// migrating onto real thread; load stack params
-		start = hoststart;
-		end = hostend;
-		hoststart = 0;
-		hostend = 0;
-	}
-	else
-	{
-		// migrating onto cothread; compute its extents we allocated them
-		start = (uintptr_t)co->stack;
-		end = start + co->stack_size;
-	}
-	__asm__("movq %0, %%gs:0x08":: "r"(end));
-	__asm__("movq %0, %%gs:0x10":: "r"(start));
+	// uint64_t start;
+	// uint64_t end;
+	// if (co_active_handle == &co_host_buffer)
+	// {
+	// 	// migrating off of real thread; save stack params
+	// 	__asm__("movq %%gs:0x08, %0": "=r"(end));
+	// 	__asm__("movq %%gs:0x10, %0": "=r"(start));
+	// 	hoststart = start;
+	// 	hostend = end;
+	// }
+	// if (handle == &co_host_buffer)
+	// {
+	// 	// migrating onto real thread; load stack params
+	// 	start = hoststart;
+	// 	end = hostend;
+	// 	hoststart = 0;
+	// 	hostend = 0;
+	// }
+	// else
+	// {
+	// 	// migrating onto cothread; compute its extents we allocated them
+	// 	start = (uintptr_t)co->stack;
+	// 	end = start + co->stack_size;
+	// }
+	// __asm__("movq %0, %%gs:0x08":: "r"(end));
+	// __asm__("movq %0, %%gs:0x10":: "r"(start));
 
 	register cothread_impl* co_previous_handle = co_active_handle;
 	co_swap(co_active_handle = co, co_previous_handle);

waterbox/linguard/Makefile

@@ -1,13 +0,0 @@
-ifeq (,$(findstring Linux,$(shell uname)))
-$(error This must be built with linux)
-endif
-
-linguard.so: linguard.c
-	gcc -Wall -O2 -s -o $@ $< -shared -fPIC
-
-install: linguard.so
-	cp $< ../../output
-clean:
-	rm linguard.so
-
-.PHONY: install clean

waterbox/linguard/linguard.c

@@ -1,279 +0,0 @@
-#define _GNU_SOURCE
-#include <stdint.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <signal.h>
-#include <sys/mman.h>
-#include <stdio.h>
-#include <errno.h>
-
-#define MAX_TRIPS 64
-
-typedef struct {
-	uintptr_t start;
-	uintptr_t length;
-	uint8_t tripped[0];
-} tripwire_t;
-
-static tripwire_t* Trips[MAX_TRIPS];
-static int HandlerInstalled;
-
-static char altstack[SIGSTKSZ];
-
-static struct sigaction sa_old;
-
-static void SignalHandler(int sig, siginfo_t* info, void* ucontext)
-{
-	uintptr_t faultAddress = (uintptr_t)info->si_addr;
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (Trips[i] && faultAddress >= Trips[i]->start && faultAddress < Trips[i]->start + Trips[i]->length)
-		{
-			uintptr_t page = (faultAddress - Trips[i]->start) >> 12;
-			if (Trips[i]->tripped[page] & 1) // should change
-			{
-				if (mprotect((void*)(faultAddress & ~0xffful), 0x1000, PROT_READ | PROT_WRITE) != 0)
-				{
-					abort();
-					while (1)
-						;
-				}
-
-				Trips[i]->tripped[page] = 3; // did change
-				return;
-			}
-			else
-			{
-				break;
-			}
-		}
-	}
-
-	if (sa_old.sa_flags & SA_SIGINFO)
-		sa_old.sa_sigaction(sig, info, ucontext);
-	else 
-		sa_old.sa_handler(sig);
-}
-
-static int InstallHandler()
-{
-	stack_t ss;
-	ss.ss_flags = 0;
-	ss.ss_sp = altstack;
-	ss.ss_size = sizeof(altstack);
-
-	if (sigaltstack(&ss, NULL) != 0)
-	{
-		fprintf(stderr, "sigaltstack: %i\n", errno);
-		return 0;
-	}
-
-	struct sigaction sa;
-	sa.sa_sigaction = SignalHandler;
-	sa.sa_flags = SA_ONSTACK | SA_SIGINFO;
-	sigfillset(&sa.sa_mask);
-
-	if (sigaction(SIGSEGV, &sa, &sa_old) != 0)
-	{
-		fprintf(stderr, "sigaction: %i\n", errno);
-		return 0;
-	}
-	return 1;
-}
-
-uint8_t* AddTripGuard(uintptr_t start, uintptr_t length)
-{
-	if (!HandlerInstalled)
-	{
-		if (!InstallHandler())
-			return NULL;
-		HandlerInstalled = 1;
-	}
-
-	uintptr_t npage = length >> 12;
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (!Trips[i])
-		{
-			Trips[i] = calloc(1, sizeof(*Trips[i]) + npage);
-			if (!Trips[i])
-				return NULL;
-			Trips[i]->start = start;
-			Trips[i]->length = length;
-			return &Trips[i]->tripped[0];
-		}
-	}
-	return NULL;
-}
-
-int64_t RemoveTripGuard(uintptr_t start, uintptr_t length)
-{
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (Trips[i] && Trips[i]->start == start && Trips[i]->length == length)
-		{
-			free(Trips[i]);
-			Trips[i] = NULL;
-			return 1;
-		}
-	}
-	return 0;
-}
-
-uint8_t* ExamineTripGuard(uintptr_t start, uintptr_t length)
-{
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (Trips[i] && Trips[i]->start == start && Trips[i]->length == length)
-			return &Trips[i]->tripped[0];
-	}
-	return NULL;
-}
-
-
-/*
-
-#define _GNU_SOURCE
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <unistd.h>
-#include <sys/types.h>
-#include <errno.h>
-#include <string.h>
-#include <stdint.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/syscall.h>
-#include <sys/types.h>
-#include <linux/userfaultfd.h>
-#include <pthread.h>
-#include <sys/ioctl.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-			struct uffdio_writeprotect wp;
-			wp.range.start = start;
-			wp.range.len = length;
-			wp.mode = 0;
-			if (ioctl(fd, UFFDIO_WRITEPROTECT, &wp) == -1)
-			{
-				free(Trips[i]);
-				Trips[i] = NULL;
-				return NULL;
-			}
-
-static void v(const char* msg, int value)
-{
-	if (v < 0)
-	{
-		printf("ERROR: %s %i\n", msg, errno);
-		exit(1);
-	}
-}
-
-const uint64_t addr = 0x36f00000000ul;
-const uint64_t size = 0x100000ul;
-
-static void* threadproc(void* arg)
-{
-	int wpfd = (int)(long)arg;
-	// should be able to register once for a large range,
-	// then wp smaller ranges
-	struct uffdio_register uffdio_register;
-	uffdio_register.range.start = addr;
-	uffdio_register.range.len = size;
-
-	uffdio_register.mode = UFFDIO_REGISTER_MODE_WP;
-
-	v("ioctl:UFFDIO_REGISTER", ioctl(wpfd, UFFDIO_REGISTER, &uffdio_register));
-	v("uffdio_register.ioctls", (uffdio_register.ioctls & UFFD_API_RANGE_IOCTLS) == UFFD_API_RANGE_IOCTLS ? 0 : -1);
-
-	// wp
-	{
-		struct uffdio_writeprotect wp;
-		wp.range.start = addr;
-		wp.range.len = size;
-		wp.mode = UFFDIO_WRITEPROTECT_MODE_WP;
-		v("ioctl:UFFDIO_WRITEPROTECT", ioctl(wpfd, UFFDIO_WRITEPROTECT, &wp));
-	}
-
-	while (1)
-	{
-		struct uffd_msg msg;
-	
-		int nb = read(wpfd, &msg, sizeof(msg));
-		if (nb == -1)
-		{
-			if (errno == EAGAIN)
-				continue;
-			v("read", errno);
-		}
-		if (nb != sizeof(msg))
-			v("sizeof(msg)", -1);
-		if (msg.event & UFFD_EVENT_PAGEFAULT)
-		{
-			printf("==> Event is pagefault on %p flags 0x%llx write? 0x%llx wp? 0x%llx\n"
-				, (void *)msg.arg.pagefault.address
-				, msg.arg.pagefault.flags
-				, msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE
-				, msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP
-				);
-		}
-		if (msg.arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP)
-		{
-			//  send write unlock
-			struct uffdio_writeprotect wp;
-			wp.range.start = addr;
-			wp.range.len = size;
-			wp.mode = 0;
-			printf("sending !UFFDIO_WRITEPROTECT event to userfaultfd\n");
-			v("ioctl:UFFDIO_WRITEPROTECT", ioctl(wpfd, UFFDIO_WRITEPROTECT, &wp));
-		}
-	}
-	return NULL;
-}
-
-int main(void)
-{
-	int fd = memfd_create("pewps", MFD_CLOEXEC);
-	v("memfd_create", fd);
-	printf("fd: %i\n", fd);
-	v("ftruncate", ftruncate(fd, size));
-	char* ptr = mmap((void*)addr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_SHARED | MAP_FIXED, fd, 0);
-	if (ptr == MAP_FAILED || ptr != (void*)addr)
-		v("mmap", (int)(long)ptr);
-	v("mprotect", mprotect(ptr, size, PROT_READ | PROT_WRITE));
-
-	int wpfd = syscall(SYS_userfaultfd, O_CLOEXEC);
-	v("SYS_userfaultfd", wpfd);
-
-	struct uffdio_api uffdio_api;
-	uffdio_api.api = UFFD_API;
-	uffdio_api.features = 0;
-	v("ioctl:UFFDIO_API", ioctl(wpfd, UFFDIO_API, &uffdio_api));
-
-	v("uffdio_api.api", uffdio_api.api == UFFD_API ? 0 : -1);
-
-
-	pthread_t thr;
-	v("pthread_create", pthread_create(&thr, NULL, threadproc, (void*)(long)wpfd));
-
-	for (uint64_t a = addr; a < addr + size; a += 4096)
-	{
-		sleep(1);
-		printf("Gonna read");
-		if (*(uint64_t*)a == 77777)
-		{
-			printf("Lucky!");
-		}
-		sleep(1);
-		printf("Gonna write");
-		strcpy((void*)a, "string in mem area");
-		printf("%s\n", (char*)a);
-	}
-}
-
-*/

waterbox/musl

@@ -1 +1 @@
-Subproject commit 1020370fb2850b33c5bb8c016dfe822551faf4c6
+Subproject commit fc0fd09fa3aad2b9a30d5825caab0259304c5a45

waterbox/waterboxhost/Cargo.toml

@@ -8,6 +8,9 @@ rust = "nightly"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[profile.release]
+lto = true
+
 [dependencies]
 bitflags = "1.2.1"
 page_size = "0.4.2"

waterbox/waterboxhost/README.md

@@ -0,0 +1,37 @@
+# Waterboxhost
+
+This is the native support code for Waterbox.  It's intended to be consumed as a shared library from the host environment
+with a C api.  For most work with Waterbox cores, you don't need to get into this at all.
+
+## API
+The public api is mostly all in `src/cinterface.rs` and has basic documentation on it.  Bare minimum sequence of calls to
+get going:
+
+0. (Optional) In a release environment, turn off certain checks to speed things up
+	`wbx_set_always_evict_blocks()`
+1. Create an environment, and load the ELF into it
+	`wbx_create_host()`
+	`wbx_activate_host()`
+2. Connect exports from the guest executable to your host system
+	`wbx_get_proc_addr()`
+3. Run the guest system's init, using function pointers it exposed through wbx_get_proc_addr()
+4. Get ready to take savestates
+	`wbx_seal()`
+5. Run emulation, using frameadvance or other advance functions exposed by the guest through wbx_get_proc_addr()
+6. Save and load states as needed
+	`wbx_save_state()`
+	`wbx_load_state()`
+7. Tear down the environment when done with it.  (One shot processes that are about to exit can skip this; the OS will clean everything up)
+	`wbx_deactivate_host()`
+	`wbx_destroy_host()`
+
+If you're keeping around multiple hosts that may compete for the same address space, use `wbx_activate_host` and `wbx_deactivate_host`
+to switch between them.  If you'd like to expose files to the virtual filesystem, see `wbx_mount_file` and `wbx_unmount_file`
+
+## Building
+
+Standard rust build infrastructure is used and can be installed with `rustup`.  At the moment, we're using the `nightly-x86_64-pc-windows-gnu`
+chain on Windows, and the `nightly-x86_64-unknown-linux-gnu` chain on linux.  I don't know much about crosspiling, but presumably that will work.
+The linux chain works fine in WSL, anyway.  When used in a Windows environment with the right default chain, `build-release.bat` will build
+waterboxhost.dll and copy it to the right place.  When used in a Linux (or WSL) environment with the right default chain, `build-release.sh`
+will build libwaterboxhost.so and copy it to the right place.

waterbox/waterboxhost/build-debug.bat

@@ -0,0 +1,2 @@
+@cargo b
+@copy target\debug\waterboxhost.dll ..\..\output\dll

waterbox/waterboxhost/build-debug.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+cargo b
+cp target/debug/libwaterboxhost.so ../../Assets

waterbox/waterboxhost/build-release.bat

@@ -0,0 +1,2 @@
+@cargo b --release
+@copy target\release\waterboxhost.dll ..\..\output\dll

waterbox/waterboxhost/build-release.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+cargo b --release
+cp target/release/libwaterboxhost.so ../../Assets

waterbox/waterboxhost/src/cinterface.rs

@@ -1,17 +1,12 @@
 use crate::*;
 use host::{ActivatedWaterboxHost, WaterboxHost};
-use std::{os::raw::c_char, ffi::CStr};
+use std::{os::raw::c_char, io, ffi::{/*CString, */CStr}};
 
 /// The memory template for a WaterboxHost.  Don't worry about
 /// making every size as small as possible, since the savestater handles sparse regions
 /// well enough.  All values should be PAGESIZE aligned.
 #[repr(C)]
 pub struct MemoryLayoutTemplate {
-	/// Absolute pointer to the start of the mapped space
-	pub start: usize,
-	/// Memory space for the elf executable.  The elf must be non-relocatable and
-	/// all loaded segments must fit within [start..start + elf_size]
-	pub elf_size: usize,
 	/// Memory space to serve brk(2)
 	pub sbrk_size: usize,
 	/// Memory space to serve alloc_sealed(3)
@@ -27,19 +22,14 @@ pub struct MemoryLayoutTemplate {
 }
 impl MemoryLayoutTemplate {
 	/// checks a memory layout for validity
-	pub fn make_layout(&self) -> anyhow::Result<WbxSysLayout> {
-		let start = align_down(self.start);
-		let elf_size = align_up(self.elf_size);
+	pub fn make_layout(&self, elf_addr: AddressRange) -> anyhow::Result<WbxSysLayout> {
 		let sbrk_size = align_up(self.sbrk_size);
 		let sealed_size = align_up(self.sealed_size);
 		let invis_size = align_up(self.invis_size);
 		let plain_size = align_up(self.plain_size);
 		let mmap_size = align_up(self.mmap_size);
 		let mut res = unsafe { std::mem::zeroed::<WbxSysLayout>() };
-		res.elf = AddressRange {
-			start,
-			size: elf_size
-		};
+		res.elf = elf_addr.align_expand();
 		res.sbrk = AddressRange {
 			start: res.elf.end(),
 			size: sbrk_size
@@ -57,10 +47,10 @@ impl MemoryLayoutTemplate {
 			size: plain_size
 		};
 		res.mmap = AddressRange {
-			start: res.invis.end(),
+			start: res.plain.end(),
 			size: mmap_size
 		};
-		if start >> 32 != (res.mmap.end() - 1) >> 32 {
+		if res.elf.start >> 32 != (res.mmap.end() - 1) >> 32 {
 			Err(anyhow!("HostMemoryLayout must fit into a single 4GiB region!"))
 		} else {
 			Ok(res)
@@ -94,50 +84,63 @@ impl<T> Return<T> {
 	}
 }
 
-/// stream writer
-#[repr(C)]
-pub struct CWriter {
+/// write bytes.  Return 0 on success, or < 0 on failure.
+/// Must write all provided bytes in one call or fail, not permitted to write less (unlike reader).
+pub type WriteCallback = extern fn(userdata: usize, data: *const u8, size: usize) -> i32;
+struct CWriter {
 	/// will be passed to callback
 	pub userdata: usize,
-	/// write bytes.  Return number of bytes written on success, or < 0 on failure.
-	/// Permitted to write less than the provided number of bytes.
-	pub callback: extern fn(userdata: usize, data: *const u8, size: usize) -> isize,
+	pub callback: WriteCallback,
 }
 impl Write for CWriter {
-	fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+	fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
 		let res = (self.callback)(self.userdata, buf.as_ptr(), buf.len());
 		if res < 0 {
-			Err(std::io::Error::new(std::io::ErrorKind::Other, "Callback signaled abnormal failure"))
+			Err(io::Error::new(io::ErrorKind::Other, "Callback signaled abnormal failure"))
 		} else {
-			Ok(res as usize)
+			Ok(buf.len())
 		}
 	}
-	fn flush(&mut self) -> std::io::Result<()> {
+	fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
+		self.write(buf)?;
+		Ok(())
+	}
+	fn flush(&mut self) -> io::Result<()> {
 		Ok(())
 	}
 }
 
-/// stream reader
-#[repr(C)]
-pub struct CReader {
-	/// will be passed to callback
+/// Read bytes into the buffer.  Return number of bytes read on success, or < 0 on failure.
+/// permitted to read less than the provided buffer size, but must always read at least 1
+/// byte if EOF is not reached.  If EOF is reached, should return 0.
+pub type ReadCallback = extern fn(userdata: usize, data: *mut u8, size: usize) -> isize;
+struct CReader {
 	pub userdata: usize,
-	/// Read bytes into the buffer.  Return number of bytes read on success, or < 0 on failure.
-	/// permitted to read less than the provided buffer size, but must always read at least 1
-	/// byte if EOF is not reached.  If EOF is reached, should return 0.
-	pub callback: extern fn(userdata: usize, data: *mut u8, size: usize) -> isize,
+	pub callback: ReadCallback,
 }
 impl Read for CReader {
-	fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
+	fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
 		let res = (self.callback)(self.userdata, buf.as_mut_ptr(), buf.len());
 		if res < 0 {
-			Err(std::io::Error::new(std::io::ErrorKind::Other, "Callback signaled abnormal failure"))
+			Err(io::Error::new(io::ErrorKind::Other, "Callback signaled abnormal failure"))
 		} else {
 			Ok(res as usize)
 		}
 	}
 }
 
+// #[repr(C)]
+// pub struct MissingFileCallback {
+// 	pub userdata: usize,
+// 	pub callback: extern fn(userdata: usize, name: *const c_char) -> *mut MissingFileResult,
+// }
+
+// #[repr(C)]
+// pub struct MissingFileResult {
+// 	pub reader: CReader,
+// 	pub writable: bool,
+// }
+
 fn arg_to_str(arg: *const c_char) -> anyhow::Result<String> {
 	let cs = unsafe { CStr::from_ptr(arg as *const c_char) };
 	match cs.to_str() {
@@ -148,17 +151,21 @@ fn arg_to_str(arg: *const c_char) -> anyhow::Result<String> {
 
 fn read_whole_file(reader: &mut CReader) -> anyhow::Result<Vec<u8>> {
 	let mut res = Vec::<u8>::new();
-	std::io::copy(reader, &mut res)?;
+	io::copy(reader, &mut res)?;
 	Ok(res)
 }
 
 /// Given a guest executable and a memory layout, create a new host environment.  All data will be immediately consumed from the reader,
 /// which will not be used after this call.
 #[no_mangle]
-pub extern fn wbx_create_host(layout: &MemoryLayoutTemplate, module_name: *const c_char, wbx: &mut CReader, ret: &mut Return<*mut WaterboxHost>) {
+pub extern fn wbx_create_host(layout: &MemoryLayoutTemplate, module_name: *const c_char, callback: ReadCallback, userdata: usize, ret: &mut Return<*mut WaterboxHost>) {
+	let mut reader = CReader {
+		userdata,
+		callback
+	};
 	let res = (|| {
-		let data = read_whole_file(wbx)?;
-		WaterboxHost::new(&data[..], &arg_to_str(module_name)?[..], layout)
+		let data = read_whole_file(&mut reader)?;
+		WaterboxHost::new(data, &arg_to_str(module_name)?[..], layout)
 	})();
 	ret.put(res.map(|boxed| Box::into_raw(boxed)));
 }
@@ -227,9 +234,13 @@ pub extern fn wbx_seal(obj: &mut ActivatedWaterboxHost, ret: &mut Return<()>) {
 /// when save_state is called, and can only be used for transient operations.  If a file is readable, it can appear in savestates,
 /// but it must exist in every savestate and the exact sequence of add_file calls must be consistent from savestate to savestate.
 #[no_mangle]
-pub extern fn wbx_mount_file(obj: &mut ActivatedWaterboxHost, name: *const c_char, reader: &mut CReader, writable: bool, ret: &mut Return<()>) {
+pub extern fn wbx_mount_file(obj: &mut ActivatedWaterboxHost, name: *const c_char, callback: ReadCallback, userdata: usize, writable: bool, ret: &mut Return<()>) {
+	let mut reader = CReader {
+		userdata,
+		callback
+	};
 	let res: anyhow::Result<()> = (|| {
-		obj.mount_file(arg_to_str(name)?, read_whole_file(reader)?, writable)?;
+		obj.mount_file(arg_to_str(name)?, read_whole_file(&mut reader)?, writable)?;
 		Ok(())
 	})();
 	ret.put(res);
@@ -238,28 +249,88 @@ pub extern fn wbx_mount_file(obj: &mut ActivatedWaterboxHost, name: *const c_cha
 /// Remove a file previously added.  Writer is optional; if provided, the contents of the file at time of removal will be dumped to it.
 /// It is an error to remove a file which is currently open in the guest.
 #[no_mangle]
-pub extern fn wbx_unmount_file(obj: &mut ActivatedWaterboxHost, name: *const c_char, writer: Option<&mut CWriter>, ret: &mut Return<()>) {
+pub extern fn wbx_unmount_file(obj: &mut ActivatedWaterboxHost, name: *const c_char, callback_opt: Option<WriteCallback>, userdata: usize, ret: &mut Return<()>) {
 	let res: anyhow::Result<()> = (|| {
 		let data = obj.unmount_file(&arg_to_str(name)?)?;
-		if let Some(w) = writer {
-			std::io::copy(&mut &data[..], w)?;
+		if let Some(callback) = callback_opt {
+			let mut writer = CWriter {
+				userdata,
+				callback
+			};
+			io::copy(&mut &data[..], &mut writer)?;
 		}
 		Ok(())
 	})();
 	ret.put(res);
 }
 
+/// Set (or clear, with None) a callback to be called whenever the guest tries to load a nonexistant file.
+/// The callback will be provided with the name of the requested load, and can either return null to signal the waterbox
+/// to return ENOENT to the guest, or a struct to immediately load that file.  You may not call any wbx methods
+/// in the callback.  If the MissingFileResult is provided, it will be consumed immediately and will have the same effect
+/// as wbx_mount_file().  You may free resources associated with the MissingFileResult whenever control next returns to your code.
+// #[no_mangle]
+// pub extern fn wbx_set_missing_file_callback(obj: &mut ActivatedWaterboxHost, mfc_o: Option<&MissingFileCallback>) {
+// 	match mfc_o {
+// 		None => obj.set_missing_file_callback(None),
+// 		Some(mfc) => {
+// 			let userdata = mfc.userdata;
+// 			let callback = mfc.callback;
+// 			obj.set_missing_file_callback(Some(Box::new(move |name| {
+// 				let namestr = CString::new(name).unwrap();
+// 				let mfr = callback(userdata, namestr.as_ptr() as *const c_char);
+// 				if mfr == 0 as *mut MissingFileResult {
+// 					return None
+// 				}
+// 				unsafe {
+// 					let data = read_whole_file(&mut (*mfr).reader);
+// 					match data {
+// 						Ok(d) => Some(fs::MissingFileResult {
+// 							data: d,
+// 							writable: (*mfr).writable
+// 						}),
+// 						Err(_) => None,
+// 					}
+// 				}
+// 			})));
+// 		}
+// 	}
+// }
+
 /// Save state.  Must not be called before seal.  Must not be called with any writable files mounted.
 /// Must always be called with the same sequence and contents of readonly files.
 #[no_mangle]
-pub extern fn wbx_save_state(obj: &mut ActivatedWaterboxHost, writer: &mut CWriter, ret: &mut Return<()>) {
-	ret.put(obj.save_state(writer));
+pub extern fn wbx_save_state(obj: &mut ActivatedWaterboxHost, callback: WriteCallback, userdata: usize, ret: &mut Return<()>) {
+	let mut writer = CWriter {
+		userdata,
+		callback
+	};
+	let res: anyhow::Result<()> = (|| {
+		obj.save_state(&mut writer)?;
+		Ok(())
+	})();
+	ret.put(res);
 }
 /// Load state.  Must not be called before seal.  Must not be called with any writable files mounted.
 /// Must always be called with the same sequence and contents of readonly files that were in the save state.
 /// Must be called with the same wbx executable and memory layout as in the savestate.
 /// Errors generally poison the environment; sorry!
 #[no_mangle]
-pub extern fn wbx_load_state(obj: &mut ActivatedWaterboxHost, reader: &mut CReader, ret: &mut Return<()>) {
-	ret.put(obj.load_state(reader));
+pub extern fn wbx_load_state(obj: &mut ActivatedWaterboxHost, callback: ReadCallback, userdata: usize, ret: &mut Return<()>) {
+	let mut reader = CReader {
+		userdata,
+		callback
+	};
+	ret.put(obj.load_state(&mut reader));
+}
+
+/// Control whether the host automatically evicts blocks from memory when they are not active.  For the best performance,
+/// this should be set to false.  Set to true to help catch dangling pointer issues.  Will be ignored (and forced to true)
+/// if waterboxhost was built in debug mode.  This is a single global setting.
+#[no_mangle]
+pub extern fn wbx_set_always_evict_blocks(_val: bool) {
+	#[cfg(not(debug_assertions))]
+	{
+		unsafe { ALWAYS_EVICT_BLOCKS = _val; }
+	}
 }

waterbox/waterboxhost/src/elf.rs

@@ -1,5 +1,5 @@
 use goblin;
-use goblin::elf64::{sym::*, section_header::*};
+use goblin::{elf::Elf, elf64::{sym::*, section_header::*}};
 use crate::*;
 use crate::memory_block::ActivatedMemoryBlock;
 use crate::memory_block::Protection;
@@ -30,26 +30,26 @@ pub struct ElfLoader {
 	import_area: AddressRange,
 }
 impl ElfLoader {
-	pub fn new(data: &[u8],
-		module_name: &str,
-		layout: &WbxSysLayout,
-		b: &mut ActivatedMemoryBlock
-	) -> anyhow::Result<ElfLoader> {
-		let wbx = goblin::elf::Elf::parse(data)?;
-
+	pub fn elf_addr(wbx: &Elf) -> AddressRange {
 		let start = wbx.program_headers.iter()
+			.filter(|x| x.p_vaddr != 0)
 			.map(|x| x.vm_range().start)
 			.min()
 			.unwrap();
 		let end = wbx.program_headers.iter()
+			.filter(|x| x.p_vaddr != 0)
 			.map(|x| x.vm_range().end)
 			.max()
 			.unwrap();
-		if start < layout.elf.start || end > layout.elf.end() {
-			return Err(anyhow!("{} from {}..{} did not fit in the provided region", module_name, start, end))
-		}
-
-		println!("Mouting `{}` @{:x}", module_name, start);
+		return AddressRange { start, size: end - start };
+	}
+	pub fn new(wbx: &Elf, data: &[u8],
+		module_name: &str,
+		layout: &WbxSysLayout,
+		b: &mut ActivatedMemoryBlock
+	) -> anyhow::Result<ElfLoader> {
+		println!("Mouting `{}` @{:x}", module_name, layout.elf.start);
+		println!("  Sections:");
 
 		let mut sections = Vec::new();	
 
@@ -58,8 +58,9 @@ impl ElfLoader {
 				Some(Ok(s)) => s,
 				_ => "<anon>"
 			};
-			println!("  @{:x} {}{}{} `{}` {} bytes",
+			println!("    @{:x}:{:x} {}{}{} `{}` {} bytes",
 				section.sh_addr,
+				section.sh_addr + section.sh_size,
 				if section.sh_flags & (SHF_ALLOC as u64) != 0 { "R" } else { " " },
 				if section.sh_flags & (SHF_WRITE as u64) != 0 { "W" } else { " " },
 				if section.sh_flags & (SHF_EXECINSTR as u64) != 0 { "X" } else { " " },
@@ -112,10 +113,20 @@ impl ElfLoader {
 		{
 			let invis_opt = sections.iter().find(|x| x.name == ".invis");
 			if let Some(invis) = invis_opt {
-				let any_below = sections.iter().any(|x| x.addr.align_expand().end() > invis.addr.align_expand().start);
-				let any_above = sections.iter().any(|x| x.addr.align_expand().start < invis.addr.align_expand().end());
-				if any_below || any_above {
-					return Err(anyhow!("Overlap between .invis and other sections -- check linkscript."));
+				for s in sections.iter() {
+					if s.addr.align_expand().start < invis.addr.align_expand().start {
+						if s.addr.align_expand().end() > invis.addr.align_expand().start {
+							return Err(anyhow!("When aligned, {} partially overlaps .invis from below -- check linkscript.", s.name))
+						}
+					} else if s.addr.align_expand().start > invis.addr.align_expand().start {
+						if invis.addr.align_expand().end() > s.addr.align_expand().start {
+							return Err(anyhow!("When aligned, {} partially overlaps .invis from above -- check linkscript.", s.name))
+						}
+					} else {
+						if s.name != ".invis" {
+							return Err(anyhow!("When aligned, {} partially overlays .invis -- check linkscript", s.name))
+						}
+					}
 				}
 				b.mark_invisible(invis.addr.align_expand())?;
 			}
@@ -123,7 +134,8 @@ impl ElfLoader {
 
 		b.mark_invisible(layout.invis)?;
 
-		for segment in wbx.program_headers.iter() {
+		println!("  Segments:");
+		for segment in wbx.program_headers.iter().filter(|x| x.p_vaddr != 0) {
 			let addr = AddressRange {
 				start: segment.vm_range().start,
 				size: segment.vm_range().end - segment.vm_range().start
@@ -136,12 +148,22 @@ impl ElfLoader {
 				(_, true, false) => Protection::RW,
 				(_, true, true) => Protection::RWX
 			};
-			b.mmap_fixed(prot_addr, prot)?;
+			println!("    %{:x}:{:x} {}{}{} {} bytes",
+				addr.start,
+				addr.end(),
+				if segment.is_read() { "R" } else { " " },
+				if segment.is_write() { "W" } else { " " },
+				if segment.is_executable() { "X" } else { " " },
+				addr.size
+			);
+			// TODO:  Using no_replace false here because the linker puts eh_frame_hdr in a separate segment that overlaps the other RO segment???
+			b.mmap_fixed(prot_addr, Protection::RW, false)?;
 			unsafe {
 				let src = &data[segment.file_range()];
 				let dst = AddressRange { start: addr.start, size: segment.file_range().end - segment.file_range().start }.slice_mut();
 				dst.copy_from_slice(src);
 			}
+			b.mprotect(prot_addr, prot)?;
 		}
 
 		Ok(ElfLoader {
@@ -152,18 +174,21 @@ impl ElfLoader {
 			import_area
 		})
 	}
-	pub fn seal(&self, b: &mut ActivatedMemoryBlock) {
+	pub fn pre_seal(&mut self, b: &mut ActivatedMemoryBlock) {
+		self.run_proc(b, "co_clean");
+		self.run_proc(b, "ecl_seal");
 		for section in self.sections.iter() {
 			if section_name_is_readonly(section.name.as_str()) {
-				b.mprotect(section.addr, Protection::R).unwrap();
+				b.mprotect(section.addr.align_expand(), Protection::R).unwrap();
 			}
 		}
+		self.clear_syscalls(b);
 	}
 	pub fn connect_syscalls(&mut self, _b: &mut ActivatedMemoryBlock, sys: &WbxSysArea) {
 		let addr = self.import_area;
 		unsafe { *(addr.start as *mut WbxSysArea) = *sys; }
 	}
-	pub fn clear_syscalls(&mut self, _b: &mut ActivatedMemoryBlock) {
+	fn clear_syscalls(&mut self, _b: &mut ActivatedMemoryBlock) {
 		let addr = self.import_area;
 		unsafe { addr.zero(); }
 	}
@@ -173,11 +198,11 @@ impl ElfLoader {
 			std::mem::transmute::<usize, extern "win64" fn() -> ()>(self.entry_point)();
 		}
 	}
-	pub fn co_clean(&mut self, _b: &mut ActivatedMemoryBlock) {
-		match self.get_proc_addr("co_clean") {
+	fn run_proc(&mut self, _b: &mut ActivatedMemoryBlock, name: &str) {
+		match self.get_proc_addr(name) {
 			0 => (),
 			ptr => {
-				println!("Calling co_clean()");
+				println!("Calling {}()", name);
 				unsafe {
 					std::mem::transmute::<usize, extern "win64" fn() -> ()>(ptr)();
 				}

waterbox/waterboxhost/src/fs/mod.rs

@@ -90,8 +90,15 @@ impl IStateable for MountedFile {
 	}
 }
 
+// pub type MissingFileCallback = Box<dyn FnMut(&str) -> Option<MissingFileResult>>;
+// pub struct MissingFileResult {
+// 	pub data: Vec<u8>,
+// 	pub writable: bool,
+// }
+
 pub struct FileSystem {
 	files: Vec<MountedFile>,
+	// missing_file_callback: Option<MissingFileCallback>,
 }
 impl FileSystem {
 	pub fn new() -> FileSystem {
@@ -113,8 +120,13 @@ impl FileSystem {
 					obj: Box::new(SysOutObj { host_handle: Box::new(std::io::stderr()) })
 				},
 			],
+			// missing_file_callback: None,
 		}
 	}
+	/// Set (or clear, with None) a callback to be called whenever the guest tries to load a nonexistant file
+	// pub fn set_missing_file_callback(&mut self, cb: Option<MissingFileCallback>) {
+	// 	self.missing_file_callback = cb;
+	// }
 	/// Accept a file from the outside world.  Writable files may never appear in a savestate,
 	/// and readonly files must not be added or removed from savestate to savestate, so all uses
 	/// are either transient or read only resources that last for the life of emulation.
@@ -145,9 +157,15 @@ impl FileSystem {
 		}
 		Ok(self.files.remove(idx).obj.unmount())
 	}
+
+	// fn try_missing_file_cb(&mut self, name: &str) -> Option<()> {
+	// 	self.missing_file_callback
+	// 		.as_mut()
+	// 		.and_then(|cb| cb(name))
+	// 		.map(|res| self.mount(name.to_string(), res.data, res.writable).unwrap())
+	// }
 	/// Implements a subset of open(2)
 	pub fn open(&mut self, name: &str, flags: i32, _mode: i32) -> Result<FileDescriptor, SyscallError> {
-		// TODO: Missing file callback
 		let fd = {
 			let mut i = 0;
 			loop {
@@ -157,7 +175,26 @@ impl FileSystem {
 				i += 1;
 			}
 		};
-		let file = match self.files.iter_mut().find(|f| f.name == name) {
+		let file_opt = {
+			// let mut did_cb = false;
+			loop {
+				match self.files.iter_mut().find(|f| f.name == name) {
+					Some(f) => break Some(f),
+					// None if !did_cb => {
+					// 	match self.try_missing_file_cb(name) {
+					// 		Some(()) => {
+					// 			did_cb = true;
+					// 			continue
+					// 		},
+					// 		None => break None,
+					// 	}
+					// }
+					_ => break None,
+				}
+			}
+		};
+
+		let file = match file_opt {
 			Some(f) => f,
 			None => return Err(ENOENT)
 		};

waterbox/waterboxhost/src/gdb.rs

@@ -1 +1,94 @@
-// TODO https://sourceware.org/gdb/current/onlinedocs/gdb/Declarations.html#Declarations
+// https://sourceware.org/gdb/current/onlinedocs/gdb/Declarations.html#Declarations
+#![allow(non_camel_case_types)]
+#![allow(non_upper_case_globals)]
+
+use std::{sync::Mutex, ptr::null_mut};
+use lazy_static::lazy_static;
+
+const JIT_NOACTION: u32 = 0;
+const JIT_REGISTER_FN: u32 = 1;
+const JIT_UNREGISTER_FN: u32 = 2;
+
+#[repr(C)]
+struct jit_code_entry {
+	next_entry: *mut jit_code_entry,
+	prev_entry: *mut jit_code_entry,
+	symfile_addr: *const u8,
+	symfile_size: u64
+}
+unsafe impl Send for jit_code_entry {}
+unsafe impl Sync for jit_code_entry {}
+
+#[repr(C)]
+struct jit_descriptor {
+	version: u32,
+	action_flag: u32,
+	relevant_entry: *mut jit_code_entry,
+	first_entry: *mut jit_code_entry,
+}
+unsafe impl Send for jit_descriptor {}
+unsafe impl Sync for jit_descriptor {}
+
+#[no_mangle]
+#[inline(never)]
+extern fn __jit_debug_register_code() {}
+
+#[no_mangle]
+static mut __jit_debug_descriptor: jit_descriptor = jit_descriptor {
+	version: 1,
+	action_flag: JIT_NOACTION,
+	relevant_entry: null_mut(), // 0 as *mut jit_code_entry,
+	first_entry: null_mut(), // 0 as *mut jit_code_entry
+};
+
+
+lazy_static! {
+	static ref LOCK: Mutex<()> = Mutex::new(());
+}
+
+/// unsafe:  the data should be valid until a matching unregister call
+pub unsafe fn register(data: &[u8]) {
+	let _guard = LOCK.lock().unwrap();
+	let entry = Box::into_raw(Box::new(jit_code_entry {
+		next_entry: null_mut(),
+		prev_entry: null_mut(),
+		symfile_addr: &data[0],
+		symfile_size: data.len() as u64
+	}));
+	if __jit_debug_descriptor.first_entry == null_mut() {
+		__jit_debug_descriptor.first_entry = entry;
+	} else {
+		let mut tail = __jit_debug_descriptor.first_entry;
+		while (*tail).next_entry != null_mut() {
+			tail = (*tail).next_entry;
+		}
+		(*tail).next_entry = entry;
+		(*entry).prev_entry = tail;
+	}
+	__jit_debug_descriptor.relevant_entry = entry;
+	__jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
+	__jit_debug_register_code();
+}
+
+/// unsafe: undefined if not exactly matching a register call
+pub unsafe fn deregister(data: &[u8]) {
+	let _guard = LOCK.lock().unwrap();
+
+	let mut entry = __jit_debug_descriptor.first_entry;
+	while (*entry).symfile_addr != &data[0] {
+		entry = (*entry).next_entry;
+	}
+	if (*entry).next_entry != null_mut() {
+		(*(*entry).next_entry).prev_entry = (*entry).prev_entry;
+	}
+	if (*entry).prev_entry != null_mut() {
+		(*(*entry).prev_entry).next_entry = (*entry).next_entry;
+	} else {
+		__jit_debug_descriptor.first_entry = (*entry).next_entry;
+	}
+	__jit_debug_descriptor.relevant_entry = entry;
+	__jit_debug_descriptor.action_flag = JIT_UNREGISTER_FN;
+	__jit_debug_register_code();
+
+	Box::from_raw(entry);
+}

waterbox/waterboxhost/src/host.rs

@@ -2,9 +2,10 @@ use crate::*;
 use crate::{memory_block::ActivatedMemoryBlock, syscall_defs::*};
 use memory_block::{MemoryBlock, Protection};
 use std::{os::raw::c_char, ffi::CStr};
-use fs::{FileDescriptor, FileSystem};
+use fs::{FileDescriptor, FileSystem/*, MissingFileCallback*/};
 use elf::ElfLoader;
 use cinterface::MemoryLayoutTemplate;
+use goblin::elf::Elf;
 
 pub struct WaterboxHost {
 	fs: FileSystem,
@@ -14,15 +15,19 @@ pub struct WaterboxHost {
 	memory_block: Box<MemoryBlock>,
 	active: bool,
 	sealed: bool,
+	image_file: Vec<u8>,
 }
 impl WaterboxHost {
-	pub fn new(wbx: &[u8], module_name: &str, layout_template: &MemoryLayoutTemplate) -> anyhow::Result<Box<WaterboxHost>> {
-		let layout = layout_template.make_layout()?;
+	pub fn new(image_file: Vec<u8>, module_name: &str, layout_template: &MemoryLayoutTemplate) -> anyhow::Result<Box<WaterboxHost>> {
+		let wbx = Elf::parse(&image_file[..])?;
+		let elf_addr = ElfLoader::elf_addr(&wbx);
+		let layout = layout_template.make_layout(elf_addr)?;
 		let mut memory_block = MemoryBlock::new(layout.all());
 		let mut b = memory_block.enter();
-		let elf = ElfLoader::new(wbx, module_name, &layout, &mut b)?;
+		let elf = ElfLoader::new(&wbx, &image_file[..], module_name, &layout, &mut b)?;
 		let fs = FileSystem::new();
 		drop(b);
+		unsafe { gdb::register(&image_file[..]) }
 		let mut res = Box::new(WaterboxHost {
 			fs,
 			program_break: layout.sbrk.start,
@@ -31,10 +36,10 @@ impl WaterboxHost {
 			memory_block,
 			active: false,
 			sealed: false,
+			image_file,
 		});
 
 		let mut active = res.activate();
-		active.h.elf.connect_syscalls(&mut active.b, &mut active.sys);
 		active.h.elf.native_init(&mut active.b);
 		drop(active);
 
@@ -62,10 +67,16 @@ impl WaterboxHost {
 			sys
 		});
 		res.sys.syscall.ud = res.as_mut() as *mut ActivatedWaterboxHost as usize;
+		res.h.elf.connect_syscalls(&mut res.b, &res.sys);
 		res.h.active = true;
 		res
 	}
 }
+impl Drop for WaterboxHost {
+	fn drop(&mut self) {
+		unsafe { gdb::deregister(&self.image_file[..]) }
+	}
+}
 
 const TAG: u64 = 0xd01487803948acff;
 pub struct ActivatedWaterboxHost<'a> {
@@ -95,11 +106,9 @@ impl<'a> ActivatedWaterboxHost<'a> {
 		if self.h.sealed {
 			return Err(anyhow!("Already sealed!"))
 		}
-		self.h.elf.clear_syscalls(&mut self.b);
-		self.h.elf.seal(&mut self.b);
-		self.h.elf.connect_syscalls(&mut self.b, &self.sys);
-		self.h.elf.co_clean(&mut self.b);
+		self.h.elf.pre_seal(&mut self.b);
 		self.b.seal();
+		self.h.elf.connect_syscalls(&mut self.b, &self.sys);
 		self.h.sealed = true;
 		Ok(())
 	}
@@ -109,6 +118,9 @@ impl<'a> ActivatedWaterboxHost<'a> {
 	pub fn unmount_file(&mut self, name: &str) -> anyhow::Result<Vec<u8>> {
 		self.h.fs.unmount(name)
 	}
+	// pub fn set_missing_file_callback(&mut self, cb: Option<MissingFileCallback>) {
+	// 	self.h.fs.set_missing_file_callback(cb);
+	// }
 }
 
 const SAVE_START_MAGIC: &str = "ActivatedWaterboxHost_v1";
@@ -192,10 +204,10 @@ fn arg_to_statbuff<'a>(arg: usize) -> &'a mut KStat {
 }
 
 pub extern "win64" fn syscall(nr: SyscallNumber, ud: usize, a1: usize, a2: usize, a3: usize, a4: usize, _a5: usize, _a6: usize) -> SyscallReturn {
-	let mut h = gethost(ud);
+	let h = gethost(ud);
 	match nr {
 		NR_MMAP => {
-			let prot = arg_to_prot(a3)?;
+			let mut prot = arg_to_prot(a3)?;
 			let flags = a4;
 			if flags & MAP_ANONYMOUS == 0 {
 				// anonymous + private is easy
@@ -207,8 +219,16 @@ pub extern "win64" fn syscall(nr: SyscallNumber, ud: usize, a1: usize, a2: usize
 				// various unsupported flags
 				return syscall_err(EOPNOTSUPP)
 			}
+			if flags & MAP_STACK != 0 {
+				if prot == Protection::RW {
+					prot = Protection::RWStack;
+				} else {
+					return syscall_err(EINVAL) // stacks must be readable and writable
+				}
+			}
+			let no_replace = flags & MAP_FIXED_NOREPLACE != 0;
 			let arena_addr = h.sys.layout.mmap;
-			let res = h.b.mmap(AddressRange { start: a1, size: a2 }, prot, arena_addr)?;
+			let res = h.b.mmap(AddressRange { start: a1, size: a2 }, prot, arena_addr, no_replace)?;
 			syscall_ok(res)
 		},
 		NR_MREMAP => {
@@ -297,10 +317,17 @@ pub extern "win64" fn syscall(nr: SyscallNumber, ud: usize, a1: usize, a2: usize
 			let old = h.h.program_break;
 			let res = if a1 != align_down(a1) {
 				old
-			} else if a1 < addr.start || a1 > addr.end() {
+			} else if a1 < addr.start {
+				if a1 == 0 {
+					println!("Initializing heap sbrk at {:x}:{:x}", addr.start, addr.end());
+				}
 				old
+			} else if a1 > addr.end() {
+				eprintln!("Failed to satisfy allocation of {} bytes on sbrk heap", a1 - old);
+				old	
 			} else if a1 > old {
-				h.b.mmap_fixed(AddressRange { start: old, size: a1 - old }, Protection::RW).unwrap();
+				h.b.mmap_fixed(AddressRange { start: old, size: a1 - old }, Protection::RW, true).unwrap();
+				println!("Allocated {} bytes on sbrk heap, usage {}/{}", a1 - old, a1 - addr.start, addr.size);
 				a1
 			} else {
 				old

waterbox/waterboxhost/src/lib.rs

@@ -20,6 +20,7 @@ mod elf;
 mod fs;
 mod host;
 mod cinterface;
+mod gdb;
 
 pub trait IStateable {
 	fn save_state(&mut self, stream: &mut dyn Write) -> anyhow::Result<()>;
@@ -76,7 +77,7 @@ fn align_down(p: usize) -> usize {
 	p & !PAGEMASK
 }
 fn align_up(p: usize) -> usize {
-	((p - 1) | PAGEMASK) + 1
+	((p.wrapping_sub(1)) | PAGEMASK).wrapping_add(1)
 }
 
 /// Information about memory layout injected into the guest application
@@ -115,6 +116,10 @@ pub struct WbxSysArea {
 	pub syscall: WbxSysSyscall,
 }
 
+/// Always remove memoryblocks from active ram when possible, to help debug dangling pointers.
+/// Severe performance consequences.
+static mut ALWAYS_EVICT_BLOCKS: bool = true;
+
 #[cfg(test)]
 mod tests {
 	#[test]

waterbox/waterboxhost/src/memory_block/mod.rs

@@ -316,8 +316,7 @@ impl MemoryBlock {
 	unsafe fn deactivate(&mut self, mut guard: BlockGuard) {
 		// self.trace("deactivate");
 		assert!(self.active);
-		#[cfg(debug_assertions)]
-		{
+		if ALWAYS_EVICT_BLOCKS {
 			// in debug mode, forcibly evict to catch dangling pointers
 			let other_opt = guard.deref_mut();
 			match *other_opt {
@@ -533,10 +532,9 @@ impl<'block> ActivatedMemoryBlock<'block> {
 	}
 
 	/// implements a subset of mmap(2) for anonymous, fixed address mappings
-	pub fn mmap_fixed(&mut self, addr: AddressRange, prot: Protection) -> SyscallResult {
+	pub fn mmap_fixed(&mut self, addr: AddressRange, prot: Protection, no_replace: bool) -> SyscallResult {
 		let mut range = self.b.validate_range(addr)?;
-		if range.iter().any(|p| p.status != PageAllocation::Free) {
-			// assume MAP_FIXED_NOREPLACE at all times
+		if no_replace && range.iter().any(|p| p.status != PageAllocation::Free) {
 			return Err(EEXIST)
 		}
 		MemoryBlock::set_protections(&mut range, PageAllocation::Allocated(prot));
@@ -639,14 +637,14 @@ impl<'block> ActivatedMemoryBlock<'block> {
 		self.munmap_impl(addr, false)
 	}
 
-	pub fn mmap(&mut self, addr: AddressRange, prot: Protection, arena_addr: AddressRange) -> Result<usize, SyscallError> {
+	pub fn mmap(&mut self, addr: AddressRange, prot: Protection, arena_addr: AddressRange, no_replace: bool) -> Result<usize, SyscallError> {
 		if addr.size == 0 {
 			return Err(EINVAL)
 		}
 		if addr.start == 0 {
 			self.mmap_movable(addr.size, prot, arena_addr)
 		} else {
-			self.mmap_fixed(addr, prot)?;
+			self.mmap_fixed(addr, prot, no_replace)?;
 			Ok(addr.start)
 		}
 	}
@@ -757,10 +755,23 @@ impl<'block>  IStateable for ActivatedMemoryBlock<'block> {
 		self.b.get_stack_dirty();
 		self.b.addr.save_state(stream)?;
 
+		unsafe {
+			let mut statii = Vec::new();
+			let mut dirtii = Vec::new();
+			statii.reserve_exact(self.b.pages.len());
+			dirtii.reserve_exact(self.b.pages.len());
+			for p in self.b.pages.iter() {
+				statii.push(p.status);
+				dirtii.push(p.dirty);
+			}
+			stream.write_all(std::mem::transmute(&statii[..]))?;
+			stream.write_all(std::mem::transmute(&dirtii[..]))?;
+		}
+
 		for (paddr, p) in self.b.page_range().iter_with_addr() {
-			bin::write(stream, &p.status)?;
+			// bin::write(stream, &p.status)?;
 			if !p.invisible {
-				bin::write(stream, &p.dirty)?;
+				// bin::write(stream, &p.dirty)?;
 				if p.dirty {
 					unsafe {
 						if !p.status.readable() {
@@ -795,10 +806,18 @@ impl<'block>  IStateable for ActivatedMemoryBlock<'block> {
 		unsafe {
 			pal::protect(self.b.addr, Protection::RW);
 
+			let mut statii = vec![PageAllocation::Free; self.b.pages.len()];
+			let mut dirtii = vec![false; self.b.pages.len()];
+			stream.read_exact(std::mem::transmute(&mut statii[..]))?;
+			stream.read_exact(std::mem::transmute(&mut dirtii[..]))?;
+
+			let mut index = 0usize;
 			for (paddr, p) in self.b.page_range().iter_mut_with_addr() {
-				let status = bin::readval::<PageAllocation>(stream)?;
+				let status = statii[index];
+				// let status = bin::readval::<PageAllocation>(stream)?;
 				if !p.invisible {
-					let dirty = bin::readval::<bool>(stream)?;
+					let dirty = dirtii[index];
+					// let dirty = bin::readval::<bool>(stream)?;
 					match (p.dirty, dirty) {
 						(false, false) => (),
 						(false, true) => {
@@ -821,6 +840,7 @@ impl<'block>  IStateable for ActivatedMemoryBlock<'block> {
 					p.dirty = dirty;
 				}
 				p.status = status;
+				index += 1;
 			}
 
 			self.b.refresh_all_protections();

waterbox/waterboxhost/src/memory_block/tests.rs

@@ -31,7 +31,7 @@ fn test_dirty() -> TestResult {
 		let addr = AddressRange { start: 0x36f00000000, size: 0x10000 };
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
-		g.mmap_fixed(addr, Protection::RW)?;
+		g.mmap_fixed(addr, Protection::RW, true)?;
 		let ptr = g.b.addr.slice_mut();
 		ptr[0x2003] = 5;
 		assert!(g.b.pages[2].dirty);
@@ -46,7 +46,7 @@ fn test_offset() -> TestResult {
 		let addr = AddressRange { start: 0x36f00000000, size: 0x20000 };
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
-		g.mmap_fixed(AddressRange { start: 0x36f00003000, size: 0x1000 }, Protection::RW)?;
+		g.mmap_fixed(AddressRange { start: 0x36f00003000, size: 0x1000 }, Protection::RW, true)?;
 		let ptr = g.b.addr.slice_mut();
 		ptr[0x3663] = 12;
 		assert!(g.b.pages[3].dirty);
@@ -61,14 +61,19 @@ fn test_stk_norm() -> TestResult {
 		let addr = AddressRange { start: 0x36200000000, size: 0x10000 };
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
-		g.mmap_fixed(addr, Protection::RWStack)?;
+		g.mmap_fixed(addr, Protection::RWStack, true)?;
 		let ptr = g.b.addr.slice_mut();
 		ptr[0xeeee] = 0xee;
 		ptr[0x44] = 0x44;
+
+		g.b.get_stack_dirty();
+
 		assert!(g.b.pages[0].dirty);
 		assert!(g.b.pages[14].dirty);
 		assert_eq!(ptr[0x8000], 0);
 
+		g.b.get_stack_dirty();
+
 		// This is an unfair test, but it's just documenting the current limitations of the system.
 		// Ideally, page 8 would be clean because we read from it but did not write to it.
 		// Due to limitations of RWStack tracking on windows, it is dirty.
@@ -89,7 +94,7 @@ fn test_stack() -> TestResult {
 		let addr = AddressRange { start: 0x36f00000000, size: 0x10000 };
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
-		g.mmap_fixed(addr, Protection::RW)?;
+		g.mmap_fixed(addr, Protection::RW, true)?;
 		let ptr = g.b.addr.slice_mut();
 		let mut i = 0;
 
@@ -105,6 +110,9 @@ fn test_stack() -> TestResult {
 		let tmp_rsp = addr.end();
 		let res = transmute::<usize, extern "sysv64" fn(rsp: usize) -> u8>(addr.start)(tmp_rsp);
 		assert_eq!(res, 42);
+
+		g.b.get_stack_dirty();
+
 		assert!(g.b.pages[0].dirty);
 		assert!(!g.b.pages[1].dirty);
 		assert!(!g.b.pages[14].dirty);
@@ -124,7 +132,7 @@ fn test_state_basic() -> TestResult {
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
 		let ptr = g.b.addr.slice_mut();
-		g.mmap_fixed(addr, Protection::RW)?;
+		g.mmap_fixed(addr, Protection::RW, true)?;
 		ptr[0x0000] = 20;
 		ptr[0x1000] = 40;
 		ptr[0x2000] = 60;
@@ -172,7 +180,7 @@ fn test_state_unreadable() -> TestResult {
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
 		let ptr = g.b.addr.slice_mut();
-		g.mmap_fixed(addr, Protection::RW)?;
+		g.mmap_fixed(addr, Protection::RW, true)?;
 		g.seal();
 
 		ptr[200] = 200;
@@ -222,7 +230,7 @@ fn test_thready_stack() -> TestResult {
 				let mut g = b.enter();
 
 				blocker.wait();
-				g.mmap_fixed(addr, Protection::RWX)?;
+				g.mmap_fixed(addr, Protection::RWX, true)?;
 				g.mprotect(AddressRange { start: addr.start + PAGESIZE, size: PAGESIZE }, Protection::RWStack)?;
 
 				let ptr = g.b.addr.slice_mut();
@@ -236,12 +244,15 @@ fn test_thready_stack() -> TestResult {
 				ptr[i] = 0xc3 ; // ret 
 
 				g.seal();
-	
+
 				assert!(!g.b.pages[0].dirty);
 				assert!(!g.b.pages[1].dirty);
 				let tmp_rsp = addr.end();
 				let res = transmute::<usize, extern "sysv64" fn(rsp: usize) -> u8>(addr.start)(tmp_rsp);
 				assert_eq!(res, 42);
+
+				g.b.get_stack_dirty();
+	
 				assert!(!g.b.pages[0].dirty);
 				assert!(g.b.pages[1].dirty);
 
@@ -266,7 +277,7 @@ fn test_state_invisible() -> TestResult {
 		let mut b = MemoryBlock::new(addr);
 		let mut g = b.enter();
 		let ptr = g.b.addr.slice_mut();
-		g.mmap_fixed(addr, Protection::RW)?;
+		g.mmap_fixed(addr, Protection::RW, true)?;
 		ptr[0x0055] = 11;
 		ptr[0x1055] = 22;
 		g.mark_invisible(AddressRange { start: 0x36400001000, size: 0x2000 })?;
@@ -316,7 +327,7 @@ fn test_dontneed() -> TestResult {
 		g.seal();
 		let ptr = g.b.addr.slice_mut();
 
-		g.mmap_fixed(addr, Protection::RW)?;
+		g.mmap_fixed(addr, Protection::RW, true)?;
 		for i in 0..addr.size {
 			ptr[i] = i as u8;
 		}
@@ -341,7 +352,7 @@ fn test_remap_nomove() -> TestResult {
 	let mut b = MemoryBlock::new(addr);
 	let mut g = b.enter();
 
-	g.mmap_fixed(AddressRange { start: addr.start, size: 0x4000 }, Protection::RWX)?;
+	g.mmap_fixed(AddressRange { start: addr.start, size: 0x4000 }, Protection::RWX, true)?;
 	g.mremap_nomove(AddressRange { start: addr.start, size: 0x4000 }, 0x6000)?;
 	assert_eq!(g.b.pages[3].status, PageAllocation::Allocated(Protection::RWX));
 	assert_eq!(g.b.pages[5].status, PageAllocation::Allocated(Protection::RWX));
@@ -379,12 +390,12 @@ fn test_mremap_move_expand() -> TestResult {
 		let ptr = g.b.addr.slice_mut();
 
 		let initial_addr = AddressRange { start: 0x36800002000, size: 0x1000 };
-		g.mmap_fixed(initial_addr, Protection::RW)?;
+		g.mmap_fixed(initial_addr, Protection::RW, true)?;
 		ptr[0x2004] = 11;
 		let p1 = g.mremap_maymove(initial_addr, 0x2000, addr)?;
 		assert_eq!(p1, addr.start);
 		assert_eq!(ptr[4], 11);
-		g.mmap_fixed(initial_addr, Protection::RW)?;
+		g.mmap_fixed(initial_addr, Protection::RW, true)?;
 		assert_eq!(ptr[0x2004], 0);
 	}
 	Ok(())
@@ -399,12 +410,12 @@ fn test_mremap_move_shrink() -> TestResult {
 		let ptr = g.b.addr.slice_mut();
 
 		let initial_addr = AddressRange { start: 0x36900001000, size: 0x3000 };
-		g.mmap_fixed(initial_addr, Protection::RW)?;
+		g.mmap_fixed(initial_addr, Protection::RW, true)?;
 		ptr[0x1004] = 11;
 		let p1 = g.mremap_maymove(initial_addr, 0x1000, addr)?;
 		assert_eq!(p1, addr.start);
 		assert_eq!(ptr[4], 11);
-		g.mmap_fixed(initial_addr, Protection::RW)?;
+		g.mmap_fixed(initial_addr, Protection::RW, true)?;
 		assert_eq!(ptr[0x1004], 0);
 	}
 	Ok(())

waterbox/waterboxhost/src/memory_block/tripguard.rs

@@ -55,6 +55,7 @@ unsafe fn trip(addr: usize) -> TripResult {
 	let page_start_addr = addr & !PAGEMASK;
 	let page = &mut memory_block.pages[(addr - memory_block.addr.start) >> PAGESHIFT];
 	if !page.status.writable() {
+		std::intrinsics::breakpoint();
 		return TripResult::NotHandled
 	}
 	page.maybe_snapshot(page_start_addr);
@@ -62,6 +63,7 @@ unsafe fn trip(addr: usize) -> TripResult {
 	if pal::protect(AddressRange { start: page_start_addr, size: PAGESIZE }, page.native_prot()) {
 		TripResult::Handled
 	} else {
+		std::intrinsics::breakpoint();
 		std::process::abort();
 	}
 }
@@ -79,7 +81,22 @@ mod trip_pal {
 			let flags = p_record.ExceptionInformation[0];
 			match p_record.ExceptionCode {
 				STATUS_ACCESS_VIOLATION if (flags & 1) != 0 => (), // write exception
-				STATUS_GUARD_PAGE_VIOLATION => (), // guard exception
+				STATUS_GUARD_PAGE_VIOLATION => {
+					// guard exception
+
+					// If this is ours, it's from a cothread stack.  If we're on a cothread
+					// stack right now, we need to return without taking our lock because
+					// the stack used in handler() and everything it calls might move onto
+					// another page and trip again which would deadlock us.  (Alternatively,
+					// this might be the second trip in a row while already servicing another one.)
+					// This does not cause determinism issues because of get_stack_dirty() and the
+					// windows specific code in set_protections().
+
+					// The only problem here is that we might be swallowing a completely unrelated guard
+					// exception by returning before checking whether it was in a waterbox block;
+					// in which case we'll find out what we broke eventually.
+					return EXCEPTION_CONTINUE_EXECUTION 
+				},
 				_ => return EXCEPTION_CONTINUE_SEARCH
 			}
 			let fault_address = p_record.ExceptionInformation[1] as usize;
@@ -125,13 +142,20 @@ mod trip_pal {
 			}
 		}
 		unsafe {
+			// TODO: sigaltstack is per thread, so this won't work
+			// At the same time, one seems to be set up automatically on each thread, so this isn't needed.
+			// let ss = stack_t {
+			// 	ss_flags: 0,
+			// 	ss_sp: Box::into_raw(Box::new(zeroed::<[u8; SIGSTKSZ]>())) as *mut c_void,
+			// 	ss_size: SIGSTKSZ
+			// };
+			// let mut ss_old = stack_t {
+			// 	ss_flags: 0,
+			// 	ss_sp: 0 as *mut c_void,
+			// 	ss_size: 0
+			// };
+			// assert!(sigaltstack(&ss, &mut ss_old) == 0, "sigaltstack failed");
 			SA_OLD = Some(Box::new(zeroed()));
-			let ss = stack_t {
-				ss_flags: 0,
-				ss_sp: Box::into_raw(Box::new(zeroed::<[u8; SIGSTKSZ]>)) as *mut c_void,
-				ss_size: SIGSTKSZ
-			};
-			assert!(sigaltstack(&ss, 0 as *mut stack_t) == 0, "sigaltstack failed");
 			let mut sa = sigaction {
 				sa_mask: zeroed(),
 				sa_sigaction: transmute::<SaSigaction, usize>(handler),

waterbox/waterboxhost/src/syscall_defs.rs

@@ -64,7 +64,7 @@ macro_rules! lookup {
 		$(pub const $N: $T = $T($E);)+
 		pub fn $P(val: &$T) -> &'static str {
 			match val {
-				$($T($E) => stringify!($E),)+
+				$($T($E) => stringify!($N),)+
 				_ => "????"
 			}
 		}

waterbox/winguard/Makefile

@@ -1,13 +0,0 @@
-ifeq (,$(findstring MINGW,$(shell uname)))
-$(error This must be built with mingw)
-endif
-
-winguard.dll: winguard.c
-	gcc -O2 -s -o $@ $< -shared
-
-install: winguard.dll
-	cp $< ../../output/dll
-clean:
-	rm winguard.dll
-
-.PHONY: install clean

waterbox/winguard/winguard.c

@@ -1,101 +0,0 @@
-#include <windows.h>
-#include <stdint.h>
-#include <stdlib.h>
-
-#define MAX_TRIPS 64
-
-typedef struct {
-	uintptr_t start;
-	uintptr_t length;
-	uint8_t tripped[0];
-} tripwire_t;
-
-static tripwire_t* Trips[MAX_TRIPS];
-static int HandlerInstalled;
-
-static LONG VectoredHandler(struct _EXCEPTION_POINTERS* p_info)
-{
-	EXCEPTION_RECORD* p_record = p_info->ExceptionRecord;
-
-	// CONTEXT* p_context = p_info->ContextRecord;
-	DWORD64 flags = p_record->ExceptionInformation[0];
-
-	if (p_record->ExceptionCode != STATUS_ACCESS_VIOLATION // only trigger on access violations...
-		|| !(flags & 1)) // ...due to a write attempts
-		return EXCEPTION_CONTINUE_SEARCH;
-	
-	uintptr_t faultAddress = (uintptr_t)p_record->ExceptionInformation[1];
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (Trips[i] && faultAddress >= Trips[i]->start && faultAddress < Trips[i]->start + Trips[i]->length)
-		{
-			uintptr_t page = (faultAddress - Trips[i]->start) >> 12;
-			if (Trips[i]->tripped[page] & 1) // should change
-			{
-				DWORD oldprot;
-				if (!VirtualProtect((void*)faultAddress, 1, PAGE_READWRITE, &oldprot))
-				{
-					RaiseFailFastException(NULL, NULL, 0);
-					while (1)
-						;
-				}
-				Trips[i]->tripped[page] = 3; // did change
-				return EXCEPTION_CONTINUE_EXECUTION;
-			}
-			else
-			{
-				return EXCEPTION_CONTINUE_SEARCH;
-			}
-		}
-	}
-	return EXCEPTION_CONTINUE_SEARCH;
-}
-
-__declspec(dllexport) uint8_t* AddTripGuard(uintptr_t start, uintptr_t length)
-{
-	if (!HandlerInstalled)
-	{
-		if (!AddVectoredExceptionHandler(1 /* CALL_FIRST */, VectoredHandler))
-			return NULL;
-		HandlerInstalled = 1;
-	}
-
-	uintptr_t npage = length >> 12;
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (!Trips[i])
-		{
-			Trips[i] = calloc(1, sizeof(*Trips[i]) + npage);
-			if (!Trips[i])
-				return NULL;
-			Trips[i]->start = start;
-			Trips[i]->length = length;
-			return &Trips[i]->tripped[0];
-		}
-	}
-	return NULL;
-}
-
-__declspec(dllexport) int64_t RemoveTripGuard(uintptr_t start, uintptr_t length)
-{
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (Trips[i] && Trips[i]->start == start && Trips[i]->length == length)
-		{
-			free(Trips[i]);
-			Trips[i] = NULL;
-			return 1;
-		}
-	}
-	return 0;
-}
-
-__declspec(dllexport) uint8_t* ExamineTripGuard(uintptr_t start, uintptr_t length)
-{
-	for (int i = 0; i < MAX_TRIPS; i++)
-	{
-		if (Trips[i] && Trips[i]->start == start && Trips[i]->length == length)
-			return &Trips[i]->tripped[0];
-	}
-	return NULL;
-}