Waterbox: Stack Marshalling (#2209)

Waterbox guest code now runs on a stack inside the guest memory space. This removes some potential opportunities for nondeterminism and makes future porting of libco-enabled cores easier.

src/BizHawk.BizInvoke/BizInvoker.cs

@@ -75,13 +75,14 @@ namespace BizHawk.BizInvoke
 		public static T GetInvoker<T>(IImportResolver dll, ICallingConventionAdapter adapter)
 			where T : class
 		{
+			var nonTrivialAdapter = adapter.GetType() != CallingConventionAdapters.Native.GetType();
 			InvokerImpl impl;
 			lock (Impls)
 			{
 				var baseType = typeof(T);
 				if (!Impls.TryGetValue(baseType, out impl))
 				{
-					impl = CreateProxy(baseType, false);
+					impl = CreateProxy(baseType, false, nonTrivialAdapter);
 					Impls.Add(baseType, impl);
 				}
 			}
@@ -98,13 +99,14 @@ namespace BizHawk.BizInvoke
 		public static T GetInvoker<T>(IImportResolver dll, IMonitor monitor, ICallingConventionAdapter adapter)
 			where T : class
 		{
+			var nonTrivialAdapter = adapter.GetType() != CallingConventionAdapters.Native.GetType();
 			InvokerImpl impl;
 			lock (Impls)
 			{
 				var baseType = typeof(T);
 				if (!Impls.TryGetValue(baseType, out impl))
 				{
-					impl = CreateProxy(baseType, true);
+					impl = CreateProxy(baseType, true, nonTrivialAdapter);
 					Impls.Add(baseType, impl);
 				}
 			}
@@ -117,7 +119,7 @@ namespace BizHawk.BizInvoke
 			return (T)impl.Create(dll, monitor, adapter);
 		}
 
-		private static InvokerImpl CreateProxy(Type baseType, bool monitor)
+		private static InvokerImpl CreateProxy(Type baseType, bool monitor, bool nonTrivialAdapter)
 		{
 			if (baseType.IsSealed)
 			{
@@ -180,7 +182,7 @@ namespace BizHawk.BizInvoke
 				var entryPointName = mi.Attr.EntryPoint ?? mi.Info.Name;
 
 				var hook = mi.Attr.Compatibility
-					? ImplementMethodDelegate(type, mi.Info, mi.Attr.CallingConvention, entryPointName, monitorField)
+					? ImplementMethodDelegate(type, mi.Info, mi.Attr.CallingConvention, entryPointName, monitorField, nonTrivialAdapter)
 					: ImplementMethodCalli(type, mi.Info, mi.Attr.CallingConvention, entryPointName, monitorField, adapterField);
 
 				postCreateHooks.Add(hook);
@@ -204,7 +206,8 @@ namespace BizHawk.BizInvoke
 		/// create a method implementation that uses GetDelegateForFunctionPointer internally
 		/// </summary>
 		private static Action<object, IImportResolver, ICallingConventionAdapter> ImplementMethodDelegate(
-			TypeBuilder type, MethodInfo baseMethod, CallingConvention nativeCall, string entryPointName, FieldInfo monitorField)
+			TypeBuilder type, MethodInfo baseMethod, CallingConvention nativeCall, string entryPointName, FieldInfo monitorField,
+			bool nonTrivialAdapter)
 		{
 			// create the delegate type
 			var delegateType = BizInvokeUtilities.CreateDelegateType(baseMethod, nativeCall, type, out var delegateInvoke);
@@ -215,9 +218,13 @@ namespace BizHawk.BizInvoke
 
 			if (paramTypes.Concat(new[] { returnType }).Any(typeof(Delegate).IsAssignableFrom))
 			{
-				// this isn't a problem if CallingConventionAdapters.Waterbox is a no-op
+				// this isn't a problem if CallingConventionAdapters.Waterbox is a no-op, but it is otherwise:  we don't
-				if (CallingConventionAdapters.Waterbox.GetType() != CallingConventionAdapters.Native.GetType())
+				// have a custom marshaller set up so the user needs to manually pump the callingconventionadapter
-					throw new InvalidOperationException("Compatibility call mode cannot use ICallingConventionAdapters for automatically marshalled delegate types!");
+				if (nonTrivialAdapter)
+				{
+					throw new InvalidOperationException(
+						"Compatibility call mode cannot use ICallingConventionAdapters for automatically marshalled delegate types!");
+				}
 			}
 
 			// define a field on the class to hold the delegate

src/BizHawk.BizInvoke/CallingConventionAdapter.cs

@@ -12,10 +12,36 @@ namespace BizHawk.BizInvoke
 	/// </summary>
 	public interface ICallingConventionAdapter
 	{
+		/// <summary>
+		/// Like Marshal.GetFunctionPointerForDelegate(), but wraps a thunk around the returned native pointer
+		/// to adjust the calling convention appropriately
+		/// </summary>
 		IntPtr GetFunctionPointerForDelegate(Delegate d);
+		/// <summary>
+		/// Like Marshal.GetFunctionPointerForDelegate, but only the unmanaged thunk-to-thunk part, with no
+		/// managed wrapper involved.  Called "arrival" because it is to be used when the foreign code is calling
+		/// back into host code.
+		/// </summary>
+		/// <returns></returns>
 		IntPtr GetArrivalFunctionPointer(IntPtr p, ParameterInfo pp, object lifetime);
 
+		/// <summary>
+		/// Like Marshal.GetDelegateForFunctionPointer(), but wraps a thunk around the passed native pointer
+		/// to adjust the calling convention appropriately
+		/// </summary>
+		/// <param name="p"></param>
+		/// <param name="delegateType"></param>
+		/// <returns></returns>
 		Delegate GetDelegateForFunctionPointer(IntPtr p, Type delegateType);
+		/// <summary>
+		/// Like Marshal.GetDelegateForFunctionPointer(), but only the unmanaged thunk-to-thunk part, with no
+		/// managed wrapper involved.static  Called "departure" beause it is to be used when first leaving host
+		/// code for foreign code.
+		/// </summary>
+		/// <param name="p"></param>
+		/// <param name="pp"></param>
+		/// <param name="lifetime"></param>
+		/// <returns></returns>
 		IntPtr GetDepartureFunctionPointer(IntPtr p, ParameterInfo pp, object lifetime);
 	}
 
@@ -50,6 +76,23 @@ namespace BizHawk.BizInvoke
 		}
 	}
 
+	/// <summary>
+	/// Abstract over some waterbox functionality, sort of.  TODO: Would this ever make sense for anything else,
+	/// or maybe it's just actually another CallingConventionAdapter and we're composing them?
+	/// </summary>
+	public interface ICallbackAdjuster
+	{
+		/// <summary>
+		/// Returns a thunk over an arrival callback, for the given slot number.  Slots don't have much of
+		/// any meaning to CallingConvention Adapter; it's just a unique key associated with the callback.
+		/// </summary>
+		IntPtr GetCallbackProcAddr(IntPtr exitPoint, int slot);
+		/// <summary>
+		/// Returns a thunk over a departure point.
+		/// </summary>
+		IntPtr GetCallinProcAddr(IntPtr entryPoint);
+	}
+
 	public static class CallingConventionAdapters
 	{
 		private class NativeConvention : ICallingConventionAdapter
@@ -80,205 +123,111 @@ namespace BizHawk.BizInvoke
 		/// </summary>
 		public static ICallingConventionAdapter Native { get; } = new NativeConvention();
 
-		static CallingConventionAdapters()
+		/// <summary>
+		/// waterbox calling convention, including thunk handling for stack marshalling
+		/// </summary>
+		public static ICallingConventionAdapter MakeWaterbox(IEnumerable<Delegate> slots, ICallbackAdjuster waterboxHost)
 		{
-			Waterbox = OSTailoredCode.IsUnixHost
+			return new WaterboxAdapter(slots, waterboxHost);
-				? new NativeConvention()
+		}
-				: (ICallingConventionAdapter)new MsHostSysVGuest();
+		/// <summary>
-				// ? (ICallingConventionAdapter)new SysVHostMsGuest()
+		/// waterbox calling convention, including thunk handling for stack marshalling.  Can only do callins, not callouts
-				// : new NativeConvention();
+		/// </summary>
+		public static ICallingConventionAdapter MakeWaterboxDepartureOnly(ICallbackAdjuster waterboxHost)
+		{
+			return new WaterboxAdapter(null, waterboxHost);
 		}
 
 		/// <summary>
-		/// convention appropriate for waterbox guests
+		/// Get a waterbox calling convention adapater, except no wrapping is done for stack marshalling and callback support.
+		/// This is very unsafe; any attempts by the guest to call syscalls will crash, and stack hygiene will be all wrong.
+		/// DO NOT USE THIS.
 		/// </summary>
-		public static ICallingConventionAdapter Waterbox { get; }
+		/// <returns></returns>
-
+		public static ICallingConventionAdapter GetWaterboxUnsafeUnwrapped()
-		private class SysVHostMsGuest : ICallingConventionAdapter
 		{
-			private const ulong Placeholder = 0xdeadbeeffeedface;
+			return WaterboxAdapter.WaterboxWrapper;
-			private const byte Padding = 0x06;
-			private const int BlockSize = 256;
-			private static readonly byte[][] Depart =
-			{
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x28, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x28, 0xc3, 0x0f, 0x1f, 0x80, 0x00, 0x00, 0x00, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x28, 0x48, 0x89, 0xf9, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x28, 0xc3, 0x0f, 0x1f, 0x40, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x28, 0x48, 0x89, 0xf9, 0x48, 0x89, 0xf2, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x28, 0xc3, 0x90, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x28, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf9, 0x48, 0x89, 0xf2, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x28, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x1f, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x28, 0x49, 0x89, 0xd0, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf2, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x89, 0xf9, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x28, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x10, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x38, 0xc3, 0x0f, 0x1f, 0x44, 0x00, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x08, 0x41, 0x51, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x38, 0xc3, 0x0f, 0x1f, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x10, 0xff, 0x74, 0x24, 0x18, 0x41, 0x51, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x48, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x1f, 0x40, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x08, 0xff, 0x74, 0x24, 0x18, 0xff, 0x74, 0x24, 0x18, 0x41, 0x51, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x48, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
-			};
-			private static readonly byte[][] Arrive =
-			{
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x56, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x1f, 0x40, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x56, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x90, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x56, 0x48, 0x89, 0xd6, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x0f, 0x1f, 0x44, 0x00, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x0f, 0x1f, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x4c, 0x8b, 0x84, 0x24, 0xe0, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x4c, 0x8b, 0x8c, 0x24, 0xe8, 0x00, 0x00, 0x00, 0x4c, 0x8b, 0x84, 0x24, 0xe0, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x0f, 0x1f, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xb8, 0x00, 0x00, 0x00, 0x48, 0x8b, 0x84, 0x24, 0x00, 0x01, 0x00, 0x00, 0x0f, 0x29, 0x74, 0x24, 0x10, 0x4c, 0x8b, 0x8c, 0x24, 0xf8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x20, 0x4c, 0x8b, 0x84, 0x24, 0xf0, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x89, 0x04, 0x24, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x0f, 0x28, 0x74, 0x24, 0x10, 0x0f, 0x28, 0x7c, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x28, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xb8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x1f, 0x40, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xb8, 0x00, 0x00, 0x00, 0x48, 0x8b, 0x84, 0x24, 0x08, 0x01, 0x00, 0x00, 0x0f, 0x29, 0x74, 0x24, 0x10, 0x4c, 0x8b, 0x8c, 0x24, 0xf8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x20, 0x4c, 0x8b, 0x84, 0x24, 0xf0, 0x00, 0x00, 0x00, 0x48, 0x89, 0x44, 0x24, 0x08, 0x48, 0x8b, 0x84, 0x24, 0x00, 0x01, 0x00, 0x00, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x89, 0x04, 0x24, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x0f, 0x28, 0x74, 0x24, 0x10, 0x0f, 0x28, 0x7c, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x28, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xb8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x90, },
-			};
-			private static readonly int[] DepartPlaceholderIndices;
-			private static readonly int[] ArrivePlaceholderIndices;
-
-			private static int FindPlaceholderIndex(byte[] data)
-			{
-				return Enumerable.Range(0, data.Length - 7)
-					.Single(i => BitConverter.ToUInt64(data, i) == Placeholder);
 		}
 
-			static SysVHostMsGuest()
+		private class WaterboxAdapter : ICallingConventionAdapter
 		{
-				DepartPlaceholderIndices = Depart.Select(FindPlaceholderIndex).ToArray();
+			private class ReferenceEqualityComparer : IEqualityComparer<Delegate>
-				ArrivePlaceholderIndices = Arrive.Select(FindPlaceholderIndex).ToArray();
+			{
-				if (Depart.Any(b => b.Length > BlockSize) || Arrive.Any(b => b.Length > BlockSize))
+				public bool Equals(Delegate x, Delegate y)
-					throw new InvalidOperationException();
+				{
+					return x == y;
 				}
 
-			private readonly MemoryBlock _memory;
+				public int GetHashCode(Delegate obj)
-			private readonly object _sync = new object();
-			private readonly WeakReference[] _refs;
-
-			public SysVHostMsGuest()
 				{
-				int size = 4 * 1024 * 1024;
+					return System.Runtime.CompilerServices.RuntimeHelpers.GetHashCode(obj);
-				_memory = MemoryBlock.Create(0x36a00000000, (ulong)size);
+				}
-				_memory.Activate();
-				_refs = new WeakReference[size / BlockSize];
 			}
 
-			private int FindFreeIndex()
+			internal static readonly ICallingConventionAdapter WaterboxWrapper;
+			static WaterboxAdapter()
 			{
-				for (int i = 0; i < _refs.Length; i++)
+				WaterboxWrapper = OSTailoredCode.IsUnixHost
-				{
+					? new NativeConvention()
-					if (_refs[i] == null || !_refs[i].IsAlive)
+					: (ICallingConventionAdapter)new MsHostSysVGuest();
-						return i;
-				}
-				throw new InvalidOperationException("Out of Thunk memory");
 			}
 
-			private int FindUsedIndex(object lifetime)
+			private readonly Dictionary<Delegate, int> _slots;
-			{
+			private readonly ICallbackAdjuster _waterboxHost;
-				for (int i = 0; i < _refs.Length; i++)
-				{
-					if (_refs[i]?.Target == lifetime)
-						return i;
-				}
-				return -1;
-			}
 
-			private static void VerifyParameter(Type type)
+			public WaterboxAdapter(IEnumerable<Delegate> slots, ICallbackAdjuster waterboxHost)
 			{
-				if (type == typeof(float) || type == typeof(double))
+				if (slots != null)
-					throw new NotSupportedException("floating point not supported");
+				{
-				if (type == typeof(void) || type.IsPrimitive || type.IsEnum)
+					_slots = slots.Select((cb, i) => new { cb, i })
-					return;
+						.ToDictionary(a => a.cb, a => a.i, new ReferenceEqualityComparer());
-				if (type.IsPointer || typeof(Delegate).IsAssignableFrom(type))
-					return;
-				if (type.IsByRef || type.IsClass)
-					return;
-				throw new NotSupportedException($"Unknown type {type}. Possibly supported?");
-			}
-
-			private static int VerifyDelegateSignature(ParameterInfo pp)
-			{
-				VerifyParameter(pp.ReturnType);
-				foreach (var ppp in pp.ParameterTypes)
-					VerifyParameter(ppp);
-				var ret = pp.ParameterTypes.Count;
-				if (ret >= Arrive.Length)
-					throw new InvalidOperationException("Too many parameters to marshal!");
-				return ret;
-			}
-
-			private void WriteThunk(byte[] data, int placeholderIndex, IntPtr p, int index)
-			{
-				_memory.Protect(_memory.Start, _memory.Size, MemoryBlock.Protection.RW);
-				var ss = _memory.GetStream(_memory.Start + (ulong)index * BlockSize, BlockSize, true);
-				ss.Write(data, 0, data.Length);
-				for (int i = data.Length; i < BlockSize; i++)
-					ss.WriteByte(Padding);
-				ss.Position = placeholderIndex;
-				var bw = new BinaryWriter(ss);
-				bw.Write((long)p);
-				_memory.Protect(_memory.Start, _memory.Size, MemoryBlock.Protection.RX);
-			}
-
-			private IntPtr GetThunkAddress(int index)
-			{
-				return Z.US(_memory.Start + (ulong)index * BlockSize);
-			}
-
-			private void SetLifetime(int index, object lifetime)
-			{
-				if (_refs[index] == null)
-					_refs[index] = new WeakReference(lifetime);
-				else
-					_refs[index].Target = lifetime;
-			}
-
-			public IntPtr GetFunctionPointerForDelegate(Delegate d)
-			{
-				// for this call only, the expectation is that it can be called multiple times
-				// on the same delegate and not leak extra memory, so the result has to be cached
-				lock (_sync)
-				{
-					var index = FindUsedIndex(d);
-					if (index != -1)
-					{
-						return GetThunkAddress(index);
-					}
-					else
-					{
-						return GetArrivalFunctionPointer(
-							Marshal.GetFunctionPointerForDelegate(d), new ParameterInfo(d.GetType()), d);
-					}
 				}
+				_waterboxHost = waterboxHost;
 			}
 
 			public IntPtr GetArrivalFunctionPointer(IntPtr p, ParameterInfo pp, object lifetime)
 			{
-				lock (_sync)
+				if (_slots == null)
 				{
-					var index = FindFreeIndex();
+					throw new InvalidOperationException("This calling convention adapter was created for departure only!  Pass known delegate slots when constructing to enable arrival");
-					var count = VerifyDelegateSignature(pp);
-					WriteThunk(Arrive[count], ArrivePlaceholderIndices[count], p, index);
-					SetLifetime(index, lifetime);
-					return GetThunkAddress(index);
 				}
+				if (!(lifetime is Delegate d))
+				{
+					throw new ArgumentException("For this calling convention adapter, lifetimes must be delegate so guest slot can be inferred");
+				}
+				if (!_slots.TryGetValue(d, out var slot))
+				{
+					throw new InvalidOperationException("All callback delegates must be registered at load");
+				}
+				return _waterboxHost.GetCallbackProcAddr(WaterboxWrapper.GetArrivalFunctionPointer(p, pp, lifetime), slot);
 			}
 
 			public Delegate GetDelegateForFunctionPointer(IntPtr p, Type delegateType)
 			{
-				lock (_sync)
+				p = _waterboxHost.GetCallinProcAddr(p);
-				{
+				return WaterboxWrapper.GetDelegateForFunctionPointer(p, delegateType);
-					var index = FindFreeIndex();
-					var count = VerifyDelegateSignature(new ParameterInfo(delegateType));
-					WriteThunk(Depart[count], DepartPlaceholderIndices[count], p, index);
-					var ret = Marshal.GetDelegateForFunctionPointer(GetThunkAddress(index), delegateType);
-					SetLifetime(index, ret);
-					return ret;
-				}
 			}
 
 			public IntPtr GetDepartureFunctionPointer(IntPtr p, ParameterInfo pp, object lifetime)
 			{
-				lock (_sync)
+				p = _waterboxHost.GetCallinProcAddr(p);
+				return WaterboxWrapper.GetDepartureFunctionPointer(p, pp, lifetime);
+			}
+
+			public IntPtr GetFunctionPointerForDelegate(Delegate d)
 			{
-					var index = FindFreeIndex();
+				if (_slots == null)
-					var count = VerifyDelegateSignature(pp);
+				{
-					WriteThunk(Depart[count], DepartPlaceholderIndices[count], p, index);
+					throw new InvalidOperationException("This calling convention adapter was created for departure only!  Pass known delegate slots when constructing to enable arrival");
-					SetLifetime(index, lifetime);
+				}
-					return GetThunkAddress(index);
+				if (!_slots.TryGetValue(d, out var slot))
+				{
+					throw new InvalidOperationException("All callback delegates must be registered at load");
+				}
+				return _waterboxHost.GetCallbackProcAddr(WaterboxWrapper.GetFunctionPointerForDelegate(d), slot);
 			}
 		}
 
-		}
 		private class MsHostSysVGuest : ICallingConventionAdapter
 		{
 			private const ulong Placeholder = 0xdeadbeeffeedface;
@@ -293,8 +242,6 @@ namespace BizHawk.BizInvoke
 				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x28, 0x49, 0x89, 0xd0, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf2, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x89, 0xf9, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x28, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
 				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0x83, 0xec, 0x10, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x38, 0xc3, 0x0f, 0x1f, 0x44, 0x00, 0x00, },
 				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x08, 0x41, 0x51, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x38, 0xc3, 0x0f, 0x1f, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x10, 0xff, 0x74, 0x24, 0x18, 0x41, 0x51, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x48, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x1f, 0x40, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x48, 0x83, 0xec, 0x08, 0xff, 0x74, 0x24, 0x18, 0xff, 0x74, 0x24, 0x18, 0x41, 0x51, 0x49, 0x89, 0xc9, 0x48, 0x89, 0xf9, 0x41, 0x50, 0x49, 0x89, 0xd0, 0x48, 0x89, 0xf2, 0x48, 0x83, 0xec, 0x20, 0xff, 0xd0, 0x48, 0x83, 0xc4, 0x48, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
 			};
 			private static readonly byte[][] Depart =
 			{
@@ -305,8 +252,6 @@ namespace BizHawk.BizInvoke
 				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x0f, 0x1f, 0x00, },
 				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x4c, 0x8b, 0x84, 0x24, 0xe0, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, },
 				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xa8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x34, 0x24, 0x4c, 0x8b, 0x8c, 0x24, 0xe8, 0x00, 0x00, 0x00, 0x4c, 0x8b, 0x84, 0x24, 0xe0, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0xff, 0xd0, 0x0f, 0x28, 0x34, 0x24, 0x0f, 0x28, 0x7c, 0x24, 0x10, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x60, 0x44, 0x0f, 0x28, 0x6c, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0x90, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xa8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x0f, 0x1f, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xb8, 0x00, 0x00, 0x00, 0x48, 0x8b, 0x84, 0x24, 0x00, 0x01, 0x00, 0x00, 0x0f, 0x29, 0x74, 0x24, 0x10, 0x4c, 0x8b, 0x8c, 0x24, 0xf8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x20, 0x4c, 0x8b, 0x84, 0x24, 0xf0, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x89, 0x04, 0x24, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x0f, 0x28, 0x74, 0x24, 0x10, 0x0f, 0x28, 0x7c, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x28, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xb8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x1f, 0x40, 0x00, },
-				new byte[] { 0xf3, 0x0f, 0x1e, 0xfa, 0x57, 0x48, 0x89, 0xcf, 0x4c, 0x89, 0xc9, 0x56, 0x48, 0x89, 0xd6, 0x4c, 0x89, 0xc2, 0x48, 0x81, 0xec, 0xb8, 0x00, 0x00, 0x00, 0x48, 0x8b, 0x84, 0x24, 0x08, 0x01, 0x00, 0x00, 0x0f, 0x29, 0x74, 0x24, 0x10, 0x4c, 0x8b, 0x8c, 0x24, 0xf8, 0x00, 0x00, 0x00, 0x0f, 0x29, 0x7c, 0x24, 0x20, 0x4c, 0x8b, 0x84, 0x24, 0xf0, 0x00, 0x00, 0x00, 0x48, 0x89, 0x44, 0x24, 0x08, 0x48, 0x8b, 0x84, 0x24, 0x00, 0x01, 0x00, 0x00, 0x44, 0x0f, 0x29, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x29, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x29, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x29, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x29, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x29, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x29, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x89, 0x04, 0x24, 0x48, 0xb8, 0xce, 0xfa, 0xed, 0xfe, 0xef, 0xbe, 0xad, 0xde, 0xff, 0xd0, 0x0f, 0x28, 0x74, 0x24, 0x10, 0x0f, 0x28, 0x7c, 0x24, 0x20, 0x44, 0x0f, 0x28, 0x44, 0x24, 0x30, 0x44, 0x0f, 0x28, 0x4c, 0x24, 0x40, 0x44, 0x0f, 0x28, 0x54, 0x24, 0x50, 0x44, 0x0f, 0x28, 0x5c, 0x24, 0x60, 0x44, 0x0f, 0x28, 0xac, 0x24, 0x80, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0x64, 0x24, 0x70, 0x44, 0x0f, 0x28, 0xb4, 0x24, 0x90, 0x00, 0x00, 0x00, 0x44, 0x0f, 0x28, 0xbc, 0x24, 0xa0, 0x00, 0x00, 0x00, 0x48, 0x81, 0xc4, 0xb8, 0x00, 0x00, 0x00, 0x5e, 0x5f, 0xc3, 0x66, 0x90, },
 			};
 			private static readonly int[] DepartPlaceholderIndices;
 			private static readonly int[] ArrivePlaceholderIndices;

src/BizHawk.Emulation.Cores/Consoles/Nintendo/SNES/LibsnesApi.cs

@@ -59,7 +59,7 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 			_readonlyFiles.Add(name);
 		}
 
-		public LibsnesApi(string dllPath, CoreComm comm)
+		public LibsnesApi(string dllPath, CoreComm comm, IEnumerable<Delegate> allCallbacks)
 		{
 			_exe = new WaterboxHost(new WaterboxOptions
 			{
@@ -78,7 +78,7 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 				// Marshal checks that function pointers passed to GetDelegateForFunctionPointer are
 				// _currently_ valid when created, even though they don't need to be valid until
 				// the delegate is later invoked.  so GetInvoker needs to be acquired within a lock.
-				_core = BizInvoker.GetInvoker<CoreImpl>(_exe, _exe, CallingConventionAdapters.Waterbox);
+				_core = BizInvoker.GetInvoker<CoreImpl>(_exe, _exe, CallingConventionAdapters.MakeWaterbox(allCallbacks, _exe));
 				_comm = (CommStruct*)_core.DllInit().ToPointer();
 			}
 		}
@@ -188,6 +188,7 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 		snes_trace_t traceCallback;
 
 		public void QUERY_set_video_refresh(snes_video_refresh_t video_refresh) { this.video_refresh = video_refresh; }
+		// not used??
 		public void QUERY_set_input_poll(snes_input_poll_t input_poll) { this.input_poll = input_poll; }
 		public void QUERY_set_input_state(snes_input_state_t input_state) { this.input_state = input_state; }
 		public void QUERY_set_input_notify(snes_input_notify_t input_notify) { this.input_notify = input_notify; }

src/BizHawk.Emulation.Cores/Consoles/Nintendo/SNES/LibsnesCore.IStatable.cs

@@ -19,12 +19,6 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 			IsLagFrame = reader.ReadBoolean();
 			LagCount = reader.ReadInt32();
 			Frame = reader.ReadInt32();
-			// refresh all callbacks now
-			Api.QUERY_set_video_refresh(snes_video_refresh);
-			Api.QUERY_set_input_poll(snes_input_poll);
-			Api.QUERY_set_input_state(snes_input_state);
-			Api.QUERY_set_input_notify(snes_input_notify);
-			Api.QUERY_set_audio_sample(_soundcb);
 		}
 
 		public byte[] SaveStateBinary()

src/BizHawk.Emulation.Cores/Consoles/Nintendo/SNES/LibsnesCore.cs

@@ -60,8 +60,27 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 			_settings = (SnesSettings)settings ?? new SnesSettings();
 			_syncSettings = (SnesSyncSettings)syncSettings ?? new SnesSyncSettings();
 
+			_videocb = snes_video_refresh;
+			_inputpollcb = snes_input_poll;
+			_inputstatecb = snes_input_state;
+			_inputnotifycb = snes_input_notify;
+			_scanlineStartCb = snes_scanlineStart;
+			_tracecb = snes_trace;
+			_soundcb = snes_audio_sample;
+			_pathrequestcb = snes_path_request;
+
 			// TODO: pass profile here
-			Api = new LibsnesApi(CoreComm.CoreFileProvider.DllPath(), CoreComm)
+			Api = new LibsnesApi(CoreComm.CoreFileProvider.DllPath(), CoreComm, new Delegate[]
+			{
+				_videocb,
+				_inputpollcb,
+				_inputstatecb,
+				_inputnotifycb,
+				_scanlineStartCb,
+				_tracecb,
+				_soundcb,
+				_pathrequestcb
+			})
 			{
 				ReadHook = ReadHook,
 				ExecHook = ExecHook,
@@ -78,12 +97,7 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 
 			Api.CMD_init(_syncSettings.RandomizedInitialState);
 
-			Api.QUERY_set_path_request(snes_path_request);
+			Api.QUERY_set_path_request(_pathrequestcb);
-
-			_scanlineStartCb = snes_scanlineStart;
-			_tracecb = snes_trace;
-
-			_soundcb = snes_audio_sample;
 
 			// start up audio resampler
 			InitAudio();
@@ -174,15 +188,21 @@ namespace BizHawk.Emulation.Cores.Nintendo.SNES
 			}
 
 			Api.QUERY_set_path_request(null);
-			Api.QUERY_set_video_refresh(snes_video_refresh);
+			Api.QUERY_set_video_refresh(_videocb);
-			Api.QUERY_set_input_poll(snes_input_poll);
+			Api.QUERY_set_input_poll(_inputpollcb);
-			Api.QUERY_set_input_state(snes_input_state);
+			Api.QUERY_set_input_state(_inputstatecb);
-			Api.QUERY_set_input_notify(snes_input_notify);
+			Api.QUERY_set_input_notify(_inputnotifycb);
 			Api.QUERY_set_audio_sample(_soundcb);
 			Api.Seal();
 			RefreshPalette();
 		}
 
+		private readonly LibsnesApi.snes_video_refresh_t _videocb;
+		private readonly LibsnesApi.snes_input_poll_t _inputpollcb;
+		private readonly LibsnesApi.snes_input_state_t _inputstatecb;
+		private readonly LibsnesApi.snes_input_notify_t _inputnotifycb;
+		private readonly LibsnesApi.snes_path_request_t _pathrequestcb;
+
 		internal CoreComm CoreComm { get; }
 
 		private readonly string _baseRomPath = "";

src/BizHawk.Emulation.Cores/Consoles/Sega/PicoDrive/PicoDrive.cs

@@ -45,6 +45,8 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.PicoDrive
 
 			_syncSettings = syncSettings ?? new SyncSettings();
 
+			_cdcallback = CDRead;
+
 			_core = PreInit<LibPicoDrive>(new WaterboxOptions
 			{
 				Filename = "picodrive.wbx",
@@ -55,7 +57,7 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.PicoDrive
 				PlainHeapSizeKB = 64,
 				SkipCoreConsistencyCheck = comm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxCoreConsistencyCheck),
 				SkipMemoryConsistencyCheck = comm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxMemoryConsistencyCheck),
-			});
+			}, new Delegate[] { _cdcallback });
 
 			if (has32xBios)
 			{
@@ -72,7 +74,6 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.PicoDrive
 				_exe.AddReadonlyFile(gpgx.GPGX.GetCDData(cd), "toc");
 				_cd = cd;
 				_cdReader = new DiscSectorReader(_cd);
-				_cdcallback = CDRead;
 				_core.SetCDReadCallback(_cdcallback);
 				DriveLightEnabled = true;
 			}

src/BizHawk.Emulation.Cores/Consoles/Sega/gpgx64/GPGX.cs

@@ -31,6 +31,12 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.gpgx
 
 		public GPGX(CoreComm comm, GameInfo game, byte[] rom, IEnumerable<Disc> cds, object settings, object syncSettings)
 		{
+			LoadCallback = new LibGPGX.load_archive_cb(load_archive);
+			_inputCallback = new LibGPGX.input_cb(input_callback);
+			InitMemCallbacks(); // ExecCallback, ReadCallback, WriteCallback
+			CDCallback = new LibGPGX.CDCallback(CDCallbackProc);
+			cd_callback_handle = new LibGPGX.cd_read_cb(CDRead);
+
 			ServiceProvider = new BasicServiceProvider(this);
 			// this can influence some things internally (autodetect romtype, etc)
 			string romextension = "GEN";
@@ -57,23 +63,26 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.gpgx
 				SkipMemoryConsistencyCheck = comm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxMemoryConsistencyCheck),
 			});
 
+			var callingConventionAdapter = CallingConventionAdapters.MakeWaterbox(new Delegate[]
+			{
+				LoadCallback, _inputCallback, ExecCallback, ReadCallback, WriteCallback,
+				CDCallback, cd_callback_handle,
+			}, _elf);
+
 			using (_elf.EnterExit())
 			{
-				Core = BizInvoker.GetInvoker<LibGPGX>(_elf, _elf, CallingConventionAdapters.Waterbox);
+				Core = BizInvoker.GetInvoker<LibGPGX>(_elf, _elf, callingConventionAdapter);
 				_syncSettings = (GPGXSyncSettings)syncSettings ?? new GPGXSyncSettings();
 				_settings = (GPGXSettings)settings ?? new GPGXSettings();
 
 				CoreComm = comm;
 
-				LoadCallback = new LibGPGX.load_archive_cb(load_archive);
-
 				_romfile = rom;
 
 				if (cds != null)
 				{
 					_cds = cds.ToArray();
 					_cdReaders = cds.Select(c => new DiscSectorReader(c)).ToArray();
-					cd_callback_handle = new LibGPGX.cd_read_cb(CDRead);
 					Core.gpgx_set_cdd_callback(cd_callback_handle);
 					DriveLightEnabled = true;
 				}
@@ -110,16 +119,11 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.gpgx
 
 				SetMemoryDomains();
 
-				_inputCallback = new LibGPGX.input_cb(input_callback);
 				Core.gpgx_set_input_callback(_inputCallback);
 
 				// process the non-init settings now
 				PutSettings(_settings);
 
-				//TODO - this hits performance, we need to make it controllable
-				CDCallback = new LibGPGX.CDCallback(CDCallbackProc);
-
-				InitMemCallbacks();
 				KillMemCallbacks();
 
 				_tracer = new GPGXTraceBuffer(this, _memoryDomains, this);
@@ -164,7 +168,7 @@ namespace BizHawk.Emulation.Cores.Consoles.Sega.gpgx
 
 		private bool _disposed = false;
 
-		LibGPGX.load_archive_cb LoadCallback = null;
+		LibGPGX.load_archive_cb LoadCallback;
 
 		LibGPGX.InputData input = new LibGPGX.InputData();
 

src/BizHawk.Emulation.Cores/Waterbox/NymaCore.cs

@@ -32,6 +32,10 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			IDictionary<string, (string SystemID, string FirmwareID)> firmwares = null)
 			where T : LibNymaCore
 		{
+			_settingsQueryDelegate = SettingsQuery;
+			_cdTocCallback = CDTOCCallback;
+			_cdSectorCallback = CDSectorCallback;
+
 			var t = PreInit<T>(new WaterboxOptions
 			{
 				Filename = wbxFilename,
@@ -43,9 +47,8 @@ namespace BizHawk.Emulation.Cores.Waterbox
 				MmapHeapSizeKB = 1024 * 48,
 				SkipCoreConsistencyCheck = CoreComm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxCoreConsistencyCheck),
 				SkipMemoryConsistencyCheck = CoreComm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxMemoryConsistencyCheck),
-			});
+			}, new Delegate[] { _settingsQueryDelegate, _cdTocCallback, _cdSectorCallback });
 			_nyma = t;
-			_settingsQueryDelegate = new LibNymaCore.FrontendSettingQuery(SettingsQuery);
 
 			using (_exe.EnterExit())
 			{
@@ -94,8 +97,6 @@ namespace BizHawk.Emulation.Cores.Waterbox
 				{
 					_disks = discs;
 					_diskReaders = _disks.Select(d => new DiscSectorReader(d) { Policy = _diskPolicy }).ToArray();
-					_cdTocCallback = CDTOCCallback;
-					_cdSectorCallback = CDSectorCallback;
 					_nyma.SetCDCallbacks(_cdTocCallback, _cdSectorCallback);
 					var didInit = _nyma.InitCd(_disks.Length);
 					if (!didInit)
@@ -181,7 +182,7 @@ namespace BizHawk.Emulation.Cores.Waterbox
 					// if (deterministic)
 					// 	throw new InvalidOperationException("Internal error: Core set a frame thread proc in deterministic mode");
 					Console.WriteLine($"Setting up waterbox thread for {_frameThreadPtr}");
-					_frameThreadStart = CallingConventionAdapters.Waterbox.GetDelegateForFunctionPointer<Action>(_frameThreadPtr);
+					_frameThreadStart = CallingConventionAdapters.GetWaterboxUnsafeUnwrapped().GetDelegateForFunctionPointer<Action>(_frameThreadPtr);
 				}
 			}
 

src/BizHawk.Emulation.Cores/Waterbox/WaterboxCore.cs

@@ -5,6 +5,7 @@ using System;
 using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices;
+using System.Collections.Generic;
 
 namespace BizHawk.Emulation.Cores.Waterbox
 {
@@ -43,14 +44,17 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			_inputCallback = InputCallbacks.Call;
 		}
 
-		protected T PreInit<T>(WaterboxOptions options)
+		protected T PreInit<T>(WaterboxOptions options, IEnumerable<Delegate> allExtraDelegates = null)
 			where T : LibWaterboxCore
 		{
 			options.Path ??= CoreComm.CoreFileProvider.DllPath();
 			_exe = new WaterboxHost(options);
+			var delegates = new Delegate[] { _inputCallback }.AsEnumerable();
+			if (allExtraDelegates != null)
+				delegates = delegates.Concat(allExtraDelegates);
 			using (_exe.EnterExit())
 			{
-				var ret = BizInvoker.GetInvoker<T>(_exe, _exe, CallingConventionAdapters.Waterbox);
+				var ret = BizInvoker.GetInvoker<T>(_exe, _exe, CallingConventionAdapters.MakeWaterbox(delegates, _exe));
 				_core = ret;
 				return ret;
 			}

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHost.cs

@@ -68,7 +68,7 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		public bool SkipMemoryConsistencyCheck { get; set; } = false;
 	}
 
-	public unsafe class WaterboxHost : IMonitor, IImportResolver, IBinaryStateable, IDisposable
+	public unsafe class WaterboxHost : IMonitor, IImportResolver, IBinaryStateable, IDisposable, ICallbackAdjuster
 	{
 		private IntPtr _nativeHost;
 		private IntPtr _activatedNativeHost;
@@ -135,7 +135,7 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			using (this.EnterExit())
 			{
 				var retobj = new ReturnData();
-				NativeImpl.wbx_get_proc_addr(_activatedNativeHost, entryPoint, retobj);
+				NativeImpl.wbx_get_proc_addr_raw(_activatedNativeHost, entryPoint, retobj);
 				return retobj.GetDataOrThrow();
 			}
 		}
@@ -153,6 +153,26 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			}
 		}
 
+		public IntPtr GetCallbackProcAddr(IntPtr exitPoint, int slot)
+		{
+			using (this.EnterExit())
+			{
+				var retobj = new ReturnData();
+				NativeImpl.wbx_get_callback_addr(_activatedNativeHost, exitPoint, slot, retobj);
+				return retobj.GetDataOrThrow();
+			}
+		}
+
+		public IntPtr GetCallinProcAddr(IntPtr entryPoint)
+		{
+			using (this.EnterExit())
+			{
+				var retobj = new ReturnData();
+				NativeImpl.wbx_get_callin_addr(_activatedNativeHost, entryPoint, retobj);
+				return retobj.GetDataOrThrow();
+			}
+		}
+
 		public void Seal()
 		{
 			using (this.EnterExit())

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHostNative.cs

@@ -146,14 +146,37 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// </summary>
 		[BizImport(CallingConvention.Cdecl)]
 		public abstract void wbx_deactivate_host(IntPtr /*ActivatedWaterboxHost*/ obj, ReturnData /*void*/ ret);
-
 		/// <summary>
-		/// Returns the address of an exported function from the guest executable.  This pointer is only valid
+		/// Returns a thunk suitable for calling an exported function from the guest executable.  This pointer is only valid
-		/// while the host is active.  A missing proc is not an error and simply returns 0.
+		/// while the host is active.  A missing proc is not an error and simply returns 0.  The guest function must be,
+		/// and the returned callback will be, sysv abi, and will only pass up to 6 int/ptr args and no other arg types.
 		/// </summary>
 		[BizImport(CallingConvention.Cdecl)]
 		public abstract void wbx_get_proc_addr(IntPtr /*ActivatedWaterboxHost*/ obj, string name, ReturnData /*UIntPtr*/ ret);
-
+		/// <summary>
+		/// Returns a thunk suitable for calling an arbitrary entry point into the guest executable.  This pointer is only valid
+		/// while the host is active.  wbx_get_proc_addr already calls this internally on pointers it returns, so this call is
+		/// only needed if the guest exposes callin pointers that aren't named exports (for instance, if a function returns
+		/// a pointer to another function).
+		/// </summary>
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_get_callin_addr(IntPtr /*ActivatedWaterboxHost*/ obj, IntPtr ptr, ReturnData /*UIntPtr*/ ret);
+		/// <summary>
+		/// Returns the raw address of a function exported from the guest.  `wbx_get_proc_addr()` is equivalent to
+		/// `wbx_get_callin_addr(wbx_get_proc_addr_raw()).  Most things should not use this directly, as the returned
+		/// pointer will not have proper stack hygiene and will crash on syscalls from the guest.
+		/// </summary>
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_get_proc_addr_raw(IntPtr /*ActivatedWaterboxHost*/ obj, string name, ReturnData /*UIntPtr*/ ret);
+		/// <summary>
+		/// Returns a function pointer suitable for passing to the guest to allow it to call back while active.
+		/// Slot number is an integer that is used to keep pointers consistent across runs:  If the host is loaded
+		/// at a different address, and some external function `foo` moves from run to run, things will still work out
+		/// in the guest because `foo` was bound to the same slot and a particular slot gives a consistent pointer.
+		/// The returned thunk will be, and the callback must be, sysv abi and will only pass up to 6 int/ptr args and no other arg types.
+		/// </summary>
+		[BizImport(CallingConvention.Cdecl)]
+		public abstract void wbx_get_callback_addr(IntPtr /*ActivatedWaterboxHost*/ obj, IntPtr callback, int slot, ReturnData /*UIntPtr*/ ret);
 		/// <summary>
 		/// Calls the seal operation, which is a one time action that prepares the host to save states.
 		/// </summary>

src/BizHawk.Emulation.Cores/Waterbox/WaterboxMemoryDomain.cs

@@ -17,7 +17,7 @@ namespace BizHawk.Emulation.Cores.Waterbox
 
 		public MemoryArea Definition { get; }
 
-		public static WaterboxMemoryDomain Create(MemoryArea m, IMonitor monitor)
+		public static WaterboxMemoryDomain Create(MemoryArea m, WaterboxHost monitor)
 		{
 			return m.Flags.HasFlag(MemoryDomainFlags.FunctionHook)
 				? (WaterboxMemoryDomain)new WaterboxMemoryDomainFunc(m, monitor)
@@ -138,9 +138,10 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			public IntPtr GetProcAddrOrZero(string entryPoint) => _p;
 		}
 
-		public static MemoryDomainAccessStub Create(IntPtr p, IMonitor monitor)
+		public static MemoryDomainAccessStub Create(IntPtr p, WaterboxHost host)
 		{
-			return BizInvoker.GetInvoker<MemoryDomainAccessStub>(new StubResolver(p), monitor, CallingConventionAdapters.Waterbox);
+			return BizInvoker.GetInvoker<MemoryDomainAccessStub>(
+				new StubResolver(p), host, CallingConventionAdapters.MakeWaterboxDepartureOnly(host));
 		}
 	}
 
@@ -148,7 +149,7 @@ namespace BizHawk.Emulation.Cores.Waterbox
 	{
 		private readonly MemoryDomainAccessStub _access;
 
-		internal WaterboxMemoryDomainFunc(MemoryArea m, IMonitor monitor)
+		internal WaterboxMemoryDomainFunc(MemoryArea m, WaterboxHost monitor)
 			: base(m, monitor)
 		{
 			if (!m.Flags.HasFlag(MemoryDomainFlags.FunctionHook))

waterbox/Notes on Debugging.md

@@ -31,6 +31,8 @@ Bring lots of tools, and lots of self loathing.
 	han SIGSEGV nos nopr
 	```
 	But if the real exception you're trying to break on is a SIGSEGV, this leaves you defenseless.
+	You probably want to use the `no-dirty-detection` feature in waterboxhost to turn off these
+	SIGSEGVs for some kinds of debugging.
 * Also understands symbols for waterboxhost.dll, since that was actually built with MINGW.
 * `b rust_panic` to examine rust unwinds before they explode your computer.
 * Breakpoints on symbols in the wbx file just don't work a lot of the time.

waterbox/emulibc/emulibc.c

@@ -2,28 +2,21 @@
 #include <stdio.h>
 #include <sys/mman.h>
 
-// Keep this in sync with "syscall.h"!!
+// Keep this in sync with the rust code!!
 struct __AddressRange {
 	unsigned long start;
 	unsigned long size;
 };
 struct __WbxSysLayout {
 	struct __AddressRange elf;
+	struct __AddressRange main_thread;
 	struct __AddressRange sbrk;
 	struct __AddressRange sealed;
 	struct __AddressRange invis;
 	struct __AddressRange plain;
 	struct __AddressRange mmap;
 };
-struct __WbxSysSyscall {
+__attribute__((section(".invis"))) __attribute__((visibility("default"))) struct __WbxSysLayout __wbxsysinfo;
-	long ud;
-	void* syscall;
-};
-struct __WbxSysArea {
-	struct __WbxSysLayout layout;
-	struct __WbxSysSyscall syscall;
-};
-extern struct __WbxSysArea __wbxsysarea;
 
 void* alloc_helper(size_t size, const struct __AddressRange* range, unsigned long* current, const char* name)
 {
@@ -63,19 +56,19 @@ void* alloc_helper(size_t size, const struct __AddressRange* range, unsigned lon
 static unsigned long __sealed_current;
 void* alloc_sealed(size_t size)
 {
-	return alloc_helper(size, &__wbxsysarea.layout.sealed, &__sealed_current, "sealed");
+	return alloc_helper(size, &__wbxsysinfo.sealed, &__sealed_current, "sealed");
 }
 
 static unsigned long __invisible_current;
 void* alloc_invisible(size_t size)
 {
-	return alloc_helper(size, &__wbxsysarea.layout.invis, &__invisible_current, "invisible");
+	return alloc_helper(size, &__wbxsysinfo.invis, &__invisible_current, "invisible");
 }
 
 static unsigned long __plain_current;
 void* alloc_plain(size_t size)
 {
-	return alloc_helper(size, &__wbxsysarea.layout.plain, &__plain_current, "plain");
+	return alloc_helper(size, &__wbxsysinfo.plain, &__plain_current, "plain");
 }
 
 // TODO: This existed before we even had stdio support.  Retire?
@@ -88,7 +81,7 @@ ECL_EXPORT void ecl_seal()
 {
 	if (__sealed_current)
 	{
-		if (mprotect((void*)__wbxsysarea.layout.sealed.start, (__sealed_current - __wbxsysarea.layout.sealed.start + 0xfff) & ~0xffful, PROT_READ) != 0)
+		if (mprotect((void*)__wbxsysinfo.sealed.start, (__sealed_current - __wbxsysinfo.sealed.start + 0xfff) & ~0xffful, PROT_READ) != 0)
 			__asm__("int3");
 	}
 }

waterbox/musl

@@ -1 +1 @@
-Subproject commit 0aad5d72332a52b2ebd659b56a98fd117aed8d8f
+Subproject commit 59ad9c6250e6efcc8aae9d6aac127b518d40923a

waterbox/waterboxhost/Cargo.toml

@@ -4,9 +4,10 @@ version = "0.1.0"
 authors = ["nattthebear <goyuken@gmail.com>"]
 edition = "2018"
 publish = false
-rust = "nightly"
+#rust = "nightly"
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+no-dirty-detection = []
 
 [profile.release]
 lto = true

waterbox/waterboxhost/build-debug-no-dirty-detection.bat

@@ -0,0 +1,2 @@
+@cargo b --features "no-dirty-detection"
+@copy target\debug\waterboxhost.dll ..\..\output\dll

waterbox/waterboxhost/src/cinterface.rs

@@ -1,6 +1,7 @@
 use crate::*;
 use host::{ActivatedWaterboxHost, WaterboxHost};
 use std::{os::raw::c_char, io, ffi::{/*CString, */CStr}};
+use context::ExternalCallback;
 
 /// The memory template for a WaterboxHost.  Don't worry about
 /// making every size as small as possible, since the savestater handles sparse regions
@@ -23,34 +24,26 @@ pub struct MemoryLayoutTemplate {
 impl MemoryLayoutTemplate {
 	/// checks a memory layout for validity
 	pub fn make_layout(&self, elf_addr: AddressRange) -> anyhow::Result<WbxSysLayout> {
-		let sbrk_size = align_up(self.sbrk_size);
-		let sealed_size = align_up(self.sealed_size);
-		let invis_size = align_up(self.invis_size);
-		let plain_size = align_up(self.plain_size);
-		let mmap_size = align_up(self.mmap_size);
 		let mut res = unsafe { std::mem::zeroed::<WbxSysLayout>() };
 		res.elf = elf_addr.align_expand();
-		res.sbrk = AddressRange {
+
-			start: res.elf.end(),
+		let mut end = res.elf.end();
-			size: sbrk_size
+		let mut add_area = |size| {
+			let a = AddressRange {
+				start: end,
+				size: align_up(size)
 			};
-		res.sealed = AddressRange {
+			end = a.end();
-			start: res.sbrk.end(),
+			a
-			size: sealed_size
 		};
-		res.invis = AddressRange {
+		res.main_thread = add_area(1 << 20);
-			start: res.sealed.end(),
+		res.sbrk = add_area(self.sbrk_size);
-			size: invis_size
+		res.sealed = add_area(self.sealed_size);
-		};
+		res.invis = add_area(self.invis_size);
-		res.plain = AddressRange {
+		res.plain = add_area(self.plain_size);
-			start: res.invis.end(),
+		res.mmap = add_area(self.mmap_size);
-			size: plain_size
+
-		};
+		if res.all().start >> 32 != (res.all().end() - 1) >> 32 {
-		res.mmap = AddressRange {
-			start: res.plain.end(),
-			size: mmap_size
-		};
-		if res.elf.start >> 32 != (res.mmap.end() - 1) >> 32 {
 			Err(anyhow!("HostMemoryLayout must fit into a single 4GiB region!"))
 		} else {
 			Ok(res)
@@ -209,19 +202,52 @@ pub extern fn wbx_deactivate_host(obj: *mut ActivatedWaterboxHost, ret: &mut Ret
 	ret.put(Ok(()));
 }
 
-/// Returns the address of an exported function from the guest executable.  This pointer is only valid
+/// Returns a thunk suitable for calling an exported function from the guest executable.  This pointer is only valid
-/// while the host is active.  A missing proc is not an error and simply returns 0.
+/// while the host is active.  A missing proc is not an error and simply returns 0.  The guest function must be,
+/// and the returned callback will be, sysv abi, and will only pass up to 6 int/ptr args and no other arg types.
 #[no_mangle]
 pub extern fn wbx_get_proc_addr(obj: &mut ActivatedWaterboxHost, name: *const c_char, ret: &mut Return<usize>) {
 	match arg_to_str(name) {
 		Ok(s) => {
-			ret.put(Ok(obj.get_proc_addr(&s)));
+			ret.put(obj.get_proc_addr(&s));
 		},
 		Err(e) => {
 			ret.put(Err(e))
 		}
 	}
 }
+/// Returns a thunk suitable for calling an arbitrary entry point into the guest executable.  This pointer is only valid
+/// while the host is active.  wbx_get_proc_addr already calls this internally on pointers it returns, so this call is
+/// only needed if the guest exposes callin pointers that aren't named exports (for instance, if a function returns
+/// a pointer to another function).
+#[no_mangle]
+pub extern fn wbx_get_callin_addr(obj: &mut ActivatedWaterboxHost, ptr: usize, ret: &mut Return<usize>) {
+	ret.put(obj.get_external_callin_ptr(ptr));
+}
+/// Returns the raw address of a function exported from the guest.  `wbx_get_proc_addr()` is equivalent to
+/// `wbx_get_callin_addr(wbx_get_proc_addr_raw()).  Most things should not use this directly, as the returned
+/// pointer will not have proper stack hygiene and will crash on syscalls from the guest.
+#[no_mangle]
+pub extern fn wbx_get_proc_addr_raw(obj: &mut ActivatedWaterboxHost, name: *const c_char, ret: &mut Return<usize>) {
+	match arg_to_str(name) {
+		Ok(s) => {
+			ret.put(obj.get_proc_addr_raw(&s));
+		},
+		Err(e) => {
+			ret.put(Err(e))
+		}
+	}
+}
+
+/// Returns a function pointer suitable for passing to the guest to allow it to call back while active.
+/// Slot number is an integer that is used to keep pointers consistent across runs:  If the host is loaded
+/// at a different address, and some external function `foo` moves from run to run, things will still work out
+/// in the guest because `foo` was bound to the same slot and a particular slot gives a consistent pointer.
+/// The returned thunk will be, and the callback must be, sysv abi and will only pass up to 6 int/ptr args and no other arg types.
+#[no_mangle]
+pub extern fn wbx_get_callback_addr(obj: &mut ActivatedWaterboxHost, callback: ExternalCallback, slot: usize, ret: &mut Return<usize>) {
+	ret.put(obj.get_external_callback_ptr(callback, slot));
+}
 
 /// Calls the seal operation, which is a one time action that prepares the host to save states.
 #[no_mangle]

waterbox/waterboxhost/src/context/Makefile

@@ -0,0 +1,2 @@
+interop.bin: interop.s
+	nasm -f bin -Wall -o $@ $<

waterbox/waterboxhost/src/context/interop.s

@@ -0,0 +1,86 @@
+bits 64
+org 0x35f00000000
+
+struc Context
+	.host_rsp resq 1
+	.guest_rsp resq 1
+	.dispatch_syscall resq 1
+	.host_ptr resq 1
+	.extcall_slots resq 64
+endstruc
+
+; sets up guest stack and calls a function
+; r11 - guest entry point
+; r10 - address of context structure
+; regular arg registers are 0..6 args passed through to guest
+call_guest_impl:
+	mov [gs:0x18], r10
+	mov [r10 + Context.host_rsp], rsp
+	mov rsp, [r10 + Context.guest_rsp]
+	call r11 ; stack hygiene note - this host address is saved on the guest stack
+	mov r10, [gs:0x18]
+	mov [r10 + Context.guest_rsp], rsp ; restore stack so next call using same Context will work
+	mov rsp, [r10 + Context.host_rsp]
+	mov r11, 0
+	mov [gs:0x18], r11
+	ret
+align 64, int3
+
+; alternative to guest call thunks for functions with 0 args
+; rdi - guest entry point
+; rsi - address of context structure
+call_guest_simple:
+	mov r11, rdi
+	mov r10, rsi
+	jmp call_guest_impl
+align 64, int3
+
+; called by guest when it wishes to make a syscall
+; must be loaded at fixed address, as that address is burned into guest executables
+; rax - syscall number
+; regular arg registers are 0..6 args to the syscall
+guest_syscall:
+	mov r10, [gs:0x18]
+	mov [r10 + Context.guest_rsp], rsp
+	mov rsp, [r10 + Context.host_rsp]
+	sub rsp, 8 ; align
+	mov r11, [r10 + Context.host_ptr]
+	push r11 ; arg 8 to dispatch_syscall: host
+	push rax ; arg 7 to dispatch_syscall: nr
+	mov rax, [r10 + Context.dispatch_syscall]
+	call rax
+	mov r10, [gs:0x18]
+	mov rsp, [r10 + Context.guest_rsp]
+	ret
+align 64, int3
+
+; called by individual extcall thunks when the guest wishes to make an external call
+; (very similar to guest_syscall)
+; rax - slot number
+; regular arg registers are 0..6 args to the extcall
+guest_extcall_impl:
+	mov r10, [gs:0x18]
+	mov [r10 + Context.guest_rsp], rsp
+	mov rsp, [r10 + Context.host_rsp]
+	mov r11, [r10 + Context.extcall_slots + rax * 8] ; get slot ptr
+	sub rsp, 8 ; align
+	call r11
+	mov r10, [gs:0x18]
+	mov rsp, [r10 + Context.guest_rsp]
+	ret
+align 64, int3
+
+; individual thunks to each of 64 call slots
+; should be in fixed locations for memory hygiene in the core, since they may be stored there for some time
+
+%macro guest_extcall_thunk 1
+	mov rax, %1
+	jmp guest_extcall_impl
+	align 16, int3
+%endmacro
+
+%assign j 0
+%rep 64
+	guest_extcall_thunk j
+	%assign j j+1
+%endrep

waterbox/waterboxhost/src/context/mod.rs

@@ -0,0 +1,91 @@
+use lazy_static::lazy_static;
+use crate::*;
+use memory_block::{Protection, pal};
+use host::{ActivatedWaterboxHost};
+use syscall_defs::SyscallNumber;
+
+pub mod thunks;
+
+// manually match these up with interop.s
+const ORG: usize = 0x35f00000000;
+
+const CALL_GUEST_IMPL_ADDR: usize = ORG;
+const CALL_GUEST_SIMPLE_ADDR: usize = ORG + 64;
+
+pub const CALLBACK_SLOTS: usize = 64;
+/// Retrieves a function pointer suitable for sending to the guest that will cause
+/// the host to callback to `slot` when called
+pub fn get_callback_ptr(slot: usize) -> usize{
+	assert!(slot < CALLBACK_SLOTS);
+	ORG + 0x100 + slot * 16
+}
+
+fn init_interop_area() -> AddressRange {
+	unsafe {
+		let bytes = include_bytes!("interop.bin");
+		let addr = pal::map_anon(
+			AddressRange { start: ORG, size: bytes.len() }.align_expand(),
+			Protection::RW).unwrap();
+		addr.slice_mut()[0..bytes.len()].copy_from_slice(bytes);
+		pal::protect(addr, Protection::RX).unwrap();
+		addr
+	}
+}
+
+lazy_static! {
+	static ref INTEROP_AREA: AddressRange = init_interop_area();
+}
+
+// https://github.com/rust-lang/rust/issues/53605
+#[repr(C)]
+union FuncCast<T: Copy> {
+	pub p: usize,
+	pub f: T,
+}
+
+/// Enter waterbox code with a function that takes 0 arguments
+/// Returns the function's return value
+const CALL_GUEST_SIMPLE: FuncCast<extern "sysv64" fn(entry_point: usize, context: &mut Context) -> usize> = FuncCast { p: CALL_GUEST_SIMPLE_ADDR };
+/// Enter waterbox code with a function that takes 0 arguments
+/// Returns the function's return value
+pub fn call_guest_simple(entry_point: usize, context: &mut Context) -> usize{
+	unsafe { (CALL_GUEST_SIMPLE.f)(entry_point, context) }
+}
+
+
+pub type ExternalCallback = extern "sysv64" fn(
+	a1: usize, a2: usize, a3: usize, a4: usize, a5: usize, a6: usize) -> usize;
+pub type SyscallCallback = extern "sysv64" fn(
+	a1: usize, a2: usize, a3: usize, a4: usize, a5: usize, a6: usize, nr: SyscallNumber, h: &mut ActivatedWaterboxHost) -> SyscallReturn;
+
+/// Layout must be synced with interop.s
+#[repr(C)]
+pub struct Context {
+	pub host_rsp: usize,
+	pub guest_rsp: usize,
+	pub dispatch_syscall: SyscallCallback,
+	pub host_ptr: usize,
+	pub extcall_slots: [Option<ExternalCallback>; 64],
+}
+
+/// Prepares this host thread to be allowed to call guest code.  Noop if already called.
+/// Only needs to happen once per host thread
+pub fn prepare_thread() {
+	// not per-thread setup, but setup that needs to happen anyway
+	// todo: lazy_static isn't really the right idea here since we discard the value
+	assert_eq!(INTEROP_AREA.start, ORG);
+
+	// We stomp over [gs:0x18] and use it for our own mini-TLS to track the stack marshalling
+	// On windows, that's a (normally unused and free for the plundering?) field in TIB
+	// On linux, that register is not normally in use, so we put some bytes there and then use it
+	#[cfg(unix)]
+	unsafe {
+		use libc::*;
+		let mut gs = 0usize;
+		assert_eq!(syscall(SYS_arch_prctl, 0x1004 /*ARCH_GET_GS*/, &gs), 0);
+		if gs == 0 {
+			gs = Box::into_raw(Box::new([0usize; 4])) as usize;
+			assert_eq!(syscall(SYS_arch_prctl, 0x1001 /*ARCH_SET_GS*/, gs), 0);
+		}
+	}
+}

waterbox/waterboxhost/src/context/thunks.rs

@@ -0,0 +1,79 @@
+use crate::*;
+use memory_block::{Protection, pal};
+use context::{CALL_GUEST_IMPL_ADDR, Context};
+use std::collections::HashMap;
+
+// each generated thunk should look like this:
+// 49 ba .. .. .. .. .. .. .. ..             mov r10, <host_context_ptr>
+// 49 bb .. .. .. .. .. .. .. ..             mov r11, <guest_entry_point>
+// 48 b8 .. .. .. .. .. .. .. ..             mov rax, <call_guest_impl>
+// ff e0                                     jmp rax
+
+const THUNK_SIZE: usize = 32;
+
+pub struct ThunkManager {
+	memory: AddressRange,
+	lookup: HashMap<usize, usize>,
+}
+impl ThunkManager {
+	pub fn new() -> anyhow::Result<ThunkManager> {
+		let addr = pal::map_anon(AddressRange { start: 0, size: PAGESIZE }, Protection::RWX)?;
+		Ok(ThunkManager {
+			memory: addr,
+			lookup: HashMap::new(),
+		})
+	}
+	/// Generates a thunk for calling into waterbox.
+	/// Only valid so long as this ThunkManager is alive and set_context_ptr is kept up to date
+	pub fn get_thunk_for_proc(&mut self, guest_entry_point: usize, context: *mut Context) -> anyhow::Result<usize> {
+		match self.lookup.get(&guest_entry_point) {
+			Some(p) => return Ok(*p),
+			None => ()
+		}
+		let offset = self.lookup.len() * THUNK_SIZE;
+		let p = self.memory.start + offset;
+		if p >= self.memory.end() {
+			Err(anyhow!("No room for another thunk!"))
+		} else {
+			unsafe {
+				let dest = &mut &mut self.memory.slice_mut()[offset..offset + THUNK_SIZE];
+				// mov r10, <host_context_ptr>
+				bin::writeval::<u8>(dest, 0x49)?;
+				bin::writeval::<u8>(dest, 0xba)?;
+				bin::writeval(dest, context as usize)?;
+
+				// mov r11, <guest_entry_point>
+				bin::writeval::<u8>(dest, 0x49)?;
+				bin::writeval::<u8>(dest, 0xbb)?;
+				bin::writeval(dest, guest_entry_point as usize)?;
+
+				// mov rax, <call_guest_impl>
+				bin::writeval::<u8>(dest, 0x48)?;
+				bin::writeval::<u8>(dest, 0xb8)?;
+				bin::writeval(dest, CALL_GUEST_IMPL_ADDR)?;
+
+				// jmp rax
+				bin::writeval::<u8>(dest, 0xff)?;
+				bin::writeval::<u8>(dest, 0xe0)?;
+
+				self.lookup.insert(guest_entry_point, p);
+				Ok(p)
+			}
+		}
+	}
+	// /// updates context value for all previously created thunks
+	// pub fn update_context_ptr(&mut self, context: *mut Context) -> anyhow::Result<()> {
+	// 	unsafe {
+	// 		let slice = self.memory.slice_mut();
+	// 		for i in 0..self.lookup.len() {
+	// 			bin::writeval(&mut &mut slice[i * THUNK_SIZE + 2..i * THUNK_SIZE + 10], context as usize)?;
+	// 		}
+	// 	}
+	// 	Ok(())
+	// }
+}
+impl Drop for ThunkManager {
+	fn drop(&mut self) {
+		unsafe { pal::unmap_anon(self.memory).unwrap(); }
+	}
+}

waterbox/waterboxhost/src/elf.rs

@@ -6,7 +6,7 @@ use crate::memory_block::Protection;
 use std::collections::HashMap;
 
 /// Special system import area
-const IMPORTS_OBJECT_NAME: &str = "__wbxsysarea";
+const INFO_OBJECT_NAME: &str = "__wbxsysinfo";
 
 /// Section names that are not marked as readonly, but we'll make them readonly anyway
 fn section_name_is_readonly(name: &str) -> bool {
@@ -27,7 +27,6 @@ pub struct ElfLoader {
 	exports: HashMap<String, AddressRange>,
 	entry_point: usize,
 	hash: Vec<u8>,
-	import_area: AddressRange,
 }
 impl ElfLoader {
 	pub fn elf_addr(wbx: &Elf) -> AddressRange {
@@ -82,7 +81,7 @@ impl ElfLoader {
 		}
 
 		let mut exports = HashMap::new();
-		let mut import_area_opt = None;
+		let mut info_area_opt = None;
 
 		for sym in wbx.syms.iter() {
 			let name = match wbx.strtab.get(sym.st_name) {
@@ -95,21 +94,11 @@ impl ElfLoader {
 					AddressRange { start: sym.st_value as usize, size: sym.st_size as usize }
 				);
 			}
-			if name == IMPORTS_OBJECT_NAME {
+			if name == INFO_OBJECT_NAME {
-				import_area_opt = Some(AddressRange { start: sym.st_value as usize, size: sym.st_size as usize });
+				info_area_opt = Some(AddressRange { start: sym.st_value as usize, size: sym.st_size as usize });
 			}
 		}
 
-		let import_area = match import_area_opt {
-			Some(i) => {
-				if i.size != std::mem::size_of::<WbxSysArea>() {
-					return Err(anyhow!("Symbol {} is the wrong size", IMPORTS_OBJECT_NAME))
-				}
-				i
-			},
-			None => return Err(anyhow!("Symbol {} is missing", IMPORTS_OBJECT_NAME))
-		};
-
 		{
 			let invis_opt = sections.iter().find(|x| x.name == ".invis");
 			if let Some(invis) = invis_opt {
@@ -166,48 +155,40 @@ impl ElfLoader {
 			b.mprotect(prot_addr, prot)?;
 		}
 
+		match info_area_opt {
+			Some(i) => {
+				if i.size != std::mem::size_of::<WbxSysLayout>() {
+					return Err(anyhow!("Symbol {} is the wrong size", INFO_OBJECT_NAME))
+				}
+				unsafe { *(i.start as *mut WbxSysLayout) = *layout; }
+			},
+			// info area can legally be missing if the core calls no emulibc functions
+			// None => return Err(anyhow!("Symbol {} is missing", INFO_OBJECT_NAME))
+			None => ()
+		};
+
+		// Main thread area.  TODO:  Should this happen here?
+		b.mmap_fixed(layout.main_thread, Protection::RWStack, true)?;
+		b.mprotect(AddressRange { start: layout.main_thread.start, size: PAGESIZE * 4 }, Protection::None)?;
+		b.mark_invisible(layout.main_thread)?;
+
 		Ok(ElfLoader {
 			sections,
 			exports,
 			entry_point: wbx.entry as usize,
-			hash: bin::hash(data),
+			hash: bin::hash(data)
-			import_area
 		})
 	}
-	pub fn pre_seal(&mut self, b: &mut ActivatedMemoryBlock) {
+	pub fn seal(&mut self, b: &mut ActivatedMemoryBlock) {
-		self.run_proc(b, "co_clean");
-		self.run_proc(b, "ecl_seal");
 		for section in self.sections.iter() {
 			if section_name_is_readonly(section.name.as_str()) {
 				b.mprotect(section.addr.align_expand(), Protection::R).unwrap();
 			}
 		}
-		self.clear_syscalls(b);
-	}
-	pub fn connect_syscalls(&mut self, _b: &mut ActivatedMemoryBlock, sys: &WbxSysArea) {
-		let addr = self.import_area;
-		unsafe { *(addr.start as *mut WbxSysArea) = *sys; }
-	}
-	fn clear_syscalls(&mut self, _b: &mut ActivatedMemoryBlock) {
-		let addr = self.import_area;
-		unsafe { addr.zero(); }
-	}
-	pub fn native_init(&mut self, _b: &mut ActivatedMemoryBlock) {
-		println!("Calling _start()");
-		unsafe {
-			std::mem::transmute::<usize, extern "sysv64" fn() -> ()>(self.entry_point)();
-		}
-	}
-	fn run_proc(&mut self, _b: &mut ActivatedMemoryBlock, name: &str) {
-		match self.get_proc_addr(name) {
-			0 => (),
-			ptr => {
-				println!("Calling {}()", name);
-				unsafe {
-					std::mem::transmute::<usize, extern "sysv64" fn() -> ()>(ptr)();
-				}
-			},
 	}
+
+	pub fn entry_point(&self) -> usize {
+		self.entry_point
 	}
 	pub fn get_proc_addr(&self, proc: &str) -> usize {
 		match self.exports.get(proc) {

waterbox/waterboxhost/src/host.rs

@@ -6,6 +6,7 @@ use fs::{FileDescriptor, FileSystem/*, MissingFileCallback*/};
 use elf::ElfLoader;
 use cinterface::MemoryLayoutTemplate;
 use goblin::elf::Elf;
+use context::{CALLBACK_SLOTS, Context, ExternalCallback, thunks::ThunkManager};
 
 pub struct WaterboxHost {
 	fs: FileSystem,
@@ -16,9 +17,12 @@ pub struct WaterboxHost {
 	active: bool,
 	sealed: bool,
 	image_file: Vec<u8>,
+	context: Context,
+	thunks: ThunkManager,
 }
 impl WaterboxHost {
 	pub fn new(image_file: Vec<u8>, module_name: &str, layout_template: &MemoryLayoutTemplate) -> anyhow::Result<Box<WaterboxHost>> {
+		let thunks = ThunkManager::new()?;
 		let wbx = Elf::parse(&image_file[..])?;
 		let elf_addr = ElfLoader::elf_addr(&wbx);
 		let layout = layout_template.make_layout(elf_addr)?;
@@ -37,10 +41,19 @@ impl WaterboxHost {
 			active: false,
 			sealed: false,
 			image_file,
+			context: Context {
+				host_rsp: 0,
+				guest_rsp: layout.main_thread.end(),
+				dispatch_syscall: syscall,
+				host_ptr: 0,
+				extcall_slots: [None; 64],
+			},
+			thunks,
 		});
 
 		let mut active = res.activate();
-		active.h.elf.native_init(&mut active.b);
+		println!("Calling _start()");
+		active.run_guest_simple(active.h.elf.entry_point());
 		drop(active);
 
 		Ok(res)
@@ -51,24 +64,15 @@ impl WaterboxHost {
 	}
 
 	pub fn activate(&mut self) -> Box<ActivatedWaterboxHost> {
+		context::prepare_thread();
 		let h = unsafe { &mut *(self as *mut WaterboxHost) };
 		let b = self.memory_block.enter();
-		let sys = WbxSysArea {
-			layout: self.layout,
-			syscall: WbxSysSyscall {
-				ud: 0,
-				syscall,
-			}
-		};
 		let mut res = Box::new(ActivatedWaterboxHost {
-			tag: TAG,
 			h,
 			b,
-			sys
 		});
-		res.sys.syscall.ud = res.as_mut() as *mut ActivatedWaterboxHost as usize;
-		res.h.elf.connect_syscalls(&mut res.b, &res.sys);
 		res.h.active = true;
+		res.h.context.host_ptr = res.as_mut() as *mut ActivatedWaterboxHost as usize;
 		res
 	}
 }
@@ -78,22 +82,40 @@ impl Drop for WaterboxHost {
 	}
 }
 
-const TAG: u64 = 0xd01487803948acff;
 pub struct ActivatedWaterboxHost<'a> {
-	tag: u64,
 	h: &'a mut WaterboxHost,
 	b: ActivatedMemoryBlock<'a>,
-	sys: WbxSysArea,
 }
 impl<'a> Drop for ActivatedWaterboxHost<'a> {
 	fn drop(&mut self) {
 		self.h.active = false;
+		self.h.context.host_ptr = 0;
 	}
 }
 
 impl<'a> ActivatedWaterboxHost<'a> {
-	pub fn get_proc_addr(&self, name: &str) -> usize {
+	pub fn get_external_callback_ptr(&mut self, callback: ExternalCallback, slot: usize) -> anyhow::Result<usize> {
-		self.h.elf.get_proc_addr(name)
+		if slot >= CALLBACK_SLOTS {
+			Err(anyhow!("slot must be less than {}", CALLBACK_SLOTS))
+		} else {
+			self.h.context.extcall_slots[slot] = Some(callback);
+			Ok(context::get_callback_ptr(slot))
+		}
+	}
+	pub fn get_external_callin_ptr(&mut self, ptr: usize) -> anyhow::Result<usize> {
+		self.h.thunks.get_thunk_for_proc(ptr, &mut self.h.context as *mut Context)
+	}
+	pub fn get_proc_addr(&mut self, name: &str) -> anyhow::Result<usize> {
+		let ptr = self.h.elf.get_proc_addr(name);
+		if ptr == 0 {
+			Ok(0)
+		} else {
+			self.h.thunks.get_thunk_for_proc(ptr, &mut self.h.context as *mut Context)
+		}
+	}
+	pub fn get_proc_addr_raw(&mut self, name: &str) -> anyhow::Result<usize> {
+		let ptr = self.h.elf.get_proc_addr(name);
+		Ok(ptr)
 	}
 	fn check_sealed(&self) -> anyhow::Result<()> {
 		if !self.h.sealed {
@@ -106,9 +128,21 @@ impl<'a> ActivatedWaterboxHost<'a> {
 		if self.h.sealed {
 			return Err(anyhow!("Already sealed!"))
 		}
-		self.h.elf.pre_seal(&mut self.b);
+
+		fn run_proc(h: &mut ActivatedWaterboxHost, name: &str) {
+			match h.h.elf.get_proc_addr(name) {
+				0 => (),
+				ptr => {
+					println!("Calling {}()", name);
+					h.run_guest_simple(ptr);
+				},
+			}
+		}
+		run_proc(self, "co_clean");
+		run_proc(self, "ecl_seal");
+
+		self.h.elf.seal(&mut self.b);
 		self.b.seal();
-		self.h.elf.connect_syscalls(&mut self.b, &self.sys);
 		self.h.sealed = true;
 		Ok(())
 	}
@@ -121,6 +155,11 @@ impl<'a> ActivatedWaterboxHost<'a> {
 	// pub fn set_missing_file_callback(&mut self, cb: Option<MissingFileCallback>) {
 	// 	self.h.fs.set_missing_file_callback(cb);
 	// }
+
+	/// Run a guest entry point that takes no arguments
+	pub fn run_guest_simple(&mut self, entry_point: usize) {
+		context::call_guest_simple(entry_point, &mut self.h.context);
+	}
 }
 
 const SAVE_START_MAGIC: &str = "ActivatedWaterboxHost_v1";
@@ -144,7 +183,6 @@ impl<'a> IStateable for ActivatedWaterboxHost<'a> {
 		self.h.elf.load_state(stream)?;
 		self.b.load_state(stream)?;
 		bin::verify_magic(stream, SAVE_END_MAGIC)?;
-		self.h.elf.connect_syscalls(&mut self.b, &self.sys);
 		Ok(())
 	}
 }
@@ -155,15 +193,6 @@ fn unimp(nr: SyscallNumber) -> SyscallResult {
 	Err(ENOSYS)
 }
 
-fn gethost<'a>(ud: usize) -> &'a mut ActivatedWaterboxHost<'a> {
-	let res = unsafe { &mut *(ud as *mut ActivatedWaterboxHost) };
-	if res.tag != TAG {
-		unsafe { std::intrinsics::breakpoint() }
-		std::process::abort();
-	}
-	res
-}
-
 fn arg_to_prot(arg: usize) -> Result<Protection, SyscallError> {
 	use Protection::*;
 	if arg != arg & (PROT_READ | PROT_WRITE | PROT_EXEC) {
@@ -203,8 +232,10 @@ fn arg_to_statbuff<'a>(arg: usize) -> &'a mut KStat {
 	unsafe { &mut *(arg as *mut KStat) }
 }
 
-pub extern "sysv64" fn syscall(nr: SyscallNumber, ud: usize, a1: usize, a2: usize, a3: usize, a4: usize, _a5: usize, _a6: usize) -> SyscallReturn {
+extern "sysv64" fn syscall(
-	let h = gethost(ud);
+	a1: usize, a2: usize, a3: usize, a4: usize, _a5: usize, _a6: usize,
+	nr: SyscallNumber, h: &mut ActivatedWaterboxHost
+) -> SyscallReturn {
 	match nr {
 		NR_MMAP => {
 			let mut prot = arg_to_prot(a3)?;
@@ -227,12 +258,12 @@ pub extern "sysv64" fn syscall(nr: SyscallNumber, ud: usize, a1: usize, a2: usiz
 				}
 			}
 			let no_replace = flags & MAP_FIXED_NOREPLACE != 0;
-			let arena_addr = h.sys.layout.mmap;
+			let arena_addr = h.h.layout.mmap;
 			let res = h.b.mmap(AddressRange { start: a1, size: a2 }, prot, arena_addr, no_replace)?;
 			syscall_ok(res)
 		},
 		NR_MREMAP => {
-			let arena_addr = h.sys.layout.mmap;
+			let arena_addr = h.h.layout.mmap;
 			let res = h.b.mremap(AddressRange { start: a1, size: a2 }, a3, arena_addr)?;
 			syscall_ok(res)
 		},
@@ -313,7 +344,7 @@ pub extern "sysv64" fn syscall(nr: SyscallNumber, ud: usize, a1: usize, a2: usiz
 		},
 		NR_BRK => {
 			// TODO: This could be done on the C side
-			let addr = h.sys.layout.sbrk;
+			let addr = h.h.layout.sbrk;
 			let old = h.h.program_break;
 			let res = if a1 != align_down(a1) {
 				old

waterbox/waterboxhost/src/lib.rs

@@ -7,7 +7,7 @@
 
 use std::io::{Read, Write};
 use anyhow::anyhow;
-use syscall_defs::{SyscallNumber, SyscallReturn};
+use syscall_defs::{SyscallReturn};
 
 const PAGESIZE: usize = 0x1000;
 const PAGEMASK: usize = 0xfff;
@@ -21,6 +21,7 @@ mod fs;
 mod host;
 mod cinterface;
 mod gdb;
+mod context;
 
 pub trait IStateable {
 	fn save_state(&mut self, stream: &mut dyn Write) -> anyhow::Result<()>;
@@ -85,6 +86,7 @@ fn align_up(p: usize) -> usize {
 #[derive(Copy, Clone)]
 pub struct WbxSysLayout {
 	pub elf: AddressRange,
+	pub main_thread: AddressRange,
 	pub sbrk: AddressRange,
 	pub sealed: AddressRange,
 	pub invis: AddressRange,
@@ -100,22 +102,6 @@ impl WbxSysLayout {
 	}
 }
 
-/// Information for making syscalls injected into the guest application
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct WbxSysSyscall {
-	pub ud: usize,
-	pub syscall: extern "sysv64" fn(nr: SyscallNumber, ud: usize, a1: usize, a2: usize, a3: usize, a4: usize, a5: usize, a6: usize) -> SyscallReturn,
-}
-
-/// Data that is injected into the guest application
-#[repr(C)]
-#[derive(Copy, Clone)]
-pub struct WbxSysArea {
-	pub layout: WbxSysLayout,
-	pub syscall: WbxSysSyscall,
-}
-
 /// Always remove memoryblocks from active ram when possible, to help debug dangling pointers.
 /// Severe performance consequences.
 static mut ALWAYS_EVICT_BLOCKS: bool = true;

waterbox/waterboxhost/src/memory_block/mod.rs

@@ -1,5 +1,5 @@
 mod pageblock;
-mod pal;
+pub mod pal;
 mod tripguard;
 mod tests;
 
@@ -239,13 +239,19 @@ impl MemoryBlock {
 		if addr.start >> 32 != (addr.end() - 1) >> 32 {
 			panic!("MemoryBlock must fit into a single 4G region!");
 		}
+
 		let npage = addr.size >> PAGESHIFT;
 		let mut pages = Vec::new();
 		pages.reserve_exact(npage);
 		for _ in 0..npage {
 			pages.push(Page::new());
 		}
-		let handle = pal::open(addr.size).unwrap();
+		#[cfg(feature = "no-dirty-detection")]
+		for p in pages.iter_mut() {
+			p.dirty = true;
+		}
+
+		let handle = pal::open_handle(addr.size).unwrap();
 		let lock_index = (addr.start >> 32) as u32;
 		// add the lock_index stuff now, so we won't have to check for it later on activate / drop
 		lock_list::maybe_add(lock_index);
@@ -337,14 +343,14 @@ impl MemoryBlock {
 
 	unsafe fn swapin(&mut self) {
 		// self.trace("swapin");
-		assert!(pal::map(&self.handle, self.addr));
+		pal::map_handle(&self.handle, self.addr).unwrap();
 		tripguard::register(self);
 		self.refresh_all_protections();
 	}
 	unsafe fn swapout(&mut self) {
 		// self.trace("swapout");
 		self.get_stack_dirty();
-		assert!(pal::unmap(self.addr));
+		pal::unmap_handle(self.addr).unwrap();
 		tripguard::unregister(self);
 	}
 
@@ -400,7 +406,7 @@ impl MemoryBlock {
 
 		for c in chunks {
 			unsafe {
-				assert!(pal::protect(c.addr, c.prot));
+				pal::protect(c.addr, c.prot).unwrap();
 			}
 		}
 	}
@@ -471,7 +477,7 @@ impl Drop for MemoryBlock {
 			None => ()
 		}
 		let h = std::mem::replace(&mut self.handle, pal::bad());
-		unsafe { pal::close(h); }
+		unsafe { let _ = pal::close_handle(h); }
 	}
 }
 
@@ -587,7 +593,7 @@ impl<'block> ActivatedMemoryBlock<'block> {
 			old_status.push(p.status);
 		}
 		unsafe {
-			pal::protect(src_addr, Protection::R);
+			pal::protect(src_addr, Protection::R).unwrap();
 			old_data.copy_from_slice(src_addr.slice());
 		}
 		ActivatedMemoryBlock::free_pages_impl(&mut src, false);
@@ -605,7 +611,7 @@ impl<'block> ActivatedMemoryBlock<'block> {
 		let nbcopy = std::cmp::min(addr.size, new_size);
 		let npcopy = nbcopy >> PAGESHIFT;
 		unsafe {
-			pal::protect(dest.addr(), Protection::RW);
+			pal::protect(dest.addr(), Protection::RW).unwrap();
 			dest.addr().slice_mut()[0..nbcopy].copy_from_slice(&old_data[0..nbcopy]);
 		}
 		for (status, pdst) in old_status.iter().zip(dest.iter_mut()) {
@@ -668,9 +674,10 @@ impl<'block> ActivatedMemoryBlock<'block> {
 		// the expectation is that they will start out as zero filled.  accordingly, the most
 		// sensible way to do this is to zero them now
 		unsafe {
-			pal::protect(addr, Protection::RW);
+			pal::protect(addr, Protection::RW).unwrap();
 			addr.zero();
 			// simple state size optimization: we can undirty pages in this case depending on the initial state
+			#[cfg(not(feature = "no-dirty-detection"))]
 			for p in range.iter_mut() {
 				p.dirty = !p.invisible && match p.snapshot {
 					Snapshot::ZeroFilled => false,
@@ -726,6 +733,15 @@ impl<'block> ActivatedMemoryBlock<'block> {
 				p.snapshot = Snapshot::None;
 			}
 		}
+		#[cfg(feature = "no-dirty-detection")]
+		unsafe {
+			pal::protect(self.b.addr, Protection::R).unwrap();
+			for (a, p) in self.b.page_range().iter_mut_with_addr() {
+				p.dirty = true;
+				p.maybe_snapshot(a.start);
+			}
+		}
+
 		self.b.refresh_all_protections();
 		self.b.sealed = true;
 		self.b.hash = {
@@ -775,11 +791,11 @@ impl<'block>  IStateable for ActivatedMemoryBlock<'block> {
 				if p.dirty {
 					unsafe {
 						if !p.status.readable() {
-							assert!(pal::protect(paddr, Protection::R));
+							pal::protect(paddr, Protection::R).unwrap();
 						}
 						stream.write_all(paddr.slice())?;
 						if !p.status.readable() {
-							assert!(pal::protect(paddr, Protection::None));
+							pal::protect(paddr, Protection::None).unwrap();
 						}
 					}
 				}
@@ -804,7 +820,7 @@ impl<'block>  IStateable for ActivatedMemoryBlock<'block> {
 		}
 
 		unsafe {
-			pal::protect(self.b.addr, Protection::RW);
+			pal::protect(self.b.addr, Protection::RW).unwrap();
 
 			let mut statii = vec![PageAllocation::Free; self.b.pages.len()];
 			let mut dirtii = vec![false; self.b.pages.len()];

waterbox/waterboxhost/src/memory_block/pageblock.rs

@@ -1,6 +1,6 @@
-use std::ptr::{null_mut, NonNull};
+use std::ptr::NonNull;
-use core::ffi::c_void;
 use crate::*;
+use memory_block::{Protection, pal};
 
 /// wraps the allocation of a single PAGESIZE bytes of ram, and is safe-ish to call within a signal handler
 #[derive(Debug)]
@@ -11,13 +11,9 @@ pub struct PageBlock {
 impl PageBlock {
 	pub fn new() -> PageBlock {
 		unsafe {
-			let ptr = alloc();
+			let addr = pal::map_anon(AddressRange { start: 0, size: PAGESIZE }, Protection::RW).unwrap();
-			if ptr == null_mut() {
-				panic!("PageBlock could not allocate memory!");
-			} else {
 			PageBlock {
-					ptr: NonNull::new_unchecked(ptr as *mut u8),
+				ptr: NonNull::new_unchecked(addr.start as *mut u8),
-				}
 			}
 		}
 	}
@@ -43,47 +39,9 @@ impl PageBlock {
 impl Drop for PageBlock {
 	fn drop(&mut self) {
 		unsafe {
-			let res = free(self.ptr.as_ptr() as *mut c_void);
+			pal::unmap_anon(AddressRange { start: self.ptr.as_ptr() as usize, size: PAGESIZE }).unwrap();
-			if !res {
-				panic!("PageBlock could not free memory!");
 		}
 	}
-	}
-}
-
-#[cfg(windows)]
-use winapi::um::memoryapi::*;
-#[cfg(windows)]
-use winapi::um::winnt::*;
-#[cfg(windows)]
-unsafe fn alloc() -> *mut c_void {
-	VirtualAlloc(null_mut(), PAGESIZE, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE) as *mut c_void
-}
-#[cfg(windows)]
-unsafe fn free(ptr: *mut c_void) -> bool {
-	match VirtualFree(ptr as *mut winapi::ctypes::c_void, 0, MEM_RELEASE) {
-		0 => false,
-		_ => true
-	}
-}
-
-#[cfg(unix)]
-use libc::*;
-#[cfg(unix)]
-unsafe fn alloc() -> *mut c_void {
-	let ptr = mmap(null_mut(), PAGESIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
-	match ptr {
-		MAP_FAILED => null_mut(),
-		_ => ptr
-	}
-}
-#[cfg(unix)]
-unsafe fn free(ptr: *mut c_void) -> bool {
-	let res = munmap(ptr, PAGESIZE);
-	match res {
-		0 => true,
-		_ => false
-	}
 }
 
 #[cfg(test)]

waterbox/waterboxhost/src/memory_block/pal.rs

@@ -17,14 +17,19 @@ mod win {
 	use std::ptr::{null, null_mut};
 	use winapi::ctypes::c_void;
 
-	fn error() {
+	fn error() -> anyhow::Error {
-		unsafe {
+		anyhow!("WinApi failure code: {}", unsafe { winapi::um::errhandlingapi::GetLastError() })
-			let err = winapi::um::errhandlingapi::GetLastError();
+	}
-			eprintln!("WinApi failure code: {}", err);
+	fn ret(code: i32) -> anyhow::Result<()> {
+		match code {
+			0 => Err(error()),
+			_ => Ok(())
 		}
 	}
 
-	pub fn open(size: usize) -> Option<Handle> {
+	/// Open a file (not backed by the fs) for memory mapping
+	/// Caller must close_handle() later or else leak
+	pub fn open_handle(size: usize) -> anyhow::Result<Handle> {
 		unsafe {
 			let res = CreateFileMappingW(
 				INVALID_HANDLE_VALUE,
@@ -35,23 +40,28 @@ mod win {
 				null()
 			);
 			if res == null_mut() {
-				error();
+				Err(error())
-				None
 			} else {
-				Some(Handle(res as usize))
+				Ok(Handle(res as usize))
 			}
 		}
 	}
 
-	pub unsafe fn close(handle: Handle) -> bool {
+	/// close a handle returned by open_handle()
-		CloseHandle(handle.0 as *mut c_void) != 0
+	/// Unsafe:  Only call with handle returned by open_handle().  Do not call when that handle is mapped
+	pub unsafe fn close_handle(handle: Handle) -> anyhow::Result<()> {
+		ret(CloseHandle(handle.0 as *mut c_void))
 	}
 
+	/// Return a trapping handle.  Calling just about anything with this is bad
 	pub fn bad() -> Handle {
 		return Handle(INVALID_HANDLE_VALUE as usize);
 	}
 
-	pub fn map(handle: &Handle, addr: AddressRange) -> bool {
+	/// Map a handle into an address range
+	/// Probably shouldn't call with addr.size > handle's alloced size
+	/// Leaks if not later unmapped
+	pub fn map_handle(handle: &Handle, addr: AddressRange) -> anyhow::Result<()> {
 		unsafe {
 			let res = MapViewOfFileEx(
 				handle.0 as *mut c_void,
@@ -62,29 +72,58 @@ mod win {
 				addr.start as *mut c_void
 			);
 			if res == addr.start as *mut c_void {
-				true
+				Ok(())
 			} else {
-				error();
+				Err(error())
-				false
 			}
 		}
 	}
 
-	pub unsafe fn unmap(addr: AddressRange) -> bool {
+	/// Unmaps an address range previously mapped by map_handle
-		UnmapViewOfFile(addr.start as *mut c_void) != 0
+	/// Unsafe:  Do not call with any `addr` that does not exactly match a previous successful map_handle
+	pub unsafe fn unmap_handle(addr: AddressRange) -> anyhow::Result<()> {
+		ret(UnmapViewOfFile(addr.start as *mut c_void))
 	}
 
-	pub unsafe fn protect(addr: AddressRange, prot: Protection) -> bool {
+	fn prottoprot(prot: Protection) -> u32 {
-		let p = match prot {
+		match prot {
 			Protection::None => PAGE_NOACCESS,
 			Protection::R => PAGE_READONLY,
 			Protection::RW => PAGE_READWRITE,
 			Protection::RX => PAGE_EXECUTE_READ,
 			Protection::RWX => PAGE_EXECUTE_READWRITE,
 			Protection::RWStack => PAGE_READWRITE | PAGE_GUARD,
-		};
+		}
+	}
+
+	/// Map some anonymous bytes with no fd backing
+	/// addr.start can be 0, which means the OS chooses a location, or non-zero, which gives fixed behavior like map_handle
+	/// Returned address range will be identical in the case of non-zero, or give the actual address in the case of zero.
+	pub fn map_anon(addr: AddressRange, initial_prot: Protection) -> anyhow::Result<AddressRange> {
+		unsafe {
+			let res = VirtualAlloc(addr.start as *mut c_void, 
+				addr.size, MEM_RESERVE | MEM_COMMIT, 
+				prottoprot(initial_prot)) as usize;
+			match res {
+				0 => Err(error()),
+				p => Ok(AddressRange { start: p, size: addr.size })
+			}
+		}
+	}
+
+	/// Change memory protection on allocated bytes
+	/// This should work safely on any aligned subset of map_anon or map_handle
+	pub unsafe fn protect(addr: AddressRange, prot: Protection) -> anyhow::Result<()> {
+		let p = prottoprot(prot);
 		let mut old_protect: u32 = 0;
-		VirtualProtect(addr.start as *mut c_void, addr.size, p, &mut old_protect) != 0
+		let res = VirtualProtect(addr.start as *mut c_void, addr.size, p, &mut old_protect);
+		ret(res)
+	}
+
+	/// Unmap bytes previously mapped by map_anon
+	/// addr should exactly match the return value from map_anon (so if you mapped with start 0, you need to pass the actual start back)
+	pub unsafe fn unmap_anon(addr: AddressRange) -> anyhow::Result<()> {
+		ret(VirtualFree(addr.start as *mut c_void, 0, MEM_RELEASE))
 	}
 
 	pub struct StackTripResult {
@@ -116,39 +155,51 @@ mod nix {
 	use super::*;
 	use libc::*;
 
-	fn error() {
+	fn error() -> anyhow::Error {
 		unsafe {
 			let err = *__errno_location();
-			eprintln!("Libc failure code: {}", err);
+			anyhow!("Libc failure code: {}", err)
+		}
+	}
+	fn ret(code: i32) -> anyhow::Result<()> {
+		match code {
+			0 => Ok(()),
+			_ => Err(error()),
 		}
 	}
 
-	pub fn open(size: usize) -> Option<Handle> {
+	/// Open a file (not backed by the fs) for memory mapping
+	/// Caller must close_handle() later or else leak
+	pub fn open_handle(size: usize) -> anyhow::Result<Handle> {
 		unsafe {
 			let s = std::ffi::CString::new("MemoryBlockUnix").unwrap();
 			let fd = syscall(SYS_memfd_create, s.as_ptr(), MFD_CLOEXEC) as i32;
 			if fd == -1 {
-				error();
+				return Err(error())
-				return None
 			}
 			if ftruncate(fd, size as i64) != 0 {
-				error();
+				Err(error())
-				None
 			} else {
-				Some(Handle(fd as usize))
+				Ok(Handle(fd as usize))
 			}
 		}
 	}
 
-	pub unsafe fn close(handle: Handle) -> bool {
+	/// close a handle returned by open_handle()
-		libc::close(handle.0 as i32) == 0
+	/// Unsafe:  Only call with handle returned by open_handle().  Do not call when that handle is mapped
+	pub unsafe fn close_handle(handle: Handle) -> anyhow::Result<()> {
+		ret(libc::close(handle.0 as i32))
 	}
 
+	/// Return a trapping handle.  Calling just about anything with this is bad
 	pub fn bad() -> Handle {
 		return Handle(-1i32 as usize);
 	}
 
-	pub fn map(handle: &Handle, addr: AddressRange) -> bool {
+	/// Map a handle into an address range
+	/// Probably shouldn't call with addr.size > handle's alloced size
+	/// Leaks if not later unmapped
+	pub fn map_handle(handle: &Handle, addr: AddressRange) -> anyhow::Result<()> {
 		unsafe {
 			let res = mmap(addr.start as *mut c_void,
 				addr.size,
@@ -158,28 +209,58 @@ mod nix {
 				0
 			);
 			if res == addr.start as *mut c_void {
-				true
+				Ok(())
 			} else {
-				error();
+				Err(error())
-				false
 			}
 		}
 	}
 
-	pub unsafe fn unmap(addr: AddressRange) -> bool {
+	/// Unmaps an address range previously mapped by map_handle
-		munmap(addr.start as *mut c_void, addr.size) == 0
+	/// Unsafe:  Do not call with any `addr` that does not exactly match a previous successful map_handle
+	pub unsafe fn unmap_handle(addr: AddressRange) -> anyhow::Result<()> {
+		ret(munmap(addr.start as *mut c_void, addr.size))
 	}
 
-	pub unsafe fn protect(addr: AddressRange, prot: Protection) -> bool {
+	fn prottoprot(prot: Protection) -> i32 {
-		let p = match prot {
+		match prot {
 			Protection::None => PROT_NONE,
 			Protection::R => PROT_READ,
 			Protection::RW => PROT_READ | PROT_WRITE,
 			Protection::RX => PROT_READ | PROT_EXEC,
 			Protection::RWX => PROT_READ | PROT_WRITE | PROT_EXEC,
-			Protection::RWStack => panic!("RWStack should not be passed to pal layer"),
+			Protection::RWStack => panic!("RWStack should not be passed to nix pal layer"),
-		};
+		}
-		mprotect(addr.start as *mut c_void, addr.size, p) == 0
+	}
+
+	/// Map some anonymous bytes with no fd backing
+	/// addr.start can be 0, which means the OS chooses a location, or non-zero, which gives fixed behavior like map_handle
+	/// Returned address range will be identical in the case of non-zero, or give the actual address in the case of zero.
+	pub fn map_anon(addr: AddressRange, initial_prot: Protection) -> anyhow::Result<AddressRange> {
+		unsafe {
+			let mut flags = MAP_PRIVATE | MAP_ANONYMOUS;
+			if addr.start != 0 {
+				flags |= MAP_FIXED | MAP_FIXED_NOREPLACE;
+			}
+			let ptr = mmap(addr.start as *mut c_void, addr.size, prottoprot(initial_prot), flags, -1, 0);
+			match ptr {
+				MAP_FAILED => Err(error()),
+				p => Ok(AddressRange { start: p as usize, size: addr.size })
+			}
+		}
+	}
+
+	/// Change memory protection on allocated bytes
+	/// This should work safely on any aligned subset of map_anon or map_handle
+	pub unsafe fn protect(addr: AddressRange, prot: Protection) -> anyhow::Result<()> {
+		let p = prottoprot(prot);
+		ret(mprotect(addr.start as *mut c_void, addr.size, p))
+	}
+
+	/// Unmap bytes previously mapped by map_anon
+	/// addr should exactly match the return value from map_anon (so if you mapped with start 0, you need to pass the actual start back)
+	pub unsafe fn unmap_anon(addr: AddressRange) -> anyhow::Result<()> {
+		ret(munmap(addr.start as *mut c_void, addr.size))
 	}
 }
 
@@ -189,40 +270,58 @@ mod tests {
 	use std::mem::transmute;
 	
 	#[test]
-	fn basic_test() {
+	fn basic_test() -> anyhow::Result<()> {
 		assert!(crate::PAGESIZE == 0x1000);
 		// can't test the fault states (RWStack, R prohibits write, etc.) without cooperation of tripguard, so do that elsewhere
 		unsafe {
 			let size = 0x20000usize;
 			let start = 0x36a00000000usize;
 			let addr = AddressRange { start, size };
-			let handle = open(size).unwrap();
+			let handle = open_handle(size).unwrap();
 
-			assert!(map(&handle, addr));
+			map_handle(&handle, addr)?;
-			assert!(protect(addr, Protection::RW));
+			protect(addr, Protection::RW)?;
 			*((start + 0x14795) as *mut u8) = 42;
-			assert!(unmap(addr));
+			unmap_handle(addr)?;
 
-			assert!(map(&handle, addr));
+			map_handle(&handle, addr)?;
-			assert!(protect(addr, Protection::R));
+			protect(addr, Protection::R)?;
 			assert_eq!(*((start + 0x14795) as *const u8), 42);
-			assert!(unmap(addr));
+			unmap_handle(addr)?;
 
-			assert!(map(&handle, addr));
+			map_handle(&handle, addr)?;
-			assert!(protect(addr, Protection::RW));
+			protect(addr, Protection::RW)?;
 			*(start as *mut u8) = 0xc3; // RET
-			assert!(protect(addr, Protection::RX));
+			protect(addr, Protection::RX)?;
 			transmute::<usize, extern fn() -> ()>(start)();
-			assert!(protect(addr, Protection::RWX));
+			protect(addr, Protection::RWX)?;
 			*(start as *mut u8) = 0x90; // NOP
 			*((start + 1) as *mut u8) = 0xb0; // MOV AL
 			*((start + 2) as *mut u8) = 0x7b; // 123
 			*((start + 3) as *mut u8) = 0xc3; // RET
 			let i = transmute::<usize, extern fn() -> u8>(start)();
 			assert_eq!(i, 123);
-			assert!(unmap(addr));
+			unmap_handle(addr)?;
 
-			assert!(close(handle));
+			close_handle(handle)?;
 		}
+		Ok(())
+	}
+
+	#[test]
+	fn test_map_anon() -> anyhow::Result<()> {
+		unsafe {
+			let addr_in = AddressRange { start: 0x34100000000, size: 0x20000 };
+			let addr = map_anon(addr_in, Protection::RW)?;
+			assert_eq!(addr.start, addr_in.start);
+			unmap_anon(addr)?;
+		}
+		unsafe {
+			let addr_in = AddressRange { start: 0, size: 0x20000 };
+			let addr = map_anon(addr_in, Protection::RW)?;
+			addr.slice_mut()[0] = 13;
+			unmap_anon(addr)?;
+		}
+		Ok(())
 	}
 }

waterbox/waterboxhost/src/memory_block/tripguard.rs

@@ -60,12 +60,13 @@ unsafe fn trip(addr: usize) -> TripResult {
 	}
 	page.maybe_snapshot(page_start_addr);
 	page.dirty = true;
-	if pal::protect(AddressRange { start: page_start_addr, size: PAGESIZE }, page.native_prot()) {
+	match pal::protect(AddressRange { start: page_start_addr, size: PAGESIZE }, page.native_prot()) {
-		TripResult::Handled
+		Ok(()) => TripResult::Handled,
-	} else {
+		_ => {
 			std::intrinsics::breakpoint();
 			std::process::abort();
 		}
+	}
 }
 
 #[cfg(windows)]