Waterbox: Add pseudo-thread implementation and experimental DobieStation (PS2) core (#2263)

Waterbox supports threads now, but they're not real threads on the host side because that's complicated and can be nondeterministic. Instead, everything is scheduled to share one host thread. This means that scheduling is actually cooperative and certain patterns of spinlocks and other nonsense can fail to work at all, but "regular" code probably will.

With this, add DobieStation PS2 core. This core was selected because it has threads and is otherwise simple to port; easy to build and a good core/frontend separation. It's not a wonderful core however, with low speed (made abysmally lower by our lack of real threads) and low compatibility, so it remains a curiosity for now.

.gitmodules

@@ -32,3 +32,6 @@
 [submodule "waterbox/llvm-project"]
 	path = waterbox/llvm-project
 	url = git@github.com:llvm/llvm-project.git
+[submodule "waterbox/dobie/dobiestation"]
+	path = waterbox/dobie/dobiestation
+	url = git@github.com:nattthebear/DobieStation.git

src/BizHawk.Client.Common/RomLoader.cs

@@ -266,6 +266,9 @@ namespace BizHawk.Client.Common
 					case DiscType.SonyPSP:
 						game.System = "PSP";
 						break;
+					case DiscType.SonyPS2:
+						game.System = "PS2";
+						break;
 					case DiscType.MegaCD:
 						game.System = "GEN";
 						break;

src/BizHawk.Emulation.Common/Database/FirmwareDatabase.cs

@@ -277,6 +277,14 @@ namespace BizHawk.Emulation.Common
 			Firmware("PCFX", "SCSIROM", "fx-scsi.rom");
 			var fxscsi = File("65482A23AC5C10A6095AEE1DB5824CCA54EAD6E5", 512 * 1024, "PCFX_fx-scsi.rom", "PCFX SCSI ROM");
 			Option("PCFX", "SCSIROM", fxscsi);
+
+			Firmware("PS2", "BIOS", "PS2 Bios");
+			Option("PS2", "BIOS", File("fbd54bfc020af34008b317dcb80b812dd29b3759", 4 * 1024 * 1024, "ps2-0230j-20080220.bin", "PS2 Bios"));
+			Option("PS2", "BIOS", File("8361d615cc895962e0f0838489337574dbdc9173", 4 * 1024 * 1024, "ps2-0220a-20060905.bin", "PS2 Bios"));
+			Option("PS2", "BIOS", File("da5aacead2fb55807d6d4e70b1f10f4fdcfd3281", 4 * 1024 * 1024, "ps2-0220e-20060905.bin", "PS2 Bios"));
+			Option("PS2", "BIOS", File("3baf847c1c217aa71ac6d298389c88edb3db32e2", 4 * 1024 * 1024, "ps2-0220j-20060905.bin", "PS2 Bios"));
+			Option("PS2", "BIOS", File("f9229fe159d0353b9f0632f3fdc66819c9030458", 4 * 1024 * 1024, "ps2-0230a-20080220.bin", "PS2 Bios"), FirmwareOptionStatus.Ideal);
+			Option("PS2", "BIOS", File("9915b5ba56798f4027ac1bd8d10abe0c1c9c326a", 4 * 1024 * 1024, "ps2-0230e-20080220.bin", "PS2 Bios"));
 		}
 
 		// adds a defined firmware ID to the database

src/BizHawk.Emulation.Cores/Consoles/Nintendo/VB/LibVirtualBoyee.cs

@@ -81,7 +81,6 @@ namespace BizHawk.Emulation.Cores.Consoles.Nintendo.VB
 			}
 		}
 
-
 		[BizImport(CC)]
 		public abstract bool Load(byte[] rom, int length, NativeSyncSettings settings);
 

src/BizHawk.Emulation.Cores/Consoles/Sony/PS2/DobieStation.cs

@@ -0,0 +1,202 @@
+using System;
+using System.Runtime.InteropServices;
+using BizHawk.Common;
+using BizHawk.Emulation.Common;
+using BizHawk.Emulation.Cores.Waterbox;
+using BizHawk.Emulation.DiscSystem;
+
+namespace BizHawk.Emulation.Cores.Sony.PS2
+{
+	[Core("DobieStation", "PSI", true, false, "fa33778b056aa32", "https://github.com/PSI-Rockin/DobieStation", false)]
+	public unsafe class DobieStation : WaterboxCore, ISettable<object, DobieStation.DobieSyncSettings>
+	{
+		private readonly LibDobieStation _core;
+		[CoreConstructor("PS2")]
+		public DobieStation(CoreLoadParameters<object, DobieSyncSettings> lp)
+			:base(lp.Comm, new Configuration
+			{
+				MaxWidth = 640,
+				MaxHeight = 480,
+				DefaultWidth = 640,
+				DefaultHeight = 480,
+				DefaultFpsNumerator = 294912000,
+				DefaultFpsDenominator = 4920115,
+				MaxSamples = 1024,
+				SystemId = "PS2"
+			})
+		{
+			if (lp.Discs.Count != 1)
+			{
+				throw new InvalidOperationException("Must load a CD or DVD with PS2 core!");
+			}
+			ControllerDefinition = DualShock;
+			_syncSettings = lp.SyncSettings ?? new DobieSyncSettings();
+			_syncSettingsActual = lp.SyncSettings ?? new DobieSyncSettings();
+
+			_disc = new DiscSectorReader(lp.Discs[0].DiscData);
+			_cdCallback = ReadCd;
+			_core = PreInit<LibDobieStation>(new WaterboxOptions
+			{
+				Filename = "dobie.wbx",
+				SbrkHeapSizeKB = 4 * 1024,
+				SealedHeapSizeKB = 4 * 1024,
+				InvisibleHeapSizeKB = 4 * 1024,
+				PlainHeapSizeKB = 256,
+				MmapHeapSizeKB = 2 * 1024 * 1024,
+				SkipCoreConsistencyCheck = lp.Comm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxCoreConsistencyCheck),
+				SkipMemoryConsistencyCheck = lp.Comm.CorePreferences.HasFlag(CoreComm.CorePreferencesFlags.WaterboxMemoryConsistencyCheck),
+			}, new[] { _cdCallback });
+
+			var bios = lp.Comm.CoreFileProvider.GetFirmware("PS2", "BIOS", true);
+			_exe.AddReadonlyFile(new byte[0x840000], "MEMCARD0");
+
+			var worked = _core.Initialize(bios,
+				(ulong)(lp.Discs[0].DiscData.Session1.Tracks[2].LBA - lp.Discs[0].DiscData.Session1.Tracks[1].LBA) * 2048,
+				_cdCallback,
+				_syncSettingsActual.GetNativeSettings()
+			);
+
+			if (!worked)
+			{
+				throw new InvalidOperationException("Initialize failed!");
+			}
+
+			_exe.RemoveReadonlyFile("MEMCARD0");
+
+			PostInit();
+
+			_resampler = new SpeexResampler((SpeexResampler.Quality)6, 480, 441, 48000, 44100, null, this);
+			_serviceProvider.Register<ISoundProvider>(_resampler);
+		}
+
+		private SpeexResampler _resampler;
+
+		public override void Dispose()
+		{
+			base.Dispose();
+			if (_resampler != null)
+			{
+				_resampler.Dispose();
+				_resampler = null;
+			}
+		}
+
+		private readonly LibDobieStation.CdCallback _cdCallback;
+		private DiscSectorReader _disc;
+		private void ReadCd(ulong sector, byte* dest)
+		{
+			var tmp = new byte[2048];
+			_disc.ReadLBA_2048((int)sector, tmp, 0);
+			Marshal.Copy(tmp, 0, (IntPtr)dest, 2048);
+		}
+
+		protected override LibWaterboxCore.FrameInfo FrameAdvancePrep(IController controller, bool render, bool rendersound)
+		{
+			var ret = new LibDobieStation.FrameInfo();
+			for (int i = 0; i < DualShock.BoolButtons.Count; i++)
+			{
+				if (controller.IsPressed(DualShock.BoolButtons[i]))
+				{
+					ret.Buttons |= 1u << i;
+				}
+			}
+			for (int i = 0; i < DualShock.Axes.Count; i++)
+			{
+				ret.Axes |= (uint)controller.AxisValue(DualShock.Axes[i]) << (i * 8);
+			}
+			return ret;
+		}
+		protected override void FrameAdvancePost()
+		{
+			// DobieStation core kicks back 0 values sometimes.  Not sure what they mean, no image produced?
+			// Easiest to just fix them here.
+			if (BufferWidth == 0)
+				BufferWidth = 640;
+			if (BufferHeight == 0)
+				BufferHeight = 480;
+		}
+
+		public object GetSettings() => new object();
+		public PutSettingsDirtyBits PutSettings(object o) => PutSettingsDirtyBits.None;
+
+		private DobieSyncSettings _syncSettings;
+		private readonly DobieSyncSettings _syncSettingsActual;
+
+		public DobieSyncSettings GetSyncSettings()
+		{
+			return _syncSettings.Clone();
+		}
+
+		public PutSettingsDirtyBits PutSyncSettings(DobieSyncSettings o)
+		{
+			_syncSettings = o;
+			return DobieSyncSettings.NeedsReboot(_syncSettings, _syncSettingsActual)
+				? PutSettingsDirtyBits.RebootCore
+				: PutSettingsDirtyBits.None;
+		}
+
+		private static readonly ControllerDefinition DualShock = new ControllerDefinition
+		{
+			Name = "PS2 DualShock",
+			BoolButtons =
+			{
+				"SELECT",
+				"L3",
+				"R3",
+				"START",
+				"UP",
+				"RIGHT",
+				"DOWN",
+				"LEFT",
+				"L2",
+				"R2",
+				"L1",
+				"R1",
+				"TRIANGLE",
+				"CIRCLE",
+				"CROSS",
+				"SQUARE",
+			},
+			Axes =
+			{
+				{ "RIGHT X", new AxisSpec(RangeExtensions.MutableRangeTo(0, 255), 128) },
+				{ "RIGHT Y", new AxisSpec(RangeExtensions.MutableRangeTo(0, 255), 128) },
+				{ "LEFT X", new AxisSpec(RangeExtensions.MutableRangeTo(0, 255), 128) },
+				{ "LEFT Y", new AxisSpec(RangeExtensions.MutableRangeTo(0, 255), 128) },
+			}
+		};
+
+		public class DobieSyncSettings
+		{
+			public enum CpuMode
+			{
+				Jit,
+				Interpreter
+			}
+
+			public CpuMode EEMode { get; set;}
+			public CpuMode VU0Mode { get; set; }
+			public CpuMode VU1Mode { get; set; }
+
+			public static bool NeedsReboot(DobieSyncSettings x, DobieSyncSettings y)
+			{
+				return !DeepEquality.DeepEquals(x, y); 
+			}
+
+			public DobieSyncSettings Clone()
+			{
+				return (DobieSyncSettings)MemberwiseClone();
+			}
+
+			public LibDobieStation.SyncSettings GetNativeSettings()
+			{
+				return new LibDobieStation.SyncSettings
+				{
+					EEJit = EEMode == CpuMode.Jit,
+					VU0Jit = VU0Mode == CpuMode.Jit,
+					VU1Jit = VU1Mode == CpuMode.Jit
+				};
+			}
+		}
+	}
+}

src/BizHawk.Emulation.Cores/Consoles/Sony/PS2/LibDobieStation.cs

@@ -0,0 +1,29 @@
+using System.Runtime.InteropServices;
+using BizHawk.BizInvoke;
+using BizHawk.Emulation.Cores.Waterbox;
+
+namespace BizHawk.Emulation.Cores.Sony.PS2
+{
+	public abstract class LibDobieStation : LibWaterboxCore
+	{
+		[StructLayout(LayoutKind.Sequential)]
+		public new class FrameInfo : LibWaterboxCore.FrameInfo
+		{
+			public uint Buttons;
+			public uint Axes;
+		}
+
+		[StructLayout(LayoutKind.Sequential)]
+		public class SyncSettings
+		{
+			public bool EEJit;
+			public bool VU0Jit;
+			public bool VU1Jit;
+		}
+
+		public unsafe delegate void CdCallback(ulong sector, byte* dest);
+
+		[BizImport(CC)]
+		public abstract bool Initialize(byte[] bios, ulong cdLength, CdCallback cdCallback, SyncSettings syncSettings);
+	}
+}

src/BizHawk.Emulation.DiscSystem/DiscIdentifier.cs

@@ -97,7 +97,12 @@ namespace BizHawk.Emulation.DiscSystem
 		/// <summary>
 		/// Sega Dreamcast
 		/// </summary>
-		Dreamcast
+		Dreamcast,
+
+		/// <summary>
+		/// Yes, that one
+		/// </summary>
+		SonyPS2,
 	}
 
 	public class DiscIdentifier
@@ -217,6 +222,14 @@ namespace BizHawk.Emulation.DiscSystem
 					return DiscType.Amiga;
 				}
 
+				if (iso.Root.Children.TryGetValue("SYSTEM.CNF;1", out var cnf))
+				{
+					if (SectorContains("BOOT2", (int)cnf.Offset))
+						return DiscType.SonyPS2;
+					else if (SectorContains("BOOT", (int)cnf.Offset))
+						return DiscType.SonyPSX;
+				}
+
 				// NeoGeoCD Check
 				var absTxt = iso.Root.Children.Where(kvp => kvp.Key.Contains("ABS.TXT")).Select(kvp => kvp.Value).FirstOrDefault();
 				if (absTxt != null && SectorContains("abstracted by snk", Convert.ToInt32(absTxt.Offset))) return DiscType.NeoGeoCD;

waterbox/dobie/Makefile

@@ -0,0 +1,107 @@
+CXXFLAGS := \
+	-Wall -Werror=int-to-pointer-cast \
+	-std=c++14 -fomit-frame-pointer -fno-rtti \
+	-Wno-reorder -Wno-unused-value \
+	-Idobiestation/ext/libdeflate \
+	-Dprivate=public
+
+CCFLAGS := \
+	-Wall -Wundef -Wpedantic -Wdeclaration-after-statement -Wmissing-prototypes -Wstrict-prototypes -Wvla \
+	-D_ANSI_SOURCE \
+	-Idobiestation/ext/libdeflate \
+	-Idobiestation/ext/libdeflate/common
+
+TARGET = dobie.wbx
+
+CORE_SRCS = \
+	audio/utils.cpp \
+	ee/bios_hle.cpp \
+	ee/cop0.cpp \
+	ee/cop1.cpp \
+	ee/cop2.cpp \
+	ee/dmac.cpp \
+	ee/ee_jit.cpp \
+	ee/ee_jit64.cpp \
+	ee/ee_jit64_cop2.cpp \
+	ee/ee_jit64_fpu.cpp \
+	ee/ee_jit64_fpu_avx.cpp \
+	ee/ee_jit64_gpr.cpp \
+	ee/ee_jit64_mmi.cpp \
+	ee/ee_jittrans.cpp \
+	ee/emotion.cpp \
+	ee/emotion_fpu.cpp \
+	ee/emotion_mmi.cpp \
+	ee/emotion_special.cpp \
+	ee/emotion_vu0.cpp \
+	ee/emotionasm.cpp \
+	ee/emotiondisasm.cpp \
+	ee/emotioninterpreter.cpp \
+	ee/intc.cpp \
+	ee/ipu/chromtable.cpp \
+	ee/ipu/codedblockpattern.cpp \
+	ee/ipu/dct_coeff.cpp \
+	ee/ipu/dct_coeff_table0.cpp \
+	ee/ipu/dct_coeff_table1.cpp \
+	ee/ipu/ipu.cpp \
+	ee/ipu/ipu_fifo.cpp \
+	ee/ipu/lumtable.cpp \
+	ee/ipu/mac_addr_inc.cpp \
+	ee/ipu/mac_b_pic.cpp \
+	ee/ipu/mac_i_pic.cpp \
+	ee/ipu/mac_p_pic.cpp \
+	ee/ipu/motioncode.cpp \
+	ee/ipu/vlc_table.cpp \
+	ee/timers.cpp \
+	ee/vif.cpp \
+	ee/vu.cpp \
+	ee/vu_disasm.cpp \
+	ee/vu_interpreter.cpp \
+	ee/vu_jit.cpp \
+	ee/vu_jit64.cpp \
+	ee/vu_jittrans.cpp \
+	emulator.cpp \
+	errors.cpp \
+	gif.cpp \
+	gs.cpp \
+	gscontext.cpp \
+	gsmem.cpp \
+	gsregisters.cpp \
+	gsthread.cpp \
+	iop/cdvd/bincuereader.cpp \
+	iop/cdvd/cdvd.cpp \
+	iop/cdvd/cso_reader.cpp \
+	iop/cdvd/iso_reader.cpp \
+	iop/firewire.cpp \
+	iop/gamepad.cpp \
+	iop/iop.cpp \
+	iop/iop_cop0.cpp \
+	iop/iop_dma.cpp \
+	iop/iop_intc.cpp \
+	iop/iop_interpreter.cpp \
+	iop/iop_timers.cpp \
+	iop/memcard.cpp \
+	iop/sio2.cpp \
+	iop/spu/spu.cpp \
+	iop/spu/spu_adpcm.cpp \
+	iop/spu/spu_envelope.cpp \
+	iop/spu/spu_tables.cpp \
+	jitcommon/emitter64.cpp \
+	jitcommon/ir_block.cpp \
+	jitcommon/ir_instr.cpp \
+	jitcommon/jitcache.cpp \
+	scheduler.cpp \
+	serialize.cpp \
+	sif.cpp \
+	tests/iop/alu.cpp
+
+DEFLATE_SRCS = \
+	lib/aligned_malloc.c \
+	lib/deflate_decompress.c \
+	lib/x86/cpu_features.c
+
+SRCS = \
+	$(addprefix dobiestation/src/core/,$(CORE_SRCS)) \
+	$(addprefix dobiestation/ext/libdeflate/,$(DEFLATE_SRCS)) \
+	cinterface.cpp
+
+include ../common.mak

waterbox/dobie/cinterface.cpp

@@ -0,0 +1,186 @@
+#include "../emulibc/emulibc.h"
+#include "../emulibc/waterboxcore.h"
+#include <stdint.h>
+#include "dobiestation/src/core/emulator.hpp"
+#include "dobiestation/src/core/iop/cdvd/cdvd_container.hpp"
+
+
+static size_t cd_length;
+static void (*cdcallback)(size_t sector, uint8_t* dest);
+static size_t cd_pos;
+
+class DVD: public CDVD_Container
+{
+	virtual bool open(std::string name) override
+	{
+		return true;
+	}
+	virtual void close() override {}
+	virtual size_t read(uint8_t* buff, size_t bytes)
+	{
+		uint8_t* buff_end = buff + bytes;
+		if (cd_pos % 2048 != 0)
+		{
+			auto offset = cd_pos % 2048;
+			auto nread = std::min(2048 - offset, bytes);
+			uint8_t tmp[2048];
+			cdcallback(cd_pos / 2048, tmp);
+			memcpy(buff, tmp + offset, nread);
+			buff += nread;
+			cd_pos += nread;
+		}
+		while (buff_end >= buff + 2048)
+		{
+			cdcallback(cd_pos / 2048, buff);
+			buff += 2048;
+			cd_pos += 2048;
+		}
+		if (buff_end > buff)
+		{
+			auto nread = buff_end - buff;
+			uint8_t tmp[2048];
+			cdcallback(cd_pos / 2048, tmp);
+			memcpy(buff, tmp, nread);
+			cd_pos += nread;
+		}
+		return bytes;
+	}
+	virtual void seek(size_t pos, std::ios::seekdir whence) override
+	{
+		cd_pos = pos * 2048;
+	}
+	virtual bool is_open() override
+	{
+		return true;
+	}
+	virtual size_t get_size() override
+	{
+		return cd_length;
+	}
+};
+
+
+struct MyFrameInfo: public FrameInfo
+{
+	uint32_t Buttons;
+	uint32_t Axes;
+};
+
+Emulator* emu;
+
+struct SyncSettings
+{
+	bool EEJit;
+	bool VU0Jit;
+	bool VU1Jit;
+};
+
+ECL_EXPORT bool Initialize(
+	const uint8_t* bios,
+	size_t cd_length_,
+	void (*cdcallback_)(size_t sector, uint8_t* dest),
+	const SyncSettings& syncSettings
+)
+{
+	cd_length = cd_length_;
+	cdcallback = cdcallback_;
+	emu = new Emulator();
+	emu->reset();
+	emu->set_skip_BIOS_hack(LOAD_DISC);
+	emu->load_BIOS(bios);
+	emu->load_memcard(0, "MEMCARD0");
+	if (!emu->load_CDVD_Container("", std::unique_ptr<CDVD_Container>(new DVD())))
+		return false;
+	emu->set_ee_mode(syncSettings.EEJit ? JIT : INTERPRETER);
+	printf("EE Mode: %s\n", syncSettings.EEJit ? "JIT" : "INTERPRETER");
+	emu->set_vu0_mode(syncSettings.VU0Jit ? JIT : INTERPRETER);
+	printf("VU0 Mode: %s\n", syncSettings.VU0Jit ? "JIT" : "INTERPRETER");
+	emu->set_vu1_mode(syncSettings.VU1Jit ? JIT : INTERPRETER);
+	printf("VU1 Mode: %s\n", syncSettings.VU1Jit ? "JIT" : "INTERPRETER");
+	emu->set_wav_output(true);
+	return true;
+}
+
+static int16_t* audio_pos;
+void hacky_enqueue_audio(stereo_sample s)
+{
+	*audio_pos++ = s.left;
+	*audio_pos++ = s.right;
+}
+
+bool hacky_lag_flag;
+void (*hacky_input_callback)();
+
+ECL_EXPORT void FrameAdvance(MyFrameInfo& f)
+{
+	for (auto i = 0; i < 16; i++)
+	{
+		if (f.Buttons & 1 << i)
+		{
+			emu->press_button((PAD_BUTTON)i);
+		}
+		else
+		{
+			emu->release_button((PAD_BUTTON)i);
+		}
+	}
+	for (auto i = 0; i < 4; i++)
+	{
+		emu->update_joystick((JOYSTICK)(i >> 1), (JOYSTICK_AXIS)(i & 1), f.Axes >> (i * 8));
+	}
+	audio_pos = f.SoundBuffer;
+	hacky_lag_flag = true;
+	emu->run();
+	f.Lagged = hacky_lag_flag;
+	emu->get_inner_resolution(f.Width, f.Height);
+	{
+		const uint32_t* src = emu->get_framebuffer();
+		const uint32_t* srcend = src + f.Width * f.Height;
+		uint32_t* dst = f.VideoBuffer;
+		while (src < srcend)
+		{
+			*dst = *src;
+			std::swap(((uint8_t*)dst)[2], ((uint8_t*)dst)[0]);
+			src++;
+			dst++;
+		}
+	}
+
+	f.Samples = (audio_pos - f.SoundBuffer) / 2;
+	audio_pos = nullptr;
+}
+
+static uint8_t junkus[14];
+
+ECL_EXPORT void GetMemoryAreas(MemoryArea *m)
+{
+	m[0].Data = emu->RDRAM;
+	m[0].Name = "RDRAM";
+	m[0].Size = 1024 * 1024 * 32;
+	m[0].Flags = MEMORYAREA_FLAGS_WRITABLE | MEMORYAREA_FLAGS_WORDSIZE4 | MEMORYAREA_FLAGS_PRIMARY;
+
+	m[1].Data = emu->IOP_RAM;
+	m[1].Name = "IOP_RAM";
+	m[1].Size = 1024 * 1024 * 2;
+	m[1].Flags = MEMORYAREA_FLAGS_WRITABLE | MEMORYAREA_FLAGS_WORDSIZE4;
+
+	m[2].Data = emu->BIOS;
+	m[2].Name = "BIOS";
+	m[2].Size = 1024 * 1024 * 4;
+	m[2].Flags = MEMORYAREA_FLAGS_WORDSIZE4;
+
+	m[3].Data = emu->SPU_RAM;
+	m[3].Name = "SPU_RAM";
+	m[3].Size = 1024 * 1024 * 2;
+	m[3].Flags = MEMORYAREA_FLAGS_WRITABLE | MEMORYAREA_FLAGS_WORDSIZE4;
+
+	m[4].Data = emu->memcard.mem;
+	m[4].Name = "MEMCARD0";
+	m[4].Size = 0x840000;
+	m[4].Flags = MEMORYAREA_FLAGS_WRITABLE | MEMORYAREA_FLAGS_WORDSIZE1 | MEMORYAREA_FLAGS_SAVERAMMABLE;
+}
+
+ECL_EXPORT void SetInputCallback(void (*callback)())
+{
+	hacky_input_callback = callback;
+}

waterbox/dobie/dobiestation

@@ -0,0 +1 @@
+Subproject commit b736d7d5668aa2e52705faabb1e32fbfaff4c3ce

waterbox/libcxx/configure-for-waterbox-phase-0

@@ -11,7 +11,6 @@ cmake \
 	-DCMAKE_C_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DCMAKE_CXX_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DLIBUNWIND_ENABLE_SHARED=OFF \
-	-DLIBUNWIND_ENABLE_THREADS=OFF \
 	-DLIBUNWIND_USE_COMPILER_RT=ON \
 	-DCMAKE_INSTALL_PREFIX="$SYSROOT" \
 	-DCMAKE_AR="/usr/bin/gcc-ar" \

waterbox/libcxx/configure-for-waterbox-phase-1

@@ -12,7 +12,6 @@ cmake \
 	-DCMAKE_CXX_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DLIBCXXABI_ENABLE_EXCEPTIONS=ON \
 	-DLIBCXXABI_ENABLE_PIC=OFF \
-	-DLIBCXXABI_ENABLE_THREADS=OFF \
 	-DLIBCXXABI_ENABLE_SHARED=OFF \
 	-DLIBCXXABI_BAREMETAL=ON \
 	-DLIBCXXABI_SILENT_TERMINATE=ON \

waterbox/libcxx/configure-for-waterbox-phase-2

@@ -22,7 +22,6 @@ cmake \
 	-DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
 	-DLIBCXX_ENABLE_EXCEPTIONS=ON \
 	-DLIBCXX_ENABLE_RTTI=ON \
-	-DLIBCXX_ENABLE_THREADS=OFF \
 	-DLIBCXX_HAS_MUSL_LIBC=ON \
 	-DLIBCXX_USE_COMPILER_RT=ON \
 	-DLIBCXX_INCLUDE_TESTS=OFF \

waterbox/musl

@@ -1 +1 @@
-Subproject commit 1020370fb2850b33c5bb8c016dfe822551faf4c6
+Subproject commit 6446985d83d5810ee84bab5e5c107226a22572c7

waterbox/waterboxhost/src/context/interop.s

@@ -2,6 +2,7 @@ bits 64
 org 0x35f00000000
 
 struc Context
+	.thread_area resq 1
 	.host_rsp resq 1
 	.guest_rsp resq 1
 	.dispatch_syscall resq 1
@@ -51,6 +52,7 @@ times 0x80-($-$$) int3
 ; rax - syscall number
 ; regular arg registers are 0..6 args to the syscall
 guest_syscall:
+	push rbp ; this call might be suspended and cothreaded.  the guest knows to save nonvolatiles if it needs to, except rbp
 	mov r10, [gs:0x18]
 	mov [r10 + Context.guest_rsp], rsp
 	mov rsp, [r10 + Context.host_rsp]
@@ -76,6 +78,7 @@ guest_syscall:
 
 	mov r10, [gs:0x18]
 	mov rsp, [r10 + Context.guest_rsp]
+	pop rbp
 	ret
 
 times 0x100-($-$$) int3 ; CALL_GUEST_SIMPLE_ADDR

waterbox/waterboxhost/src/context/mod.rs

@@ -64,9 +64,12 @@ pub type SyscallCallback = extern "sysv64" fn(
 /// Layout must be synced with interop.s
 #[repr(C)]
 pub struct Context {
+	/// thread pointer as set by guest libc (pthread_self, set_thread_area)
+	pub thread_area: usize,
 	/// Used internally to track the host's most recent rsp when transitioned to Waterbox code.
 	pub host_rsp: usize,
 	/// Sets the guest's starting rsp, and used internally to track the guest's most recent rsp when transitioned to extcall or syscall
+	/// can be changed by the host to return to a different guest thread
 	pub guest_rsp: usize,
 	/// syscall service function
 	pub dispatch_syscall: SyscallCallback,
@@ -79,6 +82,7 @@ impl Context {
 	/// Returns a suitably initialized context.  It's almost ready to use, but host_ptr must be set before each usage
 	pub fn new(initial_guest_rsp: usize, dispatch_syscall: SyscallCallback) -> Context {
 		Context {
+			thread_area: 0,
 			host_rsp: 0,
 			guest_rsp: initial_guest_rsp,
 			dispatch_syscall,

waterbox/waterboxhost/src/fs/mod.rs

@@ -371,12 +371,14 @@ mod tests {
 		fs.write(fd, ")".as_bytes())?;
 		fs.seek(fd, -1, SEEK_END)?;
 		fs.write(fd, "$$$$".as_bytes())?;
+		fs.seek(fd, 9, SEEK_SET)?;
+		fs.write(fd, "-".as_bytes())?;
 		let mut statbuff = Box::new(KStat::default());
 		fs.fstat(fd, statbuff.as_mut())?;
 		assert_eq!(statbuff.st_size, 11);
 		fs.close(fd)?;
 		let vec = fs.unmount("z")?;
-		assert_eq!(vec, "Qig)tes$$$$".as_bytes());
+		assert_eq!(vec, "Qig)tes$$-$".as_bytes());
 		Ok(())
 	}
 }

waterbox/waterboxhost/src/fs/regular_file.rs

@@ -84,7 +84,7 @@ impl FileObject for RegularFile {
 	fn seek(&mut self, offset: i64, whence: i32) -> Result<i64, SyscallError> {
 		let newpos = match whence {
 			SEEK_SET => {
-				0
+				offset
 			},
 			SEEK_CUR => {
 				self.position as i64 + offset

waterbox/waterboxhost/src/host.rs

@@ -7,6 +7,7 @@ use elf::ElfLoader;
 use cinterface::MemoryLayoutTemplate;
 use goblin::elf::Elf;
 use context::{CALLBACK_SLOTS, Context, ExternalCallback, thunks::ThunkManager};
+use threading::GuestThreadSet;
 
 pub struct WaterboxHost {
 	fs: FileSystem,
@@ -19,6 +20,7 @@ pub struct WaterboxHost {
 	image_file: Vec<u8>,
 	context: Context,
 	thunks: ThunkManager,
+	threads: GuestThreadSet,
 }
 impl WaterboxHost {
 	pub fn new(image_file: Vec<u8>, module_name: &str, layout_template: &MemoryLayoutTemplate) -> anyhow::Result<Box<WaterboxHost>> {
@@ -42,6 +44,7 @@ impl WaterboxHost {
 			image_file,
 			context: Context::new(layout.main_thread.end(), syscall),
 			thunks,
+			threads: GuestThreadSet::new(),
 		});
 		res.activate();
 		println!("Calling _start()");
@@ -174,6 +177,7 @@ impl IStateable for WaterboxHost {
 		bin::write(stream, &self.program_break)?;
 		self.elf.save_state(stream)?;
 		self.memory_block.save_state(stream)?;
+		self.threads.save_state(&self.context, stream)?;
 		bin::write_magic(stream, SAVE_END_MAGIC)?;
 		Ok(())
 	}
@@ -184,6 +188,7 @@ impl IStateable for WaterboxHost {
 		bin::read(stream, &mut self.program_break)?;
 		self.elf.load_state(stream)?;
 		self.memory_block.load_state(stream)?;
+		self.threads.load_state(&mut self.context, stream)?;
 		bin::verify_magic(stream, SAVE_END_MAGIC)?;
 		Ok(())
 	}
@@ -235,7 +240,7 @@ fn arg_to_statbuff<'a>(arg: usize) -> &'a mut KStat {
 }
 
 extern "sysv64" fn syscall(
-	a1: usize, a2: usize, a3: usize, a4: usize, _a5: usize, _a6: usize,
+	a1: usize, a2: usize, a3: usize, a4: usize, a5: usize, _a6: usize,
 	nr: SyscallNumber, h: &mut WaterboxHost
 ) -> SyscallReturn {
 	match nr {
@@ -332,10 +337,6 @@ extern "sysv64" fn syscall(
 		NR_LSEEK => syscall_ret_i64(h.fs.seek(arg_to_fd(a1)?, a2 as i64, a3 as i32)),
 		NR_TRUNCATE => syscall_ret(h.fs.truncate(&arg_to_str(a1)?, a2 as i64)),
 		NR_FTRUNCATE => syscall_ret(h.fs.ftruncate(arg_to_fd(a1)?, a2 as i64)),
-		// TODO: 99% sure nothing calls this
-		NR_SET_THREAD_AREA => syscall_err(ENOSYS),
-		// TODO: What calls this?
-		NR_SET_TID_ADDRESS => syscall_ok(8675309),
 		NR_CLOCK_GETTIME => {
 			let ts = a2 as *mut TimeSpec;
 			unsafe {
@@ -368,6 +369,69 @@ extern "sysv64" fn syscall(
 			h.program_break = res;
 			syscall_ok(res)
 		},
+		NR_WBX_CLONE => {
+			syscall_ret_val(
+				h.threads
+					.spawn(
+						&mut h.memory_block, 
+						a1, 
+						a2, 
+						a3,
+						a4,
+						a5 as *mut u32,
+					)
+					.map(|tid| tid as usize)
+			)
+		},
+		NR_EXIT => {
+			h.threads.exit(&mut h.context)
+		},
+		NR_FUTEX => {
+			let op = a2 as i32 & !FUTEX_PRIVATE;
+			match op {
+				FUTEX_WAIT => {
+					h.threads.futex_wait(&mut h.context, a1, a3 as u32)
+				},
+				// int *uaddr, int futex_op, int val,
+				// const struct timespec *timeout,   /* or: uint32_t val2 */
+				// int *uaddr2, int val3
+				FUTEX_WAKE => {
+					syscall_ok(h.threads.futex_wake(a1, a3 as u32))
+				},
+				FUTEX_REQUEUE => {
+					syscall_ret_val(
+						h.threads.futex_requeue(
+							a1,
+							a5,
+							a3 as u32,
+							a4 as u32
+						)
+					)
+				},
+				FUTEX_UNLOCK_PI => {
+					h.threads.futex_unlock_pi(&mut h.context, a1)
+				},
+				FUTEX_LOCK_PI => {
+					h.threads.futex_lock_pi(&mut h.context, a1)
+				},
+				_ => syscall_err(ENOSYS),
+			}
+		},
+		NR_SET_THREAD_AREA => syscall_err(ENOSYS), // musl handles this in userspace
+		NR_SET_TID_ADDRESS => syscall_ok(h.threads.set_tid_address(a1) as usize),
+		NR_GETTID => syscall_ok(h.threads.get_tid() as usize),
+		NR_RT_SIGPROCMASK => {
+			// we don't (nor ever plan to?) deliver any signals to guests, so...
+			syscall_ok(0)
+		},
+		NR_SCHED_YIELD => {
+			h.threads.yield_any(&mut h.context)
+		},
+		NR_NANOSLEEP | NR_CLOCK_NANOSLEEP => {
+			// We'll never be interrupted by signals, and isolate guest time from real time,
+			// so don't need to examine the arguments here and can just treat this as another yield
+			h.threads.yield_any(&mut h.context)
+		},
 		_ => syscall_ret(unimp(nr)),
 	}
 }

waterbox/waterboxhost/src/lib.rs

@@ -3,6 +3,7 @@
 #![feature(try_trait)]
 #![feature(core_intrinsics)]
 #![feature(asm)]
+#![feature(map_first_last)]
 
 #![allow(dead_code)]
 
@@ -23,6 +24,7 @@ mod host;
 mod cinterface;
 mod gdb;
 mod context;
+mod threading;
 
 pub trait IStateable {
 	fn save_state(&mut self, stream: &mut dyn Write) -> anyhow::Result<()>;

waterbox/waterboxhost/src/memory_block/mod.rs

@@ -11,7 +11,6 @@ use crate::syscall_defs::*;
 use itertools::Itertools;
 use std::sync::atomic::AtomicU32;
 use crate::bin;
-use sha2::{Sha256, Digest};
 
 /// Tracks one lock for each 4GB memory area
 mod lock_list {
@@ -285,6 +284,7 @@ impl MemoryBlock {
 			active_guard: None,
 			swapped_in: false,
 		});
+		println!("MemoryBlock created for address {:x}:{:x} with mirror {:x}:{:x}", addr.start, addr.end(), mirror.start, mirror.end());
 		// res.trace("new");
 		res
 	}
@@ -817,18 +817,23 @@ impl MemoryBlock {
 
 		self.refresh_all_protections();
 		self.sealed = true;
-		self.hash = {
-			let mut hasher = Sha256::new();
-			bin::write(&mut hasher, &self.addr).unwrap();
-			for p in self.pages.iter() {
-				match &p.snapshot {
-					Snapshot::None => bin::writeval(&mut hasher, 1).unwrap(),
-					Snapshot::ZeroFilled => bin::writeval(&mut hasher, 2).unwrap(),
-					Snapshot::Data(d) => { hasher.write(d.slice()).unwrap(); },
+		#[cfg(not(feature = "no-dirty-detection"))]
+		{
+			use sha2::{Sha256, Digest};
+
+			self.hash = {
+				let mut hasher = Sha256::new();
+				bin::write(&mut hasher, &self.addr).unwrap();
+				for p in self.pages.iter() {
+					match &p.snapshot {
+						Snapshot::None => bin::writeval(&mut hasher, 1).unwrap(),
+						Snapshot::ZeroFilled => bin::writeval(&mut hasher, 2).unwrap(),
+						Snapshot::Data(d) => { hasher.write(d.slice()).unwrap(); },
+					}
 				}
-			}
-			hasher.finalize()[..].to_owned()
-		};
+				hasher.finalize()[..].to_owned()
+			};
+		}
 
 		Ok(())
 	}

waterbox/waterboxhost/src/syscall_defs.rs

@@ -36,6 +36,7 @@ pub fn syscall_ok(result: usize) -> SyscallReturn {
 }
 
 #[repr(transparent)]
+#[derive(Copy, Clone)]
 pub struct SyscallReturn(pub usize);
 impl SyscallReturn {
 	pub const ERROR_THRESH: usize = -4096 as isize as usize;
@@ -653,9 +654,10 @@ lookup! { lookup_syscall: SyscallNumber {
 	NR_FSPICK = 433;
 	NR_PIDFD_OPEN = 434;
 	NR_CLONE3 = 435;
+	NR_WBX_CLONE = 2000;
 }}
 
-pub const MAP_FAILED: usize = 0xffffffffffffffff;
+pub const MAP_FAILED: usize = !0;
 
 pub const MAP_SHARED: usize = 0x01;
 pub const MAP_PRIVATE: usize = 0x02;
@@ -750,3 +752,22 @@ pub struct TimeSpec {
 	pub tv_sec: i64,
 	pub tv_nsec: i64,
 }
+
+pub const FUTEX_WAITERS: u32 = 0x80000000;
+pub const FUTEX_OWNER_DIED: u32 = 0x40000000;
+pub const FUTEX_TID_MASK: u32 = 0x3fffffff;
+
+pub const FUTEX_WAIT: i32 = 0;
+pub const FUTEX_WAKE: i32 = 1;
+pub const FUTEX_FD: i32 = 2;
+pub const FUTEX_REQUEUE: i32 = 3;
+pub const FUTEX_CMP_REQUEUE: i32 = 4;
+pub const FUTEX_WAKE_OP: i32 = 5;
+pub const FUTEX_LOCK_PI: i32 = 6;
+pub const FUTEX_UNLOCK_PI: i32 = 7;
+pub const FUTEX_TRYLOCK_PI: i32 = 8;
+pub const FUTEX_WAIT_BITSET: i32 = 9;
+
+pub const FUTEX_PRIVATE: i32 = 128;
+
+pub const FUTEX_CLOCK_REALTIME: i32 = 256;

waterbox/waterboxhost/src/threading.rs

@@ -0,0 +1,346 @@
+use crate::*;
+use std::collections::{BTreeMap, HashMap};
+use syscall_defs::*;
+use memory_block::{MemoryBlock, Protection};
+use context::Context;
+
+#[derive(Eq, PartialEq, Debug)]
+enum ThreadState {
+	Runnable,
+	Waiting,
+}
+
+struct GuestThread {
+	tid: u32,
+	state: ThreadState,
+	/// The rax value (ie, return from most recent syscall) that will be returned to the guest
+	/// when this thread is next run
+	rax: SyscallReturn,
+	/// the rsp value that will be returned to the guest when this thread is next run
+	rsp: usize,
+	/// pthread_self
+	thread_area: usize,
+	/// set_tid_address
+	tid_address: usize,
+}
+
+struct UnparkResult {
+	tid: u32,
+	has_more: bool
+}
+
+pub struct GuestThreadSet {
+	next_tid: u32,
+	threads: BTreeMap<u32, GuestThread>,
+	/// waiters for each futex
+	futicies: HashMap<usize, Vec<u32>>,
+	/// currently running thread
+	active_tid: u32,
+}
+impl GuestThreadSet {
+	pub fn new() -> GuestThreadSet {
+		let mut threads = BTreeMap::new();
+		threads.insert(1, GuestThread {
+			tid: 1,
+			state: ThreadState::Runnable,
+			rax: syscall_ok(0),
+			rsp: 0, // the real initial state of guest_rsp is preloaded into Context by WaterboxHost
+			thread_area: 0,
+			tid_address: 0,
+		});
+		GuestThreadSet {
+			next_tid: 2,
+			threads,
+			futicies: HashMap::new(),
+			active_tid: 1,
+		}
+	}
+	/// yield to anyone
+	/// If current thread is still runnable, might noop
+	/// Stows the return value to this call, and returns whatever that last thread's return call was
+	fn swap_to_next(&mut self, context: &mut Context, ret: SyscallReturn) -> SyscallReturn {
+		// TODO: Something more strategic here
+		let mut candidates = self.threads.range(self.active_tid + 1..)
+			.chain(self.threads.range(..self.active_tid + 1))
+			.filter(|kvp| kvp.1.state == ThreadState::Runnable);
+
+		match candidates.next() {
+			Some((&tid, _)) => {
+				if tid == self.active_tid {
+					// yield call failed to change thread.  totally legal; do nothing
+					ret
+				} else {
+					self.swap_to(context, tid, ret)
+				}
+			},
+			None => {
+				panic!("All threads fell asleep")
+			}
+		}
+	}
+	/// yield to a particular thread
+	/// Stows the return value to this call, and returns whatever that last thread's return call was
+	fn swap_to(&mut self, context: &mut Context, tid: u32, ret: SyscallReturn) -> SyscallReturn {
+		let old_thread = self.threads.get_mut(&self.active_tid).unwrap();
+		old_thread.rax = ret;
+		old_thread.rsp = context.guest_rsp;
+		old_thread.thread_area = context.thread_area;
+		let new_thread = self.threads.get(&tid).unwrap();
+		assert_eq!(new_thread.state, ThreadState::Runnable);
+		context.guest_rsp = new_thread.rsp;
+		context.thread_area = new_thread.thread_area;
+		self.active_tid = tid;
+		new_thread.rax
+	}
+	fn park_other(&mut self, addr: usize, tid: u32) {
+		assert_ne!(self.active_tid, tid);
+		self.futicies.entry(addr)
+			.or_insert_with(Vec::new)
+			.push(tid);
+		self.threads.get_mut(&tid).unwrap().state = ThreadState::Waiting;
+	}
+	fn park_me(&mut self, context: &mut Context, ret: SyscallReturn, addr: usize) -> SyscallReturn {
+		self.futicies.entry(addr)
+			.or_insert_with(Vec::new)
+			.push(self.active_tid);
+		self.threads.get_mut(&self.active_tid).unwrap().state = ThreadState::Waiting;
+		self.swap_to_next(context, ret)
+	}
+	fn unpark_one(&mut self, addr: usize) -> Option<UnparkResult> {
+		if let Some(queue) = self.futicies.get_mut(&addr) {
+			if queue.len() > 0 {
+				let tid = queue.remove(0);
+				self.threads.get_mut(&tid).unwrap().state = ThreadState::Runnable;
+				if queue.len() == 0 {
+					self.futicies.remove(&addr);
+					Some(UnparkResult {
+						tid,
+						has_more: false
+					})
+				} else {
+					Some(UnparkResult {
+						tid,
+						has_more: true
+					})
+				}
+			} else {
+				None
+			}
+		} else {
+			None
+		}
+	}
+
+	/// Similar to a limited subset of clone(2).
+	/// flags are hardcoded to CLONE_VM | CLONE_FS
+	/// | CLONE_FILES | CLONE_SIGHAND | CLONE_THREAD | CLONE_SYSVSEM
+	/// | CLONE_SETTLS | CLONE_PARENT_SETTID | CLONE_CHILD_CLEARTID | CLONE_DETACHED.
+	/// Child thread does not return to the same place the parent did; instead, it will begin at child_rip
+	pub fn spawn(&mut self, memory_block: &mut MemoryBlock,
+		thread_area: usize, guest_rsp: usize, guest_rip: usize, child_tid: usize, parent_tid: *mut u32
+	) -> Result<u32, SyscallError> {
+		let tid = self.next_tid;
+
+		unsafe {
+			// peek inside the pthread struct to find the full area we must mark as stack-protected
+			let pthread = std::slice::from_raw_parts(thread_area as *const usize, 14);
+			let stack_end = pthread[12];
+			let stack_size = pthread[13];
+			let stack = AddressRange { start: stack_end - stack_size, size: stack_size };
+			memory_block.mprotect(stack.align_expand(), Protection::RWStack)?;
+
+			// set up data on child_stack:  This thread will begin execution by way of a return from syscall_dispatch
+			// to guest_syscall with the specified value in rsp.  guest_syscall will then `pop rbp ; ret`, so arrange
+			// things for that
+			let child_stack = std::slice::from_raw_parts_mut((guest_rsp - 16) as *mut usize, 2);
+			child_stack[0] = 0; // initial RBP value
+			child_stack[1] = guest_rip; // `ret`
+
+			// "return" tid value to caller
+			*parent_tid = tid;
+		}
+
+		let tid = self.next_tid;
+		let thread = GuestThread {
+			tid,
+			state: ThreadState::Runnable,
+			rax: syscall_ok(0),
+			rsp: guest_rsp - 16,
+			thread_area,
+			tid_address: child_tid,
+		};
+		self.threads.insert(tid, thread);
+		self.next_tid += 1;
+		Ok(tid)
+	}
+
+	pub fn exit(&mut self, context: &mut Context) -> SyscallReturn {
+		if self.active_tid == 1 {
+			unsafe { std::intrinsics::breakpoint() }
+		}
+		let addr = self.threads.get_mut(&self.active_tid).unwrap().tid_address;
+		if addr != 0 {
+			let atom = unsafe { &mut *(addr as *mut u32) };
+			*atom = 0;
+			self.unpark_one(addr);
+		}
+		let dead_tid = self.active_tid;
+		let ret = self.swap_to_next(context, syscall_ok(0));
+		if self.active_tid == dead_tid {
+			panic!("Thread exited but no thread is available to replace it");
+		}
+		self.threads.remove(&dead_tid);
+		ret
+	}
+
+	pub fn futex_wait(&mut self, context: &mut Context, addr: usize, compare: u32) -> SyscallReturn {
+		let atom = unsafe { &mut *(addr as *mut u32) };
+		if *atom != compare {
+			syscall_err(EAGAIN)
+		} else {
+			self.park_me(context, syscall_ok(0), addr)
+		}
+	}
+	
+	pub fn futex_wake(&mut self, addr: usize, count: u32) -> usize {
+		self.futex_requeue(addr, 0, count, 0)
+			.unwrap()
+	}
+
+	pub fn futex_requeue(&mut self, addr_from: usize, addr_to: usize, mut wake_count: u32, mut requeue_count: u32) -> Result<usize, SyscallError> {
+		// NB: musl only does wake_count = 0, requeue_count = 1
+		let mut count = 0;
+
+		while wake_count > 0 || requeue_count > 0 {
+			if let Some(res) = self.unpark_one(addr_from) {
+				count += 1;
+				if wake_count > 0 {
+					wake_count -= 1;
+				} else {
+					self.park_other(addr_to, res.tid);
+					requeue_count -= 1;
+				}
+				if !res.has_more {
+					break
+				}
+			} else {
+				break
+			}
+		}
+
+		Ok(count)
+	}
+	
+	// don't handle priority inversion, or the clock information (how could we introduce a clock, anyway?)
+	// use fair handoffs for simplicity
+	pub fn futex_lock_pi(&mut self, context: &mut Context, addr: usize) -> SyscallReturn {
+		let atom = unsafe { &mut *(addr as *mut u32) };
+		if *atom == 0 {
+			*atom = self.active_tid;
+			return syscall_ok(0)
+		}
+		*atom |= FUTEX_WAITERS;
+		self.park_me(context, syscall_ok(0), addr)
+	}
+	
+	pub fn futex_unlock_pi(&mut self, context: &mut Context, addr: usize) -> SyscallReturn {
+		let atom = unsafe { &mut *(addr as *mut u32) };
+		match self.unpark_one(addr) {
+			Some(res) => {
+				if res.has_more {
+					*atom = res.tid | FUTEX_WAITERS;
+				} else {
+					*atom = res.tid;
+				}
+				self.swap_to(context, res.tid, syscall_ok(0)) // "fair" unlock
+			},
+			None => {
+				// unclear what to return here
+				*atom = 0;
+				syscall_ok(0)
+			}
+		}
+	}
+	
+	pub fn set_tid_address(&mut self, addr: usize) -> u32 {
+		let thread = self.threads.get_mut(&self.active_tid).unwrap();
+		thread.tid_address = addr;
+		thread.tid
+	}
+
+	pub fn get_tid(&self) -> u32 {
+		self.active_tid
+	}
+
+	pub fn yield_any(&mut self, context: &mut Context) -> SyscallReturn {
+		self.swap_to_next(context, syscall_ok(0))
+	}
+}
+
+const MAGIC: &str = "GuestThreadSet";
+impl GuestThreadSet {
+	pub fn save_state(&mut self, context: &Context, stream: &mut dyn Write) -> anyhow::Result<()> {
+		assert_eq!(self.active_tid, 1, "Thread hijack?");
+
+		{
+			let main_thread = self.threads.get_mut(&1).unwrap();
+			main_thread.thread_area = context.thread_area;
+			main_thread.rsp = context.guest_rsp;
+		}
+
+		bin::write_magic(stream, MAGIC)?;
+
+		bin::write(stream, &self.next_tid)?;
+		bin::write(stream, &self.active_tid)?;
+
+		bin::write(stream, &self.threads.len())?;
+		for t in self.threads.values() {
+			bin::write(stream, t)?;
+		}
+
+		bin::write(stream, &self.futicies.len())?;
+		for (addr, waiters) in self.futicies.iter() {
+			bin::write(stream, addr)?;
+			bin::write(stream, &waiters.len())?;
+			for tid in waiters.iter() {
+				bin::write(stream, tid)?;
+			}
+		}
+
+		bin::write_magic(stream, MAGIC)?;
+		Ok(())
+	}
+	pub fn load_state(&mut self, context: &mut Context, stream: &mut dyn Read) -> anyhow::Result<()> {
+		assert_eq!(self.active_tid, 1, "Thread hijack?");
+		bin::verify_magic(stream, MAGIC)?;
+
+		bin::read(stream, &mut self.next_tid)?;
+		bin::read(stream, &mut self.active_tid)?;
+
+		self.threads.clear();
+		for _ in 0..bin::readval::<usize>(stream)? {
+			let thread = bin::readval::<GuestThread>(stream)?;
+			self.threads.insert(thread.tid, thread);
+		}
+
+		self.futicies.clear();
+		for _ in 0..bin::readval::<usize>(stream)? {
+			let addr = bin::readval::<usize>(stream)?;
+			let mut waiters = Vec::new();
+			for _ in 0..bin::readval::<usize>(stream)? {
+				waiters.push(bin::readval(stream)?);
+			}
+			self.futicies.insert(addr, waiters);
+		}
+
+		bin::verify_magic(stream, MAGIC)?;
+
+		{
+			let main_thread = self.threads.get(&1).unwrap();
+			context.thread_area = main_thread.thread_area;
+			context.guest_rsp = main_thread.rsp;
+		}
+
+		Ok(())
+	}
+}