rainy day project: load waterbox cores in the first 2G and compile with mcmodel=small

5-10% speedup in some cores.

implementing this would also include changes to musl buildscript which were not committed here

src/BizHawk.Emulation.Cores/Waterbox/Swappable.cs

@@ -52,8 +52,6 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			var lockkey = (uint)(startAddress >> 32);
 
 			_lockkey = lockkey;
-			if (lockkey == 0)
-				throw new NullReferenceException();
 			_currentLockInfo = LockInfos.GetOrAdd(_lockkey, new LockInfo { Sync = new object() });
 		}
 

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHost.cs

@@ -76,7 +76,7 @@ namespace BizHawk.Emulation.Cores.Waterbox
 		/// <summary>
 		/// usual starting point for the executable
 		/// </summary>
-		public const ulong CanonicalStart = 0x0000036f00000000;
+		public const ulong CanonicalStart = 0x2000_0000;
 
 		/// <summary>
 		/// the next place where we can put a module or heap

waterbox/common.mak

@@ -15,6 +15,7 @@ ifdef NEED_LIBCO
 EMULIBC_OBJS := $(EMULIBC_OBJS) $(shell find $(WATERBOX_DIR)/libco/obj/release -type f -name '*.o')
 EMULIBC_DOBJS := $(EMULIBC_DOBJS) $(shell find $(WATERBOX_DIR)/libco/obj/debug -type f -name '*.o')
 endif
+LINKSCRIPT := $(WATERBOX_DIR)/linkscript.T
 
 print-%: ;
 	@echo $* = $($*)
@@ -22,11 +23,11 @@ print-%: ;
 #LD_PLUGIN := $(shell gcc --print-file-name=liblto_plugin.so)
 
 CC := $(SYSROOT)/bin/musl-gcc
-COMMONFLAGS := -mabi=ms -fvisibility=hidden -I$(WATERBOX_DIR)/emulibc -Wall -mcmodel=large \
+COMMONFLAGS := -mabi=ms -fvisibility=hidden -I$(WATERBOX_DIR)/emulibc -Wall -mcmodel=small \
 	-mstack-protector-guard=global -no-pie -fno-pic -fno-pie -fcf-protection=none \
 	-MD -MP
 CCFLAGS := $(CCFLAGS) $(COMMONFLAGS)
-LDFLAGS := $(LDFLAGS) -static -Wl,--eh-frame-hdr -T $(WATERBOX_DIR)/linkscript.T #-Wl,--plugin,$(LD_PLUGIN)
+LDFLAGS := $(LDFLAGS) -static -Wl,--eh-frame-hdr -T $(LINKSCRIPT) #-Wl,--no-relax #-Wl,--plugin,$(LD_PLUGIN)
 CCFLAGS_DEBUG := -O0 -g
 CCFLAGS_RELEASE := -O3 -flto
 CCFLAGS_RELEASE_ASONLY := -O3
@@ -83,10 +84,10 @@ TARGET_DEBUG := $(DOBJ_DIR)/$(TARGET)
 release: $(TARGET_RELEASE)
 debug: $(TARGET_DEBUG)
 
-$(TARGET_RELEASE): $(OBJS) $(EMULIBC_OBJS)
+$(TARGET_RELEASE): $(OBJS) $(EMULIBC_OBJS) $(LINKSCRIPT)
 	@echo ld $@
 	@$(CC) -o $@ $(LDFLAGS) $(LDFLAGS_RELEASE) $(CCFLAGS) $(CCFLAGS_RELEASE) $(OBJS) $(EMULIBC_OBJS) $(EXTRA_LIBS)
-$(TARGET_DEBUG): $(DOBJS) $(EMULIBC_DOBJS)
+$(TARGET_DEBUG): $(DOBJS) $(EMULIBC_DOBJS) $(LINKSCRIPT)
 	@echo ld $@
 	@$(CC) -o $@ $(LDFLAGS) $(LDFLAGS_DEBUG) $(CCFLAGS) $(CCFLAGS_DEBUG) $(DOBJS) $(EMULIBC_DOBJS) $(EXTRA_LIBS)
 

waterbox/libco/amd64.c

@@ -161,8 +161,39 @@ void co_delete(cothread_t handle)
 	free_thread(handle);
 }
 
+static uint64_t hoststart;
+static uint64_t hostend;
+
 void co_switch(cothread_t handle)
 {
+	uint64_t start;
+	uint64_t end;
+	if (co_active_handle == &co_host_buffer)
+	{
+		// migrating off of real thread; save stack params
+		__asm__("movq %%gs:0x08, %0": "=r"(end));
+		__asm__("movq %%gs:0x10, %0": "=r"(start));
+		hoststart = start;
+		hostend = end;
+	}
+	if (handle == &co_host_buffer)
+	{
+		// migrating onto real thread; load stack params
+		start = hoststart;
+		end = hostend;
+		hoststart = 0;
+		hostend = 0;
+	}
+	else
+	{
+		// migrating onto cothread; compute its extents we allocated them
+		cothread_impl* co = handle;
+		start = (uintptr_t)co->stack_bottom;
+		end = (uintptr_t)co->stack_top;
+	}
+	__asm__("movq %0, %%gs:0x08":: "r"(end));
+	__asm__("movq %0, %%gs:0x10":: "r"(start));
+
 	register cothread_t co_previous_handle = co_active_handle;
 	co_swap(co_active_handle = handle, co_previous_handle);
 }

waterbox/libcxx/configure-for-waterbox-phase--

@@ -10,8 +10,8 @@ sed -i -e '13c\' -e '' "$SYSROOT/lib/musl-gcc.specs"
 rm -rf build-
 mkdir build-
 cd build-
-export CFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -no-pie -fno-pic -fno-pie"
-export CXXFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -no-pie -fno-pic -fno-pie"
+export CFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -no-pie -fno-pic -fno-pie"
+export CXXFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -no-pie -fno-pic -fno-pie"
 cmake \
 	-DCMAKE_C_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DCMAKE_CXX_COMPILER="$SYSROOT/bin/musl-gcc" \

waterbox/libcxx/configure-for-waterbox-phase-0

@@ -5,8 +5,8 @@ LLVMDIR="`realpath \"$MYPATH/../../../llvm-project\"`"
 rm -rf build0
 mkdir build0
 cd build0
-export CFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie -D_WIN64 -D_LIBUNWIND_IS_BAREMETAL -D_LIBUNWIND_SUPPORT_DWARF_UNWIND"
-export CXXFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie -D_WIN64 -D_LIBUNWIND_IS_BAREMETAL -D_LIBUNWIND_SUPPORT_DWARF_UNWIND"
+export CFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie -D_WIN64 -D_LIBUNWIND_IS_BAREMETAL -D_LIBUNWIND_SUPPORT_DWARF_UNWIND"
+export CXXFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie -D_WIN64 -D_LIBUNWIND_IS_BAREMETAL -D_LIBUNWIND_SUPPORT_DWARF_UNWIND"
 cmake \
 	-DCMAKE_C_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DCMAKE_CXX_COMPILER="$SYSROOT/bin/musl-gcc" \

waterbox/libcxx/configure-for-waterbox-phase-1

@@ -5,8 +5,8 @@ LLVMDIR="`realpath \"$MYPATH/../../../llvm-project\"`"
 rm -rf build1
 mkdir build1
 cd build1
-export CFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
-export CXXFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
+export CFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
+export CXXFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
 cmake \
 	-DCMAKE_C_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DCMAKE_CXX_COMPILER="$SYSROOT/bin/musl-gcc" \

waterbox/libcxx/configure-for-waterbox-phase-2

@@ -10,8 +10,8 @@ cp -n "/usr/include/linux/version.h" "$SYSROOT/include/linux"
 rm -rf build2
 mkdir build2
 cd build2
-export CFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
-export CXXFLAGS="-mabi=ms -mcmodel=large -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
+export CFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
+export CXXFLAGS="-mabi=ms -mcmodel=small -mstack-protector-guard=global -fno-use-cxa-atexit -no-pie -fno-pic -fno-pie"
 cmake \
 	-DCMAKE_C_COMPILER="$SYSROOT/bin/musl-gcc" \
 	-DCMAKE_CXX_COMPILER="$SYSROOT/bin/musl-gcc" \

waterbox/linkscript.T

@@ -10,7 +10,7 @@ ENTRY(_start)
 SEARCH_DIR("=/usr/local/lib/x86_64-linux-gnu"); SEARCH_DIR("=/lib/x86_64-linux-gnu"); SEARCH_DIR("=/usr/lib/x86_64-linux-gnu"); SEARCH_DIR("=/usr/lib/x86_64-linux-gnu64"); SEARCH_DIR("=/usr/local/lib64"); SEARCH_DIR("=/lib64"); SEARCH_DIR("=/usr/lib64"); SEARCH_DIR("=/usr/local/lib"); SEARCH_DIR("=/lib"); SEARCH_DIR("=/usr/lib"); SEARCH_DIR("=/usr/x86_64-linux-gnu/lib64"); SEARCH_DIR("=/usr/x86_64-linux-gnu/lib");
 SECTIONS
 {
-  PROVIDE (__executable_start = SEGMENT_START("text-segment", 0x36f00000000)); . = SEGMENT_START("text-segment", 0x36f00000000) + SIZEOF_HEADERS;
+  PROVIDE (__executable_start = SEGMENT_START("text-segment", 0x20000000)); . = SEGMENT_START("text-segment", 0x20000000) + SIZEOF_HEADERS;
   .interp         : { *(.interp) }
   .note.gnu.build-id  : { *(.note.gnu.build-id) }
   .hash           : { *(.hash) }

waterbox/nyma/zlib/configure-for-waterbox

@@ -2,5 +2,5 @@
 MYPATH="`dirname \"$0\"`"
 SYSROOT="`realpath \"$MYPATH/../../sysroot\"`"
 export CC=$SYSROOT/bin/musl-gcc
-export CFLAGS="-O3 -mabi=ms -mcmodel=large -mstack-protector-guard=global -no-pie -fno-pic -fno-pie"
+export CFLAGS="-O3 -mabi=ms -mcmodel=small -mstack-protector-guard=global -no-pie -fno-pic -fno-pie"
 ./configure --static --prefix=$SYSROOT

waterbox/nyma/zlib/do-everything-for-waterbox

@@ -0,0 +1,5 @@
+#!/bin/sh
+./configure-for-waterbox
+make clean
+make libz.a
+./install-for-waterbox