Zstd Compression (#3345)

Deflate compression in rewinder is now zstd compression
Binary blobs in zip files are zstd compressed (text is uncompresed for user ease).
All wbx cores and resources are re-compressed with zstd, wbx build scripts are changed to account for this. Shaves off a bit with download size and it's faster to decompress to.

src/BizHawk.Client.Common/FrameworkZipWriter.cs

@@ -2,12 +2,16 @@
 using System.IO;
 using System.IO.Compression;
 
+using BizHawk.Emulation.Common;
+
 namespace BizHawk.Client.Common
 {
 	public class FrameworkZipWriter : IZipWriter
 	{
-		private ZipArchive _archive;
+		private ZipArchive _archive;
+		private Zstd _zstd;
 		private readonly CompressionLevel _level;
+		private readonly int _zstdCompressionLevel;
 
 		public FrameworkZipWriter(string path, int compressionLevel)
 		{
@@ -18,13 +22,27 @@ namespace BizHawk.Client.Common
 			else if (compressionLevel < 5)
 				_level = CompressionLevel.Fastest;
 			else
-				_level = CompressionLevel.Optimal;
+				_level = CompressionLevel.Optimal;
+
+			_zstd = new();
+			// compressionLevel ranges from 0 to 9
+			// normal compression level range for zstd is 1 to 19
+			_zstdCompressionLevel = compressionLevel * 2 + 1;
 		}
 
-		public void WriteItem(string name, Action<Stream> callback)
+		public void WriteItem(string name, Action<Stream> callback, bool zstdCompress)
 		{
 			using var stream = _archive.CreateEntry(name, _level).Open();
-			callback(stream);
+
+			if (zstdCompress)
+			{
+				using var z = _zstd.CreateZstdCompressionStream(stream, _zstdCompressionLevel);
+				callback(z);
+			}
+			else
+			{
+				callback(stream);
+			}
 		}
 
 		public void Dispose()
@@ -34,6 +52,12 @@ namespace BizHawk.Client.Common
 				_archive.Dispose();
 				_archive = null;
 			}
+
+			if (_zstd != null)
+			{
+				_zstd.Dispose();
+				_zstd = null;
+			}
 		}
 	}
 }

src/BizHawk.Client.Common/IZipWriter.cs

@@ -5,6 +5,6 @@ namespace BizHawk.Client.Common
 {
 	public interface IZipWriter : IDisposable
 	{
-		void WriteItem(string name, Action<Stream> callback);
+		void WriteItem(string name, Action<Stream> callback, bool zstdCompress);
 	}
 }

src/BizHawk.Client.Common/config/RewindConfig.cs

@@ -5,13 +5,13 @@
 		/// <summary>
 		/// Gets a value indicating whether or not to compress savestates before storing them
 		/// </summary>
-		bool UseCompression { get; }
-
+		bool UseCompression { get; }
+
 		/// <summary>
 		/// Gets a value indicating whether or not to delta compress savestates before storing them
 		/// </summary>
 		/// <value></value>
-		// TODO: This is in here for frontend reasons, but the buffer itself doesn't interact with this.
+		// TODO: This is in here for frontend reasons, but the buffer itself doesn't interact with this.
 		bool UseDelta { get; }
 
 		/// <summary>
@@ -22,7 +22,7 @@
 		/// <summary>
 		/// Specifies whether TargetFrameLength or TargetRewindInterval is used.
 		/// </summary>
-		public bool UseFixedRewindInterval { get; }
+		bool UseFixedRewindInterval { get; }
 
 		/// <summary>
 		/// Desired frame length (number of emulated frames you can go back before running out of buffer)
@@ -45,13 +45,13 @@
 			TempFile,
 		}
 
-		public BackingStoreType BackingStore { get; }
+		BackingStoreType BackingStore { get; }
 	}
 
 	public class RewindConfig : IRewindSettings
 	{
-		public bool UseCompression { get; set; }
-		public bool UseDelta { get; set; }
+		public bool UseCompression { get; set; } = false;
+		public bool UseDelta { get; set; } = false;
 		public bool Enabled { get; set; } = true;
 		public long BufferSize { get; set; } = 512; // in mb
 		public bool UseFixedRewindInterval { get; set; } = false;

src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManager.cs

@@ -67,10 +67,15 @@ namespace BizHawk.Client.Common
 			get
 			{
 				var (f, data) = GetStateClosestToFrame(frame);
-				if (f != frame) return NonState;
+				if (f != frame)
+				{
+					data.Dispose();
+					return NonState;
+				}
 
 				var ms = new MemoryStream();
 				data.CopyTo(ms);
+				data.Dispose();
 				return ms.ToArray();
 			}
 		}
@@ -169,7 +174,11 @@ namespace BizHawk.Client.Common
 						if (_reserveCallback(si.Frame))
 							AddToReserved(si);
 						else
-							buffer.Capture(si.Frame, s => si.GetReadStream().CopyTo(s), null, true);
+							buffer.Capture(si.Frame, s => 
+							{
+								using var rs = si.GetReadStream();
+								rs.CopyTo(s);
+							}, null, true);
 					}
 					old.Dispose();
 				}
@@ -276,10 +285,10 @@ namespace BizHawk.Client.Common
 				return;
 			}
 
-			var bb = new byte[state.Size];
-			var ms = new MemoryStream(bb);
-			state.GetReadStream().CopyTo(ms);
-			_reserved.Add(state.Frame, bb);
+			var ms = new MemoryStream();
+			using var s = state.GetReadStream();
+			s.CopyTo(ms);
+			_reserved.Add(state.Frame, ms.ToArray());
 			AddStateCache(state.Frame);
 		}
 
@@ -347,7 +356,8 @@ namespace BizHawk.Client.Common
 					_recent.Capture(state.Frame,
 						s =>
 						{
-							state.GetReadStream().CopyTo(s);
+							using var rs = state.GetReadStream();
+							rs.CopyTo(s);
 							AddStateCache(state.Frame);
 						},
 						index2 =>

src/BizHawk.Client.Common/rewind/ZeldaWinder.cs

@@ -203,7 +203,8 @@ namespace BizHawk.Client.Common
 		{
 			var lengthHolder = 0;
 			var lengthHolderSpan = new Span<byte>(&lengthHolder, 4);
-			var zeldas = SpanStream.GetOrBuild(state.GetReadStream());
+			using var rs = state.GetReadStream();
+			var zeldas = SpanStream.GetOrBuild(rs);
 			zeldas.Read(lengthHolderSpan);
 			_masterLength = lengthHolder;
 			fixed (byte* buffer_ = _master)

src/BizHawk.Client.Common/rewind/Zwinder.cs

@@ -72,14 +72,16 @@ namespace BizHawk.Client.Common
 				{
 					state = _buffer.GetState(index - 1);
 				}
-				_stateSource.LoadStateBinary(new BinaryReader(state.GetReadStream()));
+				using var br = new BinaryReader(state.GetReadStream());
+				_stateSource.LoadStateBinary(br);
 				_buffer.InvalidateEnd(index);
 			}
 			else
 			{
 				// The emulator will frame advance without giving us a chance to
 				// re-capture this frame, so we shouldn't invalidate this state just yet.
-				_stateSource.LoadStateBinary(new BinaryReader(state.GetReadStream()));
+				using var br = new BinaryReader(state.GetReadStream());
+				_stateSource.LoadStateBinary(br);
 			}
 			return true;
 		}

src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs

@@ -1,10 +1,11 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.IO.Compression;
 using System.Linq;
+
 using BizHawk.BizInvoke;
 using BizHawk.Common;
+using BizHawk.Emulation.Common;
 
 namespace BizHawk.Client.Common
 {
@@ -63,6 +64,7 @@ namespace BizHawk.Client.Common
 			}
 			_allowOutOfOrderStates = settings.AllowOutOfOrderStates;
 			_states = new StateInfo[STATEMASK + 1];
+			_zstd = new();
 			_useCompression = settings.UseCompression;
 		}
 
@@ -71,6 +73,7 @@ namespace BizHawk.Client.Common
 			foreach (var d in (_disposables as IEnumerable<IDisposable>).Reverse())
 				d.Dispose();
 			_disposables.Clear();
+			_zstd.Dispose();
 		}
 
 		private readonly List<IDisposable> _disposables = new List<IDisposable>();
@@ -126,6 +129,7 @@ namespace BizHawk.Client.Common
 		private int _nextStateIndex;
 		private int HeadStateIndex => (_nextStateIndex - 1) & STATEMASK;
 
+		private readonly Zstd _zstd;
 		private readonly bool _useCompression;
 
 		/// <summary>
@@ -233,7 +237,8 @@ namespace BizHawk.Client.Common
 
 			if (_useCompression)
 			{
-				using var compressor = new DeflateStream(stream, CompressionLevel.Fastest, leaveOpen: true);
+				// TODO: expose compression level as a setting
+				using var compressor = _zstd.CreateZstdCompressionStream(stream, 1);
 				callback(compressor);
 			}
 			else
@@ -253,7 +258,7 @@ namespace BizHawk.Client.Common
 		{
 			Stream stream = new LoadStateStream(_backingStore, _states[index].Start, _states[index].Size, _sizeMask);
 			if (_useCompression)
-				stream = new DeflateStream(stream, CompressionMode.Decompress, leaveOpen: true);
+				stream = _zstd.CreateZstdDecompressionStream(stream);
 			return stream;
 		}
 

src/BizHawk.Client.Common/savestates/BinaryStateLump.cs

@@ -8,7 +8,7 @@ namespace BizHawk.Client.Common
 	public class BinaryStateLump
 	{
 		[Name("BizState 1", "0")]
-		public static BinaryStateLump Versiontag { get; private set; }
+		public static BinaryStateLump ZipVersion { get; private set; }
 		[Name("BizVersion", "txt")]
 		public static BinaryStateLump BizVersion { get; private set; }
 		[Name("Core", "bin")]

src/BizHawk.Client.Common/savestates/SavestateFile.cs

@@ -50,7 +50,6 @@ namespace BizHawk.Client.Common
 			// the old method of text savestate save is now gone.
 			// a text savestate is just like a binary savestate, but with a different core lump
 			using var bs = new ZipStateSaver(filename, config.CompressionLevelNormal);
-			bs.PutVersionLumps();
 
 			using (new SimpleTime("Save Core"))
 			{

src/BizHawk.Client.Common/savestates/ZipStateLoader.cs

@@ -4,6 +4,8 @@ using System.IO;
 using System.IO.Compression;
 using System.Linq;
 
+using BizHawk.Emulation.Common;
+
 namespace BizHawk.Client.Common
 {
 	public class ZipStateLoader : IDisposable
@@ -12,9 +14,11 @@ namespace BizHawk.Client.Common
 		private Version _ver;
 		private bool _isDisposed;
 		private Dictionary<string, ZipArchiveEntry> _entriesByName;
+		private readonly Zstd _zstd;
 
 		private ZipStateLoader()
 		{
+			_zstd = new();
 		}
 
 		public void Dispose()
@@ -30,11 +34,12 @@ namespace BizHawk.Client.Common
 				if (disposing)
 				{
 					_zip.Dispose();
+					_zstd.Dispose();
 				}
 			}
 		}
 
-		private void ReadVersion(Stream s, long length)
+		private void ReadZipVersion(Stream s, long length)
 		{
 			// the "BizState 1.0" tag contains an integer in it describing the sub version.
 			if (length == 0)
@@ -85,7 +90,16 @@ namespace BizHawk.Client.Common
 			{
 				ret._zip = new ZipArchive(new FileStream(filename, FileMode.Open, FileAccess.Read), ZipArchiveMode.Read);
 				ret.PopulateEntries();
-				if (!isMovieLoad && !ret.GetLump(BinaryStateLump.Versiontag, false, ret.ReadVersion))
+				if (isMovieLoad)
+				{
+					if (!ret.GetLump(BinaryStateLump.ZipVersion, false, ret.ReadZipVersion, false))
+					{
+						// movies before 1.0.2 did not include the BizState 1.0 file, don't strictly error in this case
+						ret._ver = new Version(1, 0, 0);
+						Console.WriteLine("Read a zipstate of version {0}", ret._ver);
+					}
+				}
+				else if (!ret.GetLump(BinaryStateLump.ZipVersion, false, ret.ReadZipVersion, false))
 				{
 					ret._zip.Dispose();
 					return null;
@@ -102,14 +116,24 @@ namespace BizHawk.Client.Common
 		/// <param name="lump">lump to retrieve</param>
 		/// <param name="abort">pass true to throw exception instead of returning false</param>
 		/// <param name="callback">function to call with the desired stream</param>
+		/// <param name="isZstdCompressed">lump is zstd compressed</param>
 		/// <returns>true iff stream was loaded</returns>
 		/// <exception cref="Exception">stream not found and <paramref name="abort"/> is <see langword="true"/></exception>
-		public bool GetLump(BinaryStateLump lump, bool abort, Action<Stream, long> callback)
+		public bool GetLump(BinaryStateLump lump, bool abort, Action<Stream, long> callback, bool isZstdCompressed = true)
 		{
 			if (_entriesByName.TryGetValue(lump.ReadName, out var e))
 			{
-				using var zs = e.Open();
-				callback(zs, e.Length);
+				using var zs = e.Open();
+
+				if (isZstdCompressed && _ver.Build > 1)
+				{
+					using var z = _zstd.CreateZstdDecompressionStream(zs);
+					callback(z, e.Length);
+				}
+				else
+				{
+					callback(zs, e.Length);
+				}
 
 				return true;
 			}
@@ -129,7 +153,7 @@ namespace BizHawk.Client.Common
 			=> GetLump(lump, abort, (s, length) => callback(new(s), length));
 
 		public bool GetLump(BinaryStateLump lump, bool abort, Action<TextReader> callback)
-			=> GetLump(lump, abort, (s, _) => callback(new StreamReader(s)));
+			=> GetLump(lump, abort, (s, _) => callback(new StreamReader(s)), false);
 
 		/// <exception cref="Exception">couldn't find Binary or Text savestate</exception>
 		public void GetCoreState(Action<BinaryReader, long> callbackBinary, Action<TextReader> callbackText)

src/BizHawk.Client.Common/savestates/ZipStateSaver.cs

@@ -10,54 +10,52 @@ namespace BizHawk.Client.Common
 		private readonly IZipWriter _zip;
 		private bool _isDisposed;
 
-		private static void WriteVersion(Stream s)
+		private static void WriteZipVersion(Stream s)
 		{
-			var sw = new StreamWriter(s);
-			sw.WriteLine("1"); // version 1.0.1
-			sw.Flush();
+			using var sw = new StreamWriter(s);
+			sw.WriteLine("2"); // version 1.0.2
 		}
 
 		private static void WriteEmuVersion(Stream s)
 		{
-			var sw = new StreamWriter(s);
+			using var sw = new StreamWriter(s);
 			sw.WriteLine(VersionInfo.GetEmuVersion());
-			sw.Flush();
 		}
 
 		public ZipStateSaver(string path, int compressionLevel)
 		{
-			_zip = new FrameworkZipWriter(path, compressionLevel);
+			_zip = new FrameworkZipWriter(path, compressionLevel);
+
+			// we put these in every zip, so we know where they came from
+			// a bit redundant for movie files given their headers, but w/e
+			PutLump(BinaryStateLump.ZipVersion, WriteZipVersion, false);
+			PutLump(BinaryStateLump.BizVersion, WriteEmuVersion, false);
 		}
 
-		public void PutVersionLumps()
+		public void PutLump(BinaryStateLump lump, Action<Stream> callback, bool zstdCompress = true)
 		{
-			PutLump(BinaryStateLump.Versiontag, WriteVersion);
-			PutLump(BinaryStateLump.BizVersion, WriteEmuVersion);
-		}
-
-		public void PutLump(BinaryStateLump lump, Action<Stream> callback)
-		{
-			_zip.WriteItem(lump.WriteName, callback);
+			_zip.WriteItem(lump.WriteName, callback, zstdCompress);
 		}
 
 		public void PutLump(BinaryStateLump lump, Action<BinaryWriter> callback)
 		{
 			PutLump(lump, s =>
-			{
-				var bw = new BinaryWriter(s);
-				callback(bw);
+			{
+				var bw = new BinaryWriter(s);
+				callback(bw);
 				bw.Flush();
 			});
 		}
 
 		public void PutLump(BinaryStateLump lump, Action<TextWriter> callback)
 		{
+			// don't zstd compress text, as it's annoying for users
 			PutLump(lump, s =>
 			{
 				TextWriter tw = new StreamWriter(s);
 				callback(tw);
 				tw.Flush();
-			});
+			}, false);
 		}
 
 		public void Dispose()

src/BizHawk.Client.EmuHawk/config/RewindConfig.Designer.cs

@@ -127,7 +127,7 @@
 			this.UseCompression.Name = "UseCompression";
 			this.UseCompression.Size = new System.Drawing.Size(324, 17);
 			this.UseCompression.TabIndex = 5;
-			this.UseCompression.Text = "Use zlib compression (economizes buffer usage at cost of CPU)";
+			this.UseCompression.Text = "Use zstd compression (economizes buffer usage at cost of CPU)";
 			this.UseCompression.UseVisualStyleBackColor = true;
 			this.UseCompression.CheckedChanged += new System.EventHandler(this.UseCompression_CheckedChanged);
 			// 
@@ -603,29 +603,29 @@
 		private System.Windows.Forms.GroupBox groupBox4;
 		private BizHawk.WinForms.Controls.LocLabelEx RewindFramesUsedLabel;
 		private BizHawk.WinForms.Controls.LocLabelEx label7;
-        private BizHawk.WinForms.Controls.LocLabelEx ApproxFramesLabel;
-        private BizHawk.WinForms.Controls.LocLabelEx label8;
-        private BizHawk.WinForms.Controls.LocLabelEx EstTimeLabel;
-				private BizHawk.WinForms.Controls.LocLabelEx label11;
-				private System.Windows.Forms.GroupBox groupBox6;
-				private System.Windows.Forms.RadioButton rbStatesText;
-				private System.Windows.Forms.RadioButton rbStatesBinary;
-				private System.Windows.Forms.ToolTip toolTip1;
-				private System.Windows.Forms.TrackBar trackBarCompression;
-				private System.Windows.Forms.NumericUpDown nudCompression;
-				private System.Windows.Forms.Button btnResetCompression;
-				private System.Windows.Forms.GroupBox groupBox7;
-				private BizHawk.WinForms.Controls.LocLabelEx label12;
-				private BizHawk.WinForms.Controls.LocLabelEx KbLabel;
-				private System.Windows.Forms.NumericUpDown BigScreenshotNumeric;
-				private System.Windows.Forms.CheckBox LowResLargeScreenshotsCheckbox;
-				private BizHawk.WinForms.Controls.LocLabelEx label13;
-				private BizHawk.WinForms.Controls.LocLabelEx label14;
-				private System.Windows.Forms.CheckBox ScreenshotInStatesCheckbox;
-				private BizHawk.WinForms.Controls.LocLabelEx label15;
-				private BizHawk.WinForms.Controls.LocLabelEx label16;
-				private System.Windows.Forms.CheckBox BackupSavestatesCheckbox;
-				private BizHawk.WinForms.Controls.LocLabelEx label20;
+		private BizHawk.WinForms.Controls.LocLabelEx ApproxFramesLabel;
+		private BizHawk.WinForms.Controls.LocLabelEx label8;
+		private BizHawk.WinForms.Controls.LocLabelEx EstTimeLabel;
+		private BizHawk.WinForms.Controls.LocLabelEx label11;
+		private System.Windows.Forms.GroupBox groupBox6;
+		private System.Windows.Forms.RadioButton rbStatesText;
+		private System.Windows.Forms.RadioButton rbStatesBinary;
+		private System.Windows.Forms.ToolTip toolTip1;
+		private System.Windows.Forms.TrackBar trackBarCompression;
+		private System.Windows.Forms.NumericUpDown nudCompression;
+		private System.Windows.Forms.Button btnResetCompression;
+		private System.Windows.Forms.GroupBox groupBox7;
+		private BizHawk.WinForms.Controls.LocLabelEx label12;
+		private BizHawk.WinForms.Controls.LocLabelEx KbLabel;
+		private System.Windows.Forms.NumericUpDown BigScreenshotNumeric;
+		private System.Windows.Forms.CheckBox LowResLargeScreenshotsCheckbox;
+		private BizHawk.WinForms.Controls.LocLabelEx label13;
+		private BizHawk.WinForms.Controls.LocLabelEx label14;
+		private System.Windows.Forms.CheckBox ScreenshotInStatesCheckbox;
+		private BizHawk.WinForms.Controls.LocLabelEx label15;
+		private BizHawk.WinForms.Controls.LocLabelEx label16;
+		private System.Windows.Forms.CheckBox BackupSavestatesCheckbox;
+		private BizHawk.WinForms.Controls.LocLabelEx label20;
 		private System.Windows.Forms.NumericUpDown TargetFrameLengthNumeric;
 		private System.Windows.Forms.NumericUpDown TargetRewindIntervalNumeric;
 		private System.Windows.Forms.CheckBox cbDeltaCompression;

src/BizHawk.Client.EmuHawk/tools/TAStudio/TAStudio.cs

@@ -910,6 +910,7 @@ namespace BizHawk.Client.EmuHawk
 			{
 				LoadState(closestState);
 			}
+			closestState.Value.Dispose();
 
 			if (fromLua)
 			{

src/BizHawk.Emulation.Common/zstd/LibZstd.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Runtime.InteropServices;
+
+using BizHawk.BizInvoke;
+
+namespace BizHawk.Emulation.Common
+{
+	public abstract class LibZstd
+	{
+		private const CallingConvention cc = CallingConvention.Cdecl;
+
+		[BizImport(cc)]
+		public abstract uint ZSTD_isError(ulong code);
+
+		[BizImport(cc)]
+		public abstract IntPtr ZSTD_getErrorName(ulong code);
+
+		[BizImport(cc)]
+		public abstract int ZSTD_minCLevel();
+
+		[BizImport(cc)]
+		public abstract int ZSTD_maxCLevel();
+
+		[StructLayout(LayoutKind.Sequential)]
+		public struct StreamBuffer
+		{
+			public IntPtr Ptr;
+			public ulong Size;
+			public ulong Pos;
+		}
+
+		[BizImport(cc)]
+		public abstract IntPtr ZSTD_createCStream();
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_freeCStream(IntPtr zcs);
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_initCStream(IntPtr zcs, int compressionLevel);
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_compressStream(IntPtr zcs, ref StreamBuffer output, ref StreamBuffer input);
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_flushStream(IntPtr zcs, ref StreamBuffer output);
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_endStream(IntPtr zcs, ref StreamBuffer output);
+
+		[BizImport(cc)]
+		public abstract IntPtr ZSTD_createDStream();
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_freeDStream(IntPtr zds);
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_initDStream(IntPtr zds);
+
+		[BizImport(cc)]
+		public abstract ulong ZSTD_decompressStream(IntPtr zds, ref StreamBuffer output, ref StreamBuffer input);
+	}
+}

src/BizHawk.Emulation.Common/zstd/Zstd.cs

@@ -0,0 +1,484 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+
+using BizHawk.BizInvoke;
+using BizHawk.Common;
+
+namespace BizHawk.Emulation.Common
+{
+	public sealed class Zstd : IDisposable
+	{
+		private sealed class ZstdCompressionStreamContext : IDisposable
+		{
+			public readonly IntPtr Zcs;
+
+			public readonly byte[] InputBuffer;
+			public readonly GCHandle InputHandle;
+			public LibZstd.StreamBuffer Input;
+
+			public readonly byte[] OutputBuffer;
+			public readonly GCHandle OutputHandle;
+			public LibZstd.StreamBuffer Output;
+
+			// TODO: tweak these sizes
+
+			// 4 MB input buffer
+			public const int INPUT_BUFFER_SIZE = 1024 * 1024 * 4;
+			// 1 MB output buffer
+			public const int OUTPUT_BUFFER_SIZE = 1024 * 1024 * 1;
+
+			public bool InUse;
+
+			public ZstdCompressionStreamContext()
+			{
+				Zcs = _lib.ZSTD_createCStream();
+
+				InputBuffer = new byte[INPUT_BUFFER_SIZE];
+				InputHandle = GCHandle.Alloc(InputBuffer, GCHandleType.Pinned);
+
+				Input = new()
+				{
+					Ptr = InputHandle.AddrOfPinnedObject(),
+					Size = 0,
+					Pos = 0,
+				};
+
+				OutputBuffer = new byte[OUTPUT_BUFFER_SIZE];
+				OutputHandle = GCHandle.Alloc(OutputBuffer, GCHandleType.Pinned);
+
+				Output = new()
+				{
+					Ptr = OutputHandle.AddrOfPinnedObject(),
+					Size = OUTPUT_BUFFER_SIZE,
+					Pos = 0,
+				};
+
+				InUse = false;
+			}
+
+			private bool _disposed = false;
+
+			public void Dispose()
+			{
+				if (!_disposed)
+				{
+					_lib.ZSTD_freeCStream(Zcs);
+					InputHandle.Free();
+					OutputHandle.Free();
+					_disposed = true;
+				}
+			}
+
+			public void InitContext(int compressionLevel)
+			{
+				if (InUse)
+				{
+					throw new InvalidOperationException("Cannot init context still in use!");
+				}
+
+				_lib.ZSTD_initCStream(Zcs, compressionLevel);
+				Input.Size = Input.Pos = Output.Pos = 0;
+				InUse = true;
+			}
+		}
+
+		private sealed class ZstdCompressionStream : Stream
+		{
+			private readonly Stream _baseStream;
+			private readonly ZstdCompressionStreamContext _ctx;
+
+			public ZstdCompressionStream(Stream baseStream, ZstdCompressionStreamContext ctx)
+			{
+				_baseStream = baseStream;
+				_ctx = ctx;
+			}
+
+			private bool _disposed = false;
+
+			protected override void Dispose(bool disposing)
+			{
+				if (disposing && !_disposed)
+				{
+					Flush();
+					while (true)
+					{
+						var n = _lib.ZSTD_endStream(_ctx.Zcs, ref _ctx.Output);
+						CheckError(n);
+						InternalFlush();
+						if (n == 0)
+						{
+							break;
+						}
+					}
+					_ctx.InUse = false;
+					_disposed = true;
+				}
+			}
+
+			public override bool CanRead
+				=> false;
+
+			public override bool CanSeek
+				=> false;
+
+			public override bool CanWrite
+				=> true;
+
+			public override long Length
+				=> throw new NotImplementedException();
+
+			public override long Position 
+			{
+				get => throw new NotImplementedException();
+				set => throw new NotImplementedException();
+			}
+
+			private void InternalFlush()
+			{
+				_baseStream.Write(_ctx.OutputBuffer, 0, (int)_ctx.Output.Pos);
+				_ctx.Output.Pos = 0;
+			}
+
+			public override void Flush()
+			{
+				while (_ctx.Input.Pos < _ctx.Input.Size)
+				{
+					CheckError(_lib.ZSTD_compressStream(_ctx.Zcs, ref _ctx.Output, ref _ctx.Input));
+					while (true)
+					{
+						if (_ctx.Output.Pos == ZstdCompressionStreamContext.OUTPUT_BUFFER_SIZE)
+						{
+							InternalFlush();
+						}
+
+						var n = _lib.ZSTD_flushStream(_ctx.Zcs, ref _ctx.Output);
+						CheckError(n);
+						if (n == 0)
+						{
+							InternalFlush();
+							break;
+						}
+					}
+				}
+
+				_ctx.Input.Pos = _ctx.Input.Size = 0;
+			}
+
+			public override int Read(byte[] buffer, int offset, int count)
+				=> throw new NotImplementedException();
+
+			public override long Seek(long offset, SeekOrigin origin)
+				=> throw new NotImplementedException();
+
+			public override void SetLength(long value)
+				=> throw new NotImplementedException();
+
+			public override void Write(byte[] buffer, int offset, int count)
+			{
+				while (count > 0)
+				{
+					if (_ctx.Input.Size == ZstdCompressionStreamContext.INPUT_BUFFER_SIZE)
+					{
+						Flush();
+					}
+
+					var n = Math.Min(count, (int)(ZstdCompressionStreamContext.INPUT_BUFFER_SIZE - _ctx.Input.Size));
+					Marshal.Copy(buffer, offset, _ctx.Input.Ptr + (int)_ctx.Input.Size, n);
+					offset += n;
+					_ctx.Input.Size += (ulong)n;
+					count -= n;
+				}
+			}
+		}
+
+		private sealed class ZstdDecompressionStreamContext : IDisposable
+		{
+			public readonly IntPtr Zds;
+
+			public readonly byte[] InputBuffer;
+			public readonly GCHandle InputHandle;
+			public LibZstd.StreamBuffer Input;
+
+			public readonly byte[] OutputBuffer;
+			public readonly GCHandle OutputHandle;
+			public LibZstd.StreamBuffer Output;
+
+			// TODO: tweak these sizes
+
+			// 1 MB input buffer
+			public const int INPUT_BUFFER_SIZE = 1024 * 1024 * 1;
+			// 4 MB output buffer
+			public const int OUTPUT_BUFFER_SIZE = 1024 * 1024 * 4;
+
+			public bool InUse;
+
+			public ZstdDecompressionStreamContext()
+			{
+				Zds = _lib.ZSTD_createDStream();
+
+				InputBuffer = new byte[INPUT_BUFFER_SIZE];
+				InputHandle = GCHandle.Alloc(InputBuffer, GCHandleType.Pinned);
+
+				Input = new()
+				{
+					Ptr = InputHandle.AddrOfPinnedObject(),
+					Size = 0,
+					Pos = 0,
+				};
+
+				OutputBuffer = new byte[OUTPUT_BUFFER_SIZE];
+				OutputHandle = GCHandle.Alloc(OutputBuffer, GCHandleType.Pinned);
+
+				Output = new()
+				{
+					Ptr = OutputHandle.AddrOfPinnedObject(),
+					Size = OUTPUT_BUFFER_SIZE,
+					Pos = 0,
+				};
+
+				InUse = false;
+			}
+
+			private bool _disposed = false;
+
+			public void Dispose()
+			{
+				if (!_disposed)
+				{
+					_lib.ZSTD_freeDStream(Zds);
+					InputHandle.Free();
+					OutputHandle.Free();
+					_disposed = true;
+				}
+			}
+
+			public void InitContext()
+			{
+				if (InUse)
+				{
+					throw new InvalidOperationException("Cannot init context still in use!");
+				}
+
+				_lib.ZSTD_initDStream(Zds);
+				Input.Size = Input.Pos = Output.Pos = 0;
+				InUse = true;
+			}
+		}
+
+		private sealed class ZstdDecompressionStream : Stream
+		{
+			private readonly Stream _baseStream;
+			private readonly ZstdDecompressionStreamContext _ctx;
+
+			public ZstdDecompressionStream(Stream baseStream, ZstdDecompressionStreamContext ctx)
+			{
+				_baseStream = baseStream;
+				_ctx = ctx;
+			}
+
+			private bool _disposed = false;
+
+			protected override void Dispose(bool disposing)
+			{
+				if (disposing && !_disposed)
+				{
+					_ctx.InUse = false;
+					_disposed = true;
+				}
+			}
+
+			public override bool CanRead
+				=> true;
+
+			public override bool CanSeek
+				=> false;
+
+			public override bool CanWrite
+				=> false;
+
+			public override long Length
+				=> _baseStream.Length; // FIXME: this wrong but this is only used in a > 0 check so I guess it works?
+
+			public override long Position
+			{
+				get => throw new NotImplementedException();
+				set => throw new NotImplementedException();
+			}
+
+			public override void Flush()
+				=> throw new NotImplementedException();
+
+			private ulong _outputConsumed = 0;
+
+			public override int Read(byte[] buffer, int offset, int count)
+			{
+				var n = count;
+				while (n > 0)
+				{
+					var inputConsumed = _baseStream.Read(_ctx.InputBuffer,
+						(int)_ctx.Input.Size, (int)(ZstdDecompressionStreamContext.INPUT_BUFFER_SIZE - _ctx.Input.Size));
+					_ctx.Input.Size += (ulong)inputConsumed;
+					// avoid interop in case compression cannot be done
+					if (_ctx.Output.Pos < ZstdDecompressionStreamContext.OUTPUT_BUFFER_SIZE
+						&& _ctx.Input.Pos < _ctx.Input.Size)
+					{
+						CheckError(_lib.ZSTD_decompressStream(_ctx.Zds, ref _ctx.Output, ref _ctx.Input));
+					}
+					var outputToConsume = Math.Min(n, (int)(_ctx.Output.Pos - _outputConsumed));
+					Marshal.Copy(_ctx.Output.Ptr + (int)_outputConsumed, buffer, offset, outputToConsume);
+					_outputConsumed += (ulong)outputToConsume;
+					offset += outputToConsume;
+					n -= outputToConsume;
+
+					if (_outputConsumed == ZstdDecompressionStreamContext.OUTPUT_BUFFER_SIZE)
+					{
+						// all the buffer is consumed, kick these back to the beginning
+						_ctx.Output.Pos = _outputConsumed = 0;
+					}
+
+					if (_ctx.Input.Pos == ZstdDecompressionStreamContext.INPUT_BUFFER_SIZE)
+					{
+						// ditto here
+						_ctx.Input.Pos = _ctx.Input.Size = 0;
+					}
+
+					// couldn't consume anything, get out
+					// (decompression must be complete at this point)
+					if (inputConsumed == 0 && outputToConsume == 0)
+					{
+						break;
+					}
+				}
+
+				return count - n;
+			}
+
+			public override long Seek(long offset, SeekOrigin origin)
+				=> throw new NotImplementedException();
+
+			public override void SetLength(long value)
+				=> throw new NotImplementedException();
+
+			public override void Write(byte[] buffer, int offset, int count)
+				=> throw new NotImplementedException();
+		}
+
+		private static readonly LibZstd _lib;
+
+		public static int MinCompressionLevel { get; }
+
+		public static int MaxCompressionLevel { get; }
+
+		static Zstd()
+		{
+			var resolver = new DynamicLibraryImportResolver(
+				OSTailoredCode.IsUnixHost ? "libzstd.so.1" : "libzstd.dll", hasLimitedLifetime: false);
+			_lib = BizInvoker.GetInvoker<LibZstd>(resolver, CallingConventionAdapters.Native);
+
+			MinCompressionLevel = _lib.ZSTD_minCLevel();
+			MaxCompressionLevel = _lib.ZSTD_maxCLevel();
+		}
+
+		private readonly ZstdCompressionStreamContext _compressionStreamContext;
+		private readonly ZstdDecompressionStreamContext _decompressionStreamContext;
+
+		public Zstd()
+		{
+			_compressionStreamContext = new();
+			_decompressionStreamContext = new();
+		}
+
+		private bool _disposed = false;
+
+		public void Dispose()
+		{
+			if (!_disposed)
+			{
+				_compressionStreamContext.Dispose();
+				_decompressionStreamContext.Dispose();
+				_disposed = true;
+			}
+		}
+
+		private static void CheckError(ulong code)
+		{
+			if (_lib.ZSTD_isError(code) != 0)
+			{
+				throw new Exception($"ZSTD ERROR: {Marshal.PtrToStringAnsi(_lib.ZSTD_getErrorName(code))}");
+			}
+		}
+
+		/// <summary>
+		/// Creates a zstd compression stream.
+		/// This stream uses a shared context as to avoid buffer allocation spam.
+		/// It is absolutely important to call Dispose() / use using on returned stream.
+		/// If this is not done, the shared context will remain in use,
+		/// and the proceeding attempt to initialize it will throw.
+		/// Also, of course, do not attempt to create multiple streams at once.
+		/// Only 1 stream at a time is allowed per Zstd instance.
+		/// </summary>
+		/// <param name="stream">the stream to write compressed data</param>
+		/// <param name="compressionLevel">compression level, bounded by MinCompressionLevel and MaxCompressionLevel</param>
+		/// <returns>zstd compression stream</returns>
+		/// <exception cref="ArgumentOutOfRangeException">compressionLevel is too small or too big</exception>
+		public Stream CreateZstdCompressionStream(Stream stream, int compressionLevel)
+		{
+			if (compressionLevel < MinCompressionLevel || compressionLevel > MaxCompressionLevel)
+			{
+				throw new ArgumentOutOfRangeException(nameof(compressionLevel));
+			}
+
+			_compressionStreamContext.InitContext(compressionLevel);
+			return new ZstdCompressionStream(stream, _compressionStreamContext);
+		}
+
+		/// <summary>
+		/// Creates a zstd decompression stream.
+		/// This stream uses a shared context as to avoid buffer allocation spam.
+		/// It is absolutely important to call Dispose() / use using on returned stream.
+		/// If this is not done, the shared context will remain in use,
+		/// and the proceeding attempt to initialize it will throw.
+		/// Also, of course, do not attempt to create multiple streams at once.
+		/// Only 1 stream at a time is allowed per Zstd instance.
+		/// </summary>
+		/// <param name="stream">a stream with zstd compressed data to decompress</param>
+		/// <returns>zstd decompression stream</returns>
+		public Stream CreateZstdDecompressionStream(Stream stream)
+		{
+			_decompressionStreamContext.InitContext();
+			return new ZstdDecompressionStream(stream, _decompressionStreamContext);
+		}
+
+		/// <summary>
+		/// Decompresses src stream and returns a memory stream with the decompressed contents.
+		/// Context creation and disposing is handled internally in this function, unlike the non-static ones.
+		/// This is useful in cases where you are not doing repeated decompressions,
+		/// so keeping a Zstd instance around is not as useful.
+		/// </summary>
+		/// <param name="src">stream with zstd compressed data to decompress</param>
+		/// <returns>MemoryStream with the decompressed contents of src</returns>
+		/// <exception cref="InvalidOperationException">src does not have a ZSTD header</exception>
+		public static MemoryStream DecompressZstdStream(Stream src)
+		{
+			// check for ZSTD header
+			var tmp = new byte[4];
+			if (src.Read(tmp, 0, 4) != 4)
+			{
+				throw new InvalidOperationException("Unexpected end of stream");
+			}
+			if (tmp[0] != 0x28 || tmp[1] != 0xB5 || tmp[2] != 0x2F || tmp[3] != 0xFD)
+			{
+				throw new InvalidOperationException("ZSTD header not present");
+			}
+			src.Seek(0, SeekOrigin.Begin);
+
+			using var dctx = new ZstdDecompressionStreamContext();
+			dctx.InitContext();
+			using var dstream = new ZstdDecompressionStream(src, dctx);
+			var ret = new MemoryStream();
+			dstream.CopyTo(ret);
+			return ret;
+		}
+	}
+}

src/BizHawk.Emulation.Cores/Computers/AmstradCPC/AmstradCPC.cs

@@ -119,21 +119,21 @@ namespace BizHawk.Emulation.Cores.Computers.AmstradCPC
 			{
 				// CPC 464 ROMS
 				case "OS464ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.OS_464_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.OS_464_ROM.Value)).ToArray();
 					break;
 				case "BASIC1-0ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.CPC_BASIC_1_0_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.CPC_BASIC_1_0_ROM.Value)).ToArray();
 					break;
 
 				// CPC 6128 ROMS
 				case "OS6128ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.CPC_OS_6128_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.CPC_OS_6128_ROM.Value)).ToArray();
 					break;
 				case "BASIC1-1ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.CPC_BASIC_1_1_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.CPC_BASIC_1_1_ROM.Value)).ToArray();
 					break;
 				case "AMSDOS0-5ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.CPC_AMSDOS_0_5_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.CPC_AMSDOS_0_5_ROM.Value)).ToArray();
 					break;
 				default:
 					embeddedFound = false;

src/BizHawk.Emulation.Cores/Computers/SinclairSpectrum/ZXSpectrum.cs

@@ -187,17 +187,17 @@ namespace BizHawk.Emulation.Cores.Computers.SinclairSpectrum
 			switch (names.FirstOrDefault())
 			{
 				case "48ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.ZX_48_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.ZX_48_ROM.Value)).ToArray();
 					break;
 				case "128ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.ZX_128_ROM.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.ZX_128_ROM.Value)).ToArray();
 					break;
 				case "PLUS2ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.ZX_plus2_rom.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.ZX_plus2_rom.Value)).ToArray();
 					break;
 				case "PLUS2AROM":
 				case "PLUS3ROM":
-					embeddedRom = Util.DecompressGzipFile(new MemoryStream(Resources.ZX_plus2a_rom.Value));
+					embeddedRom = Zstd.DecompressZstdStream(new MemoryStream(Resources.ZX_plus2a_rom.Value)).ToArray();
 					break;
 				default:
 					embeddedFound = false;

src/BizHawk.Emulation.Cores/Consoles/Nintendo/Ares64/Ares64.cs

@@ -79,7 +79,7 @@ namespace BizHawk.Emulation.Cores.Consoles.Nintendo.Ares64
 			if (_settings.Deinterlacer == LibAres64.DeinterlacerType.Bob)
 				loadFlags |= LibAres64.LoadFlags.BobDeinterlace;
 
-			var pif = Util.DecompressGzipFile(new MemoryStream(pal ? Resources.PIF_PAL_ROM.Value : Resources.PIF_NTSC_ROM.Value));
+			var pif = Zstd.DecompressZstdStream(new MemoryStream(pal ? Resources.PIF_PAL_ROM.Value : Resources.PIF_NTSC_ROM.Value)).ToArray();
 
 			var gbRoms = new byte[][] { null, null, null, null };
 			var numGbRoms = lp.Roms.Count - 1;

src/BizHawk.Emulation.Cores/Consoles/Nintendo/NDS/MelonDS.cs

@@ -247,7 +247,7 @@ namespace BizHawk.Emulation.Cores.Consoles.Nintendo.NDS
 				titleId <<= 8;
 				titleId |= ware[0x237 - i];
 			}
-			using var zip = new ZipArchive(new MemoryStream(Util.DecompressGzipFile(new MemoryStream(Resources.TMDS.Value))), ZipArchiveMode.Read, false);
+			using var zip = new ZipArchive(Zstd.DecompressZstdStream(new MemoryStream(Resources.TMDS.Value)), ZipArchiveMode.Read, false);
 			using var tmd = zip.GetEntry($"{titleId:x16}.tmd")?.Open() ?? throw new Exception($"Cannot find TMD for title ID {titleId:x16}, please report");
 			var ret = new byte[tmd.Length];
 			tmd.Read(ret, 0, (int)tmd.Length);

src/BizHawk.Emulation.Cores/Consoles/Nintendo/SameBoy/SameBoy.cs

@@ -72,9 +72,9 @@ namespace BizHawk.Emulation.Cores.Nintendo.Sameboy
 			}
 			else
 			{
-				bios = Util.DecompressGzipFile(new MemoryStream(IsCgb
+				bios = Zstd.DecompressZstdStream(new MemoryStream(IsCgb
 					? _syncSettings.ConsoleMode is SameboySyncSettings.GBModel.GB_MODEL_AGB ? Resources.SameboyAgbBoot.Value : Resources.SameboyCgbBoot.Value
-					: Resources.SameboyDmgBoot.Value));
+					: Resources.SameboyDmgBoot.Value)).ToArray();
 			}
 
 			DeterministicEmulation = false;

src/BizHawk.Emulation.Cores/Properties/Resources.cs

@@ -5,26 +5,26 @@ using BizHawk.Common.IOExtensions;
 namespace BizHawk.Emulation.Cores.Properties {
 	internal static class Resources {
 		/// <param name="embedPath">Dir separator is '<c>.</c>'. Path is relative to <c>&lt;NS></c>.</param>
-		private static byte[] ReadEmbeddedByteArray(string embedPath) => Emulation.Cores.ReflectionCache.EmbeddedResourceStream($"Resources.{embedPath}").ReadAllBytes();
+		private static byte[] ReadEmbeddedByteArray(string embedPath) => ReflectionCache.EmbeddedResourceStream($"Resources.{embedPath}").ReadAllBytes();
 
-		internal static readonly Lazy<byte[]> CPC_AMSDOS_0_5_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("CPC_AMSDOS_0.5.ROM.gz"));
-		internal static readonly Lazy<byte[]> CPC_BASIC_1_0_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("CPC_BASIC_1.0.ROM.gz"));
-		internal static readonly Lazy<byte[]> CPC_BASIC_1_1_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("CPC_BASIC_1.1.ROM.gz"));
-		internal static readonly Lazy<byte[]> CPC_OS_6128_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("CPC_OS_6128.ROM.gz"));
-		internal static readonly Lazy<byte[]> OS_464_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("OS_464.ROM.gz"));
-		internal static readonly Lazy<byte[]> FastCgbBoot = new Lazy<byte[]>(() => ReadEmbeddedByteArray("cgb_boot.rom.gz"));
-		internal static readonly Lazy<byte[]> FastAgbBoot = new Lazy<byte[]>(() => ReadEmbeddedByteArray("agb_boot.rom.gz"));
-		internal static readonly Lazy<byte[]> FastDmgBoot = new Lazy<byte[]>(() => ReadEmbeddedByteArray("dmg_boot.rom.gz"));
-		internal static readonly Lazy<byte[]> SameboyCgbBoot = new Lazy<byte[]>(() => ReadEmbeddedByteArray("sameboy_cgb_boot.rom.gz"));
-		internal static readonly Lazy<byte[]> SameboyAgbBoot = new Lazy<byte[]>(() => ReadEmbeddedByteArray("sameboy_agb_boot.rom.gz"));
-		internal static readonly Lazy<byte[]> SameboyDmgBoot = new Lazy<byte[]>(() => ReadEmbeddedByteArray("sameboy_dmg_boot.rom.gz"));
-		internal static readonly Lazy<byte[]> SgbCartPresent_SPC = new Lazy<byte[]>(() => ReadEmbeddedByteArray("sgb-cart-present.spc.gz"));
-		internal static readonly Lazy<byte[]> ZX_128_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("128.ROM.gz"));
-		internal static readonly Lazy<byte[]> ZX_48_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("48.ROM.gz"));
-		internal static readonly Lazy<byte[]> ZX_plus2_rom = new Lazy<byte[]>(() => ReadEmbeddedByteArray("plus2.rom.gz"));
-		internal static readonly Lazy<byte[]> ZX_plus2a_rom = new Lazy<byte[]>(() => ReadEmbeddedByteArray("plus2a.rom.gz"));
-		internal static readonly Lazy<byte[]> TMDS = new Lazy<byte[]>(() => ReadEmbeddedByteArray("tmds.zip.gz"));
-		internal static readonly Lazy<byte[]> PIF_PAL_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("pif.pal.rom.gz"));
-		internal static readonly Lazy<byte[]> PIF_NTSC_ROM = new Lazy<byte[]>(() => ReadEmbeddedByteArray("pif.ntsc.rom.gz"));
+		internal static readonly Lazy<byte[]> CPC_AMSDOS_0_5_ROM = new(() => ReadEmbeddedByteArray("CPC_AMSDOS_0.5.ROM.zst"));
+		internal static readonly Lazy<byte[]> CPC_BASIC_1_0_ROM = new(() => ReadEmbeddedByteArray("CPC_BASIC_1.0.ROM.zst"));
+		internal static readonly Lazy<byte[]> CPC_BASIC_1_1_ROM = new(() => ReadEmbeddedByteArray("CPC_BASIC_1.1.ROM.zst"));
+		internal static readonly Lazy<byte[]> CPC_OS_6128_ROM = new(() => ReadEmbeddedByteArray("CPC_OS_6128.ROM.zst"));
+		internal static readonly Lazy<byte[]> OS_464_ROM = new(() => ReadEmbeddedByteArray("OS_464.ROM.zst"));
+		internal static readonly Lazy<byte[]> FastCgbBoot = new(() => ReadEmbeddedByteArray("cgb_boot.rom.zst"));
+		internal static readonly Lazy<byte[]> FastAgbBoot = new(() => ReadEmbeddedByteArray("agb_boot.rom.zst"));
+		internal static readonly Lazy<byte[]> FastDmgBoot = new(() => ReadEmbeddedByteArray("dmg_boot.rom.zst"));
+		internal static readonly Lazy<byte[]> SameboyCgbBoot = new(() => ReadEmbeddedByteArray("sameboy_cgb_boot.rom.zst"));
+		internal static readonly Lazy<byte[]> SameboyAgbBoot = new(() => ReadEmbeddedByteArray("sameboy_agb_boot.rom.zst"));
+		internal static readonly Lazy<byte[]> SameboyDmgBoot = new(() => ReadEmbeddedByteArray("sameboy_dmg_boot.rom.zst"));
+		internal static readonly Lazy<byte[]> SgbCartPresent_SPC = new(() => ReadEmbeddedByteArray("sgb-cart-present.spc.zst"));
+		internal static readonly Lazy<byte[]> ZX_128_ROM = new(() => ReadEmbeddedByteArray("128.ROM.zst"));
+		internal static readonly Lazy<byte[]> ZX_48_ROM = new(() => ReadEmbeddedByteArray("48.ROM.zst"));
+		internal static readonly Lazy<byte[]> ZX_plus2_rom = new(() => ReadEmbeddedByteArray("plus2.rom.zst"));
+		internal static readonly Lazy<byte[]> ZX_plus2a_rom = new(() => ReadEmbeddedByteArray("plus2a.rom.zst"));
+		internal static readonly Lazy<byte[]> TMDS = new(() => ReadEmbeddedByteArray("tmds.zip.zst"));
+		internal static readonly Lazy<byte[]> PIF_PAL_ROM = new(() => ReadEmbeddedByteArray("pif.pal.rom.zst"));
+		internal static readonly Lazy<byte[]> PIF_NTSC_ROM = new(() => ReadEmbeddedByteArray("pif.ntsc.rom.zst"));
 	}
 }

src/BizHawk.Emulation.Cores/Waterbox/WaterboxHost.cs

@@ -108,12 +108,12 @@ namespace BizHawk.Emulation.Cores.Waterbox
 			var moduleName = opt.Filename;
 
 			var path = Path.Combine(opt.Path, moduleName);
-			var gzpath = path + ".gz";
+			var zstpath = path + ".zst";
 			byte[] data;
-			if (File.Exists(gzpath))
+			if (File.Exists(zstpath))
 			{
-				using var fs = new FileStream(gzpath, FileMode.Open, FileAccess.Read);
-				data = Util.DecompressGzipFile(fs);
+				using var fs = new FileStream(zstpath, FileMode.Open, FileAccess.Read);
+				data = Zstd.DecompressZstdStream(fs).ToArray();
 			}
 			else
 			{

waterbox/common.mak

@@ -94,14 +94,14 @@ $(TARGET_DEBUG): $(DOBJS) $(EMULIBC_DOBJS) $(LINKSCRIPT)
 
 install: $(TARGET_RELEASE)
 	@cp -f $< $(OUTPUTDLL_DIR)
-	@gzip --stdout --best $< > $(OUTPUTDLL_DIR)/$(TARGET).gz
-	@cp $(OUTPUTDLL_DIR)/$(TARGET).gz $(OUTPUTDLLCOPY_DIR)/$(TARGET).gz || true
+	@zstd --stdout --ultra -22 --threads=0 $< > $(OUTPUTDLL_DIR)/$(TARGET).zst
+	@cp $(OUTPUTDLL_DIR)/$(TARGET).zst $(OUTPUTDLLCOPY_DIR)/$(TARGET).zst || true
 	@echo Release build of $(TARGET) installed.
 
 install-debug: $(TARGET_DEBUG)
 	@cp -f $< $(OUTPUTDLL_DIR)
-	@gzip --stdout --best $< > $(OUTPUTDLL_DIR)/$(TARGET).gz
-	@cp $(OUTPUTDLL_DIR)/$(TARGET).gz $(OUTPUTDLLCOPY_DIR)/$(TARGET).gz || true
+	@zstd --stdout --ultra -22 --threads=0 $< > $(OUTPUTDLL_DIR)/$(TARGET).zst
+	@cp $(OUTPUTDLL_DIR)/$(TARGET).zst $(OUTPUTDLLCOPY_DIR)/$(TARGET).zst || true
 	@echo Debug build of $(TARGET) installed.
 
 else

waterbox/readme.txt

@@ -11,7 +11,7 @@ It consists of a modified musl libc, and build scripts to tie it all together.
 	1. Install WSL2 
 	2. Install Ubuntu 20.04 LTS (https://www.microsoft.com/en-us/p/ubuntu-2004-lts/9n6svws3rx71)
 	3. Clone the bizhawk repository. You can use it through /mnt or /home if you really like 
-	4. Install build tools: sudo apt-get update && sudo apt-get install gcc g++ make cmake llvm
+	4. Install build tools: sudo apt-get update && sudo apt-get install gcc g++ make cmake llvm zstd
 	4b. (Note for future work: ideally the llvm installed above would not be required)
 
 	PREPARE A WIN10 VM: