Drive states2 (#2542)

* Expose new backing store type functionality for ZwinderBuffer.

* implement drive states for reserved states

* Include version numbers in Zwinder custom file formats, and for newer files rely on the separately loaded settings. With this, TempFile store types are supported when saving/loading.

src/BizHawk.Client.Common/movie/tasproj/StateDictionary.cs

@@ -0,0 +1,124 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Text;
+using System.IO;
+using BizHawk.Common;
+
+namespace BizHawk.Client.Common
+{
+	class TempFileStateDictionary : IDictionary<int, byte[]>, IDisposable
+	{ 
+		private Dictionary<int, Stream> _streams = new Dictionary<int, Stream>();
+
+		public byte[] this[int key]
+		{
+			get
+			{
+				byte[] bytes = new byte[_streams[key].Length];
+				_streams[key].Seek(0, SeekOrigin.Begin);
+				_streams[key].Read(bytes, 0, bytes.Length);
+				return bytes;
+			}
+			set => SetState(key, new MemoryStream(value));
+		}
+
+		public void SetState(int frame, Stream stream)
+		{
+			if (!_streams.ContainsKey(frame))
+			{
+				string filename =  TempFileManager.GetTempFilename("State");
+				_streams[frame] = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose);
+			}
+			else
+				_streams[frame].Seek(0, SeekOrigin.Begin);
+
+			_streams[frame].SetLength(stream.Length);
+			stream.CopyTo(_streams[frame]);
+		}
+
+		public ICollection<int> Keys => _streams.Keys;
+
+		public ICollection<byte[]> Values => throw new NotImplementedException();
+
+		public int Count => _streams.Count;
+
+		public bool IsReadOnly => false;
+
+		public void Add(int key, byte[] value)
+		{
+			this[key] = value;
+		}
+
+		public void Add(KeyValuePair<int, byte[]> item)
+		{
+			this[item.Key] = item.Value;
+		}
+
+		public void Clear()
+		{
+			foreach (var kvp in _streams)
+				kvp.Value.Dispose();
+
+			_streams.Clear();
+		}
+
+		public bool Contains(KeyValuePair<int, byte[]> item)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool ContainsKey(int key)
+		{
+			return _streams.ContainsKey(key);
+		}
+
+		public void CopyTo(KeyValuePair<int, byte[]>[] array, int arrayIndex)
+		{
+			throw new NotImplementedException();
+		}
+
+		public IEnumerator<KeyValuePair<int, byte[]>> GetEnumerator()
+		{
+			foreach (var kvp in _streams)
+				yield return new KeyValuePair<int, byte[]>(kvp.Key, this[kvp.Key]);
+		}
+
+		public bool Remove(int key)
+		{
+			if (ContainsKey(key))
+			{
+				_streams[key].Dispose();
+				return _streams.Remove(key);
+			}
+			else
+				return false;
+		}
+
+		public bool Remove(KeyValuePair<int, byte[]> item)
+		{
+			throw new NotImplementedException();
+		}
+
+		public bool TryGetValue(int key, out byte[] value)
+		{
+			if (!ContainsKey(key))
+			{
+				value = null;
+				return false;
+			}
+			else
+			{
+				value = this[key];
+				return true;
+			}
+		}
+
+		IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
+
+		public void Dispose()
+		{
+			Clear();
+		}
+	}
+}

src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManager.cs

@@ -22,7 +22,7 @@ namespace BizHawk.Client.Common
 		// These never decay, but can be invalidated, they are for reserved states
 		// such as markers and branches, but also we naturally evict states from recent to reserved, based
 		// on _ancientInterval
-		private Dictionary<int, byte[]> _reserved = new Dictionary<int, byte[]>();
+		private IDictionary<int, byte[]> _reserved;
 
 		// When recent states are evicted this interval is used to determine if we need to reserve the state
 		// We always want to keep some states throughout the movie
@@ -50,13 +50,16 @@ namespace BizHawk.Client.Common
 			}
 		}
 
-		private ZwinderStateManager(ZwinderBuffer current, ZwinderBuffer recent, ZwinderBuffer gapFiller, int ancientInterval, Func<int, bool> reserveCallback)
+		private ZwinderStateManager(ZwinderBuffer current, ZwinderBuffer recent, ZwinderBuffer gapFiller, Func<int, bool> reserveCallback, ZwinderStateManagerSettings settings)
 		{
 			_current = current;
 			_recent = recent;
 			_gapFiller = gapFiller;
-			_ancientInterval = ancientInterval;
 			_reserveCallback = reserveCallback;
+			Settings = settings;
+			_ancientInterval = settings.AncientStateInterval;
+			// init the reserved dictionary
+			RebuildReserved();
 		}
 		
 		public byte[] this[int frame]
@@ -79,6 +82,7 @@ namespace BizHawk.Client.Common
 
 		public void UpdateSettings(ZwinderStateManagerSettings settings, bool keepOldStates = false)
 		{
+			bool makeNewReserved = Settings?.AncientStoreType != settings.AncientStoreType;
 			Settings = settings;
 
 			_current = UpdateBuffer(_current, settings.Current(), keepOldStates);
@@ -108,19 +112,49 @@ namespace BizHawk.Client.Common
 			}
 			else
 			{
-				List<int> framesToRemove = new List<int>();
+				if (_reserved != null)
-				foreach (int f in _reserved.Keys)
 				{
-					if (f != 0 && !_reserveCallback(f))
+					List<int> framesToRemove = new List<int>();
-						framesToRemove.Add(f);
+					foreach (int f in _reserved.Keys)
+					{
+						if (f != 0 && !_reserveCallback(f))
+							framesToRemove.Add(f);
+					}
+					foreach (int f in framesToRemove)
+						EvictReserved(f);
 				}
-				foreach (int f in framesToRemove)
-					EvictReserved(f);
 			}
 
+			if (makeNewReserved)
+				RebuildReserved();
+
 			_ancientInterval = settings.AncientStateInterval;
 			RebuildStateCache();
 		}
+
+		private void RebuildReserved()
+		{
+			IDictionary<int, byte[]> newReserved;
+			switch (Settings.AncientStoreType)
+			{
+				case IRewindSettings.BackingStoreType.Memory:
+					newReserved = new Dictionary<int, byte[]>();
+					break;
+				case IRewindSettings.BackingStoreType.TempFile:
+					newReserved = new TempFileStateDictionary();
+					break;
+				default:
+					throw new ArgumentException("Unsupported store type for reserved states.");
+			}
+			if (_reserved != null)
+			{
+				foreach (var kvp in _reserved)
+					newReserved.Add(kvp.Key, kvp.Value);
+				(_reserved as TempFileStateDictionary)?.Dispose();
+			}
+			_reserved = newReserved;
+		}
+
 		private ZwinderBuffer UpdateBuffer(ZwinderBuffer buffer, RewindConfig newConfig, bool keepOldStates)
 		{
 			if (buffer == null) // just make a new one, plain and simple
@@ -498,16 +532,17 @@ namespace BizHawk.Client.Common
 
 		public static ZwinderStateManager Create(BinaryReader br, ZwinderStateManagerSettings settings, Func<int, bool> reserveCallback)
 		{
-			var current = ZwinderBuffer.Create(br);
+			// Initial format had no version number, but I think it's a safe bet no valid file has buffer size 2^56 or more so this should work.
-			var recent = ZwinderBuffer.Create(br);
+			int version = br.ReadByte();
-			var gaps = ZwinderBuffer.Create(br);
 
-			var ancientInterval = br.ReadInt32();
+			var current = ZwinderBuffer.Create(br, settings.Current(), version == 0);
+			var recent = ZwinderBuffer.Create(br, settings.Recent());
+			var gaps = ZwinderBuffer.Create(br, settings.GapFiller());
 
-			var ret = new ZwinderStateManager(current, recent, gaps, ancientInterval, reserveCallback)
+			if (version == 0)
-			{
+				settings.AncientStateInterval = br.ReadInt32();
-				Settings = settings
+
-			};
+			var ret = new ZwinderStateManager(current, recent, gaps, reserveCallback, settings);
 
 			var ancientCount = br.ReadInt32();
 			for (var i = 0; i < ancientCount; i++)
@@ -525,12 +560,13 @@ namespace BizHawk.Client.Common
 
 		public void SaveStateHistory(BinaryWriter bw)
 		{
+			// version
+			bw.Write((byte)1);
+
 			_current.SaveStateBinary(bw);
 			_recent.SaveStateBinary(bw);
 			_gapFiller.SaveStateBinary(bw);
 
-			bw.Write(_ancientInterval);
-
 			bw.Write(_reserved.Count);
 			foreach (var s in _reserved)
 			{

src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManagerSettings.cs

@@ -12,16 +12,20 @@ namespace BizHawk.Client.Common
 			CurrentUseCompression = settings.CurrentUseCompression;
 			CurrentBufferSize = settings.CurrentBufferSize;
 			CurrentTargetFrameLength = settings.CurrentTargetFrameLength;
+			CurrentStoreType = settings.CurrentStoreType;
 
 			RecentUseCompression = settings.RecentUseCompression;
 			RecentBufferSize = settings.RecentBufferSize;
 			RecentTargetFrameLength = settings.RecentTargetFrameLength;
+			RecentStoreType = settings.RecentStoreType;
 
 			GapsUseCompression = settings.GapsUseCompression;
 			GapsBufferSize = settings.GapsBufferSize;
 			GapsTargetFrameLength = settings.GapsTargetFrameLength;
+			GapsStoreType = settings.GapsStoreType;
 
 			AncientStateInterval = settings.AncientStateInterval;
+			AncientStoreType = settings.AncientStoreType;
 		}
 
 		/// <summary>
@@ -41,6 +45,10 @@ namespace BizHawk.Client.Common
 		[TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)]
 		public int CurrentTargetFrameLength { get; set; } = 500;
 
+		[DisplayName("Current - Storage Type")]
+		[Description("Where to keep the buffer.")]
+		public IRewindSettings.BackingStoreType CurrentStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory;
+
 		/// <summary>
 		/// Buffer settings when navigating directly before the Current buffer
 		/// </summary>
@@ -58,6 +66,10 @@ namespace BizHawk.Client.Common
 		[TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)]
 		public int RecentTargetFrameLength { get; set; } = 2000;
 
+		[DisplayName("Recent - Storage Type")]
+		[Description("Where to keep the buffer.")]
+		public IRewindSettings.BackingStoreType RecentStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory;
+
 		/// <summary>
 		/// Priority States for special use cases
 		/// </summary>
@@ -75,11 +87,19 @@ namespace BizHawk.Client.Common
 		[TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)]
 		public int GapsTargetFrameLength { get; set; } = 125;
 
+		[DisplayName("Gaps - Storage Type")]
+		[Description("Where to keep the buffer.")]
+		public IRewindSettings.BackingStoreType GapsStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory;
+
 		[DisplayName("Ancient State Interval")]
 		[Description("Once both the Current and Recent buffers have filled, some states are put into reserved to ensure there is always a state somewhat near a desired frame to navigate to. These states never decay but are invalidated. This number should be as high as possible without being overly cumbersome to replay this many frames.")]
 		[TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)]
 		public int AncientStateInterval { get; set; } = 5000;
 
+		[DisplayName("Ancient - Storage Type")]
+		[Description("Where to keep the reserved states.")]
+		public IRewindSettings.BackingStoreType AncientStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory;
+
 		// Just to simplify some other code.
 		public RewindConfig Current()
 		{
@@ -87,7 +107,8 @@ namespace BizHawk.Client.Common
 			{
 				UseCompression = CurrentUseCompression,
 				BufferSize = CurrentBufferSize,
-				TargetFrameLength = CurrentTargetFrameLength
+				TargetFrameLength = CurrentTargetFrameLength,
+				BackingStore = CurrentStoreType
 			};
 		}
 		public RewindConfig Recent()
@@ -96,7 +117,8 @@ namespace BizHawk.Client.Common
 			{
 				UseCompression = RecentUseCompression,
 				BufferSize = RecentBufferSize,
-				TargetFrameLength = RecentTargetFrameLength
+				TargetFrameLength = RecentTargetFrameLength,
+				BackingStore = RecentStoreType
 			};
 		}
 		public RewindConfig GapFiller()
@@ -105,7 +127,8 @@ namespace BizHawk.Client.Common
 			{
 				UseCompression = GapsUseCompression,
 				BufferSize = GapsBufferSize,
-				TargetFrameLength = GapsTargetFrameLength
+				TargetFrameLength = GapsTargetFrameLength,
+				BackingStore = GapsStoreType
 			};
 		}
 	}

src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs

@@ -20,6 +20,9 @@ namespace BizHawk.Client.Common
 		*/
 		public ZwinderBuffer(IRewindSettings settings)
 		{
+			if (settings == null)
+				throw new ArgumentException("ZwinderBuffer's settings cannot be null.");
+
 			long targetSize = settings.BufferSize * 1024 * 1024;
 			if (settings.TargetFrameLength < 1)
 			{
@@ -28,6 +31,7 @@ namespace BizHawk.Client.Common
 
 			Size = 1L << (int)Math.Floor(Math.Log(targetSize, 2));
 			_sizeMask = Size - 1;
+			_backingStoreType = settings.BackingStore;
 			switch (settings.BackingStore)
 			{
 				case IRewindSettings.BackingStoreType.Memory:
@@ -49,7 +53,7 @@ namespace BizHawk.Client.Common
 					break;
 				}
 				default:
-					throw new Exception();
+					throw new ArgumentException("Unsupported store type for ZwinderBuffer.");
 			}
 			_targetFrameLength = settings.TargetFrameLength;
 			_states = new StateInfo[STATEMASK + 1];
@@ -104,6 +108,8 @@ namespace BizHawk.Client.Common
 		}
 
 		private readonly Stream _backingStore;
+		// this is only used to compare settings with a RewindConfig
+		private readonly IRewindSettings.BackingStoreType _backingStoreType;
 
 		private readonly StateInfo[] _states;
 		private int _firstStateIndex;
@@ -138,7 +144,8 @@ namespace BizHawk.Client.Common
 			long size = 1L << (int)Math.Floor(Math.Log(targetSize, 2));
 			return Size == size &&
 				_useCompression == settings.UseCompression &&
-				_targetFrameLength == settings.TargetFrameLength;
+				_targetFrameLength == settings.TargetFrameLength &&
+				_backingStoreType == settings.BackingStore;
 		}
 
 		private bool ShouldCapture(int frame)
@@ -263,11 +270,8 @@ namespace BizHawk.Client.Common
 
 		public void SaveStateBinary(BinaryWriter writer)
 		{
-			writer.Write(Size);
+			// version number
-			writer.Write(_sizeMask);
+			writer.Write((byte)1);
-			writer.Write(_targetFrameLength);
-			writer.Write(_useCompression);
-
 			SaveStateBodyBinary(writer);
 		}
 
@@ -313,22 +317,36 @@ namespace BizHawk.Client.Common
 			WaterboxUtils.CopySome(reader.BaseStream, _backingStore, nextByte);
 		}
 
-		public static ZwinderBuffer Create(BinaryReader reader)
+		public static ZwinderBuffer Create(BinaryReader reader, RewindConfig rewindConfig, bool hackyV0 = false)
 		{
-			var size = reader.ReadInt64();
+			ZwinderBuffer ret;
-			var sizeMask = reader.ReadInt64();
+
-			var targetFrameLength = reader.ReadInt32();
+			// Initial format had no version number, but I think it's a safe bet no valid file has buffer size 2^56 or more so this should work.
-			var useCompression = reader.ReadBoolean();
+			int version = hackyV0 ? 0 : reader.ReadByte();
-			var ret = new ZwinderBuffer(new RewindConfig
+			if (version == 0)
 			{
-				BufferSize = (int)(size >> 20),
+				byte[] sizeArr = new byte[8];
-				TargetFrameLength = targetFrameLength,
+				reader.Read(sizeArr, 1, 7);
-				UseCompression = useCompression
+				var size = BitConverter.ToInt64(sizeArr, 0);
-			});
+				var sizeMask = reader.ReadInt64();
-			if (ret.Size != size || ret._sizeMask != sizeMask)
+				var targetFrameLength = reader.ReadInt32();
-			{
+				var useCompression = reader.ReadBoolean();
-				throw new InvalidOperationException("Bad format");
+				ret = new ZwinderBuffer(new RewindConfig
+				{
+					BufferSize = (int)(size >> 20),
+					TargetFrameLength = targetFrameLength,
+					UseCompression = useCompression
+				});
+				if (ret.Size != size || ret._sizeMask != sizeMask)
+				{
+					throw new InvalidOperationException("Bad format");
+				}
 			}
+			else if (version == 1)
+				ret = new ZwinderBuffer(rewindConfig);
+			else
+				throw new InvalidOperationException("Bad format");
+
 			ret.LoadStateBodyBinary(reader);
 			return ret;
 		}

src/BizHawk.Tests/Client.Common/Movie/ZwinderStateManagerTests.cs

@@ -80,11 +80,12 @@ namespace BizHawk.Tests.Client.Common.Movie
 		[TestMethod]
 		public void SaveCreateBufferRoundTrip()
 		{
-			var buff = new ZwinderBuffer(new RewindConfig
+			RewindConfig config = new RewindConfig
 			{
 				BufferSize = 1,
 				TargetFrameLength = 10
-			});
+			};
+			var buff = new ZwinderBuffer(config);
 			var ss = new StateSource { PaddingData = new byte[500] };
 			for (var frame = 0; frame < 2090; frame++)
 			{
@@ -101,7 +102,7 @@ namespace BizHawk.Tests.Client.Common.Movie
 			var ms = new MemoryStream();
 			buff.SaveStateBinary(new BinaryWriter(ms));
 			ms.Position = 0;
-			var buff2 = ZwinderBuffer.Create(new BinaryReader(ms));
+			var buff2 = ZwinderBuffer.Create(new BinaryReader(ms), config);
 
 			Assert.AreEqual(buff.Size, buff2.Size);
 			Assert.AreEqual(buff.Used, buff2.Used);