From b3e69782dd8ace6d591a8753c8d11f5f22d7afbe Mon Sep 17 00:00:00 2001 From: SuuperW Date: Fri, 15 Jan 2021 15:02:00 -0600 Subject: [PATCH] Drive states2 (#2542) * Expose new backing store type functionality for ZwinderBuffer. * implement drive states for reserved states * Include version numbers in Zwinder custom file formats, and for newer files rely on the separately loaded settings. With this, TempFile store types are supported when saving/loading. --- .../movie/tasproj/StateDictionary.cs | 124 ++++++++++++++++++ .../movie/tasproj/ZwinderStateManager.cs | 74 ++++++++--- .../tasproj/ZwinderStateManagerSettings.cs | 29 +++- .../rewind/ZwinderBuffer.cs | 58 +++++--- .../Movie/ZwinderStateManagerTests.cs | 7 +- 5 files changed, 247 insertions(+), 45 deletions(-) create mode 100644 src/BizHawk.Client.Common/movie/tasproj/StateDictionary.cs diff --git a/src/BizHawk.Client.Common/movie/tasproj/StateDictionary.cs b/src/BizHawk.Client.Common/movie/tasproj/StateDictionary.cs new file mode 100644 index 0000000000..94dcf832e9 --- /dev/null +++ b/src/BizHawk.Client.Common/movie/tasproj/StateDictionary.cs @@ -0,0 +1,124 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Text; +using System.IO; +using BizHawk.Common; + +namespace BizHawk.Client.Common +{ + class TempFileStateDictionary : IDictionary, IDisposable + { + private Dictionary _streams = new Dictionary(); + + public byte[] this[int key] + { + get + { + byte[] bytes = new byte[_streams[key].Length]; + _streams[key].Seek(0, SeekOrigin.Begin); + _streams[key].Read(bytes, 0, bytes.Length); + return bytes; + } + set => SetState(key, new MemoryStream(value)); + } + + public void SetState(int frame, Stream stream) + { + if (!_streams.ContainsKey(frame)) + { + string filename = TempFileManager.GetTempFilename("State"); + _streams[frame] = new FileStream(filename, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 4096, FileOptions.DeleteOnClose); + } + else + _streams[frame].Seek(0, SeekOrigin.Begin); + + _streams[frame].SetLength(stream.Length); + stream.CopyTo(_streams[frame]); + } + + public ICollection Keys => _streams.Keys; + + public ICollection Values => throw new NotImplementedException(); + + public int Count => _streams.Count; + + public bool IsReadOnly => false; + + public void Add(int key, byte[] value) + { + this[key] = value; + } + + public void Add(KeyValuePair item) + { + this[item.Key] = item.Value; + } + + public void Clear() + { + foreach (var kvp in _streams) + kvp.Value.Dispose(); + + _streams.Clear(); + } + + public bool Contains(KeyValuePair item) + { + throw new NotImplementedException(); + } + + public bool ContainsKey(int key) + { + return _streams.ContainsKey(key); + } + + public void CopyTo(KeyValuePair[] array, int arrayIndex) + { + throw new NotImplementedException(); + } + + public IEnumerator> GetEnumerator() + { + foreach (var kvp in _streams) + yield return new KeyValuePair(kvp.Key, this[kvp.Key]); + } + + public bool Remove(int key) + { + if (ContainsKey(key)) + { + _streams[key].Dispose(); + return _streams.Remove(key); + } + else + return false; + } + + public bool Remove(KeyValuePair item) + { + throw new NotImplementedException(); + } + + public bool TryGetValue(int key, out byte[] value) + { + if (!ContainsKey(key)) + { + value = null; + return false; + } + else + { + value = this[key]; + return true; + } + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public void Dispose() + { + Clear(); + } + } +} diff --git a/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManager.cs b/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManager.cs index 4b69d6bc0e..c429fa3584 100644 --- a/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManager.cs +++ b/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManager.cs @@ -22,7 +22,7 @@ namespace BizHawk.Client.Common // These never decay, but can be invalidated, they are for reserved states // such as markers and branches, but also we naturally evict states from recent to reserved, based // on _ancientInterval - private Dictionary _reserved = new Dictionary(); + private IDictionary _reserved; // When recent states are evicted this interval is used to determine if we need to reserve the state // We always want to keep some states throughout the movie @@ -50,13 +50,16 @@ namespace BizHawk.Client.Common } } - private ZwinderStateManager(ZwinderBuffer current, ZwinderBuffer recent, ZwinderBuffer gapFiller, int ancientInterval, Func reserveCallback) + private ZwinderStateManager(ZwinderBuffer current, ZwinderBuffer recent, ZwinderBuffer gapFiller, Func reserveCallback, ZwinderStateManagerSettings settings) { _current = current; _recent = recent; _gapFiller = gapFiller; - _ancientInterval = ancientInterval; _reserveCallback = reserveCallback; + Settings = settings; + _ancientInterval = settings.AncientStateInterval; + // init the reserved dictionary + RebuildReserved(); } public byte[] this[int frame] @@ -79,6 +82,7 @@ namespace BizHawk.Client.Common public void UpdateSettings(ZwinderStateManagerSettings settings, bool keepOldStates = false) { + bool makeNewReserved = Settings?.AncientStoreType != settings.AncientStoreType; Settings = settings; _current = UpdateBuffer(_current, settings.Current(), keepOldStates); @@ -108,19 +112,49 @@ namespace BizHawk.Client.Common } else { - List framesToRemove = new List(); - foreach (int f in _reserved.Keys) + if (_reserved != null) { - if (f != 0 && !_reserveCallback(f)) - framesToRemove.Add(f); + List framesToRemove = new List(); + foreach (int f in _reserved.Keys) + { + if (f != 0 && !_reserveCallback(f)) + framesToRemove.Add(f); + } + foreach (int f in framesToRemove) + EvictReserved(f); } - foreach (int f in framesToRemove) - EvictReserved(f); } + if (makeNewReserved) + RebuildReserved(); + _ancientInterval = settings.AncientStateInterval; RebuildStateCache(); } + + private void RebuildReserved() + { + IDictionary newReserved; + switch (Settings.AncientStoreType) + { + case IRewindSettings.BackingStoreType.Memory: + newReserved = new Dictionary(); + break; + case IRewindSettings.BackingStoreType.TempFile: + newReserved = new TempFileStateDictionary(); + break; + default: + throw new ArgumentException("Unsupported store type for reserved states."); + } + if (_reserved != null) + { + foreach (var kvp in _reserved) + newReserved.Add(kvp.Key, kvp.Value); + (_reserved as TempFileStateDictionary)?.Dispose(); + } + _reserved = newReserved; + } + private ZwinderBuffer UpdateBuffer(ZwinderBuffer buffer, RewindConfig newConfig, bool keepOldStates) { if (buffer == null) // just make a new one, plain and simple @@ -498,16 +532,17 @@ namespace BizHawk.Client.Common public static ZwinderStateManager Create(BinaryReader br, ZwinderStateManagerSettings settings, Func reserveCallback) { - var current = ZwinderBuffer.Create(br); - var recent = ZwinderBuffer.Create(br); - var gaps = ZwinderBuffer.Create(br); + // Initial format had no version number, but I think it's a safe bet no valid file has buffer size 2^56 or more so this should work. + int version = br.ReadByte(); - var ancientInterval = br.ReadInt32(); + var current = ZwinderBuffer.Create(br, settings.Current(), version == 0); + var recent = ZwinderBuffer.Create(br, settings.Recent()); + var gaps = ZwinderBuffer.Create(br, settings.GapFiller()); - var ret = new ZwinderStateManager(current, recent, gaps, ancientInterval, reserveCallback) - { - Settings = settings - }; + if (version == 0) + settings.AncientStateInterval = br.ReadInt32(); + + var ret = new ZwinderStateManager(current, recent, gaps, reserveCallback, settings); var ancientCount = br.ReadInt32(); for (var i = 0; i < ancientCount; i++) @@ -525,12 +560,13 @@ namespace BizHawk.Client.Common public void SaveStateHistory(BinaryWriter bw) { + // version + bw.Write((byte)1); + _current.SaveStateBinary(bw); _recent.SaveStateBinary(bw); _gapFiller.SaveStateBinary(bw); - bw.Write(_ancientInterval); - bw.Write(_reserved.Count); foreach (var s in _reserved) { diff --git a/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManagerSettings.cs b/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManagerSettings.cs index 8d0abb11d9..99534c72e1 100644 --- a/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManagerSettings.cs +++ b/src/BizHawk.Client.Common/movie/tasproj/ZwinderStateManagerSettings.cs @@ -12,16 +12,20 @@ namespace BizHawk.Client.Common CurrentUseCompression = settings.CurrentUseCompression; CurrentBufferSize = settings.CurrentBufferSize; CurrentTargetFrameLength = settings.CurrentTargetFrameLength; + CurrentStoreType = settings.CurrentStoreType; RecentUseCompression = settings.RecentUseCompression; RecentBufferSize = settings.RecentBufferSize; RecentTargetFrameLength = settings.RecentTargetFrameLength; + RecentStoreType = settings.RecentStoreType; GapsUseCompression = settings.GapsUseCompression; GapsBufferSize = settings.GapsBufferSize; GapsTargetFrameLength = settings.GapsTargetFrameLength; + GapsStoreType = settings.GapsStoreType; AncientStateInterval = settings.AncientStateInterval; + AncientStoreType = settings.AncientStoreType; } /// @@ -41,6 +45,10 @@ namespace BizHawk.Client.Common [TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)] public int CurrentTargetFrameLength { get; set; } = 500; + [DisplayName("Current - Storage Type")] + [Description("Where to keep the buffer.")] + public IRewindSettings.BackingStoreType CurrentStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory; + /// /// Buffer settings when navigating directly before the Current buffer /// @@ -58,6 +66,10 @@ namespace BizHawk.Client.Common [TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)] public int RecentTargetFrameLength { get; set; } = 2000; + [DisplayName("Recent - Storage Type")] + [Description("Where to keep the buffer.")] + public IRewindSettings.BackingStoreType RecentStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory; + /// /// Priority States for special use cases /// @@ -75,11 +87,19 @@ namespace BizHawk.Client.Common [TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)] public int GapsTargetFrameLength { get; set; } = 125; + [DisplayName("Gaps - Storage Type")] + [Description("Where to keep the buffer.")] + public IRewindSettings.BackingStoreType GapsStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory; + [DisplayName("Ancient State Interval")] [Description("Once both the Current and Recent buffers have filled, some states are put into reserved to ensure there is always a state somewhat near a desired frame to navigate to. These states never decay but are invalidated. This number should be as high as possible without being overly cumbersome to replay this many frames.")] [TypeConverter(typeof(IntConverter)), Range(1, int.MaxValue)] public int AncientStateInterval { get; set; } = 5000; + [DisplayName("Ancient - Storage Type")] + [Description("Where to keep the reserved states.")] + public IRewindSettings.BackingStoreType AncientStoreType { get; set; } = IRewindSettings.BackingStoreType.Memory; + // Just to simplify some other code. public RewindConfig Current() { @@ -87,7 +107,8 @@ namespace BizHawk.Client.Common { UseCompression = CurrentUseCompression, BufferSize = CurrentBufferSize, - TargetFrameLength = CurrentTargetFrameLength + TargetFrameLength = CurrentTargetFrameLength, + BackingStore = CurrentStoreType }; } public RewindConfig Recent() @@ -96,7 +117,8 @@ namespace BizHawk.Client.Common { UseCompression = RecentUseCompression, BufferSize = RecentBufferSize, - TargetFrameLength = RecentTargetFrameLength + TargetFrameLength = RecentTargetFrameLength, + BackingStore = RecentStoreType }; } public RewindConfig GapFiller() @@ -105,7 +127,8 @@ namespace BizHawk.Client.Common { UseCompression = GapsUseCompression, BufferSize = GapsBufferSize, - TargetFrameLength = GapsTargetFrameLength + TargetFrameLength = GapsTargetFrameLength, + BackingStore = GapsStoreType }; } } diff --git a/src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs b/src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs index ecaba4dc2e..2dd914000f 100644 --- a/src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs +++ b/src/BizHawk.Client.Common/rewind/ZwinderBuffer.cs @@ -20,6 +20,9 @@ namespace BizHawk.Client.Common */ public ZwinderBuffer(IRewindSettings settings) { + if (settings == null) + throw new ArgumentException("ZwinderBuffer's settings cannot be null."); + long targetSize = settings.BufferSize * 1024 * 1024; if (settings.TargetFrameLength < 1) { @@ -28,6 +31,7 @@ namespace BizHawk.Client.Common Size = 1L << (int)Math.Floor(Math.Log(targetSize, 2)); _sizeMask = Size - 1; + _backingStoreType = settings.BackingStore; switch (settings.BackingStore) { case IRewindSettings.BackingStoreType.Memory: @@ -49,7 +53,7 @@ namespace BizHawk.Client.Common break; } default: - throw new Exception(); + throw new ArgumentException("Unsupported store type for ZwinderBuffer."); } _targetFrameLength = settings.TargetFrameLength; _states = new StateInfo[STATEMASK + 1]; @@ -104,6 +108,8 @@ namespace BizHawk.Client.Common } private readonly Stream _backingStore; + // this is only used to compare settings with a RewindConfig + private readonly IRewindSettings.BackingStoreType _backingStoreType; private readonly StateInfo[] _states; private int _firstStateIndex; @@ -138,7 +144,8 @@ namespace BizHawk.Client.Common long size = 1L << (int)Math.Floor(Math.Log(targetSize, 2)); return Size == size && _useCompression == settings.UseCompression && - _targetFrameLength == settings.TargetFrameLength; + _targetFrameLength == settings.TargetFrameLength && + _backingStoreType == settings.BackingStore; } private bool ShouldCapture(int frame) @@ -263,11 +270,8 @@ namespace BizHawk.Client.Common public void SaveStateBinary(BinaryWriter writer) { - writer.Write(Size); - writer.Write(_sizeMask); - writer.Write(_targetFrameLength); - writer.Write(_useCompression); - + // version number + writer.Write((byte)1); SaveStateBodyBinary(writer); } @@ -313,22 +317,36 @@ namespace BizHawk.Client.Common WaterboxUtils.CopySome(reader.BaseStream, _backingStore, nextByte); } - public static ZwinderBuffer Create(BinaryReader reader) + public static ZwinderBuffer Create(BinaryReader reader, RewindConfig rewindConfig, bool hackyV0 = false) { - var size = reader.ReadInt64(); - var sizeMask = reader.ReadInt64(); - var targetFrameLength = reader.ReadInt32(); - var useCompression = reader.ReadBoolean(); - var ret = new ZwinderBuffer(new RewindConfig + ZwinderBuffer ret; + + // Initial format had no version number, but I think it's a safe bet no valid file has buffer size 2^56 or more so this should work. + int version = hackyV0 ? 0 : reader.ReadByte(); + if (version == 0) { - BufferSize = (int)(size >> 20), - TargetFrameLength = targetFrameLength, - UseCompression = useCompression - }); - if (ret.Size != size || ret._sizeMask != sizeMask) - { - throw new InvalidOperationException("Bad format"); + byte[] sizeArr = new byte[8]; + reader.Read(sizeArr, 1, 7); + var size = BitConverter.ToInt64(sizeArr, 0); + var sizeMask = reader.ReadInt64(); + var targetFrameLength = reader.ReadInt32(); + var useCompression = reader.ReadBoolean(); + ret = new ZwinderBuffer(new RewindConfig + { + BufferSize = (int)(size >> 20), + TargetFrameLength = targetFrameLength, + UseCompression = useCompression + }); + if (ret.Size != size || ret._sizeMask != sizeMask) + { + throw new InvalidOperationException("Bad format"); + } } + else if (version == 1) + ret = new ZwinderBuffer(rewindConfig); + else + throw new InvalidOperationException("Bad format"); + ret.LoadStateBodyBinary(reader); return ret; } diff --git a/src/BizHawk.Tests/Client.Common/Movie/ZwinderStateManagerTests.cs b/src/BizHawk.Tests/Client.Common/Movie/ZwinderStateManagerTests.cs index 6d6e88282d..92d75ee07e 100644 --- a/src/BizHawk.Tests/Client.Common/Movie/ZwinderStateManagerTests.cs +++ b/src/BizHawk.Tests/Client.Common/Movie/ZwinderStateManagerTests.cs @@ -80,11 +80,12 @@ namespace BizHawk.Tests.Client.Common.Movie [TestMethod] public void SaveCreateBufferRoundTrip() { - var buff = new ZwinderBuffer(new RewindConfig + RewindConfig config = new RewindConfig { BufferSize = 1, TargetFrameLength = 10 - }); + }; + var buff = new ZwinderBuffer(config); var ss = new StateSource { PaddingData = new byte[500] }; for (var frame = 0; frame < 2090; frame++) { @@ -101,7 +102,7 @@ namespace BizHawk.Tests.Client.Common.Movie var ms = new MemoryStream(); buff.SaveStateBinary(new BinaryWriter(ms)); ms.Position = 0; - var buff2 = ZwinderBuffer.Create(new BinaryReader(ms)); + var buff2 = ZwinderBuffer.Create(new BinaryReader(ms), config); Assert.AreEqual(buff.Size, buff2.Size); Assert.AreEqual(buff.Used, buff2.Used);