From 00e2fea9015800eac8d432edadcfe5601b575470 Mon Sep 17 00:00:00 2001 From: CasualPokePlayer <50538166+CasualPokePlayer@users.noreply.github.com> Date: Sun, 2 Apr 2023 02:09:44 -0700 Subject: [PATCH] Create `DeltaSerializer`, uses simple RLE encoding so the serialized delta isn't huge, use it for C64 EasyFlash and Disks --- src/BizHawk.Common/DeltaSerializer.cs | 155 ++++++++++++++++++ src/BizHawk.Common/Serializer.cs | 16 ++ .../Commodore64/Cartridge/Mapper0020.cs | 4 +- .../Computers/Commodore64/Media/Disk.cs | 61 +------ .../Computers/Commodore64/SaveState.cs | 37 ----- .../Computers/Commodore64/Serial/Drive1541.cs | 7 +- 6 files changed, 185 insertions(+), 95 deletions(-) create mode 100644 src/BizHawk.Common/DeltaSerializer.cs delete mode 100644 src/BizHawk.Emulation.Cores/Computers/Commodore64/SaveState.cs diff --git a/src/BizHawk.Common/DeltaSerializer.cs b/src/BizHawk.Common/DeltaSerializer.cs new file mode 100644 index 0000000000..ddbeb6da7f --- /dev/null +++ b/src/BizHawk.Common/DeltaSerializer.cs @@ -0,0 +1,155 @@ +using System; +using System.Runtime.InteropServices; + +namespace BizHawk.Common +{ + /// + /// Serializes deltas between data, mainly for ROM like structures which are actually writable, and therefore the differences need to be saved + /// Uses simple RLE encoding to keep the size down + /// + public static class DeltaSerializer + { + public static byte[] GetDelta(ReadOnlySpan original, ReadOnlySpan data) + where T : unmanaged + { + var orignalAsBytes = MemoryMarshal.AsBytes(original); + var dataAsBytes = MemoryMarshal.AsBytes(data); + + if (orignalAsBytes.Length != dataAsBytes.Length) + { + throw new InvalidOperationException($"{nameof(orignalAsBytes.Length)} must equal {nameof(dataAsBytes.Length)}"); + } + + var index = 0; + var end = dataAsBytes.Length; + var ret = new byte[end + 4].AsSpan(); // worst case scenario size (i.e. everything is different) + var retSize = 0; + + while (index < end) + { + var blockStart = index; + while (index < end && orignalAsBytes[index] == dataAsBytes[index]) + { + index++; + } + + var same = index - blockStart; + + if (same < 4) // something changed, or we hit end of spans, count how many different bytes come after + { + var different = 0; + while (index < end && same < 8) // in case we hit end of span before, this does nothing and different is 0 + { + if (orignalAsBytes[index] != dataAsBytes[index]) + { + different++; + same = 0; // note: same is set to 0 on first iteration + } + else + { + // we don't end on hitting a same byte, only after a sufficent number of same bytes are encountered + // this would help against possibly having a few stray same bytes splattered around changes + same++; + } + + index++; + } + + if (different > 0) // only not 0 if index == end immediately + { + if (same < 4) // we have different bytes, but we hit the end of the spans before the 8 limit, and we have less than what a same block will save + { + different += same; + same = 0; + } + + different = -different; // negative is a signal that these are different bytes + MemoryMarshal.Write(ret.Slice(retSize, 4), ref different); + retSize += 4; + for (var i = blockStart; i < index - same; i++) + { + ret[retSize++] = (byte)(orignalAsBytes[i] ^ dataAsBytes[i]); + } + } + + if (same > 0) // same is 4-8, 8 indicates we hit the 8 same bytes threshold, 4-7 indicate hit end of span + { + if (same == 8) + { + while (index < end && orignalAsBytes[index] == dataAsBytes[index]) + { + index++; + } + } + + same = index - blockStart; + MemoryMarshal.Write(ret.Slice(retSize, 4), ref same); + retSize += 4; + } + } + else // count amount of same bytes in this block + { + MemoryMarshal.Write(ret.Slice(retSize, 4), ref same); + retSize += 4; + } + } + + return ret.Slice(0, retSize).ToArray(); + } + + public static void ApplyDelta(ReadOnlySpan original, Span data, ReadOnlySpan delta) + where T : unmanaged + { + var orignalAsBytes = MemoryMarshal.AsBytes(original); + var dataAsBytes = MemoryMarshal.AsBytes(data); + + if (orignalAsBytes.Length != dataAsBytes.Length) + { + throw new InvalidOperationException($"{nameof(orignalAsBytes.Length)} must equal {nameof(dataAsBytes.Length)}"); + } + + var dataPos = 0; + var dataEnd = dataAsBytes.Length; + var deltaPos = 0; + var deltaEnd = delta.Length; + + while (deltaPos < deltaEnd) + { + if (deltaEnd - deltaPos < 4) + { + throw new InvalidOperationException("Hit end of delta unexpectingly!"); + } + + var header = MemoryMarshal.Read(delta.Slice(deltaPos, 4)); + deltaPos += 4; + if (header < 0) // difference block + { + header = -header; + + if (header < dataEnd - dataPos || header < deltaEnd - deltaPos) + { + throw new InvalidOperationException("Corrupt delta header!"); + } + + for (var i = 0; i < header; i++) + { + dataAsBytes[dataPos + i] = (byte)(orignalAsBytes[dataPos + i] ^ delta[deltaPos + i]); + } + + deltaPos += header; + } + else // sameness block + { + if (header < dataEnd - dataPos) + { + throw new InvalidOperationException("Corrupt delta header!"); + } + + orignalAsBytes.Slice(dataPos, header).CopyTo(dataAsBytes.Slice(dataPos, header)); + } + + dataPos += header; + } + } + } +} diff --git a/src/BizHawk.Common/Serializer.cs b/src/BizHawk.Common/Serializer.cs index 5fb100b498..16200db7b3 100644 --- a/src/BizHawk.Common/Serializer.cs +++ b/src/BizHawk.Common/Serializer.cs @@ -748,6 +748,22 @@ namespace BizHawk.Common } } + public void SyncDelta(string name, T[] original, T[] data) + where T : unmanaged + { + if (IsReader) + { + var delta = Array.Empty(); + Sync(name, ref delta, useNull: false); + DeltaSerializer.ApplyDelta(original, data, delta); + } + else + { + var delta = DeltaSerializer.GetDelta(original, data); + Sync(name, ref delta, useNull: false); + } + } + private BinaryReader _br; private BinaryWriter _bw; private TextReader _tr; diff --git a/src/BizHawk.Emulation.Cores/Computers/Commodore64/Cartridge/Mapper0020.cs b/src/BizHawk.Emulation.Cores/Computers/Commodore64/Cartridge/Mapper0020.cs index ed4e04653a..2ff0806bfe 100644 --- a/src/BizHawk.Emulation.Cores/Computers/Commodore64/Cartridge/Mapper0020.cs +++ b/src/BizHawk.Emulation.Cores/Computers/Commodore64/Cartridge/Mapper0020.cs @@ -100,8 +100,8 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Cartridge ser.Sync("CommandLatch55", ref _commandLatchAa); ser.Sync("CommandLatchAA", ref _commandLatchAa); ser.Sync("InternalROMState", ref _internalRomState); - SaveState.SyncDelta("MediaStateA", ser, _originalMediaA, ref _banksA); - SaveState.SyncDelta("MediaStateB", ser, _originalMediaB, ref _banksB); + ser.SyncDelta("MediaStateA", _originalMediaA, _banksA); + ser.SyncDelta("MediaStateB", _originalMediaB, _banksB); DriveLightOn = _boardLed; } diff --git a/src/BizHawk.Emulation.Cores/Computers/Commodore64/Media/Disk.cs b/src/BizHawk.Emulation.Cores/Computers/Commodore64/Media/Disk.cs index 342807831a..8f60c487c9 100644 --- a/src/BizHawk.Emulation.Cores/Computers/Commodore64/Media/Disk.cs +++ b/src/BizHawk.Emulation.Cores/Computers/Commodore64/Media/Disk.cs @@ -1,6 +1,6 @@ using System; using System.Collections.Generic; - +using System.Linq; using BizHawk.Common; namespace BizHawk.Emulation.Cores.Computers.Commodore64.Media @@ -11,7 +11,7 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Media public const int FluxBitsPerTrack = 16000000 / 5; public const int FluxEntriesPerTrack = FluxBitsPerTrack / FluxBitsPerEntry; private readonly int[][] _tracks; - private readonly int[] _originalMedia; + private readonly int[][] _originalMedia; public bool Valid; public bool WriteProtected; @@ -23,7 +23,7 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Media WriteProtected = false; _tracks = new int[trackCapacity][]; FillMissingTracks(); - _originalMedia = SerializeTracks(_tracks); + _originalMedia = _tracks.Select(t => (int[])t.Clone()).ToArray(); Valid = true; } @@ -45,7 +45,7 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Media FillMissingTracks(); Valid = true; - _originalMedia = SerializeTracks(_tracks); + _originalMedia = _tracks.Select(t => (int[])t.Clone()).ToArray(); } private int[] ConvertToFluxTransitions(int density, byte[] bytes, int fluxBitOffset) @@ -136,59 +136,14 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Media return _tracks[halftrack]; } - /// - /// Combine the tracks into a single bitstream. - /// - private int[] SerializeTracks(int[][] tracks) - { - var trackCount = tracks.Length; - var result = new int[trackCount * FluxEntriesPerTrack]; - for (var i = 0; i < trackCount; i++) - { - Array.Copy(tracks[i], 0, result, i * FluxEntriesPerTrack, FluxEntriesPerTrack); - } - return result; - } - - /// - /// Split a bitstream into tracks. - /// - private int[][] DeserializeTracks(int[] data) - { - var trackCount = data.Length / FluxEntriesPerTrack; - var result = new int[trackCount][]; - for (var i = 0; i < trackCount; i++) - { - result[i] = new int[FluxEntriesPerTrack]; - Array.Copy(data, i * FluxEntriesPerTrack, result[i], 0, FluxEntriesPerTrack); - } - return result; - } - public void SyncState(Serializer ser) { ser.Sync(nameof(WriteProtected), ref WriteProtected); - // cpp: the below comment is wrong (at least now), writes are implemented (see ExecuteFlux() in Drive1541) - // I'm not yet going to uncomment this, due to the noted performance issues - // Not sure where performance issues would truly lie, suppose it's probably due to new array spam - // Something just a bit smarter would fix such issues - - // Currently nothing actually writes to _tracks and so it is always the same as _originalMedia - // So commenting out this (very slow) code for now - // If/when disk writing is implemented, Disk.cs should implement ISaveRam as a means of file storage of the new disk state - // And this code needs to be rethought to be reasonably performant - //if (ser.IsReader) - //{ - // var mediaState = new int[_originalMedia.Length]; - // SaveState.SyncDelta("MediaState", ser, _originalMedia, ref mediaState); - // _tracks = DeserializeTracks(mediaState); - //} - //else if (ser.IsWriter) - //{ - // var mediaState = SerializeTracks(_tracks); - // SaveState.SyncDelta("MediaState", ser, _originalMedia, ref mediaState); - //} + for (var i = 0; i < _tracks.Length; i++) + { + ser.SyncDelta("MediaState", _originalMedia[i], _tracks[i]); + } } } } diff --git a/src/BizHawk.Emulation.Cores/Computers/Commodore64/SaveState.cs b/src/BizHawk.Emulation.Cores/Computers/Commodore64/SaveState.cs deleted file mode 100644 index 6f3c5dd85a..0000000000 --- a/src/BizHawk.Emulation.Cores/Computers/Commodore64/SaveState.cs +++ /dev/null @@ -1,37 +0,0 @@ -using System; -using System.Collections.Generic; - -using BizHawk.Common; - -namespace BizHawk.Emulation.Cores.Computers.Commodore64 -{ - internal static class SaveState - { - private static int[] GetDelta(IList source, IList data) - { - var length = Math.Min(source.Count, data.Count); - var delta = new int[length]; - for (var i = 0; i < length; i++) - { - delta[i] = source[i] ^ data[i]; - } - - return delta; - } - - public static void SyncDelta(string name, Serializer ser, int[] source, ref int[] data) - { - int[] delta = null; - if (ser.IsWriter && data != null) - { - delta = GetDelta(source, data); - } - - ser.Sync(name, ref delta, false); - if (ser.IsReader && delta != null) - { - data = GetDelta(source, delta); - } - } - } -} diff --git a/src/BizHawk.Emulation.Cores/Computers/Commodore64/Serial/Drive1541.cs b/src/BizHawk.Emulation.Cores/Computers/Commodore64/Serial/Drive1541.cs index 4ddfcd5404..1710949cf0 100644 --- a/src/BizHawk.Emulation.Cores/Computers/Commodore64/Serial/Drive1541.cs +++ b/src/BizHawk.Emulation.Cores/Computers/Commodore64/Serial/Drive1541.cs @@ -26,7 +26,7 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Serial private int _cpuClockNum; private int _ratioDifference; private int _driveLightOffTime; - private int[] _trackImageData = new int[1]; + private int[] _trackImageData; public Func ReadIec = () => 0xFF; public Action DebuggerStep; public readonly Chip23128 DriveRom; @@ -100,8 +100,9 @@ namespace BizHawk.Emulation.Cores.Computers.Commodore64.Serial ser.Sync("SystemCpuClockNumerator", ref _cpuClockNum); ser.Sync("SystemDriveCpuRatioDifference", ref _ratioDifference); ser.Sync("DriveLightOffTime", ref _driveLightOffTime); - // feos: drop 400KB of ROM data from savestates - //ser.Sync("TrackImageData", ref _trackImageData, useNull: false); + + // set _trackImageData back to the correct reference + _trackImageData = _disk?.GetDataForTrack(_trackNumber); ser.Sync("DiskDensityCounter", ref _diskDensityCounter); ser.Sync("DiskSupplementaryCounter", ref _diskSupplementaryCounter);