From 22192e700be134f78a3b91f0838d3595f078ae08 Mon Sep 17 00:00:00 2001 From: zeromus Date: Sun, 15 Nov 2015 02:27:48 -0600 Subject: [PATCH] attempt to make it possible to store movies on disk instead of in memory --- .../BizHawk.Client.Common.csproj | 1 + BizHawk.Client.Common/config/Config.cs | 1 + .../movie/bk2/Bk2Movie.InputLog.cs | 4 +- BizHawk.Client.Common/movie/bk2/Bk2Movie.cs | 2 + BizHawk.Client.Common/movie/bk2/StringLogs.cs | 194 ++++++++++++++++++ .../conversions/MovieConversionExtensions.cs | 4 +- .../movie/tasproj/TasBranch.cs | 15 +- .../movie/tasproj/TasMovie.IO.cs | 2 +- .../movie/tasproj/TasMovie.cs | 13 +- BizHawk.Client.EmuHawk/Program.cs | 5 +- .../tools/TAStudio/BookmarksBranchesBox.cs | 2 +- 11 files changed, 223 insertions(+), 20 deletions(-) create mode 100644 BizHawk.Client.Common/movie/bk2/StringLogs.cs diff --git a/BizHawk.Client.Common/BizHawk.Client.Common.csproj b/BizHawk.Client.Common/BizHawk.Client.Common.csproj index 6c0d64b248..821672cd53 100644 --- a/BizHawk.Client.Common/BizHawk.Client.Common.csproj +++ b/BizHawk.Client.Common/BizHawk.Client.Common.csproj @@ -154,6 +154,7 @@ Bk2Movie.cs + diff --git a/BizHawk.Client.Common/config/Config.cs b/BizHawk.Client.Common/config/Config.cs index 9150f211f0..ea8487d026 100644 --- a/BizHawk.Client.Common/config/Config.cs +++ b/BizHawk.Client.Common/config/Config.cs @@ -77,6 +77,7 @@ namespace BizHawk.Client.Common public bool ForbidUD_LR = false; public bool ShowContextMenu = true; public bool EnableBackupMovies = true; + public bool MoviesOnDisk = false; public bool HotkeyConfigAutoTab = true; public bool InputConfigAutoTab = true; public bool ShowLogWindow = false; diff --git a/BizHawk.Client.Common/movie/bk2/Bk2Movie.InputLog.cs b/BizHawk.Client.Common/movie/bk2/Bk2Movie.InputLog.cs index 2deacda209..99425ac9ed 100644 --- a/BizHawk.Client.Common/movie/bk2/Bk2Movie.InputLog.cs +++ b/BizHawk.Client.Common/movie/bk2/Bk2Movie.InputLog.cs @@ -8,7 +8,7 @@ namespace BizHawk.Client.Common { public partial class Bk2Movie { - protected List _log = new List(); + protected IStringLog _log; protected string LogKey = string.Empty; public string GetInputLog() @@ -58,7 +58,7 @@ namespace BizHawk.Client.Common // We are in record mode so replace the movie log with the one from the savestate if (!Global.MovieSession.MultiTrack.IsActive) { - if (Global.Config.EnableBackupMovies && MakeBackup && _log.Any()) + if (Global.Config.EnableBackupMovies && MakeBackup && _log.Count != 0) { SaveBackup(); MakeBackup = false; diff --git a/BizHawk.Client.Common/movie/bk2/Bk2Movie.cs b/BizHawk.Client.Common/movie/bk2/Bk2Movie.cs index 12bffc939e..ddbd4f4dd0 100644 --- a/BizHawk.Client.Common/movie/bk2/Bk2Movie.cs +++ b/BizHawk.Client.Common/movie/bk2/Bk2Movie.cs @@ -27,6 +27,8 @@ namespace BizHawk.Client.Common MakeBackup = true; Header[HeaderKeys.MOVIEVERSION] = "BizHawk v2.0.0"; + + _log = StringLogUtil.MakeStringLog(); } private string _filename; diff --git a/BizHawk.Client.Common/movie/bk2/StringLogs.cs b/BizHawk.Client.Common/movie/bk2/StringLogs.cs new file mode 100644 index 0000000000..b11db3a210 --- /dev/null +++ b/BizHawk.Client.Common/movie/bk2/StringLogs.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace BizHawk.Client.Common +{ + public static class StringLogUtil + { + public static bool DefaultToDisk; + public static IStringLog MakeStringLog() + { + if (DefaultToDisk) + return new DiskStringLog(); + else return new ListStringLog(); + } + } + + public interface IStringLog : IDisposable, IEnumerable + { + void RemoveAt(int index); + int Count { get; } + void Clear(); + void Add(string str); + string this[int index] { get; set; } + void Insert(int index, string val); + void InsertRange(int index, IEnumerable collection); + void AddRange(IEnumerable collection); + void RemoveRange(int index, int count); + IStringLog Clone(); + void CopyTo(string[] array); + void CopyTo(int index, string[] array, int arrayIndex, int count); + } + + class ListStringLog : List, IStringLog + { + public IStringLog Clone() + { + ListStringLog ret = new ListStringLog(); + ret.AddRange(this); + return ret; + } + + public void Dispose() { } + } + + /// + /// A dumb-ish IStringLog with storage on disk with no provision for recovering lost space, except upon Clear() + /// The purpose here is to avoid having too complicated buggy logic or a dependency on sqlite or such. + /// It should be faster than those alternatives, but wasteful of disk space. + /// It should also be easier to add new IList-like methods than dealing with a database + /// + class DiskStringLog : IStringLog + { + List Offsets = new List(); + long cursor = 0; + BinaryWriter bw; + BinaryReader br; + + FileStream stream; + public DiskStringLog() + { + var path = Path.Combine(Path.GetTempPath(), "bizhawk.disklist-pid" + System.Diagnostics.Process.GetCurrentProcess().Id + "-" + Guid.NewGuid()); + stream = new FileStream(path, FileMode.Create, System.Security.AccessControl.FileSystemRights.FullControl, FileShare.None, 4 * 1024, FileOptions.DeleteOnClose); + bw = new BinaryWriter(stream); + br = new BinaryReader(stream); + } + + public IStringLog Clone() + { + DiskStringLog ret = new DiskStringLog(); + for (int i = 0; i < Count; i++) + ret.Add(this[i]); + return ret; + } + + public void Dispose() + { + stream.Dispose(); + } + + public int Count { get { return Offsets.Count; } } + + public void Clear() + { + stream.SetLength(0); + Offsets.Clear(); + cursor = 0; + } + + public void Add(string str) + { + Offsets.Add(stream.Position); + bw.Write(str); + bw.Flush(); + } + + public void RemoveAt(int index) + { + Offsets.RemoveAt(index); + //no garbage collection in the disk file... oh well. + } + + public string this[int index] + { + get + { + stream.Position = Offsets[index]; + return br.ReadString(); + } + set + { + stream.Position = stream.Length; + Offsets[index] = stream.Position; + bw.Write(value); + bw.Flush(); + } + } + + public void Insert(int index, string val) + { + Offsets.Insert(index, stream.Position); + bw.Write(val); + bw.Flush(); + } + + public void InsertRange(int index, IEnumerable collection) + { + foreach(var item in collection) + Insert(index++,item); + } + + public void AddRange(IEnumerable collection) + { + foreach (var item in collection) + Add(item); + } + + class Enumerator : IEnumerator + { + public DiskStringLog log; + int index; + public string Current { get { return log[index]; } } + object System.Collections.IEnumerator.Current { get { return log[index]; } } + bool System.Collections.IEnumerator.MoveNext() + { + index++; + if (index >= log.Count) + { + index = log.Count; + return false; + } + return true; + } + void System.Collections.IEnumerator.Reset() { index = 0; } + + public void Dispose() { } + } + + IEnumerator IEnumerable.GetEnumerator() + { + return new Enumerator() { log = this }; + } + + System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() + { + return new Enumerator() { log = this }; + } + + public void RemoveRange(int index, int count) + { + int end = index + count - 1; + for (int i = 0; i < count; i++) + { + RemoveAt(end); + end--; + } + } + + public void CopyTo(string[] array) + { + for (int i = 0; i < Count; i++) + array[i] = this[i]; + } + + public void CopyTo(int index, string[] array, int arrayIndex, int count) + { + for (int i = 0; i < count; i++) + array[i + arrayIndex] = this[index + i]; + } + } +} diff --git a/BizHawk.Client.Common/movie/conversions/MovieConversionExtensions.cs b/BizHawk.Client.Common/movie/conversions/MovieConversionExtensions.cs index 0ddee97697..30e94dd5a9 100644 --- a/BizHawk.Client.Common/movie/conversions/MovieConversionExtensions.cs +++ b/BizHawk.Client.Common/movie/conversions/MovieConversionExtensions.cs @@ -145,7 +145,7 @@ namespace BizHawk.Client.Common.MovieConversionExtensions tas.BinarySavestate = savestate; tas.ClearLagLog(); - List entries = old.GetLogEntries(); + var entries = old.GetLogEntries(); tas.CopyLog(entries.Skip(frame)); tas.CopyVerificationLog(old.VerificationLog); @@ -220,7 +220,7 @@ namespace BizHawk.Client.Common.MovieConversionExtensions tas.TasStateManager.Clear(); tas.ClearLagLog(); - List entries = old.GetLogEntries(); + var entries = old.GetLogEntries(); tas.CopyVerificationLog(old.VerificationLog); tas.CopyVerificationLog(entries); diff --git a/BizHawk.Client.Common/movie/tasproj/TasBranch.cs b/BizHawk.Client.Common/movie/tasproj/TasBranch.cs index b290831ff5..0e684efc06 100644 --- a/BizHawk.Client.Common/movie/tasproj/TasBranch.cs +++ b/BizHawk.Client.Common/movie/tasproj/TasBranch.cs @@ -11,8 +11,8 @@ namespace BizHawk.Client.Common public class TasBranch { public int Frame { get; set; } - public byte[] CoreData { get; set; } - public List InputLog { get; set; } + public byte[] CoreData { get; set; } + public IStringLog InputLog { get; set; } public BitmapBuffer OSDFrameBuffer { get; set; } public TasLagLog LagLog { get; set; } public TasMovieChangeLog ChangeLog { get; set; } @@ -63,9 +63,10 @@ namespace BizHawk.Client.Common }); bs.PutLump(ninput, delegate(TextWriter tw) - { - foreach (var line in b.InputLog) - tw.WriteLine(line); + { + int todo = b.InputLog.Count; + for (int i = 0; i < todo; i++) + tw.WriteLine(b.InputLog[i]); }); bs.PutLump(nframebuffer, delegate(Stream s) @@ -145,8 +146,8 @@ namespace BizHawk.Client.Common }); bl.GetLump(ninput, true, delegate(TextReader tr) - { - b.InputLog = new List(); + { + b.InputLog = StringLogUtil.MakeStringLog(); string line; while ((line = tr.ReadLine()) != null) b.InputLog.Add(line); diff --git a/BizHawk.Client.Common/movie/tasproj/TasMovie.IO.cs b/BizHawk.Client.Common/movie/tasproj/TasMovie.IO.cs index 58ce372164..51f979c27f 100644 --- a/BizHawk.Client.Common/movie/tasproj/TasMovie.IO.cs +++ b/BizHawk.Client.Common/movie/tasproj/TasMovie.IO.cs @@ -319,7 +319,7 @@ namespace BizHawk.Client.Common ChangeLog.ClearLog(); } - private static string InputLogToString(List log) + private static string InputLogToString(IStringLog log) { var sb = new StringBuilder(); foreach (var record in log) diff --git a/BizHawk.Client.Common/movie/tasproj/TasMovie.cs b/BizHawk.Client.Common/movie/tasproj/TasMovie.cs index b78016fdf1..4bfdeeb103 100644 --- a/BizHawk.Client.Common/movie/tasproj/TasMovie.cs +++ b/BizHawk.Client.Common/movie/tasproj/TasMovie.cs @@ -21,7 +21,7 @@ namespace BizHawk.Client.Common public readonly TasSession Session; private readonly TasLagLog LagLog = new TasLagLog(); private readonly Dictionary InputStateCache = new Dictionary(); - public readonly List VerificationLog = new List(); // For movies that do not begin with power-on, this is the input required to get into the initial state + public readonly IStringLog VerificationLog = StringLogUtil.MakeStringLog(); // For movies that do not begin with power-on, this is the input required to get into the initial state public readonly TasBranchCollection Branches = new TasBranchCollection(); private BackgroundWorker _progressReportWorker = null; @@ -81,7 +81,7 @@ namespace BizHawk.Client.Common } public TasLagLog TasLagLog { get { return LagLog; } } - public List InputLog { get { return _log; } } + public IStringLog InputLog { get { return _log; } } public TasMovieMarkerList Markers { get; set; } public bool BindMarkersToInput { get; set; } public bool UseInputCache { get; set; } @@ -331,7 +331,7 @@ namespace BizHawk.Client.Common } } - public List GetLogEntries() + public IStringLog GetLogEntries() { return _log; } @@ -350,7 +350,7 @@ namespace BizHawk.Client.Common { TimelineBranchFrame = null; - if (Global.Config.EnableBackupMovies && MakeBackup && _log.Any()) + if (Global.Config.EnableBackupMovies && MakeBackup && _log.Count != 0) { SaveBackup(); MakeBackup = false; @@ -500,7 +500,8 @@ namespace BizHawk.Client.Common { int? divergentPoint = DivergentPoint(_log, branch.InputLog); - _log = branch.InputLog.ToList(); + if (_log != null) _log.Dispose(); + _log = branch.InputLog.Clone(); //_changes = true; // if there are branch states, they will be loaded anyway @@ -523,7 +524,7 @@ namespace BizHawk.Client.Common } // TODO: use LogGenerators rather than string comparisons - private int? DivergentPoint(List currentLog, List newLog) + private int? DivergentPoint(IStringLog currentLog, IStringLog newLog) { int max = newLog.Count; if (currentLog.Count < newLog.Count) diff --git a/BizHawk.Client.EmuHawk/Program.cs b/BizHawk.Client.EmuHawk/Program.cs index e1ed1c6eeb..09a44a0c86 100644 --- a/BizHawk.Client.EmuHawk/Program.cs +++ b/BizHawk.Client.EmuHawk/Program.cs @@ -62,10 +62,13 @@ namespace BizHawk.Client.EmuHawk BizHawk.Client.Common.TempFileCleaner.Start(); Application.EnableVisualStyles(); Application.SetCompatibleTextRenderingDefault(false); + + HawkFile.ArchiveHandlerFactory = new SevenZipSharpArchiveHandler(); + string iniPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "config.ini"); Global.Config = ConfigService.Load(iniPath); Global.Config.ResolveDefaults(); - HawkFile.ArchiveHandlerFactory = new SevenZipSharpArchiveHandler(); + BizHawk.Client.Common.StringLogUtil.DefaultToDisk = Global.Config.MoviesOnDisk; //super hacky! this needs to be done first. still not worth the trouble to make this system fully proper for (int i = 0; i < args.Length; i++) diff --git a/BizHawk.Client.EmuHawk/tools/TAStudio/BookmarksBranchesBox.cs b/BizHawk.Client.EmuHawk/tools/TAStudio/BookmarksBranchesBox.cs index 9379ade84a..d0ee563907 100644 --- a/BizHawk.Client.EmuHawk/tools/TAStudio/BookmarksBranchesBox.cs +++ b/BizHawk.Client.EmuHawk/tools/TAStudio/BookmarksBranchesBox.cs @@ -230,7 +230,7 @@ namespace BizHawk.Client.EmuHawk { Frame = Global.Emulator.Frame, CoreData = (byte[])((Global.Emulator as IStatable).SaveStateBinary().Clone()), - InputLog = Movie.InputLog.ToList(), + InputLog = Movie.InputLog.Clone(), OSDFrameBuffer = GlobalWin.MainForm.CaptureOSD(), LagLog = Movie.TasLagLog.Clone(), ChangeLog = new TasMovieChangeLog(Movie),