attempt to make it possible to store movies on disk instead of in memory

BizHawk.Client.Common/BizHawk.Client.Common.csproj

@@ -154,6 +154,7 @@
     <Compile Include="movie\bk2\Bk2Movie.HeaderApi.cs">
       <DependentUpon>Bk2Movie.cs</DependentUpon>
     </Compile>
+    <Compile Include="movie\bk2\StringLogs.cs" />
     <Compile Include="movie\import\PXMImport.cs" />
     <Compile Include="movie\tasproj\StateManagerState.cs" />
     <Compile Include="movie\tasproj\TasBranch.cs" />

BizHawk.Client.Common/config/Config.cs

@@ -77,6 +77,7 @@ namespace BizHawk.Client.Common
 		public bool ForbidUD_LR = false;
 		public bool ShowContextMenu = true;
 		public bool EnableBackupMovies = true;
+		public bool MoviesOnDisk = false;
 		public bool HotkeyConfigAutoTab = true;
 		public bool InputConfigAutoTab = true;
 		public bool ShowLogWindow = false;

BizHawk.Client.Common/movie/bk2/Bk2Movie.InputLog.cs

@@ -8,7 +8,7 @@ namespace BizHawk.Client.Common
 {
 	public partial class Bk2Movie
 	{
-		protected List<string> _log = new List<string>();
+		protected IStringLog _log;
 		protected string LogKey = string.Empty;
 
 		public string GetInputLog()
@@ -58,7 +58,7 @@ namespace BizHawk.Client.Common
 			// We are in record mode so replace the movie log with the one from the savestate
 			if (!Global.MovieSession.MultiTrack.IsActive)
 			{
-				if (Global.Config.EnableBackupMovies && MakeBackup && _log.Any())
+				if (Global.Config.EnableBackupMovies && MakeBackup && _log.Count != 0)
 				{
 					SaveBackup();
 					MakeBackup = false;

BizHawk.Client.Common/movie/bk2/Bk2Movie.cs

@@ -27,6 +27,8 @@ namespace BizHawk.Client.Common
 			MakeBackup = true;
 
 			Header[HeaderKeys.MOVIEVERSION] = "BizHawk v2.0.0";
+
+			_log = StringLogUtil.MakeStringLog();
 		}
 
 		private string _filename;

BizHawk.Client.Common/movie/bk2/StringLogs.cs

@@ -0,0 +1,194 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace BizHawk.Client.Common
+{
+	public static class StringLogUtil
+	{
+		public static bool DefaultToDisk;
+		public static IStringLog MakeStringLog()
+		{
+			if (DefaultToDisk)
+				return new DiskStringLog();
+			else return new ListStringLog();
+		}
+	}
+
+	public interface IStringLog : IDisposable, IEnumerable<string>
+	{
+		void RemoveAt(int index);
+		int Count { get; }
+		void Clear();
+		void Add(string str);
+		string this[int index] { get; set; }
+		void Insert(int index, string val);
+		void InsertRange(int index, IEnumerable<string> collection);
+		void AddRange(IEnumerable<string> collection);
+		void RemoveRange(int index, int count);
+		IStringLog Clone();
+		void CopyTo(string[] array);
+		void CopyTo(int index, string[] array, int arrayIndex, int count);
+	}
+
+	class ListStringLog : List<string>, IStringLog
+	{
+		public IStringLog Clone()
+		{
+			ListStringLog ret = new ListStringLog();
+			ret.AddRange(this);
+			return ret;
+		}
+
+		public void Dispose() { }
+	}
+
+	/// <summary>
+	/// A dumb-ish IStringLog with storage on disk with no provision for recovering lost space, except upon Clear()
+	/// The purpose here is to avoid having too complicated buggy logic or a dependency on sqlite or such.
+	/// It should be faster than those alternatives, but wasteful of disk space.
+	/// It should also be easier to add new IList<string>-like methods than dealing with a database
+	/// </summary>
+	class DiskStringLog : IStringLog
+	{
+		List<long> Offsets = new List<long>();
+		long cursor = 0;
+		BinaryWriter bw;
+		BinaryReader br;
+
+		FileStream stream;
+		public DiskStringLog()
+		{
+			var path = Path.Combine(Path.GetTempPath(), "bizhawk.disklist-pid" + System.Diagnostics.Process.GetCurrentProcess().Id + "-" + Guid.NewGuid());
+			stream = new FileStream(path, FileMode.Create, System.Security.AccessControl.FileSystemRights.FullControl, FileShare.None, 4 * 1024, FileOptions.DeleteOnClose);
+			bw = new BinaryWriter(stream);
+			br = new BinaryReader(stream);
+		}
+
+		public IStringLog Clone()
+		{
+			DiskStringLog ret = new DiskStringLog();
+			for (int i = 0; i < Count; i++)
+				ret.Add(this[i]);
+			return ret;
+		}
+
+		public void Dispose()
+		{
+			stream.Dispose();
+		}
+
+		public int Count { get { return Offsets.Count; } }
+
+		public void Clear()
+		{
+			stream.SetLength(0);
+			Offsets.Clear();
+			cursor = 0;
+		}
+
+		public void Add(string str)
+		{
+			Offsets.Add(stream.Position);
+			bw.Write(str);
+			bw.Flush();
+		}
+
+		public void RemoveAt(int index)
+		{
+			Offsets.RemoveAt(index);
+			//no garbage collection in the disk file... oh well.
+		}
+
+		public string this[int index]
+		{
+			get
+			{
+				stream.Position = Offsets[index];
+				return br.ReadString();
+			}
+			set
+			{
+				stream.Position = stream.Length;
+				Offsets[index] = stream.Position;
+				bw.Write(value);
+				bw.Flush();
+			}
+		}
+
+		public void Insert(int index, string val)
+		{
+			Offsets.Insert(index, stream.Position);
+			bw.Write(val);
+			bw.Flush();
+		}
+
+		public void InsertRange(int index, IEnumerable<string> collection)
+		{
+			foreach(var item in collection)
+				Insert(index++,item);
+		}
+
+		public void AddRange(IEnumerable<string> collection)
+		{
+			foreach (var item in collection)
+				Add(item);
+		}
+
+		class Enumerator : IEnumerator<string>
+		{
+			public DiskStringLog log;
+			int index;
+			public string Current { get { return log[index]; } }
+			object System.Collections.IEnumerator.Current { get { return log[index]; } }
+			bool System.Collections.IEnumerator.MoveNext()
+			{
+				index++;
+				if (index >= log.Count)
+				{
+					index = log.Count;
+					return false;
+				}
+				return true;
+			}
+			void System.Collections.IEnumerator.Reset() { index = 0; }
+			
+			public void Dispose() { }
+		}
+
+		IEnumerator<string> IEnumerable<string>.GetEnumerator()
+		{
+			return new Enumerator() { log = this };
+		}
+
+		System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+		{
+			return new Enumerator() { log = this };
+		}
+
+		public void RemoveRange(int index, int count)
+		{
+			int end = index + count - 1;
+			for (int i = 0; i < count; i++)
+			{
+				RemoveAt(end);
+				end--;
+			}
+		}
+
+		public void CopyTo(string[] array)
+		{
+			for (int i = 0; i < Count; i++)
+				array[i] = this[i];
+		}
+
+		public void CopyTo(int index, string[] array, int arrayIndex, int count)
+		{
+			for (int i = 0; i < count; i++)
+				array[i + arrayIndex] = this[index + i];
+		}
+	}
+}

BizHawk.Client.Common/movie/conversions/MovieConversionExtensions.cs

@@ -145,7 +145,7 @@ namespace BizHawk.Client.Common.MovieConversionExtensions
 			tas.BinarySavestate = savestate;
 			tas.ClearLagLog();
 
-			List<string> entries = old.GetLogEntries();
+			var entries = old.GetLogEntries();
 
 			tas.CopyLog(entries.Skip(frame));
 			tas.CopyVerificationLog(old.VerificationLog);
@@ -220,7 +220,7 @@ namespace BizHawk.Client.Common.MovieConversionExtensions
 			tas.TasStateManager.Clear();
 			tas.ClearLagLog();
 
-			List<string> entries = old.GetLogEntries();
+			var entries = old.GetLogEntries();
 
 			tas.CopyVerificationLog(old.VerificationLog);
 			tas.CopyVerificationLog(entries);

BizHawk.Client.Common/movie/tasproj/TasBranch.cs

@@ -11,8 +11,8 @@ namespace BizHawk.Client.Common
 	public class TasBranch
 	{
 		public int Frame { get; set; }
-		public byte[] CoreData { get; set; }
-		public List<string> InputLog { get; set; }
+		public byte[] CoreData { get; set; }
+		public IStringLog InputLog { get; set; }
 		public BitmapBuffer OSDFrameBuffer { get; set; }
 		public TasLagLog LagLog { get; set; }
 		public TasMovieChangeLog ChangeLog { get; set; }
@@ -63,9 +63,10 @@ namespace BizHawk.Client.Common
 				});
 
 				bs.PutLump(ninput, delegate(TextWriter tw)
-				{
-					foreach (var line in b.InputLog)
-						tw.WriteLine(line);
+				{
+					int todo = b.InputLog.Count;
+					for (int i = 0; i < todo; i++)
+						tw.WriteLine(b.InputLog[i]);
 				});
 
 				bs.PutLump(nframebuffer, delegate(Stream s)
@@ -145,8 +146,8 @@ namespace BizHawk.Client.Common
 				});
 
 				bl.GetLump(ninput, true, delegate(TextReader tr)
-				{
-					b.InputLog = new List<string>();
+				{
+					b.InputLog = StringLogUtil.MakeStringLog();
 					string line;
 					while ((line = tr.ReadLine()) != null)
 						b.InputLog.Add(line);

BizHawk.Client.Common/movie/tasproj/TasMovie.IO.cs

@@ -319,7 +319,7 @@ namespace BizHawk.Client.Common
 			ChangeLog.ClearLog();
 		}
 
-		private static string InputLogToString(List<string> log)
+		private static string InputLogToString(IStringLog log)
 		{
 			var sb = new StringBuilder();
 			foreach (var record in log)

BizHawk.Client.Common/movie/tasproj/TasMovie.cs

@@ -21,7 +21,7 @@ namespace BizHawk.Client.Common
 		public readonly TasSession Session;
 		private readonly TasLagLog LagLog = new TasLagLog();
 		private readonly Dictionary<int, IController> InputStateCache = new Dictionary<int, IController>();
-		public readonly List<string> VerificationLog = new List<string>(); // For movies that do not begin with power-on, this is the input required to get into the initial state
+		public readonly IStringLog VerificationLog = StringLogUtil.MakeStringLog(); // For movies that do not begin with power-on, this is the input required to get into the initial state
 		public readonly TasBranchCollection Branches = new TasBranchCollection();
 
 		private BackgroundWorker _progressReportWorker = null;
@@ -81,7 +81,7 @@ namespace BizHawk.Client.Common
 		}
 
 		public TasLagLog TasLagLog { get { return LagLog; } }
-		public List<string> InputLog { get { return _log; } }
+		public IStringLog InputLog { get { return _log; } }
 		public TasMovieMarkerList Markers { get; set; }
 		public bool BindMarkersToInput { get; set; }
 		public bool UseInputCache { get; set; }
@@ -331,7 +331,7 @@ namespace BizHawk.Client.Common
 			}
 		}
 
-		public List<string> GetLogEntries()
+		public IStringLog GetLogEntries()
 		{
 			return _log;
 		}
@@ -350,7 +350,7 @@ namespace BizHawk.Client.Common
 			{
 				TimelineBranchFrame = null;
 
-				if (Global.Config.EnableBackupMovies && MakeBackup && _log.Any())
+				if (Global.Config.EnableBackupMovies && MakeBackup && _log.Count != 0)
 				{
 					SaveBackup();
 					MakeBackup = false;
@@ -500,7 +500,8 @@ namespace BizHawk.Client.Common
 		{
 			int? divergentPoint = DivergentPoint(_log, branch.InputLog);
 
-			_log = branch.InputLog.ToList();
+			if (_log != null) _log.Dispose();
+				_log = branch.InputLog.Clone();
 			//_changes = true;
 
 			// if there are branch states, they will be loaded anyway
@@ -523,7 +524,7 @@ namespace BizHawk.Client.Common
 		}
 
 		// TODO: use LogGenerators rather than string comparisons
-		private int? DivergentPoint(List<string> currentLog, List<string> newLog)
+		private int? DivergentPoint(IStringLog currentLog, IStringLog newLog)
 		{
 			int max = newLog.Count;
 			if (currentLog.Count < newLog.Count)

BizHawk.Client.EmuHawk/Program.cs

@@ -62,10 +62,13 @@ namespace BizHawk.Client.EmuHawk
 			BizHawk.Client.Common.TempFileCleaner.Start();
 			Application.EnableVisualStyles();
 			Application.SetCompatibleTextRenderingDefault(false);
+
+			HawkFile.ArchiveHandlerFactory = new SevenZipSharpArchiveHandler();
+		
 			string iniPath = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "config.ini");
 			Global.Config = ConfigService.Load<Config>(iniPath);
 			Global.Config.ResolveDefaults();
-			HawkFile.ArchiveHandlerFactory = new SevenZipSharpArchiveHandler();
+			BizHawk.Client.Common.StringLogUtil.DefaultToDisk = Global.Config.MoviesOnDisk;
 
 			//super hacky! this needs to be done first. still not worth the trouble to make this system fully proper
 			for (int i = 0; i < args.Length; i++)

BizHawk.Client.EmuHawk/tools/TAStudio/BookmarksBranchesBox.cs

@@ -230,7 +230,7 @@ namespace BizHawk.Client.EmuHawk
 			{
 				Frame = Global.Emulator.Frame,
 				CoreData = (byte[])((Global.Emulator as IStatable).SaveStateBinary().Clone()),
-				InputLog = Movie.InputLog.ToList(),
+				InputLog = Movie.InputLog.Clone(),
 				OSDFrameBuffer = GlobalWin.MainForm.CaptureOSD(),
 				LagLog = Movie.TasLagLog.Clone(),
 				ChangeLog = new TasMovieChangeLog(Movie),