make a new "FixedGrowthList" and use it for the StringLogs

this is mostly to workaround OOM issues with VERY large movies (>134217728 frames).
so large just the list of string references is "too big" and exceeds the .net 2GB limit.
even the on disk method fails here, as the offsets list (note, long and reference are the same size) becomes too big
this probably gives a bit of speedup for "normal" movies, as the generous 16MB growth amount would likely be plenty for the movie, although i haven't test this much
This commit is contained in:
CasualPokePlayer 2022-11-22 09:10:19 -08:00
parent b967e6028d
commit da5fae0b01
1 changed files with 86 additions and 2 deletions

View File

@ -2,7 +2,10 @@
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
using BizHawk.BizInvoke;
using BizHawk.Common;
namespace BizHawk.Client.Common
@ -67,7 +70,88 @@ namespace BizHawk.Client.Common
void CopyTo(int index, string[] array, int arrayIndex, int count);
}
internal class ListStringLog : List<string>, IStringLog
/// <summary>
/// A list that grows in fixed 16MB increments
/// This delays 2GB OOM exceptions from occurring
/// (due to .NET doubling the internal array on resize)
/// </summary>
/// <typeparam name="T">the type for the list</typeparam>
internal class FixedGrowthList<T> : IEnumerable<T>
{
private const int GROW_AMOUNT = 16 * 1024 * 1024;
private readonly List<T> _backingList = new();
public void Add(T value)
{
if (_backingList.Count == _backingList.Capacity)
{
_backingList.Capacity += GROW_AMOUNT;
}
_backingList.Add(value);
}
public void Insert(int index, T item)
{
if (_backingList.Count == _backingList.Capacity)
{
_backingList.Capacity += GROW_AMOUNT;
}
_backingList.Insert(index, item);
}
public void AddRange(IEnumerable<T> collection)
{
if (collection is ICollection<T> c)
{
var count = c.Count;
while ((_backingList.Capacity - _backingList.Count) <= count)
{
_backingList.Capacity += GROW_AMOUNT;
}
_backingList.AddRange(collection);
}
else
{
foreach (var item in collection)
{
Add(item);
}
}
}
public void InsertRange(int index, IEnumerable<T> collection)
{
if (collection is ICollection<T> c)
{
var count = c.Count;
while ((_backingList.Capacity - _backingList.Count) <= count)
{
_backingList.Capacity += GROW_AMOUNT;
}
_backingList.InsertRange(index, collection);
}
else
{
foreach (var item in collection)
{
Insert(index++, item);
}
}
}
public T this[int index] { get => _backingList[index]; set => _backingList[index] = value; }
public int Count => _backingList.Count;
public void Clear() => _backingList.Clear();
public void CopyTo(int index, T[] array, int arrayIndex, int count) => _backingList.CopyTo(index, array, arrayIndex, count);
public IEnumerator<T> GetEnumerator() => _backingList.GetEnumerator();
public void RemoveAt(int index) => _backingList.RemoveAt(index);
public void RemoveRange(int index, int count) => _backingList.RemoveRange(index, count);
IEnumerator IEnumerable.GetEnumerator() => _backingList.GetEnumerator();
}
internal class ListStringLog : FixedGrowthList<string>, IStringLog
{
public IStringLog Clone()
{
@ -88,7 +172,7 @@ namespace BizHawk.Client.Common
internal class StreamStringLog : IStringLog
{
private readonly Stream _stream;
private readonly List<long> _offsets = new List<long>();
private readonly FixedGrowthList<long> _offsets = new();
private readonly BinaryWriter _bw;
private readonly BinaryReader _br;
private readonly bool _mDisk;