make a new "FixedGrowthList" and use it for the StringLogs
this is mostly to workaround OOM issues with VERY large movies (>134217728 frames). so large just the list of string references is "too big" and exceeds the .net 2GB limit. even the on disk method fails here, as the offsets list (note, long and reference are the same size) becomes too big this probably gives a bit of speedup for "normal" movies, as the generous 16MB growth amount would likely be plenty for the movie, although i haven't test this much
This commit is contained in:
parent
b967e6028d
commit
da5fae0b01
|
@ -2,7 +2,10 @@
|
|||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Runtime.InteropServices;
|
||||
using System.Text;
|
||||
|
||||
using BizHawk.BizInvoke;
|
||||
using BizHawk.Common;
|
||||
|
||||
namespace BizHawk.Client.Common
|
||||
|
@ -67,7 +70,88 @@ namespace BizHawk.Client.Common
|
|||
void CopyTo(int index, string[] array, int arrayIndex, int count);
|
||||
}
|
||||
|
||||
internal class ListStringLog : List<string>, IStringLog
|
||||
/// <summary>
|
||||
/// A list that grows in fixed 16MB increments
|
||||
/// This delays 2GB OOM exceptions from occurring
|
||||
/// (due to .NET doubling the internal array on resize)
|
||||
/// </summary>
|
||||
/// <typeparam name="T">the type for the list</typeparam>
|
||||
internal class FixedGrowthList<T> : IEnumerable<T>
|
||||
{
|
||||
private const int GROW_AMOUNT = 16 * 1024 * 1024;
|
||||
private readonly List<T> _backingList = new();
|
||||
|
||||
public void Add(T value)
|
||||
{
|
||||
if (_backingList.Count == _backingList.Capacity)
|
||||
{
|
||||
_backingList.Capacity += GROW_AMOUNT;
|
||||
}
|
||||
|
||||
_backingList.Add(value);
|
||||
}
|
||||
|
||||
public void Insert(int index, T item)
|
||||
{
|
||||
if (_backingList.Count == _backingList.Capacity)
|
||||
{
|
||||
_backingList.Capacity += GROW_AMOUNT;
|
||||
}
|
||||
|
||||
_backingList.Insert(index, item);
|
||||
}
|
||||
|
||||
public void AddRange(IEnumerable<T> collection)
|
||||
{
|
||||
if (collection is ICollection<T> c)
|
||||
{
|
||||
var count = c.Count;
|
||||
while ((_backingList.Capacity - _backingList.Count) <= count)
|
||||
{
|
||||
_backingList.Capacity += GROW_AMOUNT;
|
||||
}
|
||||
_backingList.AddRange(collection);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (var item in collection)
|
||||
{
|
||||
Add(item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void InsertRange(int index, IEnumerable<T> collection)
|
||||
{
|
||||
if (collection is ICollection<T> c)
|
||||
{
|
||||
var count = c.Count;
|
||||
while ((_backingList.Capacity - _backingList.Count) <= count)
|
||||
{
|
||||
_backingList.Capacity += GROW_AMOUNT;
|
||||
}
|
||||
_backingList.InsertRange(index, collection);
|
||||
}
|
||||
else
|
||||
{
|
||||
foreach (var item in collection)
|
||||
{
|
||||
Insert(index++, item);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public T this[int index] { get => _backingList[index]; set => _backingList[index] = value; }
|
||||
public int Count => _backingList.Count;
|
||||
public void Clear() => _backingList.Clear();
|
||||
public void CopyTo(int index, T[] array, int arrayIndex, int count) => _backingList.CopyTo(index, array, arrayIndex, count);
|
||||
public IEnumerator<T> GetEnumerator() => _backingList.GetEnumerator();
|
||||
public void RemoveAt(int index) => _backingList.RemoveAt(index);
|
||||
public void RemoveRange(int index, int count) => _backingList.RemoveRange(index, count);
|
||||
IEnumerator IEnumerable.GetEnumerator() => _backingList.GetEnumerator();
|
||||
}
|
||||
|
||||
internal class ListStringLog : FixedGrowthList<string>, IStringLog
|
||||
{
|
||||
public IStringLog Clone()
|
||||
{
|
||||
|
@ -88,7 +172,7 @@ namespace BizHawk.Client.Common
|
|||
internal class StreamStringLog : IStringLog
|
||||
{
|
||||
private readonly Stream _stream;
|
||||
private readonly List<long> _offsets = new List<long>();
|
||||
private readonly FixedGrowthList<long> _offsets = new();
|
||||
private readonly BinaryWriter _bw;
|
||||
private readonly BinaryReader _br;
|
||||
private readonly bool _mDisk;
|
||||
|
|
Loading…
Reference in New Issue