BizHawk/BizHawk.Client.Common/SharpCompress/Common/SevenZip/ArchiveReader.cs

1592 lines
53 KiB
C#

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using SharpCompress.Compressors.LZMA;
using SharpCompress.Compressors.LZMA.Utilites;
using SharpCompress.IO;
namespace SharpCompress.Common.SevenZip
{
internal class ArchiveReader
{
internal Stream _stream;
internal Stack<DataReader> _readerStack = new Stack<DataReader>();
internal DataReader _currentReader;
internal long _streamOrigin;
internal long _streamEnding;
internal byte[] _header;
private readonly Dictionary<int, Stream> _cachedStreams = new Dictionary<int, Stream>();
internal void AddByteStream(byte[] buffer, int offset, int length)
{
_readerStack.Push(_currentReader);
_currentReader = new DataReader(buffer, offset, length);
}
internal void DeleteByteStream()
{
_currentReader = _readerStack.Pop();
}
#region Private Methods - Data Reader
internal Byte ReadByte()
{
return _currentReader.ReadByte();
}
private void ReadBytes(byte[] buffer, int offset, int length)
{
_currentReader.ReadBytes(buffer, offset, length);
}
private ulong ReadNumber()
{
return _currentReader.ReadNumber();
}
internal int ReadNum()
{
return _currentReader.ReadNum();
}
private uint ReadUInt32()
{
return _currentReader.ReadUInt32();
}
private ulong ReadUInt64()
{
return _currentReader.ReadUInt64();
}
private BlockType? ReadId()
{
ulong id = _currentReader.ReadNumber();
if (id > 25)
{
return null;
}
#if DEBUG
Log.WriteLine("ReadId: {0}", (BlockType)id);
#endif
return (BlockType)id;
}
private void SkipData(long size)
{
_currentReader.SkipData(size);
}
private void SkipData()
{
_currentReader.SkipData();
}
private void WaitAttribute(BlockType attribute)
{
for (;;)
{
BlockType? type = ReadId();
if (type == attribute)
{
return;
}
if (type == BlockType.End)
{
throw new InvalidOperationException();
}
SkipData();
}
}
private void ReadArchiveProperties()
{
while (ReadId() != BlockType.End)
{
SkipData();
}
}
#endregion
#region Private Methods - Reader Utilities
private BitVector ReadBitVector(int length)
{
var bits = new BitVector(length);
byte data = 0;
byte mask = 0;
for (int i = 0; i < length; i++)
{
if (mask == 0)
{
data = ReadByte();
mask = 0x80;
}
if ((data & mask) != 0)
{
bits.SetBit(i);
}
mask >>= 1;
}
return bits;
}
private BitVector ReadOptionalBitVector(int length)
{
byte allTrue = ReadByte();
if (allTrue != 0)
{
return new BitVector(length, true);
}
return ReadBitVector(length);
}
private void ReadNumberVector(List<byte[]> dataVector, int numFiles, Action<int, long?> action)
{
var defined = ReadOptionalBitVector(numFiles);
using (CStreamSwitch streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector);
for (int i = 0; i < numFiles; i++)
{
if (defined[i])
{
action(i, checked((long)ReadUInt64()));
}
else
{
action(i, null);
}
}
}
}
private DateTime TranslateTime(long time)
{
// FILETIME = 100-nanosecond intervals since January 1, 1601 (UTC)
return DateTime.FromFileTimeUtc(time).ToLocalTime();
}
private DateTime? TranslateTime(long? time)
{
if (time.HasValue && time.Value >= 0 && time.Value <= 2650467743999999999) //maximum Windows file time 31.12.9999
{
return TranslateTime(time.Value);
}
return null;
}
private void ReadDateTimeVector(List<byte[]> dataVector, int numFiles, Action<int, DateTime?> action)
{
ReadNumberVector(dataVector, numFiles, (index, value) => action(index, TranslateTime(value)));
}
private void ReadAttributeVector(List<byte[]> dataVector, int numFiles, Action<int, uint?> action)
{
BitVector boolVector = ReadOptionalBitVector(numFiles);
using (var streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector);
for (int i = 0; i < numFiles; i++)
{
if (boolVector[i])
{
action(i, ReadUInt32());
}
else
{
action(i, null);
}
}
}
}
#endregion
#region Private Methods
private void GetNextFolderItem(CFolder folder)
{
#if DEBUG
Log.WriteLine("-- GetNextFolderItem --");
Log.PushIndent();
#endif
try
{
int numCoders = ReadNum();
#if DEBUG
Log.WriteLine("NumCoders: " + numCoders);
#endif
folder._coders = new List<CCoderInfo>(numCoders);
int numInStreams = 0;
int numOutStreams = 0;
for (int i = 0; i < numCoders; i++)
{
#if DEBUG
Log.WriteLine("-- Coder --");
Log.PushIndent();
#endif
try
{
CCoderInfo coder = new CCoderInfo();
folder._coders.Add(coder);
byte mainByte = ReadByte();
int idSize = (mainByte & 0xF);
byte[] longId = new byte[idSize];
ReadBytes(longId, 0, idSize);
#if DEBUG
Log.WriteLine("MethodId: " + String.Join("", Enumerable.Range(0, idSize).Select(x => longId[x].ToString("x2")).ToArray()));
#endif
if (idSize > 8)
{
throw new NotSupportedException();
}
ulong id = 0;
for (int j = 0; j < idSize; j++)
{
id |= (ulong)longId[idSize - 1 - j] << (8 * j);
}
coder._methodId = new CMethodId(id);
if ((mainByte & 0x10) != 0)
{
coder._numInStreams = ReadNum();
coder._numOutStreams = ReadNum();
#if DEBUG
Log.WriteLine("Complex Stream (In: " + coder._numInStreams + " - Out: " + coder._numOutStreams + ")");
#endif
}
else
{
#if DEBUG
Log.WriteLine("Simple Stream (In: 1 - Out: 1)");
#endif
coder._numInStreams = 1;
coder._numOutStreams = 1;
}
if ((mainByte & 0x20) != 0)
{
int propsSize = ReadNum();
coder._props = new byte[propsSize];
ReadBytes(coder._props, 0, propsSize);
#if DEBUG
Log.WriteLine("Settings: " + String.Join("", coder._props.Select(bt => bt.ToString("x2")).ToArray()));
#endif
}
if ((mainByte & 0x80) != 0)
{
throw new NotSupportedException();
}
numInStreams += coder._numInStreams;
numOutStreams += coder._numOutStreams;
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
int numBindPairs = numOutStreams - 1;
folder._bindPairs = new List<CBindPair>(numBindPairs);
#if DEBUG
Log.WriteLine("BindPairs: " + numBindPairs);
Log.PushIndent();
#endif
for (int i = 0; i < numBindPairs; i++)
{
CBindPair bp = new CBindPair();
bp._inIndex = ReadNum();
bp._outIndex = ReadNum();
folder._bindPairs.Add(bp);
#if DEBUG
Log.WriteLine("#" + i + " - In: " + bp._inIndex + " - Out: " + bp._outIndex);
#endif
}
#if DEBUG
Log.PopIndent();
#endif
if (numInStreams < numBindPairs)
{
throw new NotSupportedException();
}
int numPackStreams = numInStreams - numBindPairs;
//folder.PackStreams.Reserve(numPackStreams);
if (numPackStreams == 1)
{
for (int i = 0; i < numInStreams; i++)
{
if (folder.FindBindPairForInStream(i) < 0)
{
#if DEBUG
Log.WriteLine("Single PackStream: #" + i);
#endif
folder._packStreams.Add(i);
break;
}
}
if (folder._packStreams.Count != 1)
{
throw new NotSupportedException();
}
}
else
{
#if DEBUG
Log.WriteLine("Multiple PackStreams ...");
Log.PushIndent();
#endif
for (int i = 0; i < numPackStreams; i++)
{
var num = ReadNum();
#if DEBUG
Log.WriteLine("#" + i + " - " + num);
#endif
folder._packStreams.Add(num);
}
#if DEBUG
Log.PopIndent();
#endif
}
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
private List<uint?> ReadHashDigests(int count)
{
#if DEBUG
Log.Write("ReadHashDigests:");
#endif
var defined = ReadOptionalBitVector(count);
var digests = new List<uint?>(count);
for (int i = 0; i < count; i++)
{
if (defined[i])
{
uint crc = ReadUInt32();
#if DEBUG
Log.Write(" " + crc.ToString("x8"));
#endif
digests.Add(crc);
}
else
{
#if DEBUG
Log.Write(" ########");
#endif
digests.Add(null);
}
}
#if DEBUG
Log.WriteLine();
#endif
return digests;
}
private void ReadPackInfo(out long dataOffset, out List<long> packSizes, out List<uint?> packCrCs)
{
#if DEBUG
Log.WriteLine("-- ReadPackInfo --");
Log.PushIndent();
#endif
try
{
packCrCs = null;
dataOffset = checked((long)ReadNumber());
#if DEBUG
Log.WriteLine("DataOffset: " + dataOffset);
#endif
int numPackStreams = ReadNum();
#if DEBUG
Log.WriteLine("NumPackStreams: " + numPackStreams);
#endif
WaitAttribute(BlockType.Size);
packSizes = new List<long>(numPackStreams);
#if DEBUG
Log.Write("Sizes:");
#endif
for (int i = 0; i < numPackStreams; i++)
{
var size = checked((long)ReadNumber());
#if DEBUG
Log.Write(" " + size);
#endif
packSizes.Add(size);
}
#if DEBUG
Log.WriteLine();
#endif
BlockType? type;
for (;;)
{
type = ReadId();
if (type == BlockType.End)
{
break;
}
if (type == BlockType.Crc)
{
packCrCs = ReadHashDigests(numPackStreams);
continue;
}
SkipData();
}
if (packCrCs == null)
{
packCrCs = new List<uint?>(numPackStreams);
for (int i = 0; i < numPackStreams; i++)
{
packCrCs.Add(null);
}
}
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
private void ReadUnpackInfo(List<byte[]> dataVector, out List<CFolder> folders)
{
#if DEBUG
Log.WriteLine("-- ReadUnpackInfo --");
Log.PushIndent();
#endif
try
{
WaitAttribute(BlockType.Folder);
int numFolders = ReadNum();
#if DEBUG
Log.WriteLine("NumFolders: {0}", numFolders);
#endif
using (CStreamSwitch streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector);
//folders.Clear();
//folders.Reserve(numFolders);
folders = new List<CFolder>(numFolders);
int index = 0;
for (int i = 0; i < numFolders; i++)
{
var f = new CFolder {_firstPackStreamId = index};
folders.Add(f);
GetNextFolderItem(f);
index += f._packStreams.Count;
}
}
WaitAttribute(BlockType.CodersUnpackSize);
#if DEBUG
Log.WriteLine("UnpackSizes:");
#endif
for (int i = 0; i < numFolders; i++)
{
CFolder folder = folders[i];
#if DEBUG
Log.Write(" #" + i + ":");
#endif
int numOutStreams = folder.GetNumOutStreams();
for (int j = 0; j < numOutStreams; j++)
{
long size = checked((long)ReadNumber());
#if DEBUG
Log.Write(" " + size);
#endif
folder._unpackSizes.Add(size);
}
#if DEBUG
Log.WriteLine();
#endif
}
for (;;)
{
BlockType? type = ReadId();
if (type == BlockType.End)
{
return;
}
if (type == BlockType.Crc)
{
List<uint?> crcs = ReadHashDigests(numFolders);
for (int i = 0; i < numFolders; i++)
{
folders[i]._unpackCrc = crcs[i];
}
continue;
}
SkipData();
}
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
private void ReadSubStreamsInfo(List<CFolder> folders, out List<int> numUnpackStreamsInFolders,
out List<long> unpackSizes, out List<uint?> digests)
{
#if DEBUG
Log.WriteLine("-- ReadSubStreamsInfo --");
Log.PushIndent();
#endif
try
{
numUnpackStreamsInFolders = null;
BlockType? type;
for (;;)
{
type = ReadId();
if (type == BlockType.NumUnpackStream)
{
numUnpackStreamsInFolders = new List<int>(folders.Count);
#if DEBUG
Log.Write("NumUnpackStreams:");
#endif
for (int i = 0; i < folders.Count; i++)
{
var num = ReadNum();
#if DEBUG
Log.Write(" " + num);
#endif
numUnpackStreamsInFolders.Add(num);
}
#if DEBUG
Log.WriteLine();
#endif
continue;
}
if (type == BlockType.Crc || type == BlockType.Size)
{
break;
}
if (type == BlockType.End)
{
break;
}
SkipData();
}
if (numUnpackStreamsInFolders == null)
{
numUnpackStreamsInFolders = new List<int>(folders.Count);
for (int i = 0; i < folders.Count; i++)
{
numUnpackStreamsInFolders.Add(1);
}
}
unpackSizes = new List<long>(folders.Count);
for (int i = 0; i < numUnpackStreamsInFolders.Count; i++)
{
// v3.13 incorrectly worked with empty folders
// v4.07: we check that folder is empty
int numSubstreams = numUnpackStreamsInFolders[i];
if (numSubstreams == 0)
{
continue;
}
#if DEBUG
Log.Write("#{0} StreamSizes:", i);
#endif
long sum = 0;
for (int j = 1; j < numSubstreams; j++)
{
if (type == BlockType.Size)
{
long size = checked((long)ReadNumber());
#if DEBUG
Log.Write(" " + size);
#endif
unpackSizes.Add(size);
sum += size;
}
}
unpackSizes.Add(folders[i].GetUnpackSize() - sum);
#if DEBUG
Log.WriteLine(" - rest: " + unpackSizes.Last());
#endif
}
if (type == BlockType.Size)
{
type = ReadId();
}
int numDigests = 0;
int numDigestsTotal = 0;
for (int i = 0; i < folders.Count; i++)
{
int numSubstreams = numUnpackStreamsInFolders[i];
if (numSubstreams != 1 || !folders[i].UnpackCrcDefined)
{
numDigests += numSubstreams;
}
numDigestsTotal += numSubstreams;
}
digests = null;
for (;;)
{
if (type == BlockType.Crc)
{
digests = new List<uint?>(numDigestsTotal);
List<uint?> digests2 = ReadHashDigests(numDigests);
int digestIndex = 0;
for (int i = 0; i < folders.Count; i++)
{
int numSubstreams = numUnpackStreamsInFolders[i];
CFolder folder = folders[i];
if (numSubstreams == 1 && folder.UnpackCrcDefined)
{
digests.Add(folder._unpackCrc.Value);
}
else
{
for (int j = 0; j < numSubstreams; j++, digestIndex++)
{
digests.Add(digests2[digestIndex]);
}
}
}
if (digestIndex != numDigests || numDigestsTotal != digests.Count)
{
Debugger.Break();
}
}
else if (type == BlockType.End)
{
if (digests == null)
{
digests = new List<uint?>(numDigestsTotal);
for (int i = 0; i < numDigestsTotal; i++)
{
digests.Add(null);
}
}
return;
}
else
{
SkipData();
}
type = ReadId();
}
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
private void ReadStreamsInfo(
List<byte[]> dataVector,
out long dataOffset,
out List<long> packSizes,
out List<uint?> packCrCs,
out List<CFolder> folders,
out List<int> numUnpackStreamsInFolders,
out List<long> unpackSizes,
out List<uint?> digests)
{
#if DEBUG
Log.WriteLine("-- ReadStreamsInfo --");
Log.PushIndent();
#endif
try
{
dataOffset = long.MinValue;
packSizes = null;
packCrCs = null;
folders = null;
numUnpackStreamsInFolders = null;
unpackSizes = null;
digests = null;
for (;;)
{
switch (ReadId())
{
case BlockType.End:
return;
case BlockType.PackInfo:
ReadPackInfo(out dataOffset, out packSizes, out packCrCs);
break;
case BlockType.UnpackInfo:
ReadUnpackInfo(dataVector, out folders);
break;
case BlockType.SubStreamsInfo:
ReadSubStreamsInfo(folders, out numUnpackStreamsInFolders, out unpackSizes, out digests);
break;
default:
throw new InvalidOperationException();
}
}
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
private List<byte[]> ReadAndDecodePackedStreams(long baseOffset, IPasswordProvider pass)
{
#if DEBUG
Log.WriteLine("-- ReadAndDecodePackedStreams --");
Log.PushIndent();
#endif
try
{
long dataStartPos;
List<long> packSizes;
List<uint?> packCrCs;
List<CFolder> folders;
List<int> numUnpackStreamsInFolders;
List<long> unpackSizes;
List<uint?> digests;
ReadStreamsInfo(null,
out dataStartPos,
out packSizes,
out packCrCs,
out folders,
out numUnpackStreamsInFolders,
out unpackSizes,
out digests);
dataStartPos += baseOffset;
var dataVector = new List<byte[]>(folders.Count);
int packIndex = 0;
foreach (var folder in folders)
{
long oldDataStartPos = dataStartPos;
long[] myPackSizes = new long[folder._packStreams.Count];
for (int i = 0; i < myPackSizes.Length; i++)
{
long packSize = packSizes[packIndex + i];
myPackSizes[i] = packSize;
dataStartPos += packSize;
}
var outStream = DecoderStreamHelper.CreateDecoderStream(_stream, oldDataStartPos, myPackSizes,
folder, pass);
int unpackSize = checked((int)folder.GetUnpackSize());
byte[] data = new byte[unpackSize];
outStream.ReadExact(data, 0, data.Length);
if (outStream.ReadByte() >= 0)
{
throw new InvalidOperationException("Decoded stream is longer than expected.");
}
dataVector.Add(data);
if (folder.UnpackCrcDefined)
{
if (Crc.Finish(Crc.Update(Crc.INIT_CRC, data, 0, unpackSize)) != folder._unpackCrc)
{
throw new InvalidOperationException("Decoded stream does not match expected CRC.");
}
}
}
return dataVector;
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
private void ReadHeader(ArchiveDatabase db, IPasswordProvider getTextPassword)
{
#if DEBUG
Log.WriteLine("-- ReadHeader --");
Log.PushIndent();
#endif
try
{
BlockType? type = ReadId();
if (type == BlockType.ArchiveProperties)
{
ReadArchiveProperties();
type = ReadId();
}
List<byte[]> dataVector = null;
if (type == BlockType.AdditionalStreamsInfo)
{
dataVector = ReadAndDecodePackedStreams(db._startPositionAfterHeader, getTextPassword);
type = ReadId();
}
List<long> unpackSizes;
List<uint?> digests;
if (type == BlockType.MainStreamsInfo)
{
ReadStreamsInfo(dataVector,
out db._dataStartPosition,
out db._packSizes,
out db._packCrCs,
out db._folders,
out db._numUnpackStreamsVector,
out unpackSizes,
out digests);
db._dataStartPosition += db._startPositionAfterHeader;
type = ReadId();
}
else
{
unpackSizes = new List<long>(db._folders.Count);
digests = new List<uint?>(db._folders.Count);
db._numUnpackStreamsVector = new List<int>(db._folders.Count);
for (int i = 0; i < db._folders.Count; i++)
{
var folder = db._folders[i];
unpackSizes.Add(folder.GetUnpackSize());
digests.Add(folder._unpackCrc);
db._numUnpackStreamsVector.Add(1);
}
}
db._files.Clear();
if (type == BlockType.End)
{
return;
}
if (type != BlockType.FilesInfo)
{
throw new InvalidOperationException();
}
int numFiles = ReadNum();
#if DEBUG
Log.WriteLine("NumFiles: " + numFiles);
#endif
db._files = new List<CFileItem>(numFiles);
for (int i = 0; i < numFiles; i++)
{
db._files.Add(new CFileItem());
}
BitVector emptyStreamVector = new BitVector(numFiles);
BitVector emptyFileVector = null;
BitVector antiFileVector = null;
int numEmptyStreams = 0;
for (;;)
{
type = ReadId();
if (type == BlockType.End)
{
break;
}
long size = checked((long)ReadNumber()); // TODO: throw invalid data on negative
int oldPos = _currentReader.Offset;
switch (type)
{
case BlockType.Name:
using (var streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, dataVector);
#if DEBUG
Log.Write("FileNames:");
#endif
for (int i = 0; i < db._files.Count; i++)
{
db._files[i].Name = _currentReader.ReadString();
#if DEBUG
Log.Write(" " + db._files[i].Name);
#endif
}
#if DEBUG
Log.WriteLine();
#endif
}
break;
case BlockType.WinAttributes:
#if DEBUG
Log.Write("WinAttributes:");
#endif
ReadAttributeVector(dataVector, numFiles, delegate(int i, uint? attr)
{
// Some third party implementations established an unofficial extension
// of the 7z archive format by placing posix file attributes in the high
// bits of the windows file attributes. This makes use of the fact that
// the official implementation does not perform checks on this value.
//
// Newer versions of the official 7z GUI client will try to parse this
// extension, thus acknowledging the unofficial use of these bits.
//
// For us it is safe to just discard the upper bits if they are set and
// keep the windows attributes from the lower bits (which should be set
// properly even if posix file attributes are present, in order to be
// compatible with older 7z archive readers)
//
// Note that the 15th bit is used by some implementations to indicate
// presence of the extension, but not all implementations do that so
// we can't trust that bit and must ignore it.
//
if (attr.HasValue && (attr.Value >> 16) != 0)
{
attr = attr.Value & 0x7FFFu;
}
db._files[i].Attrib = attr;
#if DEBUG
Log.Write(" " + (attr.HasValue ? attr.Value.ToString("x8") : "n/a"));
#endif
});
#if DEBUG
Log.WriteLine();
#endif
break;
case BlockType.EmptyStream:
emptyStreamVector = ReadBitVector(numFiles);
#if DEBUG
Log.Write("EmptyStream: ");
#endif
for (int i = 0; i < emptyStreamVector.Length; i++)
{
if (emptyStreamVector[i])
{
#if DEBUG
Log.Write("x");
#endif
numEmptyStreams++;
}
else
{
#if DEBUG
Log.Write(".");
#endif
}
}
#if DEBUG
Log.WriteLine();
#endif
emptyFileVector = new BitVector(numEmptyStreams);
antiFileVector = new BitVector(numEmptyStreams);
break;
case BlockType.EmptyFile:
emptyFileVector = ReadBitVector(numEmptyStreams);
#if DEBUG
Log.Write("EmptyFile: ");
for (int i = 0; i < numEmptyStreams; i++)
{
Log.Write(emptyFileVector[i] ? "x" : ".");
}
Log.WriteLine();
#endif
break;
case BlockType.Anti:
antiFileVector = ReadBitVector(numEmptyStreams);
#if DEBUG
Log.Write("Anti: ");
for (int i = 0; i < numEmptyStreams; i++)
{
Log.Write(antiFileVector[i] ? "x" : ".");
}
Log.WriteLine();
#endif
break;
case BlockType.StartPos:
#if DEBUG
Log.Write("StartPos:");
#endif
ReadNumberVector(dataVector, numFiles, delegate(int i, long? startPos)
{
db._files[i].StartPos = startPos;
#if DEBUG
Log.Write(" " + (startPos.HasValue ? startPos.Value.ToString() : "n/a"));
#endif
});
#if DEBUG
Log.WriteLine();
#endif
break;
case BlockType.CTime:
#if DEBUG
Log.Write("CTime:");
#endif
ReadDateTimeVector(dataVector, numFiles, delegate(int i, DateTime? time)
{
db._files[i].CTime = time;
#if DEBUG
Log.Write(" " + (time.HasValue ? time.Value.ToString() : "n/a"));
#endif
});
#if DEBUG
Log.WriteLine();
#endif
break;
case BlockType.ATime:
#if DEBUG
Log.Write("ATime:");
#endif
ReadDateTimeVector(dataVector, numFiles, delegate(int i, DateTime? time)
{
db._files[i].ATime = time;
#if DEBUG
Log.Write(" " + (time.HasValue ? time.Value.ToString() : "n/a"));
#endif
});
#if DEBUG
Log.WriteLine();
#endif
break;
case BlockType.MTime:
#if DEBUG
Log.Write("MTime:");
#endif
ReadDateTimeVector(dataVector, numFiles, delegate(int i, DateTime? time)
{
db._files[i].MTime = time;
#if DEBUG
Log.Write(" " + (time.HasValue ? time.Value.ToString() : "n/a"));
#endif
});
#if DEBUG
Log.WriteLine();
#endif
break;
case BlockType.Dummy:
#if DEBUG
Log.Write("Dummy: " + size);
#endif
for (long j = 0; j < size; j++)
{
if (ReadByte() != 0)
{
throw new InvalidOperationException();
}
}
break;
default:
SkipData(size);
break;
}
// since 0.3 record sizes must be correct
bool checkRecordsSize = (db._majorVersion > 0 || db._minorVersion > 2);
if (checkRecordsSize && _currentReader.Offset - oldPos != size)
{
throw new InvalidOperationException();
}
}
int emptyFileIndex = 0;
int sizeIndex = 0;
for (int i = 0; i < numFiles; i++)
{
CFileItem file = db._files[i];
file.HasStream = !emptyStreamVector[i];
if (file.HasStream)
{
file.IsDir = false;
file.IsAnti = false;
file.Size = unpackSizes[sizeIndex];
file.Crc = digests[sizeIndex];
sizeIndex++;
}
else
{
file.IsDir = !emptyFileVector[emptyFileIndex];
file.IsAnti = antiFileVector[emptyFileIndex];
emptyFileIndex++;
file.Size = 0;
file.Crc = null;
}
}
}
finally
{
#if DEBUG
Log.PopIndent();
#endif
}
}
#endregion
#region Public Methods
public void Open(Stream stream)
{
Close();
_streamOrigin = stream.Position;
_streamEnding = stream.Length;
// TODO: Check Signature!
_header = new byte[0x20];
for (int offset = 0; offset < 0x20;)
{
int delta = stream.Read(_header, offset, 0x20 - offset);
if (delta == 0)
{
throw new EndOfStreamException();
}
offset += delta;
}
_stream = stream;
}
public void Close()
{
if (_stream != null)
{
_stream.Dispose();
}
foreach (var stream in _cachedStreams.Values)
{
stream.Dispose();
}
_cachedStreams.Clear();
}
public ArchiveDatabase ReadDatabase(IPasswordProvider pass)
{
var db = new ArchiveDatabase(pass);
db.Clear();
db._majorVersion = _header[6];
db._minorVersion = _header[7];
if (db._majorVersion != 0)
{
throw new InvalidOperationException();
}
uint crcFromArchive = DataReader.Get32(_header, 8);
long nextHeaderOffset = (long)DataReader.Get64(_header, 0xC);
long nextHeaderSize = (long)DataReader.Get64(_header, 0x14);
uint nextHeaderCrc = DataReader.Get32(_header, 0x1C);
uint crc = Crc.INIT_CRC;
crc = Crc.Update(crc, nextHeaderOffset);
crc = Crc.Update(crc, nextHeaderSize);
crc = Crc.Update(crc, nextHeaderCrc);
crc = Crc.Finish(crc);
if (crc != crcFromArchive)
{
throw new InvalidOperationException();
}
db._startPositionAfterHeader = _streamOrigin + 0x20;
// empty header is ok
if (nextHeaderSize == 0)
{
db.Fill();
return db;
}
if (nextHeaderOffset < 0 || nextHeaderSize < 0 || nextHeaderSize > Int32.MaxValue)
{
throw new InvalidOperationException();
}
if (nextHeaderOffset > _streamEnding - db._startPositionAfterHeader)
{
throw new IndexOutOfRangeException();
}
_stream.Seek(nextHeaderOffset, SeekOrigin.Current);
byte[] header = new byte[nextHeaderSize];
_stream.ReadExact(header, 0, header.Length);
if (Crc.Finish(Crc.Update(Crc.INIT_CRC, header, 0, header.Length)) != nextHeaderCrc)
{
throw new InvalidOperationException();
}
using (CStreamSwitch streamSwitch = new CStreamSwitch())
{
streamSwitch.Set(this, header);
BlockType? type = ReadId();
if (type != BlockType.Header)
{
if (type != BlockType.EncodedHeader)
{
throw new InvalidOperationException();
}
var dataVector = ReadAndDecodePackedStreams(db._startPositionAfterHeader, db.PasswordProvider);
// compressed header without content is odd but ok
if (dataVector.Count == 0)
{
db.Fill();
return db;
}
if (dataVector.Count != 1)
{
throw new InvalidOperationException();
}
streamSwitch.Set(this, dataVector[0]);
if (ReadId() != BlockType.Header)
{
throw new InvalidOperationException();
}
}
ReadHeader(db, db.PasswordProvider);
}
db.Fill();
return db;
}
internal class CExtractFolderInfo
{
internal int _fileIndex;
internal int _folderIndex;
internal List<bool> _extractStatuses = new List<bool>();
internal CExtractFolderInfo(int fileIndex, int folderIndex)
{
_fileIndex = fileIndex;
_folderIndex = folderIndex;
if (fileIndex != -1)
{
_extractStatuses.Add(true);
}
}
}
private class FolderUnpackStream : Stream
{
private readonly ArchiveDatabase _db;
private readonly int _startIndex;
private readonly List<bool> _extractStatuses;
public FolderUnpackStream(ArchiveDatabase db, int p, int startIndex, List<bool> list)
{
_db = db;
_startIndex = startIndex;
_extractStatuses = list;
}
#region Stream
public override bool CanRead => true;
public override bool CanSeek => false;
public override bool CanWrite => false;
public override void Flush()
{
throw new NotSupportedException();
}
public override long Length => throw new NotSupportedException();
public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
public override int Read(byte[] buffer, int offset, int count)
{
throw new NotSupportedException();
}
public override long Seek(long offset, SeekOrigin origin)
{
throw new NotSupportedException();
}
public override void SetLength(long value)
{
throw new NotSupportedException();
}
private Stream _stream;
private long _rem;
private int _currentIndex;
private void ProcessEmptyFiles()
{
while (_currentIndex < _extractStatuses.Count && _db._files[_startIndex + _currentIndex].Size == 0)
{
OpenFile();
_stream.Dispose();
_stream = null;
_currentIndex++;
}
}
private void OpenFile()
{
int index = _startIndex + _currentIndex;
#if DEBUG
Log.WriteLine(_db._files[index].Name);
#endif
if (_db._files[index].CrcDefined)
{
_stream = new CrcCheckStream(_db._files[index].Crc.Value);
}
else
{
_stream = new MemoryStream();
}
_rem = _db._files[index].Size;
}
public override void Write(byte[] buffer, int offset, int count)
{
while (count != 0)
{
if (_stream != null)
{
int write = count;
if (write > _rem)
{
write = (int)_rem;
}
_stream.Write(buffer, offset, write);
count -= write;
_rem -= write;
offset += write;
if (_rem == 0)
{
_stream.Dispose();
_stream = null;
_currentIndex++;
ProcessEmptyFiles();
}
}
else
{
ProcessEmptyFiles();
if (_currentIndex == _extractStatuses.Count)
{
// we support partial extracting
Debugger.Break();
throw new NotSupportedException();
}
OpenFile();
}
}
}
#endregion
}
private Stream GetCachedDecoderStream(ArchiveDatabase db, int folderIndex)
{
Stream s;
if (!_cachedStreams.TryGetValue(folderIndex, out s))
{
CFolder folderInfo = db._folders[folderIndex];
int packStreamIndex = db._folders[folderIndex]._firstPackStreamId;
long folderStartPackPos = db.GetFolderStreamPos(folderInfo, 0);
List<long> packSizes = new List<long>();
for (int j = 0; j < folderInfo._packStreams.Count; j++)
{
packSizes.Add(db._packSizes[packStreamIndex + j]);
}
s = DecoderStreamHelper.CreateDecoderStream(_stream, folderStartPackPos, packSizes.ToArray(), folderInfo,
db.PasswordProvider);
_cachedStreams.Add(folderIndex, s);
}
return s;
}
public Stream OpenStream(ArchiveDatabase db, int fileIndex)
{
int folderIndex = db._fileIndexToFolderIndexMap[fileIndex];
int numFilesInFolder = db._numUnpackStreamsVector[folderIndex];
int firstFileIndex = db._folderStartFileIndex[folderIndex];
if (firstFileIndex > fileIndex || fileIndex - firstFileIndex >= numFilesInFolder)
{
throw new InvalidOperationException();
}
int skipCount = fileIndex - firstFileIndex;
long skipSize = 0;
for (int i = 0; i < skipCount; i++)
{
skipSize += db._files[firstFileIndex + i].Size;
}
Stream s = GetCachedDecoderStream(db, folderIndex);
s.Position = skipSize;
return new ReadOnlySubStream(s, db._files[fileIndex].Size);
}
public void Extract(ArchiveDatabase db, int[] indices)
{
int numItems;
bool allFilesMode = (indices == null);
if (allFilesMode)
{
numItems = db._files.Count;
}
else
{
numItems = indices.Length;
}
if (numItems == 0)
{
return;
}
List<CExtractFolderInfo> extractFolderInfoVector = new List<CExtractFolderInfo>();
for (int i = 0; i < numItems; i++)
{
int fileIndex = allFilesMode ? i : indices[i];
int folderIndex = db._fileIndexToFolderIndexMap[fileIndex];
if (folderIndex == -1)
{
extractFolderInfoVector.Add(new CExtractFolderInfo(fileIndex, -1));
continue;
}
if (extractFolderInfoVector.Count == 0 || folderIndex != extractFolderInfoVector.Last()._folderIndex)
{
extractFolderInfoVector.Add(new CExtractFolderInfo(-1, folderIndex));
}
CExtractFolderInfo efi = extractFolderInfoVector.Last();
int startIndex = db._folderStartFileIndex[folderIndex];
for (int index = efi._extractStatuses.Count; index <= fileIndex - startIndex; index++)
{
efi._extractStatuses.Add(index == fileIndex - startIndex);
}
}
foreach (CExtractFolderInfo efi in extractFolderInfoVector)
{
int startIndex;
if (efi._fileIndex != -1)
{
startIndex = efi._fileIndex;
}
else
{
startIndex = db._folderStartFileIndex[efi._folderIndex];
}
var outStream = new FolderUnpackStream(db, 0, startIndex, efi._extractStatuses);
if (efi._fileIndex != -1)
{
continue;
}
int folderIndex = efi._folderIndex;
CFolder folderInfo = db._folders[folderIndex];
int packStreamIndex = db._folders[folderIndex]._firstPackStreamId;
long folderStartPackPos = db.GetFolderStreamPos(folderInfo, 0);
List<long> packSizes = new List<long>();
for (int j = 0; j < folderInfo._packStreams.Count; j++)
{
packSizes.Add(db._packSizes[packStreamIndex + j]);
}
// TODO: If the decoding fails the last file may be extracted incompletely. Delete it?
Stream s = DecoderStreamHelper.CreateDecoderStream(_stream, folderStartPackPos, packSizes.ToArray(),
folderInfo, db.PasswordProvider);
byte[] buffer = new byte[4 << 10];
for (;;)
{
int processed = s.Read(buffer, 0, buffer.Length);
if (processed == 0)
{
break;
}
outStream.Write(buffer, 0, processed);
}
}
}
public IEnumerable<CFileItem> GetFiles(ArchiveDatabase db)
{
return db._files;
}
public int GetFileIndex(ArchiveDatabase db, CFileItem item)
{
return db._files.IndexOf(item);
}
#endregion
}
}