discsystem-finish ECM support project. random access ECM decoding is now supported.

This commit is contained in:
zeromus 2012-11-17 22:16:09 +00:00
parent a0281498d3
commit a301b29a7d
9 changed files with 304 additions and 154 deletions

View File

@ -443,6 +443,7 @@
<Compile Include="Database\CRC32.cs" />
<Compile Include="Database\Database.cs" />
<Compile Include="Database\GameInfo.cs" />
<Compile Include="DiscSystem\Blobs\Blob_ECM.cs" />
<Compile Include="DiscSystem\Blobs\Blob_WaveFile.cs">
<SubType>Code</SubType>
</Compile>

View File

@ -1,54 +1,28 @@
//The ecm file begins with 4 bytes: ECM\0
//Copyright (c) 2012 BizHawk team
//then, repeat forever processing these blocks:
// Read the block header bytes. The block header is terminated after processing a byte without 0x80 set.
// The block header contains these bits packed in the bottom 7 LSB of successive bytes:
// xNNNNNNN NNNNNNNN NNNNNNNN NNNNNNNN TTT
// N: a Number
// T: the type of the sector
// If you encounter a Number of 0xFFFFFFFF then the blocks section is finished.
// If you need a 6th byte for the block header, then the block header is erroneous
// Increment Number, since storing 0 wouldve been useless.
//Permission is hereby granted, free of charge, to any person obtaining a copy of
//this software and associated documentation files (the "Software"), to deal in
//the Software without restriction, including without limitation the rights to
//use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
//of the Software, and to permit persons to whom the Software is furnished to do
//so, subject to the following conditions:
// Now, process the block.
// Type 0:
// Read Number bytes from the ECM file and write to the output stream.
// This block isn't necessarily a multiple of any particular sector size.
// accumulate all those bytes through the EDC
// Type 1: For Number of sectors:
// Read sector bytes 12,13,14
// Read 2048 sector bytes @16
// Reconstruct sector as type 1
// accumulate 2352 sector bytes @0 through the EDC
// write 2352 sector byte @0 to the output stream
//The above copyright notice and this permission notice shall be included in all
//copies or substantial portions of the Software.
// Type 2: For Number of sectors:
// Read 2052 sector bytes @20
// Reconstruct sector as type 2
// accumulate 2336 sector bytes @16 through the EDC
// write 2336 sector bytes @16 to the output stream
//THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
//IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
//FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
//AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
//LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
//OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
//SOFTWARE.
// Type 3: For Number of sectors:
// Read 2328 sector bytes @20
// Reconstruct sector as type 3
// accumulate 2336 sector bytes @16 through the EDC
// write 2336 sector bytes @16 to the output stream
//After encountering our end marker and exiting the block processing section:
//read a 32bit little endian value, which should be the output of the EDC (just a little check to make sure the file is valid)
//That's the end of the file
//ECM File Format reading support
//
//TODO - make a background thread to validate the EDC. be sure to terminate thread when the Blob disposes
//remember: may need another stream for that. the IBlob architecture doesnt demand multithreading support
//TODO - binary search the index.
//TODO - stress test the random access system:
// pick random chunk lengths, increment counter by length, put records in list, until bin file is exhausted
// jumble records
// read all the records through ECM and not-ECM and make sure the contents match
using System;
using System.Text;
using System.IO;
@ -56,7 +30,6 @@ using System.Collections.Generic;
namespace BizHawk.DiscSystem
{
partial class Disc
{
class Blob_ECM : IBlob
@ -93,8 +66,6 @@ namespace BizHawk.DiscSystem
public void Parse(string path)
{
//List<IndexEntry> temp = new List<IndexEntry>();
stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read);
//skip header
@ -158,11 +129,9 @@ namespace BizHawk.DiscSystem
logOffset += todo * 2336;
}
else MisformedException();
//Console.WriteLine(logOffset);
}
//TODO - endian bug
//TODO - endian bug. need endian-independent binary reader with good license
var br = new BinaryReader(stream);
EDC = br.ReadInt32();
@ -189,6 +158,44 @@ namespace BizHawk.DiscSystem
return true;
}
/// <summary>
/// finds the IndexEntry for the specified logical offset
/// </summary>
int FindInIndex(long offset, int LastReadIndex)
{
//try to avoid searching the index. check the last index we we used.
for(int i=0;i<2;i++) //try 2 times
{
IndexEntry last = Index[LastReadIndex];
if (LastReadIndex == Index.Count - 1)
{
//byte_pos would have to be after the last entry
if (offset >= last.LogicalOffset)
{
return LastReadIndex;
}
}
else
{
IndexEntry next = Index[LastReadIndex + 1];
if (offset >= last.LogicalOffset && offset < next.LogicalOffset)
{
return LastReadIndex;
}
//well, maybe we just advanced one sector. just try again one sector ahead
LastReadIndex++;
}
}
//Console.WriteLine("binary searched"); //use this to check for mistaken LastReadIndex logic resulting in binary searches during sequential access
int listIndex = Index.LowerBoundBinarySearch(idx => idx.LogicalOffset, offset);
System.Diagnostics.Debug.Assert(listIndex < Index.Count);
//Console.WriteLine("byte_pos {0:X8} using index #{1} at offset {2:X8}", offset, listIndex, Index[listIndex].LogicalOffset);
return listIndex;
}
void Reconstruct(byte[] secbuf, int type)
{
//sync
@ -238,65 +245,24 @@ namespace BizHawk.DiscSystem
}
//we dont want to keep churning through this many big byte arrays while reading stuff, so we save a sector cache.
//unlikely that we'll be hitting this from multiple threads, so low chance of contention.
byte[] Read_SectorBuf = new byte[2352];
int LastReadIndex = 0;
int Read_LastIndex = 0;
public int Read(long byte_pos, byte[] buffer, int offset, int _count)
{
//Console.WriteLine("{0:X8}", byte_pos);
//if (byte_pos + _count >= 0xb47d161)
if (byte_pos == 0xb47c830)
{
int zzz = 9;
}
long remain = _count;
int completed = 0;
//we take advantage of the fact that we pretty much always read one sector at a time.
//this would be really inefficient if we only read one byte at a time.
//on the other hand, just in case, we could keep a cache of the most recently decoded sector. that would be easy and would solve that problem (if we had it)
while (remain > 0)
{
//find the IndexEntry that corresponds to this byte position
//int listIndex = Index.BinarySearch(idx => idx.LogicalOffset, byte_pos);
//TODO - binary search. no builtin binary search is good enough to return something sensible for a non-match.
//check BinarySearch extension method in Util.cs and finish it up (too complex to add in to this mess right now)
RETRY:
int listIndex = LastReadIndex;
for (; ; )
{
IndexEntry curie = Index[listIndex];
if (curie.LogicalOffset > byte_pos)
{
if (Index[listIndex - 1].LogicalOffset > byte_pos)
{
LastReadIndex = 0;
goto RETRY;
}
break;
}
listIndex++;
if (listIndex == Index.Count)
{
break;
}
}
listIndex--;
//if it wasnt found, then we didn't actually read anything
if (listIndex == -1 || listIndex == Index.Count)
{
//fix O() for this operation to not be exponential
if (LastReadIndex == 0)
return 0;
LastReadIndex = 0;
goto RETRY;
}
LastReadIndex = listIndex;
int listIndex = FindInIndex(byte_pos, Read_LastIndex);
IndexEntry ie = Index[listIndex];
Read_LastIndex = listIndex;
if (ie.Type == 0)
{
@ -334,8 +300,6 @@ namespace BizHawk.DiscSystem
{
//these are sector-based types. they have similar handling.
//lock (Read_SectorBuf) //todo
long blockOffset = byte_pos - ie.LogicalOffset;
//figure out which sector within the block we're in
@ -380,7 +344,7 @@ namespace BizHawk.DiscSystem
break;
}
//sector is decoded to 2352 bytes. Handling doesnt depend on type from here
//sector is decoded to 2352 bytes. Handling doesnt depend much on type from here
Array.Copy(Read_SectorBuf, (int)bytesAskedIntoSector + outSecOffset, buffer, offset, todo);
int done = (int)todo;
@ -389,13 +353,56 @@ namespace BizHawk.DiscSystem
completed += done;
remain -= done;
byte_pos += done;
} //not type 0
} // while(Remain)
return completed;
}
}
}
}
}
//-------------------------------------------------------------------------------------------
//The ecm file begins with 4 bytes: ECM\0
//then, repeat forever processing these blocks:
// Read the block header bytes. The block header is terminated after processing a byte without 0x80 set.
// The block header contains these bits packed in the bottom 7 LSB of successive bytes:
// xNNNNNNN NNNNNNNN NNNNNNNN NNNNNNNN TTT
// N: a Number
// T: the type of the sector
// If you encounter a Number of 0xFFFFFFFF then the blocks section is finished.
// If you need a 6th byte for the block header, then the block header is erroneous
// Increment Number, since storing 0 wouldve been useless.
// Now, process the block.
// Type 0:
// Read Number bytes from the ECM file and write to the output stream.
// This block isn't necessarily a multiple of any particular sector size.
// accumulate all those bytes through the EDC
// Type 1: For Number of sectors:
// Read sector bytes 12,13,14
// Read 2048 sector bytes @16
// Reconstruct sector as type 1
// accumulate 2352 sector bytes @0 through the EDC
// write 2352 sector byte @0 to the output stream
// Type 2: For Number of sectors:
// Read 2052 sector bytes @20
// Reconstruct sector as type 2
// accumulate 2336 sector bytes @16 through the EDC
// write 2336 sector bytes @16 to the output stream
// Type 3: For Number of sectors:
// Read 2328 sector bytes @20
// Reconstruct sector as type 3
// accumulate 2336 sector bytes @16 through the EDC
// write 2336 sector bytes @16 to the output stream
//After encountering our end marker and exiting the block processing section:
//read a 32bit little endian value, which should be the output of the EDC (just a little check to make sure the file is valid)
//That's the end of the file

View File

@ -16,15 +16,21 @@ namespace BizHawk.DiscSystem
/// </summary>
string FindAlternateExtensionFile(string path, bool caseSensitive)
{
string targetFile = Path.GetFileName(path);
string targetFragment = Path.GetFileNameWithoutExtension(path);
var di = new FileInfo(path).Directory;
var results = new List<FileInfo>();
foreach (var fi in di.GetFiles())
{
string fragment = Path.GetFileNameWithoutExtension(fi.FullName);
//match files with differing extensions
int cmp = string.Compare(fragment, targetFragment, !caseSensitive);
if(cmp != 0)
//match files with another extension added on (likely to be mygame.bin.ecm)
cmp = string.Compare(fragment, targetFile, !caseSensitive);
if (cmp == 0)
results.Add(fi);
}
if(results.Count == 0) throw new DiscReferenceException(path, "Cannot find the specified file");
if (results.Count > 1) throw new DiscReferenceException(path, "Cannot choose between multiple options");
@ -70,6 +76,7 @@ namespace BizHawk.DiscSystem
if (blobPathExt == ".mp3") cue_file.FileType = Cue.CueFileType.Wave;
if (blobPathExt == ".mpc") cue_file.FileType = Cue.CueFileType.Wave;
if (blobPathExt == ".flac") cue_file.FileType = Cue.CueFileType.Wave;
if (blobPathExt == ".ecm") cue_file.FileType = Cue.CueFileType.ECM;
if (cue_file.FileType == Cue.CueFileType.Binary || cue_file.FileType == Cue.CueFileType.Unspecified)
{
@ -82,6 +89,19 @@ namespace BizHawk.DiscSystem
blob_leftover = (int)(blob.Length - blob_length_aba * blob_sectorsize);
cue_blob = blob;
}
else if (cue_file.FileType == Cue.CueFileType.ECM)
{
if(!Blob_ECM.IsECM(blobPath))
{
throw new DiscReferenceException(blobPath, "an ECM file was specified or detected, but it isn't a valid ECM file. You've got issues. Consult your iso vendor.");
}
Blob_ECM blob = new Blob_ECM();
Blobs.Add(blob);
blob.Parse(blobPath);
cue_blob = blob;
blob_length_aba = (int)(blob.Length / blob_sectorsize);
blob_leftover = (int)(blob.Length - blob_length_aba * blob_sectorsize);
}
else if (cue_file.FileType == Cue.CueFileType.Wave)
{
Blob_WaveFile blob = new Blob_WaveFile();
@ -109,7 +129,7 @@ namespace BizHawk.DiscSystem
FFMpeg ffmpeg = new FFMpeg();
if (!ffmpeg.QueryServiceAvailable())
{
throw new InvalidOperationException("No decoding service was available (make sure ffmpeg.exe is available. even though this may be a wav, ffmpeg is used to load oddly formatted wave files)");
throw new DiscReferenceException(blobPath, "No decoding service was available (make sure ffmpeg.exe is available. even though this may be a wav, ffmpeg is used to load oddly formatted wave files. If you object to this, please send us a note and we'll see what we can do. It shouldn't be too hard.)");
}
AudioDecoder dec = new AudioDecoder();
byte[] buf = dec.AcquireWaveData(blobPath);
@ -349,7 +369,7 @@ namespace BizHawk.DiscSystem
public enum CueFileType
{
Unspecified, Binary, Wave
Unspecified, Binary, Wave, ECM
}
public class CueFile

View File

@ -112,7 +112,7 @@ namespace BizHawk.DiscSystem
Array.Copy(lba_buf, lba_within, buffer, offset, todo);
offset += todo;
length -= todo;
lba_within = 0;
disc_offset += todo;
}
}

View File

@ -76,8 +76,14 @@ namespace BizHawk.DiscSystem
int Read(byte[] buffer, int offset);
}
/// <summary>
/// Presently, an IBlob doesn't need to work multithreadedly. It's quite an onerous demand. This should probably be managed by the Disc class somehow, or by the user making another Disc.
/// </summary>
public interface IBlob : IDisposable
{
/// <summary>
/// what a weird parameter order. normally the dest buffer would be first. weird.
/// </summary>
int Read(long byte_pos, byte[] buffer, int offset, int count);
}
@ -237,11 +243,9 @@ namespace BizHawk.DiscSystem
buffer[offset + 15] = 1;
//calculate EDC and poke into the sector
uint edc = ECM.EDC_Calc(buffer, offset);
buffer[offset + 2064 + 0] = (byte)((edc >> 0) & 0xFF);
buffer[offset + 2064 + 1] = (byte)((edc >> 8) & 0xFF);
buffer[offset + 2064 + 2] = (byte)((edc >> 16) & 0xFF);
buffer[offset + 2064 + 3] = (byte)((edc >> 24) & 0xFF);
uint edc = ECM.EDC_Calc(buffer, offset, 2064);
ECM.PokeUint(buffer, 2064, edc);
//intermediate
for (int i = 0; i < 8; i++) buffer[offset + 2068 + i] = 0;
//ECC
@ -655,6 +659,11 @@ namespace BizHawk.DiscSystem
/// </summary>
public bool ReallyDumpBin;
/// <summary>
/// Dump bins to bitbucket instead of disk
/// </summary>
public bool DumpToBitbucket;
/// <summary>
/// dump a .sub.q along with bins. one day we'll want to dump the entire subcode but really Q is all thats important for debugging most things
/// </summary>
@ -775,7 +784,8 @@ namespace BizHawk.DiscSystem
progress.ProgressCurrent = 0;
progress.InfoPresent = true;
string cuePath = Path.Combine(directory, baseName + ".cue");
File.WriteAllText(cuePath, cue);
if (prefs.DumpToBitbucket) { }
else File.WriteAllText(cuePath, cue);
progress.Message = "Writing bin(s)";
progress.TaskCurrent = 1;
@ -791,12 +801,17 @@ namespace BizHawk.DiscSystem
string trackBinFile = bfd.name;
string trackBinPath = Path.Combine(directory, trackBinFile);
string subQPath = Path.ChangeExtension(trackBinPath, ".sub.q");
FileStream fsSubQ = null;
FileStream fs = new FileStream(trackBinPath, FileMode.Create, FileAccess.Write, FileShare.None);
Stream fsSubQ = null;
Stream fs;
if(prefs.DumpToBitbucket)
fs = Stream.Null;
else fs = new FileStream(trackBinPath, FileMode.Create, FileAccess.Write, FileShare.None);
try
{
if (prefs.DumpSubchannelQ)
fsSubQ = new FileStream(subQPath, FileMode.Create, FileAccess.Write, FileShare.None);
if (prefs.DumpToBitbucket)
fsSubQ = Stream.Null;
else fsSubQ = new FileStream(subQPath, FileMode.Create, FileAccess.Write, FileShare.None);
for (int i = 0; i < bfd.abas.Count; i++)
{

View File

@ -19,7 +19,6 @@
//SOFTWARE.
//CD-ROM ECC/EDC related algorithms
//Support for Neill Corlett's ECM file format (TBD)
//todo - ecm sometimes sets the sector address to 0 before computing the ECC. i cant find any documentation to support this.
//seems to only take effect for cd-xa (mode 2, form 1). need to ask about this or test further on a cd-xa test disc
@ -186,13 +185,24 @@ namespace BizHawk.DiscSystem
}
/// <summary>
/// calculates EDC checksum for bytes [0,2063] of a sector located at (offset(
/// handy for stashing the EDC somewhere with little endian
/// </summary>
public static void PokeUint(byte[] data, int offset, uint value)
{
data[offset + 0] = (byte)((value >> 0) & 0xFF);
data[offset + 1] = (byte)((value >> 8) & 0xFF);
data[offset + 2] = (byte)((value >> 16) & 0xFF);
data[offset + 3] = (byte)((value >> 24) & 0xFF);
}
/// <summary>
/// calculates EDC checksum for the range of data provided
/// see section 14.3 of yellowbook
/// </summary>
public static uint EDC_Calc(byte[] data, int offset)
public static uint EDC_Calc(byte[] data, int offset, int length)
{
uint crc = 0;
for (int i = 0; i <= 2063; i++)
for (int i = 0; i < length; i++)
{
byte b = data[offset + i];
int entry = ((int)crc ^ b) & 0xFF;
@ -202,6 +212,8 @@ namespace BizHawk.DiscSystem
return crc;
}
/// <summary>
/// returns the address from a sector. useful for saving it before zeroing it for ECC calculations
/// </summary>

View File

@ -10,42 +10,54 @@ using System.Text;
namespace BizHawk
{
public struct Tuple<T1, T2> : IEquatable<Tuple<T1, T2>>
{
readonly T1 first;
readonly T2 second;
public T1 First { get { return first; } }
public T2 Second { get { return second; } }
public Tuple(T1 o1, T2 o2)
{
first = o1;
second = o2;
}
public bool Equals(Tuple<T1, T2> other)
{
return first.Equals(other.first) &&
second.Equals(other.second);
}
public override bool Equals(object obj)
{
if (obj is Tuple<T1, T2>)
return this.Equals((Tuple<T1, T2>)obj);
else
return false;
}
public override int GetHashCode()
{
return first.GetHashCode() ^ second.GetHashCode();
}
}
public static class Extensions
{
public static int LowerBoundBinarySearch<T, TKey>(this IList<T> list, Func<T, TKey> keySelector, TKey key) where TKey : IComparable<TKey>
{
int min = 0;
int max = list.Count;
int mid = 0;
TKey midKey;
while (min < max)
{
mid = (max + min) / 2;
T midItem = list[mid];
midKey = keySelector(midItem);
int comp = midKey.CompareTo(key);
if (comp < 0)
{
min = mid + 1;
}
else if (comp > 0)
{
max = mid - 1;
}
else
{
return mid;
}
}
//did we find it exactly?
if (min == max && keySelector(list[min]).CompareTo(key) == 0)
{
return min;
}
mid = min;
//we didnt find it. return something corresponding to lower_bound semantics
if (mid == list.Count)
return max; //had to go all the way to max before giving up; lower bound is max
if (mid == 0)
return -1; //had to go all the way to min before giving up; lower bound is min
midKey = keySelector(list[mid]);
if (midKey.CompareTo(key) >= 0) return mid - 1;
else return mid;
}
public static string ToHexString(this int n, int numdigits)
{
return string.Format("{0:X" + numdigits + "}", n);

View File

@ -1,4 +1,6 @@
using System;
using System.Linq;
using System.Diagnostics;
using System.Runtime.InteropServices;
using System.Reflection;
using System.Collections.Generic;
@ -78,8 +80,89 @@ namespace BizHawk
dialog.ShowDialog();
return;
}
}
}
}
}
//code to test ECM:
//static class test
//{
// public static void Shuffle<T>(this IList<T> list, Random rng)
// {
// int n = list.Count;
// while (n > 1)
// {
// n--;
// int k = rng.Next(n + 1);
// T value = list[k];
// list[k] = list[n];
// list[n] = value;
// }
// }
// public static void Test()
// {
// var plaindisc = BizHawk.DiscSystem.Disc.FromCuePath("d:\\ecmtest\\test.cue", BizHawk.MainDiscoForm.GetCuePrefs());
// var ecmdisc = BizHawk.DiscSystem.Disc.FromCuePath("d:\\ecmtest\\ecmtest.cue", BizHawk.MainDiscoForm.GetCuePrefs());
// //var prefs = new BizHawk.DiscSystem.CueBinPrefs();
// //prefs.AnnotateCue = false;
// //prefs.OneBlobPerTrack = false;
// //prefs.ReallyDumpBin = true;
// //prefs.SingleSession = true;
// //prefs.DumpToBitbucket = true;
// //var dump = ecmdisc.DumpCueBin("test", prefs);
// //dump.Dump("test", prefs);
// //var prefs = new BizHawk.DiscSystem.CueBinPrefs();
// //prefs.AnnotateCue = false;
// //prefs.OneBlobPerTrack = false;
// //prefs.ReallyDumpBin = true;
// //prefs.SingleSession = true;
// //var dump = ecmdisc.DumpCueBin("test", prefs);
// //dump.Dump(@"D:\ecmtest\myout", prefs);
// int seed = 102;
// for (; ; )
// {
// Console.WriteLine("running seed {0}", seed);
// Random r = new Random(seed);
// seed++;
// byte[] chunkbuf_corlet = new byte[2352 * 20];
// byte[] chunkbuf_mine = new byte[2352 * 20];
// int length = ecmdisc.LBACount * 2352;
// int counter = 0;
// List<Tuple<int, int>> testChunks = new List<Tuple<int, int>>();
// while (counter < length)
// {
// int chunk = r.Next(1, 2352 * 20);
// if (r.Next(20) == 0)
// chunk /= 100;
// if (r.Next(40) == 0)
// chunk = 0;
// if (counter + chunk > length)
// chunk = length - counter;
// testChunks.Add(new Tuple<int, int>(counter, chunk));
// counter += chunk;
// }
// testChunks.Shuffle(r);
// for (int t = 0; t < testChunks.Count; t++)
// {
// //Console.WriteLine("skank");
// var item = testChunks[t];
// //Console.WriteLine("chunk {0} of {3} is {1} bytes @ {2:X8}", t, item.Item2, item.Item1, testChunks.Count);
// plaindisc.ReadLBA_2352_Flat(item.Item1, chunkbuf_corlet, 0, item.Item2);
// ecmdisc.ReadLBA_2352_Flat(item.Item1, chunkbuf_mine, 0, item.Item2);
// for (int i = 0; i < item.Item2; i++)
// if (chunkbuf_corlet[i] != chunkbuf_mine[i])
// {
// Debug.Assert(false);
// }
// }
// }
// }
//}

View File

@ -36,7 +36,7 @@ namespace BizHawk
this.Close();
}
CueBinPrefs GetCuePrefs()
public static CueBinPrefs GetCuePrefs()
{
var prefs = new DiscSystem.CueBinPrefs();
prefs.AnnotateCue = true; // TODO? checkCueProp_Annotations.Checked;