ShuriZma
/
BizHawk
mirror of https://github.com/TASEmulators/BizHawk.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

update DBMan for new discsystem stuff and framework for batch disc hashing

This commit is contained in:
zeromus 2015-07-08 02:55:55 -05:00
parent 1026c817cc
commit 9fcaf98084
4 changed files with 174 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
BizHawk.Client.DBMan/BizHawk.Client.DBMan.csproj
View File

@ -22,7 +22,7 @@
<DebugType>full</DebugType>
<Optimize>false</Optimize>
<OutputPath>..\output\</OutputPath>
<DefineConstants>DEBUG;TRACE</DefineConstants>
<DefineConstants>TRACE;DEBUG;WINDOWS</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
<UseVSHostingProcess>false</UseVSHostingProcess>
@ -32,7 +32,7 @@
<DebugType>pdbonly</DebugType>
<Optimize>true</Optimize>
<OutputPath>..\output\</OutputPath>
<DefineConstants>TRACE</DefineConstants>
<DefineConstants>TRACE;WINDOWS</DefineConstants>
<ErrorReport>prompt</ErrorReport>
<WarningLevel>4</WarningLevel>
</PropertyGroup>
@ -44,6 +44,7 @@
<Reference Include="System.Core" />
<Reference Include="System.Data" />
<Reference Include="System.Drawing" />
<Reference Include="System.Web" />
<Reference Include="System.Windows.Forms" />
</ItemGroup>
<ItemGroup>

147
BizHawk.Client.DBMan/DiscHash.cs
View File

@ -66,6 +66,33 @@ namespace BizHawk.Client.DBMan
}
}
static List<string> FindExtensionsRecurse(string dir, string extUppercaseWithDot)
{
List<string> ret = new List<string>();
Queue<string> dpTodo = new Queue<string>();
dpTodo.Enqueue(dir);
for (; ; )
{
string dpCurr;
if (dpTodo.Count == 0)
break;
dpCurr = dpTodo.Dequeue();
Parallel.ForEach(new DirectoryInfo(dpCurr).GetFiles(), (fi) =>
{
if (fi.Extension.ToUpperInvariant() == extUppercaseWithDot)
lock (ret)
ret.Add(fi.FullName);
});
Parallel.ForEach(new DirectoryInfo(dpCurr).GetDirectories(), (di) =>
{
lock (dpTodo)
dpTodo.Enqueue(di.FullName);
});
}
return ret;
}
void MyRun(string[] args)
{
string indir = null;
@ -84,17 +111,6 @@ namespace BizHawk.Client.DBMan
fpOutfile = args[i++];
}
//prepare input
var diInput = new DirectoryInfo(indir);
//prepare output
if(dpTemp == null) dpTemp = Path.GetTempFileName() + ".dir";
//delete existing files in output
foreach (var fi in new DirectoryInfo(dpTemp).GetFiles())
fi.Delete();
foreach (var di in new DirectoryInfo(dpTemp).GetDirectories())
di.Delete(true);
using(var outf = new StreamWriter(fpOutfile))
{
@ -102,84 +118,67 @@ namespace BizHawk.Client.DBMan
object olock = new object();
int ctr = 0;
var todo = FindExtensionsRecurse(indir, ".CUE");
int progress = 0;
//loop over games
var po = new ParallelOptions();
po.MaxDegreeOfParallelism = Environment.ProcessorCount - 1;
Parallel.ForEach(diInput.GetFiles("*.7z"), po, (fi) =>
//po.MaxDegreeOfParallelism = 1;
Parallel.ForEach(todo, po, (fiCue) =>
{
string name = Path.GetFileNameWithoutExtension(fiCue);
//if (!name.Contains("Arc the Lad II"))
// return;
CRC32 crc = new CRC32();
byte[] discbuf = new byte[2352*28];
int myctr;
lock (olock)
myctr = ctr++;
string mydpTemp = Path.Combine(dpTemp,myctr.ToString());
var mydiTemp = new DirectoryInfo(mydpTemp);
mydiTemp.Create();
var process = new System.Diagnostics.Process();
process.StartInfo.UseShellExecute = false;
process.StartInfo.RedirectStandardOutput = true;
process.StartInfo.FileName = "7z.exe";
process.StartInfo.Arguments = string.Format("x -o\"{1}\" \"{0}\"", fi.FullName, mydiTemp.FullName);
process.Start();
job.AddProcess(process.Handle);
//if we need it
//for (; ; )
//{
// int c = process.StandardOutput.Read();
// if (c == -1)
// break;
// Console.Write((char)c);
//}
process.WaitForExit(); //just in case
byte[] buffer2352 = new byte[2352];
//now look for the cue file
var fiCue = mydiTemp.GetFiles("*.cue").FirstOrDefault();
if (fiCue == null)
using (var disc = Disc.LoadAutomagic(fiCue))
{
Console.WriteLine("MISSING CUE FOR: " + fi.Name);
outf.WriteLine("MISSING CUE FOR: " + fi.Name);
Console.Out.Flush();
}
else
{
using (var disc = Disc.LoadAutomagic(fiCue.FullName))
{
//generate a hash with our own custom approach
//basically, the "TOC" and a few early sectors completely
crc.Add(disc.LBACount);
crc.Add(disc.Structure.Sessions[0].Tracks.Count);
foreach (var track in disc.Structure.Sessions[0].Tracks)
{
crc.Add(track.Start_ABA);
crc.Add(track.Length);
}
//ZAMMO: change to disc sector reader, maybe a new class to read multiple
disc.ReadLBA_2352_Flat(0, discbuf, 0, discbuf.Length);
crc.Add(discbuf, 0, discbuf.Length);
var dsr = new DiscSectorReader(disc);
lock (olock)
//generate a hash with our own custom approach
//basically, the TOC and a few early sectors completely
//note: be sure to hash the leadout track, "A Ressha de Ikou 4" differs only by the track 1 / disc length between 1.0 and 1.1
//crc.Add((int)disc.TOC.Session1Format);
//crc.Add(disc.TOC.FirstRecordedTrackNumber);
//crc.Add(disc.TOC.LastRecordedTrackNumber);
//for (int i = 1; i <= 100; i++)
//{
// crc.Add((int)disc.TOC.TOCItems[i].Control);
// crc.Add(disc.TOC.TOCItems[i].Exists ? 1 : 0);
// crc.Add((int)disc.TOC.TOCItems[i].LBATimestamp.Sector);
//}
//"Arc the Lad II (J) 1.0 and 1.1 conflict up to 25 sectors (so use 26)
//Tekken 3 (Europe) (Alt) and Tekken 3 (Europe) conflict in track 2 and 3 unfortunately, not sure what to do about this yet
for (int i = 0; i < 26; i++)
{
dsr.ReadLBA_2352(i, buffer2352, 0);
crc.Add(buffer2352, 0, 2352);
}
lock (olock)
{
progress++;
Console.WriteLine("{0}/{1} [{2:X8}] {3}", progress, todo.Count, crc.Result, Path.GetFileNameWithoutExtension(fiCue));
outf.WriteLine("[{0:X8}] {1}", crc.Result, name);
if (FoundHashes.ContainsKey(crc.Result))
{
Console.WriteLine("[{0:X8}] {1}", crc.Result, fi.Name);
outf.WriteLine("[{0:X8}] {1}", crc.Result, fi.Name);
if (FoundHashes.ContainsKey(crc.Result))
{
Console.WriteLine("--> COLLISION WITH: ", FoundHashes[crc.Result]);
outf.WriteLine("--> COLLISION WITH: ", FoundHashes[crc.Result]);
}
Console.Out.Flush();
Console.WriteLine("--> COLLISION WITH: {0}", FoundHashes[crc.Result]);
outf.WriteLine("--> COLLISION WITH: {0}", FoundHashes[crc.Result]);
}
else
FoundHashes[crc.Result] = name;
Console.Out.Flush();
}
}
foreach (var fsi in mydiTemp.GetFileSystemInfos())
fsi.Delete();
mydiTemp.Delete();
}); //major loop

96
BizHawk.Client.DBMan/Program.cs
View File

@ -1,12 +1,108 @@
using System;
using System.Linq;
using System.Windows.Forms;
using System.Runtime.InteropServices;
using System.Reflection;
using System.Collections.Generic;
using Community.CsharpSqlite.SQLiteClient;
using System.IO;
namespace BizHawk.Client.DBMan
{
internal static class Program
{
static Program()
{
#if WINDOWS
//http://www.codeproject.com/Articles/310675/AppDomain-AssemblyResolve-Event-Tips
// this will look in subdirectory "dll" to load pinvoked stuff
string dllDir = System.IO.Path.Combine(GetExeDirectoryAbsolute(), "dll");
SetDllDirectory(dllDir);
//but before we even try doing that, whack the MOTW from everything in that directory (thats a dll)
//otherwise, some people will have crashes at boot-up due to .net security disliking MOTW.
//some people are getting MOTW through a combination of browser used to download bizhawk, and program used to dearchive it
WhackAllMOTW(dllDir);
//in case assembly resolution fails, such as if we moved them into the dll subdiretory, this event handler can reroute to them
AppDomain.CurrentDomain.AssemblyResolve += new ResolveEventHandler(CurrentDomain_AssemblyResolve);
#endif
}
public static string GetExeDirectoryAbsolute()
{
var uri = new Uri(Assembly.GetEntryAssembly().GetName().CodeBase);
string module = uri.LocalPath + System.Web.HttpUtility.UrlDecode(uri.Fragment);
return Path.GetDirectoryName(module);
}
static Assembly CurrentDomain_AssemblyResolve(object sender, ResolveEventArgs args)
{
lock (AppDomain.CurrentDomain)
{
var asms = AppDomain.CurrentDomain.GetAssemblies();
foreach (var asm in asms)
if (asm.FullName == args.Name)
return asm;
//load missing assemblies by trying to find them in the dll directory
string dllname = new AssemblyName(args.Name).Name + ".dll";
string directory = Path.Combine(GetExeDirectoryAbsolute(), "dll");
string fname = Path.Combine(directory, dllname);
if (!File.Exists(fname)) return null;
//it is important that we use LoadFile here and not load from a byte array; otherwise mixed (managed/unamanged) assemblies can't load
return Assembly.LoadFile(fname);
}
}
//declared here instead of a more usual place to avoid dependencies on the more usual place
#if WINDOWS
[DllImport("kernel32.dll", SetLastError = true)]
static extern bool SetDllDirectory(string lpPathName);
[DllImport("kernel32.dll", EntryPoint = "DeleteFileW", SetLastError = true, CharSet = CharSet.Unicode, ExactSpelling = true)]
static extern bool DeleteFileW([MarshalAs(UnmanagedType.LPWStr)]string lpFileName);
static void RemoveMOTW(string path)
{
DeleteFileW(path + ":Zone.Identifier");
}
//for debugging purposes, this is provided. when we're satisfied everyone understands whats going on, we'll get rid of this
[DllImportAttribute("kernel32.dll", EntryPoint = "CreateFileW")]
public static extern System.IntPtr CreateFileW([InAttribute()] [MarshalAsAttribute(UnmanagedType.LPWStr)] string lpFileName, int dwDesiredAccess, int dwShareMode, [InAttribute()] int lpSecurityAttributes, int dwCreationDisposition, int dwFlagsAndAttributes, [InAttribute()] int hTemplateFile);
static void ApplyMOTW(string path)
{
int generic_write = 0x40000000;
int file_share_write = 2;
int create_always = 2;
var adsHandle = CreateFileW(path + ":Zone.Identifier", generic_write, file_share_write, 0, create_always, 0, 0);
using (var sfh = new Microsoft.Win32.SafeHandles.SafeFileHandle(adsHandle, true))
{
var adsStream = new System.IO.FileStream(sfh, FileAccess.Write);
StreamWriter sw = new StreamWriter(adsStream);
sw.Write("[ZoneTransfer]\r\nZoneId=3");
sw.Flush();
adsStream.Close();
}
}
static void WhackAllMOTW(string dllDir)
{
var todo = new Queue<DirectoryInfo>(new[] { new DirectoryInfo(dllDir) });
while (todo.Count > 0)
{
var di = todo.Dequeue();
foreach (var disub in di.GetDirectories()) todo.Enqueue(disub);
foreach (var fi in di.GetFiles("*.dll"))
RemoveMOTW(fi.FullName);
foreach (var fi in di.GetFiles("*.exe"))
RemoveMOTW(fi.FullName);
}
}
#endif
[STAThread]
static void Main(string[] args)
{

3
BizHawk.Client.DBMan/RomHasher.cs
View File

@ -180,7 +180,8 @@ namespace BizHawk.Client.DBMan
string ext = new FileInfo(file).Extension.ToLowerInvariant();
using (var disc = Disc.LoadAutomagic(file))
{
return disc.GetHash();
var hasher = new DiscHasher(disc);
return hasher.OldHash();
}
}
catch