From 314c854422b5060c4f275b2ea294c8cf057df087 Mon Sep 17 00:00:00 2001 From: Asnivor Date: Fri, 21 Sep 2018 15:23:57 +0100 Subject: [PATCH] DevTools: Added NOINTRO->GameDB parser --- .../BizHawk.Client.DBMan.csproj | 1 + .../DATTools/DATConverter.Designer.cs | 14 + BizHawk.Client.DBMan/DATTools/DATConverter.cs | 5 + .../DATTools/DATParserBase.cs | 46 ++- .../DATTools/NOINTROParser.cs | 323 ++++++++++++++++++ BizHawk.Client.DBMan/DATTools/TOSECParser.cs | 17 - 6 files changed, 381 insertions(+), 25 deletions(-) create mode 100644 BizHawk.Client.DBMan/DATTools/NOINTROParser.cs diff --git a/BizHawk.Client.DBMan/BizHawk.Client.DBMan.csproj b/BizHawk.Client.DBMan/BizHawk.Client.DBMan.csproj index dce1a00aba..e65442660a 100644 --- a/BizHawk.Client.DBMan/BizHawk.Client.DBMan.csproj +++ b/BizHawk.Client.DBMan/BizHawk.Client.DBMan.csproj @@ -61,6 +61,7 @@ DATConverter.cs + diff --git a/BizHawk.Client.DBMan/DATTools/DATConverter.Designer.cs b/BizHawk.Client.DBMan/DATTools/DATConverter.Designer.cs index 591b8b4fb2..ee8a461d78 100644 --- a/BizHawk.Client.DBMan/DATTools/DATConverter.Designer.cs +++ b/BizHawk.Client.DBMan/DATTools/DATConverter.Designer.cs @@ -40,6 +40,7 @@ this.label2 = new System.Windows.Forms.Label(); this.label3 = new System.Windows.Forms.Label(); this.button2 = new System.Windows.Forms.Button(); + this.radioNOINTRO = new System.Windows.Forms.RadioButton(); this.groupImportTypes.SuspendLayout(); this.SuspendLayout(); // @@ -62,6 +63,7 @@ // // groupImportTypes // + this.groupImportTypes.Controls.Add(this.radioNOINTRO); this.groupImportTypes.Controls.Add(this.radioTOSEC); this.groupImportTypes.Location = new System.Drawing.Point(13, 50); this.groupImportTypes.Name = "groupImportTypes"; @@ -157,6 +159,17 @@ this.button2.UseVisualStyleBackColor = true; this.button2.Click += new System.EventHandler(this.button2_Click); // + // radioNOINTRO + // + this.radioNOINTRO.AutoSize = true; + this.radioNOINTRO.Location = new System.Drawing.Point(7, 44); + this.radioNOINTRO.Name = "radioNOINTRO"; + this.radioNOINTRO.Size = new System.Drawing.Size(172, 17); + this.radioNOINTRO.TabIndex = 1; + this.radioNOINTRO.TabStop = true; + this.radioNOINTRO.Text = "NOINTRO (standard DAT only)"; + this.radioNOINTRO.UseVisualStyleBackColor = true; + // // DATConverter // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); @@ -197,5 +210,6 @@ private System.Windows.Forms.Label label2; private System.Windows.Forms.Label label3; private System.Windows.Forms.Button button2; + private System.Windows.Forms.RadioButton radioNOINTRO; } } \ No newline at end of file diff --git a/BizHawk.Client.DBMan/DATTools/DATConverter.cs b/BizHawk.Client.DBMan/DATTools/DATConverter.cs index 7fc6cafbcc..bcf1cf67b1 100644 --- a/BizHawk.Client.DBMan/DATTools/DATConverter.cs +++ b/BizHawk.Client.DBMan/DATTools/DATConverter.cs @@ -134,6 +134,11 @@ namespace BizHawk.Client.DBMan DATParser tp = new TOSECParser((SystemType)Enum.Parse(typeof(SystemType), comboBoxSystemSelect.SelectedValue.ToString())); res = tp.ParseDAT(files.ToArray()); } + else if (radioNOINTRO.Checked) + { + DATParser dp = new NOINTROParser((SystemType)Enum.Parse(typeof(SystemType), comboBoxSystemSelect.SelectedValue.ToString())); + res = dp.ParseDAT(files.ToArray()); + } string fName = "gamedb_" + GameDB.GetSystemCode((SystemType)Enum.Parse(typeof(SystemType), comboBoxSystemSelect.SelectedValue.ToString())) + diff --git a/BizHawk.Client.DBMan/DATTools/DATParserBase.cs b/BizHawk.Client.DBMan/DATTools/DATParserBase.cs index 5e2e760695..f13196463c 100644 --- a/BizHawk.Client.DBMan/DATTools/DATParserBase.cs +++ b/BizHawk.Client.DBMan/DATTools/DATParserBase.cs @@ -14,13 +14,6 @@ namespace BizHawk.Client.DBMan /// public abstract SystemType SysType { get; set; } - /// - /// Parses a single DAT file - /// - /// - /// - public abstract void ParseDAT(object dat); - /// /// Parses multiple DAT files and returns a single GamesDB format csv string /// @@ -72,24 +65,60 @@ namespace BizHawk.Client.DBMan sb.Append("\t"); // system sb.Append(d.System); + + // additional optional fields + bool[] populated = new bool[4]; + if (d.Notes != null) + populated[0] = true; + if (d.MetaData != null) + populated[1] = true; + if (d.Region != null) + populated[2] = true; + if (d.ForcedCore != null) + populated[3] = true; + + int last = 0; + for (int i = 3; i >= 0; i--) + { + if (populated[i]) + { + last = i; + break; + } + } + + int cnt = 0; + // notes if (d.Notes != null) { sb.Append("\t"); sb.Append(d.Notes); } + else if (cnt++ <= last) + { + sb.Append("\t"); + } // metadata if (d.MetaData != null) { sb.Append("\t"); sb.Append(d.MetaData); } + else if (cnt++ <= last) + { + sb.Append("\t"); + } // region if (d.Region != null) { sb.Append("\t"); sb.Append(d.Region); } + else if (cnt++ <= last) + { + sb.Append("\t"); + } // force core if (d.ForcedCore != null) { @@ -188,6 +217,7 @@ namespace BizHawk.Client.DBMan WSWAN, Lynx, VB, - UZE + UZE, + NGP } } diff --git a/BizHawk.Client.DBMan/DATTools/NOINTROParser.cs b/BizHawk.Client.DBMan/DATTools/NOINTROParser.cs new file mode 100644 index 0000000000..fb0381daa5 --- /dev/null +++ b/BizHawk.Client.DBMan/DATTools/NOINTROParser.cs @@ -0,0 +1,323 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Xml.Linq; +using System.Xml; +using System.Windows.Forms; + +namespace BizHawk.Client.DBMan +{ + public class NOINTROParser : DATParser + { + /// + /// Required to generate a GameDB file + /// + public override SystemType SysType { get; set; } + + private List xmls = new List(); + + public NOINTROParser(SystemType type) + { + SysType = type; + } + + /// + /// Parses multiple DAT files and returns a single GamesDB format csv string + /// + /// + /// + public override string ParseDAT(string[] filePath) + { + foreach (var s in filePath) + { + try + { + xmls.Add(XDocument.Load(s)); + } + catch + { + var res = MessageBox.Show("Could not parse document as valid XML:\n\n" + s + "\n\nDo you wish to continue any other processing?", "Parsing Error", MessageBoxButtons.YesNo); + if (res != DialogResult.Yes) + return ""; + } + } + + int startIndex = 0; + + // actual tosec parsing + foreach (var obj in xmls) + { + startIndex = Data.Count > 0 ? Data.Count - 1 : 0; + // get header info + var header = obj.Root.Descendants("header").First(); + var name = header.Element("name").Value; + var version = header.Element("version").Value; + var description = header.Element("description").Value + " - " + version; + + // start comment block + List comments = new List + { + "Type:\t" + "NO-INTRO", + "Source:\t" + description, + "FileGen:\t" + DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss") + " (UTC)", + }; + + AddCommentBlock(comments.ToArray()); + + // process each entry + var query = obj.Root.Descendants("game"); + foreach (var g in query) + { + GameDB item = new GameDB(); + item.Name = g.Value; + item.SHA1 = g.Elements("rom").First().Attribute("sha1").Value.ToUpper(); + item.MD5 = g.Elements("rom").First().Attribute("md5").Value.ToUpper(); + item.System = GameDB.GetSystemCode(SysType); + + ParseNOINTROFlags(item); + + Data.Add(item); + } + + // add this file's data to the stringbuilder + // first we will sort into various ROMSTATUS groups + var working = Data.Skip(startIndex).ToList(); + + var baddump = working.Where(st => st.Status == "B").OrderBy(na => na.Name).ToList(); + AddCommentBlock("Bad Dumps"); + AppendCSVData(baddump); + + var hack = working.Where(st => st.Status == "H").OrderBy(na => na.Name).ToList(); + AddCommentBlock("Hacks"); + AppendCSVData(hack); + + var over = working.Where(st => st.Status == "O").OrderBy(na => na.Name).ToList(); + AddCommentBlock("Over Dumps"); + AppendCSVData(over); + + var trans = working.Where(st => st.Status == "T").OrderBy(na => na.Name).ToList(); + AddCommentBlock("Translated"); + AppendCSVData(trans); + + var good = working.Where(st => st.Status == "" || st.Status == null).OrderBy(na => na.Name).ToList(); + AddCommentBlock("Believed Good"); + AppendCSVData(good); + } + + string result = sb.ToString(); + return sb.ToString(); + } + + /// + /// Parses all the weird TOSEC flags within the game field + /// Detailed info here: https://www.tosecdev.org/tosec-naming-convention + /// Guts of this has been reused from here: https://github.com/Asnivor/MedLaunch/blob/master/MedLaunch/_Debug/DATDB/Platforms/TOSEC/StringConverterToSec.cs + /// + /// + private void ParseNOINTROFlags(GameDB g) + { + string nameString = g.Name; + + // remove uninteresting options + string a = RemoveUnneededOptions(nameString); + + // process data contained in () + string[] d = a.ToString().Split('(', ')'); + + if (d.Length > 0) + { + // name field + } + + if (d.Length > 1) + { + if (d[1].Length >= 3) + { + // country + g.Region = d[1].Trim(); + } + } + + if (d.Length > 2) + { + // parse all other () fields + // because these are not mandatory this can be a confusing process + for (int i = 4; i < d.Length; i++) + { + string f = d[i].Trim(); + + // check for language + if (IsLanguageFlag(f) == true) + { + g.Notes = f; + continue; + } + + // version - ignore + + // check development status (not currently implemented) + if (IsDevelopmenttStatus(f) == true) + { + continue; + } + + // check copyright status (not currently implemented) + if (IsCopyrightStatus(f) == true) + { + continue; + } + + // country flag(s) + if (IsCountryFlag(f) == true) + { + g.Region = f; + continue; + } + + // language - if present add to notes + if (IsLanguageFlag(f) == true) + { + g.Notes = f; + continue; + } + + // Media Type - ignore for now + // Media Label - ignore for now + } + + // process dump info flags and other info contained in [] + if (nameString.Contains("[") && nameString.Contains("]")) + { + List e = nameString.ToString().Split('[', ']').ToList(); + // remove first entry (this is the bit before the [] entries start + e.RemoveAt(0); + // remove empty entries + e = e.Where(s => !string.IsNullOrWhiteSpace(s)).Distinct().ToList(); + + if (e.Count > 0) + { + // bizhawk currently only has a few different RomStatus values (not as many as TOSEC anyway) + // Parsing priority will be: + // RomStatus.BadDump + // RomStatus.Hack + // RomStatus.Overdump + // RomStatus.GoodDump + // RomStatus.TranslatedRom + // everything else + // all tosec cr, h, t etc.. will fall under RomStatus.Hack + + if (e.Where(str => + // bad dump + str == "b" || str.StartsWith("b ")).ToList().Count > 0) + { + // RomStatus.BadDump + g.Status = "B"; + } + else if (e.Where(str => + // BIOS + str == "BIOS" || str.StartsWith("BIOS ")).ToList().Count > 0) + { + // RomStatus.BIOS + g.Status = "I"; + } + else + { + g.Status = ""; + } + } + } + } + } + + public static bool IsDevelopmenttStatus(string s) + { + List DS = new List + { + "alpha", "beta", "preview", "pre-release", "proto" + }; + + bool b = DS.Any(s.Contains); + return b; + } + + public static bool IsCopyrightStatus(string s) + { + List CS = new List + { + "CW", "CW-R", "FW", "GW", "GW-R", "LW", "PD", "SW", "SW-R" + }; + + bool b = CS.Any(s.Contains); + return b; + } + + public static bool IsLanguageFlag(string s) + { + List LC = new List + { + "En", "Ja", "Fr", "De", "Es", "It", "Nl", "Pt", "Sv", "No", "Da", "Fi", "Zh", "Ko", "Pl" + }; + + bool b = false; + + if (!s.Contains("[") && !s.Contains("]")) + { + foreach (var x in LC) + { + if (s == x || s.StartsWith(x + ",") || s.EndsWith("," + x)) + { + b = true; + break; + } + } + + //b = LC.Any(s.Contains); + } + + return b; + } + + public static bool IsCountryFlag(string s) + { + List CC = new List + { + "World", "Australia", "Brazil", "Canada", "China", "France", "Germany", "Hong Kong", "Italy", + "Japan", "Korea", "Netherlands", "Spain", "Sweden", "USA", "Europe", "Asia" + }; + + bool b = false; + + if (!s.Contains("[") && !s.Contains("]")) + { + foreach (var x in CC) + { + if (s == x || s.StartsWith(x) || s.EndsWith(x)) + { + b = true; + break; + } + } + + //b = CC.Any(s.Contains); + } + + return b; + } + + public static string RemoveUnneededOptions(string nameString) + { + // Remove unneeded entries + string n = nameString + .Replace(" (demo) ", " ") + .Replace(" (demo-kiosk) ", " ") + .Replace(" (demo-playable) ", " ") + .Replace(" (demo-rolling) ", " ") + .Replace(" (demo-slideshow) ", " "); + + return n; + } + } +} diff --git a/BizHawk.Client.DBMan/DATTools/TOSECParser.cs b/BizHawk.Client.DBMan/DATTools/TOSECParser.cs index ec95ec90c5..948b4e579f 100644 --- a/BizHawk.Client.DBMan/DATTools/TOSECParser.cs +++ b/BizHawk.Client.DBMan/DATTools/TOSECParser.cs @@ -363,22 +363,5 @@ namespace BizHawk.Client.DBMan return n; } - - /// - /// Parses a single DAT file - /// - /// - /// - public override void ParseDAT(object xml) - { - try - { - var obj = (XDocument)xml; - } - catch (Exception ex) - { - throw new Exception(ex.Message); - } - } } }