<html> <meta charset="utf-8"> <title>PCSX2 games index compatibility updater</title> <head> <script> /* Quick and (very) dirty gameindex.dbf update tool from website compatibility db data.csv. - works in Firefox since Chrome doesn't allow loading local files. - updates compatibility info at the dbf from data.csv - add entries to the dbf if they only appear on data.csv - sorts the dbf entries according to the serial number - keeps hacks/comments/etc untouched (except when modifying compatibility info - and prints a warning) - can import external html files with games info and compare the name/region to our own data */ var CONFIG_SKIP_SORT_1 = 0; // sort after initial dbf import var CONFIG_SKIP_REGIONS_SUMMARY = 1; var CONFIG_SKIP_IMPORT_CSV = 0; var CONFIG_SKIP_ADD_MISSING_FROM_CSV = 0; var CONFIG_SKIP_LOG_CSV_MISSING = 0; // files which have compat info at the dbf but not at the csv var CONFIG_SKIP_SORT_2 = 0; // sort after adding entries from data.csv var CONFIG_SKIP_HTML_INDEX_IMPORTS = 1; var DBF_PATH = "GameIndex.dbf"; var COMPAT_PATH = "data.csv"; var HTML_INDEX_FILES = ["serials_1.htm", "serials_2.htm", "serials_3.htm"]; var COMPAT_DELIMITER = "\t"; var COMPAT_COLUMNS = {Serial: 0, Compat: 1, Name: 5, Region: 6}; var DBF_KEYS = ["Serial", "Name", "Region", "Compat"]; var DBF_DELIMITER = "\n---------------------------------------------\n"; var DBF_HEADER_END = DBF_DELIMITER + "-- Game List" + DBF_DELIMITER; function escapeHtml(str) { return str.split("&").join("&") .split("<").join("<") .split(">").join(">") .split("\"").join("""); } function $(id){ return document.getElementById(id); } function log(text) { $("logout").innerHTML += escapeHtml(text); } function logln(text) { log(text + "\n"); } function serPretty(ser) { ser = ser.toUpperCase(); return ser.substr(0,4) + "-" + ser.substr(4); } function noComment(txt) { return txt.split("//")[0].trim(); } function serNor(ser) { return noComment(ser).split("-").join("").toLowerCase(); } function pad(txt, len, ch) { txt = "" + txt; ch = ch || " "; while (txt.length < len) txt += ch; return txt; } // Synchronous. Returns "" on failure, or file content on success. // Use Firefox. chrome doesn't allow loading local files even if at the same dir as this file. function getFile(URI, failQuietly) { var result = ""; try { function reqListener () { result = this.responseText; logln("Got file '" + URI + "', length: " + result.length); } var oReq = new XMLHttpRequest(); oReq.onload = reqListener; oReq.open("get", URI, false); oReq.overrideMimeType("text/plain; charset=x-user-defined"); oReq.send(); } catch(e) { if (!failQuietly) logln("Error file '" + URI + "' (" + e + ")"); return ""; } return result; } var exportData = []; function parseDbf(dbf, startLine) { function importGame(section, firstLine) { if (!firstLine) firstLine = 0; var result = {}; var extra = []; var lines = String(section).trim().split("\n"); for (var i in lines) { var line = lines[i].trim(); if (line.indexOf("--------") == 0 ){ logln("Warning: invalid separator? : '" + line + "'"); } var lineLow = line.toLowerCase(); var foundKey = false; for (var k in DBF_KEYS) { var rest = line.substr(DBF_KEYS[k].length).trim(); if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0 && rest.indexOf("=") == 0) { // known key var key = DBF_KEYS[k]; var value = rest.substr(1).trim(); if (!(key in result)) { if (key == "Serial") { value = value.split("-").join(""); } result[key] = value; foundKey = true; break; } else { logln ("Error: rejecting DBF Line: " + (firstLine + i) + " redefine: " + DBF_KEYS[k]); } } else if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0) { // known yet without '=', warn. logln("Warning: Line " + (firstLine + i) + ", no '='? '" + line + "'"); } } if (!foundKey) { // unknown/no key (comment, patch, hack, empty, etc) extra.push(lines[i]); // because 'line' is trimmed } } // lines result["extra"] = extra; return result; } var result = {header: "", games: []}; logln("import-dbf-start"); var dbf = getFile(DBF_PATH); var dbfs = dbf.split(DBF_HEADER_END); logln("found main sections: " + dbfs.length); var gameSections = String(dbfs[1]).split(DBF_DELIMITER); if (gameSections[gameSections.length-1].length == 0) { gameSections.pop(); } logln("Found game sections: " + gameSections.length); logln("Importing games ..."); var gamesDb = []; var serialIndex= {}; var compatInfo = []; var dbfRegions = {}; for (var g in gameSections) { var game = importGame(gameSections[g]); if (!("Serial" in game) && !("Name" in game) && !("Region" in game) && !("Compat" in game) && game.extra.length == 1) { logln("Warning: Empty section (skipping)"); } else { if (!("Serial" in game) || !("Name" in game) || !("Region" in game)) { logln("Warning: Incomplete: ser="+game.Serial.substr(0,4)+"-"+game.Serial.substr(4)+" name="+game.Name+ " region="+game.Region); } gamesDb.push(game); if (game.Serial) { var ser = serNor(game.Serial); if (ser in serialIndex) { logln("Warning: duplicate serial at dbf: " + serPretty(ser)); } else { serialIndex[ser] = gamesDb.length - 1; } } if ("Compat" in game) { var c = game.Compat.substr(0,1); if (!(c in compatInfo)) compatInfo[c] = 0; compatInfo[c]++; } } if (!dbfRegions[game.Region]) dbfRegions[game.Region] = 0; dbfRegions[game.Region]++; } logln("Imported entries: " + gamesDb.length); for (var i in compatInfo) logln("Compatibility " + i + ": " + compatInfo[i] + " games"); if (!CONFIG_SKIP_REGIONS_SUMMARY) { for (var r in dbfRegions) logln("" + r + ": " + dbfRegions[r]); } function sortDbf() { gamesDb.sort(function(a, b) { // no need for equal since there are no duplicates return serNor(a.Serial) > serNor(b.Serial) ? 1 : -1; }); for (var i = 0; i < gamesDb.length; i++) { serialIndex[serNor(gamesDb[i].Serial)] = i; } } logln("Checking sort ..."); var prevg = ""; var sorted = true; for (var g in serialIndex) { if (g < prevg) { logln ("bad sort: " + serPretty(prevg) + " before " + serPretty(g)); sorted = false; } prevg = g; } if (sorted) { logln("dbf sort: no need (already sorted)"); } else { if (CONFIG_SKIP_SORT_1) { logln("skipping sort1"); } else { logln("Sorting dbf..."); sortDbf(); logln("Done Sorting dbf."); } } logln("import-dbf-end"); function dbfRegionFromCompatRegion(csvRegion) { // currently data.csv only has NTSC/PAL/JAP // we're translating to the dbf as: NTSC-U/PAL-Unk/NTSC-J respectively // can later manually fix or set more specific region data. // return undefined if unexpected region name at data.csv var trans = { "NTSC": "NTSC-U", "PAL" : "PAL-Unk", "JAP" : "NTSC-J" }; return trans[csvRegion]; } logln(""); if (CONFIG_SKIP_IMPORT_CSV) { logln("skipping data.csv import"); } else { logln("start import compatibility ..."); var compat = getFile(COMPAT_PATH); compat = compat.split("\n") logln("Found " + compat.length + " compatibility info lines"); var updateSummary = {same: 0, "new": 0, better: 0, worse: 0, missingAtCsv: 0}; var compatVerification = {}; var compatRegions = {}; for (var i in compat) { if (!compat[i].trim().length) continue; var cols = compat[i].split(COMPAT_DELIMITER); var found = false; var ser = cols[COMPAT_COLUMNS.Serial].trim().toLowerCase(); var com = Number(cols[COMPAT_COLUMNS.Compat].trim()); var reg = cols[COMPAT_COLUMNS.Region].trim(); if (!compatRegions[reg]) compatRegions[reg] = 0; compatRegions[reg]++; if (!(ser in compatVerification)) { compatVerification[ser] = com; } else { logln("Warning (skipping): duplicate compat info (" + compatVerification[ser] + "/" + com + ") for: " + serPretty(ser)); continue; } if (ser.toLowerCase() in serialIndex) { var game = gamesDb[serialIndex[ser]]; var dbser = serNor(game.Serial); var dbcom = Number(("Compat" in game) ? game.Compat.split("//")[0].trim() : "0"); if (dbser != ser) { logln("Error: internal: invalid index at serialIndex for " + serPretty(ser)); continue; } if (com != dbcom) { //logln("Updating compat from " + dbcom + " to " + com + " for " + ser + ": " + game.Name); if (("Compat" in game) && game.Compat.split("//").length > 1) { logln("Warning: removing compat comment for serial " + serPretty(ser) + " : //" + game.Compat.split("//")[1]); } game.Compat = com; if (com > dbcom) { updateSummary.better++ } else { updateSummary.worse++; logln(":( Degrading compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name); } } else { updateSummary.same++; } } else { updateSummary.new++; logln ("not at dbf: " + serPretty(ser) + " - " + (CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipping" : "adding:")); if (!CONFIG_SKIP_ADD_MISSING_FROM_CSV) { var newDbfReg = dbfRegionFromCompatRegion(reg); if (newDbfReg) { gamesDb.push({Serial: ser, Compat: com, Region: newDbfReg, Name: cols[COMPAT_COLUMNS.Name].trim(), extra: ""}); log(getExportEntry(gamesDb[gamesDb.length - 1]) + DBF_DELIMITER); } else { logln("Error: skipping due to unexpected region at data.csv: " + reg); } } } } if (compat && compat.length > 1) { if (updateSummary.new) { if (CONFIG_SKIP_SORT_2) { logln("skipping sort after add"); } else { logln("sorting new entries into the dbf..."); sortDbf(); logln("sort - done."); } } logln("-- Update summary --"); if (!CONFIG_SKIP_REGIONS_SUMMARY) { logln("data.csv regions:"); for (var r in compatRegions) logln("" + r + ": " + compatRegions[r]); } if (!CONFIG_SKIP_LOG_CSV_MISSING) { for (var csvSer in serialIndex) { var game = gamesDb[serialIndex[csvSer]]; if (game.Compat && !compatVerification[serNor(csvSer)]) { logln("serial missing from CSV: " + pad(serPretty(csvSer), 12) + " / " + pad(noComment(game.Region), 8) + " / " + game.Compat + " / " + game.Name); updateSummary.missingAtCsv++; } } } logln("Not at dbf (and " +(CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipped" : "added")+ "): " + updateSummary.new); logln("Not at csv (but have compat info at the dbf): " + updateSummary.missingAtCsv); logln("Unchanged: " + updateSummary.same); logln("Better compat: " + updateSummary.better); logln("Worse compat: " + updateSummary.worse); logln("End import compatibility."); } } logln(""); if (CONFIG_SKIP_HTML_INDEX_IMPORTS) { logln("skipping html index files"); } else { logln("Scanning html index files"); var in_incomplete = 0; var in_identical = 0; var in_diff = 0; var in_new = 0; var in_err = 0; var in_usBetter = 0; var files = HTML_INDEX_FILES; for (var f in files) { var serials = getFile(files[f]); logln("file sections: " + serials.split('<table border="1"').length); if (!serials.length) { logln("Warning: file not found. skipping."); continue; } var list = serials.split('<table border="1"')[1].split("</tr>"); logln("found rows: " + list.length); for (var l in list) { var s = list[l].split("</td>"); if (!(s.length==4)) { in_incomplete++; continue; } try { var ser = serNor(s[0].split('.htm">')[1].split("</pre>")[0]); var name = s[1].split(' 0">')[1].split("</pre>")[0]; var region = s[2].split(' 0">')[1].split("</pre>")[0]; if (ser in serialIndex) { var game = gamesDb[serialIndex[ser]]; function norName(n) { n = n.toLowerCase().split("//")[0]; var cs = [", ", ".", "-", ":"]; for (var i in cs) n = n.split(cs[i]).join(""); return n.split(" ").join("").split(" ").join("").trim(); } if (norName(game.Name) == norName(name) && norName(game.Region) == norName(region)) { in_identical++ } else { if ((norName(region).indexOf("unk")>=0 && norName(game.Region).indexOf("unk")<0)|| norName(name).indexOf("zzz")==0) in_usBetter++ else { logln("diff: " + serPretty(ser) + " DB: [" + game.Region.split("//")[0].trim() + "] " + game.Name.split("//")[0].trim()); logln(" index: [" + region + "] "+ name); logln(""); in_diff++; } } } else { in_new++; } } catch (e) { in_err++; logln(e); } } } logln("--- Html index summary: ---"); logln("errors: " + in_err); logln("incomplete data: " + in_incomplete); logln("identical: " + in_identical); logln("we're better: " + in_usBetter); logln("different: "+ in_diff); logln("not in db: " + in_new); } // start: test export dbf function getExportEntry(gameItem) { var entry = []; for (var k in DBF_KEYS) { if (DBF_KEYS[k] in gameItem) { var v = gameItem[DBF_KEYS[k]]; var fill = ""; if (DBF_KEYS[k] == "Serial") v = (v.substr(0,4) + "-" + v.substr(4)).toUpperCase(); if (DBF_KEYS[k] == "Name") fill = " "; entry.push(DBF_KEYS[k] + fill + " = " + v); } } if (gameItem.extra.length) { entry.push(gameItem.extra.join("\n")); } return entry.join("\n"); } exportData = []; for (var g in gamesDb) { exportData.push(getExportEntry(gamesDb[g])); } exportData = dbfs[0] + DBF_HEADER_END + exportData.join(DBF_DELIMITER) + DBF_DELIMITER; $("saveUpdated").href = "data:bin/plain;base64," + window.btoa(exportData); $("saveUpdatedWrapper").style.display = "inline"; // end: export } </script> </head> <body onload="javascript:parseDbf()"> Please copy GameIndex.dbf (from the bin folder) and data.csv (compat info) to the folder of this html file<br/> <span id="saveUpdatedWrapper" style="display:none;"> <a id="saveUpdated" href="" download="GameIndex.new.dbf">Save updated DBF</a> </span> <h4>Log</h4> <pre><div id="logout"></div></pre> </body> </html>