<html>
<meta charset="utf-8">
<title>PCSX2 games index compatibility updater</title>
<head>
<script>
/*
Quick and (very) dirty gameindex.dbf update tool from website compatibility db data.csv.
- works in Firefox since Chrome doesn't allow loading local files.
- updates compatibility info at the dbf from data.csv
- add entries to the dbf if they only appear on data.csv
- sorts the dbf entries according to the serial number
- keeps hacks/comments/etc untouched (except when modifying compatibility info - and prints a warning)
- can import external html files with games info and compare the name/region to our own data
*/
var CONFIG_SKIP_SORT_1 = 0; // sort after initial dbf import
var CONFIG_SKIP_REGIONS_SUMMARY = 1;
var CONFIG_SKIP_IMPORT_CSV = 0;
var CONFIG_SKIP_ADD_MISSING_FROM_CSV = 0;
var CONFIG_SKIP_LOG_CSV_MISSING = 0; // files which have compat info at the dbf but not at the csv
var CONFIG_SKIP_SORT_2 = 0; // sort after adding entries from data.csv
var CONFIG_SKIP_HTML_INDEX_IMPORTS = 1;
var DBF_PATH = "GameIndex.dbf";
var COMPAT_PATH = "data.csv";
var HTML_INDEX_FILES = ["serials_1.htm", "serials_2.htm", "serials_3.htm"];
var COMPAT_DELIMITER = "\t";
var COMPAT_COLUMNS = {Serial: 0, Compat: 1, Name: 5, Region: 6};
var DBF_KEYS = ["Serial", "Name", "Region", "Compat"];
var DBF_DELIMITER = "\n---------------------------------------------\n";
var DBF_HEADER_END = DBF_DELIMITER + "-- Game List" + DBF_DELIMITER;
function escapeHtml(str) { return str.split("&").join("&")
.split("<").join("<")
.split(">").join(">")
.split("\"").join("""); }
function $(id){ return document.getElementById(id); }
function log(text) { $("logout").innerHTML += escapeHtml(text); }
function logln(text) { log(text + "\n"); }
function serPretty(ser) { ser = ser.toUpperCase(); return ser.substr(0,4) + "-" + ser.substr(4); }
function noComment(txt) { return txt.split("//")[0].trim(); }
function serNor(ser) { return noComment(ser).split("-").join("").toLowerCase(); }
function pad(txt, len, ch) {
txt = "" + txt;
ch = ch || " ";
while (txt.length < len)
txt += ch;
return txt;
}
// Synchronous. Returns "" on failure, or file content on success.
// Use Firefox. chrome doesn't allow loading local files even if at the same dir as this file.
function getFile(URI, failQuietly) {
var result = "";
try {
function reqListener () {
result = this.responseText;
logln("Got file '" + URI + "', length: " + result.length);
}
var oReq = new XMLHttpRequest();
oReq.onload = reqListener;
oReq.open("get", URI, false);
oReq.overrideMimeType("text/plain; charset=x-user-defined");
oReq.send();
} catch(e) {
if (!failQuietly)
logln("Error file '" + URI + "' (" + e + ")");
return "";
}
return result;
}
var exportData = [];
function parseDbf(dbf, startLine) {
function importGame(section, firstLine) {
if (!firstLine) firstLine = 0;
var result = {};
var extra = [];
var lines = String(section).trim().split("\n");
for (var i in lines) {
var line = lines[i].trim();
if (line.indexOf("--------") == 0 ){
logln("Warning: invalid separator? : '" + line + "'");
}
var lineLow = line.toLowerCase();
var foundKey = false;
for (var k in DBF_KEYS) {
var rest = line.substr(DBF_KEYS[k].length).trim();
if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0 && rest.indexOf("=") == 0) {
// known key
var key = DBF_KEYS[k];
var value = rest.substr(1).trim();
if (!(key in result)) {
if (key == "Serial") {
value = value.split("-").join("");
}
result[key] = value;
foundKey = true;
break;
} else {
logln ("Error: rejecting DBF Line: " + (firstLine + i) + " redefine: " + DBF_KEYS[k]);
}
} else if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0) {
// known yet without '=', warn.
logln("Warning: Line " + (firstLine + i) + ", no '='? '" + line + "'");
}
}
if (!foundKey) {
// unknown/no key (comment, patch, hack, empty, etc)
extra.push(lines[i]); // because 'line' is trimmed
}
} // lines
result["extra"] = extra;
return result;
}
var result = {header: "", games: []};
logln("import-dbf-start");
var dbf = getFile(DBF_PATH);
var dbfs = dbf.split(DBF_HEADER_END);
logln("found main sections: " + dbfs.length);
var gameSections = String(dbfs[1]).split(DBF_DELIMITER);
if (gameSections[gameSections.length-1].length == 0) {
gameSections.pop();
}
logln("Found game sections: " + gameSections.length);
logln("Importing games ...");
var gamesDb = [];
var serialIndex= {};
var compatInfo = [];
var dbfRegions = {};
for (var g in gameSections) {
var game = importGame(gameSections[g]);
if (!("Serial" in game) && !("Name" in game) && !("Region" in game) && !("Compat" in game) && game.extra.length == 1) {
logln("Warning: Empty section (skipping)");
} else {
if (!("Serial" in game) || !("Name" in game) || !("Region" in game)) {
logln("Warning: Incomplete: ser="+game.Serial.substr(0,4)+"-"+game.Serial.substr(4)+" name="+game.Name+ " region="+game.Region);
}
gamesDb.push(game);
if (game.Serial) {
var ser = serNor(game.Serial);
if (ser in serialIndex) {
logln("Warning: duplicate serial at dbf: " + serPretty(ser));
} else {
serialIndex[ser] = gamesDb.length - 1;
}
}
if ("Compat" in game) {
var c = game.Compat.substr(0,1);
if (!(c in compatInfo)) compatInfo[c] = 0;
compatInfo[c]++;
}
}
if (!dbfRegions[game.Region])
dbfRegions[game.Region] = 0;
dbfRegions[game.Region]++;
}
logln("Imported entries: " + gamesDb.length);
for (var i in compatInfo)
logln("Compatibility " + i + ": " + compatInfo[i] + " games");
if (!CONFIG_SKIP_REGIONS_SUMMARY) {
for (var r in dbfRegions)
logln("" + r + ": " + dbfRegions[r]);
}
function sortDbf() {
gamesDb.sort(function(a, b) {
// no need for equal since there are no duplicates
return serNor(a.Serial) > serNor(b.Serial) ? 1 : -1;
});
for (var i = 0; i < gamesDb.length; i++) {
serialIndex[serNor(gamesDb[i].Serial)] = i;
}
}
logln("Checking sort ...");
var prevg = "";
var sorted = true;
for (var g in serialIndex) {
if (g < prevg) {
logln ("bad sort: " + serPretty(prevg) + " before " + serPretty(g));
sorted = false;
}
prevg = g;
}
if (sorted) {
logln("dbf sort: no need (already sorted)");
} else {
if (CONFIG_SKIP_SORT_1) {
logln("skipping sort1");
} else {
logln("Sorting dbf...");
sortDbf();
logln("Done Sorting dbf.");
}
}
logln("import-dbf-end");
function dbfRegionFromCompatRegion(csvRegion) {
// currently data.csv only has NTSC/PAL/JAP
// we're translating to the dbf as: NTSC-U/PAL-Unk/NTSC-J respectively
// can later manually fix or set more specific region data.
// return undefined if unexpected region name at data.csv
var trans = {
"NTSC": "NTSC-U",
"PAL" : "PAL-Unk",
"JAP" : "NTSC-J"
};
return trans[csvRegion];
}
logln("");
if (CONFIG_SKIP_IMPORT_CSV) {
logln("skipping data.csv import");
} else {
logln("start import compatibility ...");
var compat = getFile(COMPAT_PATH);
compat = compat.split("\n")
logln("Found " + compat.length + " compatibility info lines");
var updateSummary = {same: 0, "new": 0, better: 0, worse: 0, missingAtCsv: 0};
var compatVerification = {};
var compatRegions = {};
for (var i in compat) {
if (!compat[i].trim().length) continue;
var cols = compat[i].split(COMPAT_DELIMITER);
var found = false;
var ser = cols[COMPAT_COLUMNS.Serial].trim().toLowerCase();
var com = Number(cols[COMPAT_COLUMNS.Compat].trim());
var reg = cols[COMPAT_COLUMNS.Region].trim();
if (!compatRegions[reg])
compatRegions[reg] = 0;
compatRegions[reg]++;
if (!(ser in compatVerification)) {
compatVerification[ser] = com;
} else {
logln("Warning (skipping): duplicate compat info (" + compatVerification[ser] + "/" + com + ") for: " + serPretty(ser));
continue;
}
if (ser.toLowerCase() in serialIndex) {
var game = gamesDb[serialIndex[ser]];
var dbser = serNor(game.Serial);
var dbcom = Number(("Compat" in game) ? game.Compat.split("//")[0].trim() : "0");
if (dbser != ser) {
logln("Error: internal: invalid index at serialIndex for " + serPretty(ser));
continue;
}
if (com != dbcom) {
//logln("Updating compat from " + dbcom + " to " + com + " for " + ser + ": " + game.Name);
if (("Compat" in game) && game.Compat.split("//").length > 1) {
logln("Warning: removing compat comment for serial " + serPretty(ser) + " : //" + game.Compat.split("//")[1]);
}
game.Compat = com;
if (com > dbcom) {
updateSummary.better++
} else {
updateSummary.worse++;
logln(":( Degrading compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
}
} else {
updateSummary.same++;
}
} else {
updateSummary.new++;
logln ("not at dbf: " + serPretty(ser) + " - " + (CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipping" : "adding:"));
if (!CONFIG_SKIP_ADD_MISSING_FROM_CSV) {
var newDbfReg = dbfRegionFromCompatRegion(reg);
if (newDbfReg) {
gamesDb.push({Serial: ser, Compat: com, Region: newDbfReg, Name: cols[COMPAT_COLUMNS.Name].trim(), extra: ""});
log(getExportEntry(gamesDb[gamesDb.length - 1]) + DBF_DELIMITER);
} else {
logln("Error: skipping due to unexpected region at data.csv: " + reg);
}
}
}
}
if (compat && compat.length > 1) {
if (updateSummary.new) {
if (CONFIG_SKIP_SORT_2) {
logln("skipping sort after add");
} else {
logln("sorting new entries into the dbf...");
sortDbf();
logln("sort - done.");
}
}
logln("-- Update summary --");
if (!CONFIG_SKIP_REGIONS_SUMMARY) {
logln("data.csv regions:");
for (var r in compatRegions)
logln("" + r + ": " + compatRegions[r]);
}
if (!CONFIG_SKIP_LOG_CSV_MISSING) {
for (var csvSer in serialIndex) {
var game = gamesDb[serialIndex[csvSer]];
if (game.Compat && !compatVerification[serNor(csvSer)]) {
logln("serial missing from CSV: " + pad(serPretty(csvSer), 12) + " / " + pad(noComment(game.Region), 8) + " / " + game.Compat + " / " + game.Name);
updateSummary.missingAtCsv++;
}
}
}
logln("Not at dbf (and " +(CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipped" : "added")+ "): " + updateSummary.new);
logln("Not at csv (but have compat info at the dbf): " + updateSummary.missingAtCsv);
logln("Unchanged: " + updateSummary.same);
logln("Better compat: " + updateSummary.better);
logln("Worse compat: " + updateSummary.worse);
logln("End import compatibility.");
}
}
logln("");
if (CONFIG_SKIP_HTML_INDEX_IMPORTS) {
logln("skipping html index files");
} else {
logln("Scanning html index files");
var in_incomplete = 0;
var in_identical = 0;
var in_diff = 0;
var in_new = 0;
var in_err = 0;
var in_usBetter = 0;
var files = HTML_INDEX_FILES;
for (var f in files) {
var serials = getFile(files[f]);
logln("file sections: " + serials.split('<table border="1"').length);
if (!serials.length) {
logln("Warning: file not found. skipping.");
continue;
}
var list = serials.split('<table border="1"')[1].split("</tr>");
logln("found rows: " + list.length);
for (var l in list) {
var s = list[l].split("</td>");
if (!(s.length==4)) {
in_incomplete++;
continue;
}
try {
var ser = serNor(s[0].split('.htm">')[1].split("</pre>")[0]);
var name = s[1].split(' 0">')[1].split("</pre>")[0];
var region = s[2].split(' 0">')[1].split("</pre>")[0];
if (ser in serialIndex) {
var game = gamesDb[serialIndex[ser]];
function norName(n) {
n = n.toLowerCase().split("//")[0];
var cs = [", ", ".", "-", ":"];
for (var i in cs)
n = n.split(cs[i]).join("");
return n.split(" ").join("").split(" ").join("").trim();
}
if (norName(game.Name) == norName(name) && norName(game.Region) == norName(region)) {
in_identical++
} else {
if ((norName(region).indexOf("unk")>=0 && norName(game.Region).indexOf("unk")<0)|| norName(name).indexOf("zzz")==0)
in_usBetter++
else {
logln("diff: " + serPretty(ser) + " DB: [" + game.Region.split("//")[0].trim() + "] " + game.Name.split("//")[0].trim());
logln(" index: [" + region + "] "+ name);
logln("");
in_diff++;
}
}
} else {
in_new++;
}
} catch (e) {
in_err++;
logln(e);
}
}
}
logln("--- Html index summary: ---");
logln("errors: " + in_err);
logln("incomplete data: " + in_incomplete);
logln("identical: " + in_identical);
logln("we're better: " + in_usBetter);
logln("different: "+ in_diff);
logln("not in db: " + in_new);
}
// start: test export dbf
function getExportEntry(gameItem) {
var entry = [];
for (var k in DBF_KEYS) {
if (DBF_KEYS[k] in gameItem) {
var v = gameItem[DBF_KEYS[k]];
var fill = "";
if (DBF_KEYS[k] == "Serial") v = (v.substr(0,4) + "-" + v.substr(4)).toUpperCase();
if (DBF_KEYS[k] == "Name") fill = " ";
entry.push(DBF_KEYS[k] + fill + " = " + v);
}
}
if (gameItem.extra.length) {
entry.push(gameItem.extra.join("\n"));
}
return entry.join("\n");
}
exportData = [];
for (var g in gamesDb) {
exportData.push(getExportEntry(gamesDb[g]));
}
exportData = dbfs[0] + DBF_HEADER_END + exportData.join(DBF_DELIMITER) + DBF_DELIMITER;
$("saveUpdated").href = "data:bin/plain;base64," + window.btoa(exportData);
$("saveUpdatedWrapper").style.display = "inline";
// end: export
}
</script>
</head>
<body onload="javascript:parseDbf()">
Please copy GameIndex.dbf (from the bin folder) and data.csv (compat info) to the folder of this html file<br/>
<span id="saveUpdatedWrapper" style="display:none;">
<a id="saveUpdated" href="" download="GameIndex.new.dbf">Save updated DBF</a>
</span>
<h4>Log</h4>
<pre><div id="logout"></div></pre>
</body>
</html>