mirror of https://github.com/PCSX2/pcsx2.git
tools: Remove GameIndex.dbf-tool.
Deprecated with yaml db, maybe a new alternative can be made in the future.
This commit is contained in:
lightningterror
parent
657ce60ff8
commit
232c4b482a
|
|
@ -1,455 +0,0 @@
|
|||
<html>
|
||||
<meta charset="utf-8">
|
||||
<title>PCSX2 games index compatibility updater</title>
|
||||
<head>
|
||||
<script>
|
||||
/*
|
||||
Quick and (very) dirty gameindex.dbf update tool from website compatibility db data.csv.
|
||||
- works in Firefox since Chrome doesn't allow loading local files.
|
||||
- updates compatibility info at the dbf from data.csv
|
||||
- add entries to the dbf if they only appear on data.csv
|
||||
- sorts the dbf entries according to the serial number
|
||||
- keeps hacks/comments/etc untouched (except when modifying compatibility info - and prints a warning)
|
||||
- can import external html files with games info and compare the name/region to our own data
|
||||
*/
|
||||
var CONFIG_SKIP_SORT_1 = 0; // sort after initial dbf import
|
||||
var CONFIG_SKIP_REGIONS_SUMMARY = 0;
|
||||
var CONFIG_SKIP_IMPORT_CSV = 0;
|
||||
var CONFIG_SKIP_ADD_MISSING_FROM_CSV = 0;
|
||||
var CONFIG_SKIP_LOG_CSV_MISSING = 0; // files which have compat info at the dbf but not at the csv
|
||||
var CONFIG_SKIP_LOG_IMPROVEMENTS = 0; // the common case of data.csv having better compat than the current dbf.
|
||||
var CONFIG_SKIP_SORT_2 = 0; // sort after adding entries from data.csv
|
||||
var CONFIG_SKIP_HTML_INDEX_IMPORTS = 1;
|
||||
|
||||
var DBF_PATH = "GameIndex.dbf";
|
||||
var COMPAT_PATH = "data.csv";
|
||||
var HTML_INDEX_FILES = ["serials_1.htm", "serials_2.htm", "serials_3.htm"];
|
||||
|
||||
var COMPAT_DELIMITER = "\t";
|
||||
var COMPAT_COLUMNS = {Serial: 0, Compat: 1, Name: 5, Region: 6};
|
||||
|
||||
var DBF_KEYS = ["Serial", "Name", "Region", "Compat"];
|
||||
var DBF_DELIMITER = "\n---------------------------------------------\n";
|
||||
var DBF_HEADER_END = DBF_DELIMITER + "-- Game List" + DBF_DELIMITER;
|
||||
|
||||
|
||||
function escapeHtml(str) { return str.split("&").join("&")
|
||||
.split("<").join("<")
|
||||
.split(">").join(">")
|
||||
.split("\"").join("""); }
|
||||
function $(id){ return document.getElementById(id); }
|
||||
function log(text) { $("logout").innerHTML += escapeHtml(text); }
|
||||
function logln(text) { log(text + "\n"); }
|
||||
|
||||
function serPretty(ser) { ser = ser.toUpperCase(); return ser.substr(0,4) + "-" + ser.substr(4); }
|
||||
function noComment(txt) { return txt.split("//")[0].trim(); }
|
||||
function serNor(ser) { return noComment(ser).split("-").join("").toLowerCase(); }
|
||||
|
||||
function pad(txt, len, ch) {
|
||||
txt = "" + txt;
|
||||
ch = ch || " ";
|
||||
while (txt.length < len)
|
||||
txt += ch;
|
||||
return txt;
|
||||
}
|
||||
|
||||
// Synchronous. Returns "" on failure, or file content on success.
|
||||
// Use Firefox. chrome doesn't allow loading local files even if at the same dir as this file.
|
||||
function getFile(URI, failQuietly) {
|
||||
var result = "";
|
||||
|
||||
try {
|
||||
function reqListener () {
|
||||
result = this.responseText;
|
||||
logln("Got file '" + URI + "', length: " + result.length);
|
||||
}
|
||||
|
||||
var oReq = new XMLHttpRequest();
|
||||
oReq.onload = reqListener;
|
||||
oReq.open("get", URI, false);
|
||||
oReq.overrideMimeType("text/plain; charset=x-user-defined");
|
||||
oReq.send();
|
||||
} catch(e) {
|
||||
if (!failQuietly)
|
||||
logln("Error file '" + URI + "' (" + e + ")");
|
||||
return "";
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
var exportData = [];
|
||||
function parseDbf(dbf, startLine) {
|
||||
|
||||
function importGame(section, firstLine) {
|
||||
if (!firstLine) firstLine = 0;
|
||||
var result = {};
|
||||
var extra = [];
|
||||
var lines = String(section).trim().split("\n");
|
||||
for (var i in lines) {
|
||||
var line = lines[i].trim();
|
||||
if (line.indexOf("--------") == 0 ){
|
||||
logln("Warning: invalid separator? : '" + line + "'");
|
||||
}
|
||||
var lineLow = line.toLowerCase();
|
||||
var foundKey = false;
|
||||
|
||||
for (var k in DBF_KEYS) {
|
||||
var rest = line.substr(DBF_KEYS[k].length).trim();
|
||||
if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0 && rest.indexOf("=") == 0) {
|
||||
// known key
|
||||
var key = DBF_KEYS[k];
|
||||
var value = rest.substr(1).trim();
|
||||
if (!(key in result)) {
|
||||
if (key == "Serial") {
|
||||
value = value.split("-").join("");
|
||||
}
|
||||
result[key] = value;
|
||||
foundKey = true;
|
||||
break;
|
||||
|
||||
} else {
|
||||
logln ("Error: rejecting DBF Line: " + (firstLine + i) + " redefine: " + DBF_KEYS[k]);
|
||||
}
|
||||
} else if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0) {
|
||||
// known yet without '=', warn.
|
||||
logln("Warning: Line " + (firstLine + i) + ", no '='? '" + line + "'");
|
||||
}
|
||||
}
|
||||
|
||||
if (!foundKey) {
|
||||
// unknown/no key (comment, patch, hack, empty, etc)
|
||||
extra.push(lines[i]); // because 'line' is trimmed
|
||||
}
|
||||
} // lines
|
||||
|
||||
result["extra"] = extra;
|
||||
return result;
|
||||
}
|
||||
|
||||
var result = {header: "", games: []};
|
||||
|
||||
logln("import-dbf-start");
|
||||
var dbf = getFile(DBF_PATH);
|
||||
|
||||
var dbfs = dbf.split(DBF_HEADER_END);
|
||||
logln("found main sections: " + dbfs.length);
|
||||
|
||||
var gameSections = String(dbfs[1]).split(DBF_DELIMITER);
|
||||
if (gameSections[gameSections.length-1].length == 0) {
|
||||
gameSections.pop();
|
||||
}
|
||||
logln("Found game sections: " + gameSections.length);
|
||||
|
||||
logln("Importing games ...");
|
||||
var gamesDb = [];
|
||||
var serialIndex= {};
|
||||
var compatInfo = [];
|
||||
var dbfRegions = {};
|
||||
for (var g in gameSections) {
|
||||
var game = importGame(gameSections[g]);
|
||||
if (!("Serial" in game) && !("Name" in game) && !("Region" in game) && !("Compat" in game) && game.extra.length == 1) {
|
||||
logln("Warning: Empty section (skipping)");
|
||||
} else {
|
||||
if (!("Serial" in game) || !("Name" in game) || !("Region" in game)) {
|
||||
logln("Warning: Incomplete: ser="+game.Serial.substr(0,4)+"-"+game.Serial.substr(4)+" name="+game.Name+ " region="+game.Region);
|
||||
}
|
||||
gamesDb.push(game);
|
||||
if (game.Serial) {
|
||||
var ser = serNor(game.Serial);
|
||||
if (ser in serialIndex) {
|
||||
logln("Warning: duplicate serial at dbf: " + serPretty(ser));
|
||||
} else {
|
||||
serialIndex[ser] = gamesDb.length - 1;
|
||||
}
|
||||
}
|
||||
if ("Compat" in game) {
|
||||
var c = game.Compat.substr(0,1);
|
||||
if (!(c in compatInfo)) compatInfo[c] = 0;
|
||||
compatInfo[c]++;
|
||||
}
|
||||
}
|
||||
if (!dbfRegions[game.Region])
|
||||
dbfRegions[game.Region] = 0;
|
||||
dbfRegions[game.Region]++;
|
||||
}
|
||||
logln("Imported entries: " + gamesDb.length);
|
||||
for (var i in compatInfo)
|
||||
logln("Compatibility " + i + ": " + compatInfo[i] + " games");
|
||||
|
||||
if (!CONFIG_SKIP_REGIONS_SUMMARY) {
|
||||
for (var r in dbfRegions)
|
||||
logln("" + r + ": " + dbfRegions[r]);
|
||||
}
|
||||
|
||||
|
||||
function sortDbf() {
|
||||
gamesDb.sort(function(a, b) {
|
||||
// no need for equal since there are no duplicates
|
||||
return serNor(a.Serial) > serNor(b.Serial) ? 1 : -1;
|
||||
});
|
||||
for (var i = 0; i < gamesDb.length; i++) {
|
||||
serialIndex[serNor(gamesDb[i].Serial)] = i;
|
||||
}
|
||||
}
|
||||
|
||||
logln("Checking sort ...");
|
||||
var prevg = "";
|
||||
var sorted = true;
|
||||
for (var g in serialIndex) {
|
||||
if (g < prevg) {
|
||||
logln ("bad sort: " + serPretty(prevg) + " before " + serPretty(g));
|
||||
sorted = false;
|
||||
}
|
||||
prevg = g;
|
||||
}
|
||||
|
||||
if (sorted) {
|
||||
logln("dbf sort: no need (already sorted)");
|
||||
} else {
|
||||
if (CONFIG_SKIP_SORT_1) {
|
||||
logln("skipping sort1");
|
||||
} else {
|
||||
logln("Sorting dbf...");
|
||||
sortDbf();
|
||||
logln("Done Sorting dbf.");
|
||||
}
|
||||
}
|
||||
|
||||
logln("import-dbf-end");
|
||||
|
||||
function dbfRegionFromCompatRegion(csvRegion) {
|
||||
// currently data.csv only has NTSC/PAL/JAP
|
||||
// we're translating to the dbf as: NTSC-U/PAL-Unk/NTSC-J respectively
|
||||
// can later manually fix or set more specific region data.
|
||||
// return undefined if unexpected region name at data.csv
|
||||
var trans = {
|
||||
"NTSC": "NTSC-U",
|
||||
"PAL" : "PAL-Unk",
|
||||
"JAP" : "NTSC-J"
|
||||
};
|
||||
return trans[csvRegion];
|
||||
}
|
||||
|
||||
logln("");
|
||||
if (CONFIG_SKIP_IMPORT_CSV) {
|
||||
logln("skipping data.csv import");
|
||||
} else {
|
||||
logln("start import compatibility ...");
|
||||
var compat = getFile(COMPAT_PATH);
|
||||
compat = compat.split("\n")
|
||||
logln("Found " + compat.length + " compatibility info lines");
|
||||
var updateSummary = {same: 0, "new": 0, better: 0, worse: 0, missingAtCsv: 0};
|
||||
var compatVerification = {};
|
||||
|
||||
var compatRegions = {};
|
||||
for (var i in compat) {
|
||||
if (!compat[i].trim().length) continue;
|
||||
var cols = compat[i].split(COMPAT_DELIMITER);
|
||||
var found = false;
|
||||
var ser = cols[COMPAT_COLUMNS.Serial].trim().toLowerCase();
|
||||
var com = Number(cols[COMPAT_COLUMNS.Compat].trim());
|
||||
var reg = cols[COMPAT_COLUMNS.Region].trim();
|
||||
|
||||
if (!compatRegions[reg])
|
||||
compatRegions[reg] = 0;
|
||||
compatRegions[reg]++;
|
||||
|
||||
if (!(ser in compatVerification)) {
|
||||
compatVerification[ser] = com;
|
||||
} else {
|
||||
logln("Warning (skipping): duplicate compat info (" + compatVerification[ser] + "/" + com + ") for: " + serPretty(ser));
|
||||
continue;
|
||||
}
|
||||
if (ser.toLowerCase() in serialIndex) {
|
||||
var game = gamesDb[serialIndex[ser]];
|
||||
var dbser = serNor(game.Serial);
|
||||
var dbcom = Number(("Compat" in game) ? game.Compat.split("//")[0].trim() : "0");
|
||||
if (dbser != ser) {
|
||||
logln("Error: internal: invalid index at serialIndex for " + serPretty(ser));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (com != dbcom) {
|
||||
//logln("Updating compat from " + dbcom + " to " + com + " for " + ser + ": " + game.Name);
|
||||
if (("Compat" in game) && game.Compat.split("//").length > 1) {
|
||||
logln("Warning: removing compat comment for serial " + serPretty(ser) + " : //" + game.Compat.split("//")[1]);
|
||||
}
|
||||
game.Compat = com;
|
||||
|
||||
if (com > dbcom) {
|
||||
updateSummary.better++
|
||||
if (!CONFIG_SKIP_LOG_IMPROVEMENTS)
|
||||
logln("--> ++ Improving compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
|
||||
} else {
|
||||
updateSummary.worse++;
|
||||
logln("--> -- Degrading compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
|
||||
}
|
||||
} else {
|
||||
updateSummary.same++;
|
||||
}
|
||||
} else {
|
||||
updateSummary.new++;
|
||||
logln ("not at dbf: " + serPretty(ser) + " - " + (CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipping" : "adding:"));
|
||||
if (!CONFIG_SKIP_ADD_MISSING_FROM_CSV) {
|
||||
var newDbfReg = dbfRegionFromCompatRegion(reg);
|
||||
if (newDbfReg) {
|
||||
gamesDb.push({Serial: ser, Compat: com, Region: newDbfReg, Name: cols[COMPAT_COLUMNS.Name].trim(), extra: ""});
|
||||
log(getExportEntry(gamesDb[gamesDb.length - 1]) + DBF_DELIMITER);
|
||||
} else {
|
||||
logln("Error: skipping due to unexpected region at data.csv: " + reg);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (compat && compat.length > 1) {
|
||||
if (updateSummary.new) {
|
||||
if (CONFIG_SKIP_SORT_2) {
|
||||
logln("skipping sort after add");
|
||||
} else {
|
||||
logln("sorting new entries into the dbf...");
|
||||
sortDbf();
|
||||
logln("sort - done.");
|
||||
}
|
||||
}
|
||||
logln("-- Update summary --");
|
||||
|
||||
if (!CONFIG_SKIP_REGIONS_SUMMARY) {
|
||||
logln("data.csv regions:");
|
||||
for (var r in compatRegions)
|
||||
logln("" + r + ": " + compatRegions[r]);
|
||||
}
|
||||
if (!CONFIG_SKIP_LOG_CSV_MISSING) {
|
||||
for (var csvSer in serialIndex) {
|
||||
var game = gamesDb[serialIndex[csvSer]];
|
||||
if (game.Compat && !compatVerification[serNor(csvSer)]) {
|
||||
logln("serial missing from CSV: " + pad(serPretty(csvSer), 12) + " / " + pad(noComment(game.Region), 8) + " / " + game.Compat + " / " + game.Name);
|
||||
updateSummary.missingAtCsv++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logln("Not at dbf (and " +(CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipped" : "added")+ "): " + updateSummary.new);
|
||||
logln("Not at csv (but have compat info at the dbf): " + updateSummary.missingAtCsv);
|
||||
logln("Unchanged: " + updateSummary.same);
|
||||
logln("Better compat: " + updateSummary.better);
|
||||
logln("Worse compat: " + updateSummary.worse);
|
||||
logln("End import compatibility.");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
logln("");
|
||||
if (CONFIG_SKIP_HTML_INDEX_IMPORTS) {
|
||||
logln("skipping html index files");
|
||||
} else {
|
||||
logln("Scanning html index files");
|
||||
var in_incomplete = 0;
|
||||
var in_identical = 0;
|
||||
var in_diff = 0;
|
||||
var in_new = 0;
|
||||
var in_err = 0;
|
||||
var in_usBetter = 0;
|
||||
var files = HTML_INDEX_FILES;
|
||||
for (var f in files) {
|
||||
var serials = getFile(files[f]);
|
||||
logln("file sections: " + serials.split('<table border="1"').length);
|
||||
if (!serials.length) {
|
||||
logln("Warning: file not found. skipping.");
|
||||
continue;
|
||||
}
|
||||
var list = serials.split('<table border="1"')[1].split("</tr>");
|
||||
logln("found rows: " + list.length);
|
||||
for (var l in list) {
|
||||
var s = list[l].split("</td>");
|
||||
if (!(s.length==4)) {
|
||||
in_incomplete++;
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
var ser = serNor(s[0].split('.htm">')[1].split("</pre>")[0]);
|
||||
var name = s[1].split(' 0">')[1].split("</pre>")[0];
|
||||
var region = s[2].split(' 0">')[1].split("</pre>")[0];
|
||||
|
||||
if (ser in serialIndex) {
|
||||
var game = gamesDb[serialIndex[ser]];
|
||||
function norName(n) {
|
||||
n = n.toLowerCase().split("//")[0];
|
||||
var cs = [", ", ".", "-", ":"];
|
||||
for (var i in cs)
|
||||
n = n.split(cs[i]).join("");
|
||||
return n.split(" ").join("").split(" ").join("").trim();
|
||||
}
|
||||
if (norName(game.Name) == norName(name) && norName(game.Region) == norName(region)) {
|
||||
in_identical++
|
||||
} else {
|
||||
if ((norName(region).indexOf("unk")>=0 && norName(game.Region).indexOf("unk")<0)|| norName(name).indexOf("zzz")==0)
|
||||
in_usBetter++
|
||||
else {
|
||||
logln("diff: " + serPretty(ser) + " DB: [" + game.Region.split("//")[0].trim() + "] " + game.Name.split("//")[0].trim());
|
||||
logln(" index: [" + region + "] "+ name);
|
||||
logln("");
|
||||
in_diff++;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
in_new++;
|
||||
}
|
||||
} catch (e) {
|
||||
in_err++;
|
||||
logln(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
logln("--- Html index summary: ---");
|
||||
logln("errors: " + in_err);
|
||||
logln("incomplete data: " + in_incomplete);
|
||||
logln("identical: " + in_identical);
|
||||
logln("we're better: " + in_usBetter);
|
||||
logln("different: "+ in_diff);
|
||||
logln("not in db: " + in_new);
|
||||
}
|
||||
|
||||
|
||||
// start: test export dbf
|
||||
function getExportEntry(gameItem) {
|
||||
var entry = [];
|
||||
for (var k in DBF_KEYS) {
|
||||
if (DBF_KEYS[k] in gameItem) {
|
||||
var v = gameItem[DBF_KEYS[k]];
|
||||
var fill = "";
|
||||
if (DBF_KEYS[k] == "Serial") v = (v.substr(0,4) + "-" + v.substr(4)).toUpperCase();
|
||||
if (DBF_KEYS[k] == "Name") fill = " ";
|
||||
entry.push(DBF_KEYS[k] + fill + " = " + v);
|
||||
}
|
||||
}
|
||||
if (gameItem.extra.length) {
|
||||
entry.push(gameItem.extra.join("\n"));
|
||||
}
|
||||
return entry.join("\n");
|
||||
}
|
||||
exportData = [];
|
||||
for (var g in gamesDb) {
|
||||
exportData.push(getExportEntry(gamesDb[g]));
|
||||
}
|
||||
exportData = dbfs[0] + DBF_HEADER_END + exportData.join(DBF_DELIMITER) + DBF_DELIMITER;
|
||||
$("saveUpdated").href = "data:bin/plain;base64," + window.btoa(exportData);
|
||||
$("saveUpdatedWrapper").style.display = "inline";
|
||||
// end: export
|
||||
|
||||
}
|
||||
|
||||
</script>
|
||||
</head>
|
||||
|
||||
<body onload="javascript:parseDbf()">
|
||||
Please copy GameIndex.dbf (from the bin folder) and data.csv (compat info) to the folder of this html file<br/>
|
||||
<span id="saveUpdatedWrapper" style="display:none;">
|
||||
<a id="saveUpdated" href="" download="GameIndex.new.dbf">Save updated DBF</a>
|
||||
</span>
|
||||
|
||||
<h4>Log</h4>
|
||||
<pre><div id="logout"></div></pre>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Loading…
Reference in New Issue