<html>
<meta charset="utf-8">
<title>PCSX2 games index compatibility updater</title>
<head>
<script>
/*
Quick and (very) dirty gameindex.dbf update tool from website compatibility db data.csv.
- works in Firefox since Chrome doesn't allow loading local files.
- updates compatibility info at the dbf from data.csv
- add entries to the dbf if they only appear on data.csv
- sorts the dbf entries according to the serial number
- keeps hacks/comments/etc untouched (except when modifying compatibility info - and prints a warning)
- can import external html files with games info and compare the name/region to our own data
*/
var CONFIG_SKIP_SORT_1 = 0; // sort after initial dbf import
var CONFIG_SKIP_REGIONS_SUMMARY = 1;
var CONFIG_SKIP_IMPORT_CSV = 0;
var CONFIG_SKIP_ADD_MISSING_FROM_CSV = 0;
var CONFIG_SKIP_LOG_CSV_MISSING = 0; // files which have compat info at the dbf but not at the csv
var CONFIG_SKIP_SORT_2 = 0; // sort after adding entries from data.csv
var CONFIG_SKIP_HTML_INDEX_IMPORTS = 1;

var DBF_PATH = "GameIndex.dbf";
var COMPAT_PATH = "data.csv";
var HTML_INDEX_FILES = ["serials_1.htm", "serials_2.htm", "serials_3.htm"];

var COMPAT_DELIMITER = "\t";
var COMPAT_COLUMNS = {Serial: 0, Compat: 1, Name: 5, Region: 6};

var DBF_KEYS =    ["Serial", "Name", "Region", "Compat"];
var DBF_DELIMITER = "\n---------------------------------------------\n";
var DBF_HEADER_END = DBF_DELIMITER + "-- Game List" + DBF_DELIMITER;


function escapeHtml(str) { return str.split("&").join("&amp;")
                                     .split("<").join("&lt;")
                                     .split(">").join("&gt;")
                                     .split("\"").join("&quot;"); }
function $(id){ return document.getElementById(id); }
function log(text) { $("logout").innerHTML += escapeHtml(text); }
function logln(text) { log(text + "\n"); }

function serPretty(ser) { ser = ser.toUpperCase(); return ser.substr(0,4) + "-" + ser.substr(4); }
function noComment(txt) { return txt.split("//")[0].trim(); }
function serNor(ser) { return noComment(ser).split("-").join("").toLowerCase(); }

function pad(txt, len, ch) {
  txt = "" + txt;
  ch = ch || " ";
  while (txt.length < len)
    txt += ch;
  return txt;
}

// Synchronous. Returns "" on failure, or file content on success.
// Use Firefox. chrome doesn't allow loading local files even if at the same dir as this file.
function getFile(URI, failQuietly) {
  var result = "";

  try {
    function reqListener () {
      result = this.responseText;
      logln("Got file '" + URI + "', length: " + result.length);
    }

    var oReq = new XMLHttpRequest();
    oReq.onload = reqListener;
    oReq.open("get", URI, false);
    oReq.overrideMimeType("text/plain; charset=x-user-defined");
    oReq.send();
  } catch(e) {
      if (!failQuietly)
        logln("Error file '" + URI + "' (" + e + ")");
      return "";
  }

  return result;
}

var exportData = [];
function parseDbf(dbf, startLine) {

  function importGame(section, firstLine) {
    if (!firstLine) firstLine = 0;
    var result = {};
    var extra = [];
    var lines = String(section).trim().split("\n");
    for (var i in lines) {
      var line = lines[i].trim();
      if (line.indexOf("--------") == 0 ){
        logln("Warning: invalid separator? : '" + line + "'");
      }
      var lineLow = line.toLowerCase();
      var foundKey = false;

      for (var k in DBF_KEYS) {
        var rest = line.substr(DBF_KEYS[k].length).trim();
        if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0 && rest.indexOf("=") == 0) {
          // known key
          var key = DBF_KEYS[k];
          var value = rest.substr(1).trim();
          if (!(key in result)) {
            if (key == "Serial") {
              value = value.split("-").join("");
            }
            result[key] = value;
            foundKey = true;
            break;

          } else {
            logln ("Error: rejecting DBF Line: " + (firstLine + i) + " redefine: " + DBF_KEYS[k]);
          }
        } else if (lineLow.indexOf(DBF_KEYS[k].toLowerCase()) == 0) {
          // known yet without '=', warn.
          logln("Warning: Line " + (firstLine + i) + ", no '='? '" + line + "'");
        }
      }

      if (!foundKey) {
        // unknown/no key (comment, patch, hack, empty, etc)
        extra.push(lines[i]); // because 'line' is trimmed
      }
    } // lines

    result["extra"] = extra;
    return result;
  }

  var result = {header: "", games: []};

	logln("import-dbf-start");
  var dbf = getFile(DBF_PATH);

	var dbfs = dbf.split(DBF_HEADER_END);
  logln("found main sections: " + dbfs.length);

  var gameSections = String(dbfs[1]).split(DBF_DELIMITER);
  if (gameSections[gameSections.length-1].length == 0) {
    gameSections.pop();
  }
  logln("Found game sections: " + gameSections.length);

  logln("Importing games ...");
  var gamesDb = [];
  var serialIndex= {};
  var compatInfo = [];
  var dbfRegions = {};
  for (var g in gameSections) {
    var game = importGame(gameSections[g]);
    if (!("Serial" in game) && !("Name" in game) && !("Region" in game) && !("Compat" in game) && game.extra.length == 1) {
      logln("Warning: Empty section (skipping)");
    } else {
      if (!("Serial" in game) || !("Name" in game) || !("Region" in game)) {
        logln("Warning: Incomplete: ser="+game.Serial.substr(0,4)+"-"+game.Serial.substr(4)+" name="+game.Name+ " region="+game.Region);
      }
      gamesDb.push(game);
      if (game.Serial) {
        var ser = serNor(game.Serial);
        if (ser in serialIndex) {
          logln("Warning: duplicate serial at dbf: " + serPretty(ser));
        } else {
          serialIndex[ser] = gamesDb.length - 1;
        }
      }
      if ("Compat" in game) {
        var c = game.Compat.substr(0,1);
        if (!(c in compatInfo)) compatInfo[c] = 0;
        compatInfo[c]++;
      }
    }
    if (!dbfRegions[game.Region])
      dbfRegions[game.Region] = 0;
    dbfRegions[game.Region]++;
  }
  logln("Imported entries: " + gamesDb.length);
  for (var i in compatInfo)
    logln("Compatibility " + i + ": " + compatInfo[i] + " games");
  
  if (!CONFIG_SKIP_REGIONS_SUMMARY) {
    for (var r in dbfRegions)
      logln("" + r + ": " + dbfRegions[r]);
  }
    

  function sortDbf() {
    gamesDb.sort(function(a, b) {
      // no need for equal since there are no duplicates
      return serNor(a.Serial) > serNor(b.Serial) ? 1 : -1;
    });
    for (var i = 0; i < gamesDb.length; i++) {
      serialIndex[serNor(gamesDb[i].Serial)] = i;
    }
  }

  logln("Checking sort ...");
  var prevg = "";
  var sorted = true;
  for (var g in serialIndex) {
    if (g < prevg) {
      logln ("bad sort: " + serPretty(prevg) + " before " + serPretty(g));
      sorted = false;
    }
    prevg = g;
  }
  
  if (sorted) {
    logln("dbf sort: no need (already sorted)");
  } else {
    if (CONFIG_SKIP_SORT_1) {
      logln("skipping sort1");
    } else {
      logln("Sorting dbf...");
      sortDbf();
      logln("Done Sorting dbf.");
    }
  }

  logln("import-dbf-end");

  function dbfRegionFromCompatRegion(csvRegion) {
    // currently data.csv only has NTSC/PAL/JAP
    // we're translating to the dbf as: NTSC-U/PAL-Unk/NTSC-J respectively
    // can later manually fix or set more specific region data.
    // return undefined if unexpected region name at data.csv
    var trans = {
      "NTSC": "NTSC-U",
      "PAL" : "PAL-Unk",
      "JAP" : "NTSC-J"
    };
    return trans[csvRegion];
  }

  logln("");
  if (CONFIG_SKIP_IMPORT_CSV) {
    logln("skipping data.csv import");
  } else {
    logln("start import compatibility ...");
    var compat = getFile(COMPAT_PATH);
    compat = compat.split("\n")
    logln("Found " + compat.length + " compatibility info lines");
    var updateSummary = {same: 0, "new": 0, better: 0, worse: 0, missingAtCsv: 0};
    var compatVerification = {};

    var compatRegions = {};
    for (var i in compat) {
      if (!compat[i].trim().length) continue;
      var cols = compat[i].split(COMPAT_DELIMITER);
      var found = false;
      var ser = cols[COMPAT_COLUMNS.Serial].trim().toLowerCase();
      var com = Number(cols[COMPAT_COLUMNS.Compat].trim());
      var reg = cols[COMPAT_COLUMNS.Region].trim();

      if (!compatRegions[reg])
        compatRegions[reg] = 0;
      compatRegions[reg]++;

      if (!(ser in compatVerification)) {
        compatVerification[ser] = com;
      } else {
        logln("Warning (skipping): duplicate compat info (" + compatVerification[ser] + "/" + com + ") for: " + serPretty(ser));
        continue;
      }
      if (ser.toLowerCase() in serialIndex) {
        var game = gamesDb[serialIndex[ser]];
        var dbser = serNor(game.Serial);
        var dbcom = Number(("Compat" in game) ? game.Compat.split("//")[0].trim() : "0");
        if (dbser != ser) {
          logln("Error: internal: invalid index at serialIndex for " + serPretty(ser));
          continue;
        }
        
        if (com != dbcom) {
          //logln("Updating compat from " + dbcom + " to " + com + " for " + ser + ": " + game.Name);
          if (("Compat" in game) && game.Compat.split("//").length > 1) {
            logln("Warning: removing compat comment for serial " + serPretty(ser) + " : //" + game.Compat.split("//")[1]);
          }
          game.Compat = com;
          
          if (com > dbcom) {
            updateSummary.better++
          } else {
            updateSummary.worse++;
            logln(":( Degrading compat from " + dbcom + " to " + com + " for " + serPretty(ser) + ": " + game.Name);
          }
        } else {
          updateSummary.same++;
        }
      } else {
        updateSummary.new++;
        logln ("not at dbf: " + serPretty(ser) + " - " + (CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipping" : "adding:"));
        if (!CONFIG_SKIP_ADD_MISSING_FROM_CSV) {
          var newDbfReg = dbfRegionFromCompatRegion(reg);
          if (newDbfReg) {
            gamesDb.push({Serial: ser, Compat: com, Region: newDbfReg, Name: cols[COMPAT_COLUMNS.Name].trim(), extra: ""});
            log(getExportEntry(gamesDb[gamesDb.length - 1]) + DBF_DELIMITER);
          } else {
            logln("Error: skipping due to unexpected region at data.csv: " + reg);
          }
        }
      }
    }
    if (compat && compat.length > 1) {
      if (updateSummary.new) {
        if (CONFIG_SKIP_SORT_2) {
          logln("skipping sort after add");
        } else {
          logln("sorting new entries into the dbf...");
          sortDbf();
          logln("sort - done.");
        }
      }
      logln("-- Update summary --");

      if (!CONFIG_SKIP_REGIONS_SUMMARY) {
        logln("data.csv regions:");
        for (var r in compatRegions)
          logln("" + r + ": " + compatRegions[r]);
      }
      if (!CONFIG_SKIP_LOG_CSV_MISSING) {
        for (var csvSer in serialIndex) {
          var game = gamesDb[serialIndex[csvSer]];
          if (game.Compat && !compatVerification[serNor(csvSer)]) {
            logln("serial missing from CSV: " + pad(serPretty(csvSer), 12) + " / " + pad(noComment(game.Region), 8) + " / " + game.Compat + " / " + game.Name);
            updateSummary.missingAtCsv++;
          }
        }
      }
   
      logln("Not at dbf (and " +(CONFIG_SKIP_ADD_MISSING_FROM_CSV ? "skipped" : "added")+ "): " + updateSummary.new);
      logln("Not at csv (but have compat info at the dbf): " + updateSummary.missingAtCsv);
      logln("Unchanged: " + updateSummary.same);
      logln("Better compat: " + updateSummary.better);
      logln("Worse compat: " + updateSummary.worse);
      logln("End import compatibility.");
    }
  }


  logln("");
  if (CONFIG_SKIP_HTML_INDEX_IMPORTS) {
    logln("skipping html index files");
  } else {
    logln("Scanning html index files");
    var in_incomplete = 0;
    var in_identical = 0;
    var in_diff = 0;
    var in_new = 0;
    var in_err = 0;
    var in_usBetter = 0;
    var files = HTML_INDEX_FILES;
    for (var f in files) {
      var serials = getFile(files[f]);
      logln("file sections: " + serials.split('<table border="1"').length);
      if (!serials.length) {
        logln("Warning: file not found. skipping.");
        continue;
      }
      var list = serials.split('<table border="1"')[1].split("</tr>");
      logln("found rows: " + list.length);
      for (var l in list) {
        var s = list[l].split("</td>");
        if (!(s.length==4)) {
          in_incomplete++;
          continue;
        }
        try {
          var ser = serNor(s[0].split('.htm">')[1].split("</pre>")[0]);
          var name = s[1].split(' 0">')[1].split("</pre>")[0];
          var region = s[2].split(' 0">')[1].split("</pre>")[0];

          if (ser in serialIndex) {
            var game = gamesDb[serialIndex[ser]];
            function norName(n) {
              n = n.toLowerCase().split("//")[0];
              var cs = [", ", ".", "-", ":"];
              for (var i in cs)
                n = n.split(cs[i]).join("");
              return n.split("  ").join("").split(" ").join("").trim();
            }
            if (norName(game.Name) == norName(name) && norName(game.Region) == norName(region)) {
              in_identical++
            } else {
              if ((norName(region).indexOf("unk")>=0 && norName(game.Region).indexOf("unk")<0)|| norName(name).indexOf("zzz")==0) 
                in_usBetter++
              else {
                logln("diff: " + serPretty(ser) + " DB: [" + game.Region.split("//")[0].trim() + "] " + game.Name.split("//")[0].trim());
                logln("              index: [" + region + "] "+ name);
                logln("");
                in_diff++;
              }
            }
          } else {
            in_new++;
          }
        } catch (e) {
          in_err++;
          logln(e);
        }
      }
    }
    logln("--- Html index summary: ---");
    logln("errors: " + in_err);
    logln("incomplete data: " + in_incomplete);
    logln("identical: " + in_identical);
    logln("we're better: " + in_usBetter);
    logln("different: "+ in_diff);
    logln("not in db: " + in_new);
  }


  // start: test export dbf
  function getExportEntry(gameItem) {
    var entry = [];
    for (var k in DBF_KEYS) {
      if (DBF_KEYS[k] in gameItem) {
        var v = gameItem[DBF_KEYS[k]];
        var fill = "";
        if (DBF_KEYS[k] == "Serial") v = (v.substr(0,4) + "-" + v.substr(4)).toUpperCase();
        if (DBF_KEYS[k] == "Name") fill = "  ";
        entry.push(DBF_KEYS[k] + fill + " = " + v);
      }
    }
    if (gameItem.extra.length) {
      entry.push(gameItem.extra.join("\n"));
    }
    return entry.join("\n");
  }
  exportData = [];
  for (var g in gamesDb) {
    exportData.push(getExportEntry(gamesDb[g]));
  }
  exportData = dbfs[0] + DBF_HEADER_END + exportData.join(DBF_DELIMITER) + DBF_DELIMITER;
  $("saveUpdated").href = "data:bin/plain;base64," + window.btoa(exportData);
  $("saveUpdatedWrapper").style.display = "inline";
  // end: export

}

</script>
</head>

<body onload="javascript:parseDbf()">
Please copy GameIndex.dbf (from the bin folder) and data.csv (compat info) to the folder of this html file<br/>
<span id="saveUpdatedWrapper" style="display:none;">
<a id="saveUpdated" href="" download="GameIndex.new.dbf">Save updated DBF</a>
</span>

<h4>Log</h4>
<pre><div id="logout"></div></pre>

</body>
</html>