SignatureDB: Support for WiiTools MEGA files added

Source/Core/Core/CMakeLists.txt

@@ -198,6 +198,7 @@ set(SRCS
   PowerPC/Profiler.cpp
   PowerPC/SignatureDB/CSVSignatureDB.cpp
   PowerPC/SignatureDB/DSYSignatureDB.cpp
+  PowerPC/SignatureDB/MEGASignatureDB.cpp
   PowerPC/SignatureDB/SignatureDB.cpp
   PowerPC/JitInterface.cpp
   PowerPC/CachedInterpreter/CachedInterpreter.cpp

Source/Core/Core/Core.vcxproj

@@ -282,6 +282,7 @@
     <ClCompile Include="PowerPC\JitCommon\JitCache.cpp" />
     <ClCompile Include="PowerPC\SignatureDB\CSVSignatureDB.cpp" />
     <ClCompile Include="PowerPC\SignatureDB\DSYSignatureDB.cpp" />
+    <ClCompile Include="PowerPC\SignatureDB\MEGASignatureDB.cpp" />
     <ClCompile Include="PowerPC\SignatureDB\SignatureDB.cpp" />
     <ClCompile Include="PowerPC\JitInterface.cpp" />
     <ClCompile Include="PowerPC\MMU.cpp" />
@@ -509,6 +510,7 @@
     <ClInclude Include="PowerPC\JitCommon\JitCache.h" />
     <ClInclude Include="PowerPC\SignatureDB\CSVSignatureDB.h" />
     <ClInclude Include="PowerPC\SignatureDB\DSYSignatureDB.h" />
+    <ClInclude Include="PowerPC\SignatureDB\MEGASignatureDB.h" />
     <ClInclude Include="PowerPC\SignatureDB\SignatureDB.h" />
     <ClInclude Include="PowerPC\JitInterface.h" />
     <ClInclude Include="PowerPC\PowerPC.h" />

Source/Core/Core/Core.vcxproj.filters

@@ -728,6 +728,7 @@
     <ClCompile Include="Analytics.cpp" />
     <ClCompile Include="PowerPC\SignatureDB\CSVSignatureDB.cpp" />
     <ClCompile Include="PowerPC\SignatureDB\DSYSignatureDB.cpp" />
+    <ClCompile Include="PowerPC\SignatureDB\MEGASignatureDB.cpp" />
     <ClCompile Include="PowerPC\SignatureDB\SignatureDB.cpp" />
     <ClCompile Include="IOS\USB\Bluetooth\BTBase.cpp">
       <Filter>IOS\USB\Bluetooth</Filter>
@@ -1360,6 +1361,7 @@
     <ClInclude Include="Analytics.h" />
     <ClInclude Include="PowerPC\SignatureDB\CSVSignatureDB.h" />
     <ClInclude Include="PowerPC\SignatureDB\DSYSignatureDB.h" />
+    <ClCompile Include="PowerPC\SignatureDB\MEGASignatureDB.h" />
     <ClInclude Include="PowerPC\SignatureDB\SignatureDB.h" />
     <ClInclude Include="IOS\USB\Bluetooth\BTBase.h">
       <Filter>IOS\USB\Bluetooth</Filter>

Source/Core/Core/PowerPC/SignatureDB/MEGASignatureDB.cpp

@@ -0,0 +1,169 @@
+// Copyright 2017 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#include "Core/PowerPC/SignatureDB/MEGASignatureDB.h"
+
+#include <cstddef>
+#include <cstdlib>
+#include <fstream>
+#include <limits>
+
+#include "Common/FileUtil.h"
+#include "Common/Logging/Log.h"
+#include "Common/StringUtil.h"
+
+#include "Core/PowerPC/PPCSymbolDB.h"
+#include "Core/PowerPC/PowerPC.h"
+
+static constexpr size_t INSTRUCTION_HEXSTRING_LENGTH = 8;
+
+MEGASignatureDB::MEGASignatureDB() = default;
+MEGASignatureDB::~MEGASignatureDB() = default;
+
+bool MEGASignatureDB::Load(const std::string& file_path)
+{
+  std::string line;
+  std::ifstream ifs;
+  OpenFStream(ifs, file_path, std::ios_base::in);
+
+  if (!ifs)
+    return false;
+  for (size_t i = 1; std::getline(ifs, line); ++i)
+  {
+    std::istringstream iss(line);
+    MEGASignature sig;
+
+    if (GetCode(&sig, &iss) && GetName(&sig, &iss) && GetRefs(&sig, &iss))
+    {
+      m_signatures.push_back(sig);
+    }
+    else
+    {
+      WARN_LOG(OSHLE, "MEGA database failed to parse line %zu", i);
+    }
+  }
+  return true;
+}
+
+void MEGASignatureDB::Apply(PPCSymbolDB* symbol_db) const
+{
+  for (auto& it : symbol_db->AccessSymbols())
+  {
+    u32 hash = it.first;
+    auto& symbol = it.second;
+    for (const auto& sig : m_signatures)
+    {
+      if (Compare(symbol.address, symbol.size, sig))
+      {
+        symbol.name = sig.name;
+        INFO_LOG(OSHLE, "Found %s at %08x (size: %08x)!", sig.name.c_str(), symbol.address,
+                 symbol.size);
+        break;
+      }
+    }
+  }
+  symbol_db->Index();
+}
+
+void MEGASignatureDB::List() const
+{
+  for (const auto& entry : m_signatures)
+  {
+    DEBUG_LOG(OSHLE, "%s : %zu bytes", entry.name.c_str(), entry.code.size() * sizeof(u32));
+  }
+  INFO_LOG(OSHLE, "%zu functions known in current MEGA database.", m_signatures.size());
+}
+
+bool MEGASignatureDB::GetCode(MEGASignature* sig, std::istringstream* iss) const
+{
+  std::string code;
+  if ((*iss >> code) && (code.length() % INSTRUCTION_HEXSTRING_LENGTH) == 0)
+  {
+    for (size_t i = 0; i < code.length(); i += INSTRUCTION_HEXSTRING_LENGTH)
+    {
+      std::string instruction = code.substr(i, INSTRUCTION_HEXSTRING_LENGTH);
+      u32 num = static_cast<u32>(strtoul(instruction.c_str(), nullptr, 16));
+      if (num != 0 || instruction == "........")
+      {
+        sig->code.emplace_back(num);
+      }
+      else
+      {
+        WARN_LOG(OSHLE, "MEGA database failed to parse code");
+        return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+bool MEGASignatureDB::GetFunctionName(std::istringstream* iss, std::string* name) const
+{
+  std::string buffer;
+
+  std::getline(*iss, buffer);
+  size_t next = buffer.find(" ^");
+  *name = StripSpaces(buffer.substr(0, next));
+
+  if (name->empty())
+    return false;
+
+  if (next == std::string::npos)
+    next = buffer.length();
+  iss->str(buffer.substr(next));
+  iss->clear();
+  return true;
+}
+
+bool MEGASignatureDB::GetName(MEGASignature* sig, std::istringstream* iss) const
+{
+  std::string unknown;
+  return (*iss >> unknown) && GetFunctionName(iss, &sig->name);
+}
+
+bool MEGASignatureDB::GetRefs(MEGASignature* sig, std::istringstream* iss) const
+{
+  std::string num, ref;
+  u32 ref_count = 1;
+  while (*iss && (*iss >> num) && !num.empty())
+  {
+    num = num.substr(1);
+    const char* ptr = num.c_str();
+    char* endptr;
+    u64 offset = strtoul(ptr, &endptr, 16);
+
+    if (ptr == endptr || offset > std::numeric_limits<u32>::max())
+    {
+      WARN_LOG(OSHLE, "MEGA database failed to parse reference %u offset", ref_count);
+      return false;
+    }
+
+    if (!GetFunctionName(iss, &ref))
+    {
+      WARN_LOG(OSHLE, "MEGA database failed to parse reference %u name", ref_count);
+      return false;
+    }
+    sig->refs.emplace_back(static_cast<u32>(offset), ref);
+
+    ref_count += 1;
+    num.clear();
+    ref.clear();
+  }
+  return true;
+}
+
+bool MEGASignatureDB::Compare(u32 address, u32 size, const MEGASignature& sig) const
+{
+  if (size != sig.code.size() * sizeof(u32))
+    return false;
+
+  for (size_t i = 0; i < sig.code.size(); ++i)
+  {
+    if (sig.code[i] != 0 &&
+        PowerPC::HostRead_U32(static_cast<u32>(address + i * sizeof(u32))) != sig.code[i])
+      return false;
+  }
+  return true;
+}

Source/Core/Core/PowerPC/SignatureDB/MEGASignatureDB.h

@@ -0,0 +1,57 @@
+// Copyright 2017 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "Common/CommonTypes.h"
+
+class PPCSymbolDB;
+
+struct MEGASignatureReference
+{
+  MEGASignatureReference(u32 ref_offset, std::string ref_name)
+      : offset(ref_offset), name(std::move(ref_name))
+  {
+  }
+  u32 offset;
+  std::string name;
+};
+
+struct MEGASignature
+{
+  std::vector<u32> code;
+  std::string name;
+  std::vector<MEGASignatureReference> refs;
+};
+
+// MEGA files from Megazig's WiiTools IDA plugin
+//  -> https://github.com/Megazig/WiiTools
+//
+// Each line contains a function signature composed of:
+//  - Hexstring representation with "." acting as a wildcard
+//  - Name, represented as follow: ":0000 function_name"
+//  - References located in the hexstring at offset: "^offset reference_name"
+class MEGASignatureDB
+{
+public:
+  MEGASignatureDB();
+  ~MEGASignatureDB();
+
+  bool Load(const std::string& file_path);
+  void Apply(PPCSymbolDB* symbol_db) const;
+  void List() const;
+
+private:
+  bool GetCode(MEGASignature* sig, std::istringstream* iss) const;
+  bool GetFunctionName(std::istringstream* iss, std::string* name) const;
+  bool GetName(MEGASignature* sig, std::istringstream* iss) const;
+  bool GetRefs(MEGASignature* sig, std::istringstream* iss) const;
+  bool Compare(u32 address, u32 size, const MEGASignature& sig) const;
+
+  std::vector<MEGASignature> m_signatures;
+};

Source/Core/Core/PowerPC/SignatureDB/SignatureDB.cpp

@@ -38,7 +38,7 @@ bool SignatureDB::Save(const std::string& file_path)
 }
 
 // Adds a known function to the hash database
-u32 SignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
+/*u32 SignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
 {
   u32 hash = ComputeCodeChecksum(startAddr, startAddr + size - 4);
 
@@ -51,7 +51,7 @@ u32 SignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
     m_database[hash] = temp_dbfunc;
 
   return hash;
-}
+}*/
 
 void SignatureDB::List()
 {

Source/Core/DolphinWX/Debugger/CodeWindowFunctions.cpp

@@ -34,6 +34,7 @@
 #include "Core/PowerPC/PPCSymbolDB.h"
 #include "Core/PowerPC/PowerPC.h"
 #include "Core/PowerPC/Profiler.h"
+#include "Core/PowerPC/SignatureDB/MEGASignatureDB.h"
 #include "Core/PowerPC/SignatureDB/SignatureDB.h"
 
 #include "DolphinWX/Debugger/BreakpointWindow.h"
@@ -390,6 +391,22 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
     }
   }
   break;
+  case IDM_USE_MEGA_SIGNATURE_FILE:
+  {
+    wxString path = wxFileSelector(
+        _("Apply MEGA signature file"), File::GetSysDirectory(), wxEmptyString, wxEmptyString,
+        _("MEGA Signature File (*.mega)") + "|*.mega|" + wxGetTranslation(wxALL_FILES),
+        wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
+    if (!path.IsEmpty())
+    {
+      MEGASignatureDB db;
+      db.Load(WxStrToStr(path));
+      db.Apply(&g_symbolDB);
+      db.List();
+      NotifyMapLoaded();
+    }
+  }
+  break;
   case IDM_PATCH_HLE_FUNCTIONS:
     HLE::PatchFunctions();
     Repopulate();

Source/Core/DolphinWX/FrameTools.cpp

@@ -243,6 +243,7 @@ void CFrame::BindDebuggerMenuBarUpdateEvents()
   Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_COMBINE_SIGNATURE_FILES);
   Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_RENAME_SYMBOLS);
   Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_USE_SIGNATURE_FILE);
+  Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_USE_MEGA_SIGNATURE_FILE);
   Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_PATCH_HLE_FUNCTIONS);
 
   Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreUninitialized, IDM_JIT_NO_BLOCK_CACHE);

Source/Core/DolphinWX/Globals.h

@@ -228,6 +228,7 @@ enum
   IDM_COMBINE_SIGNATURE_FILES,
   IDM_RENAME_SYMBOLS,
   IDM_USE_SIGNATURE_FILE,
+  IDM_USE_MEGA_SIGNATURE_FILE,
   IDM_PATCH_HLE_FUNCTIONS,
 
   // JIT

Source/Core/DolphinWX/MainMenuBar.cpp

@@ -464,6 +464,10 @@ wxMenu* MainMenuBar::CreateSymbolsMenu() const
       IDM_USE_SIGNATURE_FILE, _("Apply Signat&ure File..."),
       _("Must use Generate Symbols first! Recognise names of any standard library functions "
         "used in multiple games, by loading them from a .dsy file."));
+  symbols_menu->Append(
+      IDM_USE_MEGA_SIGNATURE_FILE, _("Apply &MEGA Signature File..."),
+      _("Must use Generate Symbol Map first! Recognise names of any standard library functions "
+        "used in multiple games, by loading them from a .mega file."));
   symbols_menu->AppendSeparator();
   symbols_menu->Append(IDM_PATCH_HLE_FUNCTIONS, _("&Patch HLE Functions"));
   symbols_menu->Append(IDM_RENAME_SYMBOLS, _("&Rename Symbols from File..."));