Merge pull request #5140 from sepalani/signature_db

SignatureDB: Rewrite FormatHandler API
This commit is contained in:
Mat M 2017-05-02 19:23:46 -04:00 committed by GitHub
commit 7c3905a612
15 changed files with 198 additions and 102 deletions

View File

@ -308,7 +308,7 @@ bool CBoot::BootUp()
if (_StartupPara.bHLE_BS2 && !_StartupPara.bEnableDebugging)
{
PPCAnalyst::FindFunctions(0x80004000, 0x811fffff, &g_symbolDB);
SignatureDB db;
SignatureDB db(SignatureDB::HandlerType::DSY);
if (db.Load(File::GetSysDirectory() + TOTALDB))
{
db.Apply(&g_symbolDB);

View File

@ -97,7 +97,7 @@ bool AnalyzeFunction(u32 startAddr, Symbol& func, int max_size)
{
func.address = startAddr;
func.analyzed = true;
func.hash = SignatureDB::ComputeCodeChecksum(startAddr, addr);
func.hash = HashSignatureDB::ComputeCodeChecksum(startAddr, addr);
if (numInternalBranches == 0)
func.flags |= FFLAG_STRAIGHT;
return true;
@ -121,7 +121,7 @@ bool AnalyzeFunction(u32 startAddr, Symbol& func, int max_size)
// Let's calc the checksum and get outta here
func.address = startAddr;
func.analyzed = true;
func.hash = SignatureDB::ComputeCodeChecksum(startAddr, addr);
func.hash = HashSignatureDB::ComputeCodeChecksum(startAddr, addr);
if (numInternalBranches == 0)
func.flags |= FFLAG_STRAIGHT;
return true;
@ -137,7 +137,7 @@ bool AnalyzeFunction(u32 startAddr, Symbol& func, int max_size)
func.size *= 4; // into bytes
func.address = startAddr;
func.analyzed = 1;
func.hash = SignatureDB::ComputeCodeChecksum(startAddr, addr);
func.hash = HashSignatureDB::ComputeCodeChecksum(startAddr, addr);
if (numInternalBranches == 0)
func.flags |= FFLAG_STRAIGHT;
return true;

View File

@ -72,7 +72,7 @@ void PPCSymbolDB::AddKnownSymbol(u32 startAddr, u32 size, const std::string& nam
Symbol* tempfunc = &iter->second;
tempfunc->name = name;
tempfunc->function_name = GetStrippedFunctionName(name);
tempfunc->hash = SignatureDB::ComputeCodeChecksum(startAddr, startAddr + size - 4);
tempfunc->hash = HashSignatureDB::ComputeCodeChecksum(startAddr, startAddr + size - 4);
tempfunc->type = type;
tempfunc->size = size;
}

View File

@ -13,7 +13,7 @@
// CSV separated with tabs
// Checksum | Size | Symbol | [Object Location |] Object Name
bool CSVSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& database) const
bool CSVSignatureDB::Load(const std::string& file_path)
{
std::string line;
std::ifstream ifs;
@ -32,7 +32,7 @@ bool CSVSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& dat
{
if (std::getline(iss, symbol, '\t') && std::getline(iss, object_location, '\t'))
std::getline(iss, object_name);
SignatureDB::DBFunc func;
HashSignatureDB::DBFunc func;
func.name = symbol;
func.size = size;
// Doesn't have an object location
@ -45,7 +45,7 @@ bool CSVSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& dat
func.object_location = object_location;
func.object_name = object_name;
}
database[checksum] = func;
m_database[checksum] = func;
}
else
{
@ -56,7 +56,7 @@ bool CSVSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& dat
return true;
}
bool CSVSignatureDB::Save(const std::string& file_path, const SignatureDB::FuncDB& database) const
bool CSVSignatureDB::Save(const std::string& file_path) const
{
File::IOFile f(file_path, "w");
@ -65,7 +65,7 @@ bool CSVSignatureDB::Save(const std::string& file_path, const SignatureDB::FuncD
ERROR_LOG(OSHLE, "CSV database save failed");
return false;
}
for (const auto& func : database)
for (const auto& func : m_database)
{
// The object name/location are unused for the time being.
// To be implemented.

View File

@ -6,10 +6,10 @@
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
class CSVSignatureDB final : public SignatureDBFormatHandler
class CSVSignatureDB final : public HashSignatureDB
{
public:
~CSVSignatureDB() = default;
bool Load(const std::string& file_path, SignatureDB::FuncDB& database) const override;
bool Save(const std::string& file_path, const SignatureDB::FuncDB& database) const override;
bool Load(const std::string& file_path) override;
bool Save(const std::string& file_path) const override;
};

View File

@ -22,7 +22,7 @@ struct FuncDesc
};
} // namespace
bool DSYSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& database) const
bool DSYSignatureDB::Load(const std::string& file_path)
{
File::IOFile f(file_path, "rb");
@ -38,16 +38,16 @@ bool DSYSignatureDB::Load(const std::string& file_path, SignatureDB::FuncDB& dat
f.ReadArray(&temp, 1);
temp.name[sizeof(temp.name) - 1] = 0;
SignatureDB::DBFunc func;
HashSignatureDB::DBFunc func;
func.name = temp.name;
func.size = temp.size;
database[temp.checksum] = func;
m_database[temp.checksum] = func;
}
return true;
}
bool DSYSignatureDB::Save(const std::string& file_path, const SignatureDB::FuncDB& database) const
bool DSYSignatureDB::Save(const std::string& file_path) const
{
File::IOFile f(file_path, "wb");
@ -56,9 +56,9 @@ bool DSYSignatureDB::Save(const std::string& file_path, const SignatureDB::FuncD
ERROR_LOG(OSHLE, "Database save failed");
return false;
}
u32 fcount = static_cast<u32>(database.size());
u32 fcount = static_cast<u32>(m_database.size());
f.WriteArray(&fcount, 1);
for (const auto& entry : database)
for (const auto& entry : m_database)
{
FuncDesc temp;
memset(&temp, 0, sizeof(temp));

View File

@ -6,10 +6,10 @@
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
class DSYSignatureDB final : public SignatureDBFormatHandler
class DSYSignatureDB final : public HashSignatureDB
{
public:
~DSYSignatureDB() = default;
bool Load(const std::string& file_path, SignatureDB::FuncDB& database) const override;
bool Save(const std::string& file_path, const SignatureDB::FuncDB& database) const override;
bool Load(const std::string& file_path) override;
bool Save(const std::string& file_path) const override;
};

View File

@ -119,6 +119,11 @@ bool Compare(u32 address, u32 size, const MEGASignature& sig)
MEGASignatureDB::MEGASignatureDB() = default;
MEGASignatureDB::~MEGASignatureDB() = default;
void MEGASignatureDB::Clear()
{
m_signatures.clear();
}
bool MEGASignatureDB::Load(const std::string& file_path)
{
std::ifstream ifs;
@ -145,6 +150,12 @@ bool MEGASignatureDB::Load(const std::string& file_path)
return true;
}
bool MEGASignatureDB::Save(const std::string& file_path) const
{
ERROR_LOG(OSHLE, "MEGA database save unsupported yet.");
return false;
}
void MEGASignatureDB::Apply(PPCSymbolDB* symbol_db) const
{
for (auto& it : symbol_db->AccessSymbols())
@ -164,6 +175,17 @@ void MEGASignatureDB::Apply(PPCSymbolDB* symbol_db) const
symbol_db->Index();
}
void MEGASignatureDB::Populate(const PPCSymbolDB* func_db, const std::string& filter)
{
ERROR_LOG(OSHLE, "MEGA database can't be populated yet.");
}
bool MEGASignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
{
ERROR_LOG(OSHLE, "Can't add symbol to MEGA database yet.");
return false;
}
void MEGASignatureDB::List() const
{
for (const auto& entry : m_signatures)

View File

@ -8,6 +8,7 @@
#include <vector>
#include "Common/CommonTypes.h"
#include "Core/PowerPC/SignatureDB/SignatureDB.h"
class PPCSymbolDB;
@ -35,16 +36,22 @@ struct MEGASignature
// - Hexstring representation with "." acting as a wildcard
// - Name, represented as follow: ":0000 function_name"
// - References located in the hexstring at offset: "^offset reference_name"
class MEGASignatureDB
class MEGASignatureDB : public SignatureDBFormatHandler
{
public:
MEGASignatureDB();
~MEGASignatureDB();
void Clear();
bool Load(const std::string& file_path);
void Apply(PPCSymbolDB* symbol_db) const;
bool Save(const std::string& file_path) const;
void List() const;
void Apply(PPCSymbolDB* symbol_db) const;
void Populate(const PPCSymbolDB* func_db, const std::string& filter = "");
bool Add(u32 startAddr, u32 size, const std::string& name);
private:
std::vector<MEGASignature> m_signatures;
};

View File

@ -16,29 +16,78 @@
// Format Handlers
#include "Core/PowerPC/SignatureDB/CSVSignatureDB.h"
#include "Core/PowerPC/SignatureDB/DSYSignatureDB.h"
#include "Core/PowerPC/SignatureDB/MEGASignatureDB.h"
std::unique_ptr<SignatureDBFormatHandler>
SignatureDB::CreateFormatHandler(const std::string& file_path)
SignatureDB::SignatureDB(SignatureDB::HandlerType handler)
: m_handler(std::move(CreateFormatHandler(handler)))
{
}
SignatureDB::SignatureDB(const std::string& file_path) : SignatureDB(GetHandlerType(file_path))
{
}
SignatureDB::HandlerType SignatureDB::GetHandlerType(const std::string& file_path)
{
if (StringEndsWith(file_path, ".csv"))
return SignatureDB::HandlerType::CSV;
if (StringEndsWith(file_path, ".mega"))
return SignatureDB::HandlerType::MEGA;
return SignatureDB::HandlerType::DSY;
}
std::unique_ptr<SignatureDBFormatHandler>
SignatureDB::CreateFormatHandler(SignatureDB::HandlerType handler) const
{
switch (handler)
{
default:
case SignatureDB::HandlerType::DSY:
return std::make_unique<DSYSignatureDB>();
case SignatureDB::HandlerType::CSV:
return std::make_unique<CSVSignatureDB>();
return std::make_unique<DSYSignatureDB>();
case SignatureDB::HandlerType::MEGA:
return std::make_unique<MEGASignatureDB>();
}
}
void SignatureDB::Clear()
{
m_handler->Clear();
}
bool SignatureDB::Load(const std::string& file_path)
{
auto handler = CreateFormatHandler(file_path);
return handler->Load(file_path, m_database);
return m_handler->Load(file_path);
}
bool SignatureDB::Save(const std::string& file_path)
bool SignatureDB::Save(const std::string& file_path) const
{
auto handler = CreateFormatHandler(file_path);
return handler->Save(file_path, m_database);
return m_handler->Save(file_path);
}
void SignatureDB::List() const
{
m_handler->List();
}
void SignatureDB::Populate(const PPCSymbolDB* func_db, const std::string& filter)
{
m_handler->Populate(func_db, filter);
}
void SignatureDB::Apply(PPCSymbolDB* func_db) const
{
m_handler->Apply(func_db);
}
bool SignatureDB::Add(u32 start_addr, u32 size, const std::string& name)
{
return m_handler->Add(start_addr, size, name);
}
// Adds a known function to the hash database
/*u32 SignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
bool HashSignatureDB::Add(u32 startAddr, u32 size, const std::string& name)
{
u32 hash = ComputeCodeChecksum(startAddr, startAddr + size - 4);
@ -48,12 +97,14 @@ bool SignatureDB::Save(const std::string& file_path)
FuncDB::iterator iter = m_database.find(hash);
if (iter == m_database.end())
{
m_database[hash] = temp_dbfunc;
return true;
}
return false;
}
return hash;
}*/
void SignatureDB::List()
void HashSignatureDB::List() const
{
for (const auto& entry : m_database)
{
@ -63,12 +114,12 @@ void SignatureDB::List()
INFO_LOG(OSHLE, "%zu functions known in current database.", m_database.size());
}
void SignatureDB::Clear()
void HashSignatureDB::Clear()
{
m_database.clear();
}
void SignatureDB::Apply(PPCSymbolDB* symbol_db)
void HashSignatureDB::Apply(PPCSymbolDB* symbol_db) const
{
for (const auto& entry : m_database)
{
@ -92,13 +143,13 @@ void SignatureDB::Apply(PPCSymbolDB* symbol_db)
symbol_db->Index();
}
void SignatureDB::Initialize(PPCSymbolDB* symbol_db, const std::string& prefix)
void HashSignatureDB::Populate(const PPCSymbolDB* symbol_db, const std::string& filter)
{
for (const auto& symbol : symbol_db->Symbols())
{
if ((prefix.empty() && (!symbol.second.name.empty()) &&
if ((filter.empty() && (!symbol.second.name.empty()) &&
symbol.second.name.substr(0, 3) != "zz_" && symbol.second.name.substr(0, 1) != ".") ||
((!prefix.empty()) && symbol.second.name.substr(0, prefix.size()) == prefix))
((!filter.empty()) && symbol.second.name.substr(0, filter.size()) == filter))
{
DBFunc temp_dbfunc;
temp_dbfunc.name = symbol.second.name;
@ -108,7 +159,7 @@ void SignatureDB::Initialize(PPCSymbolDB* symbol_db, const std::string& prefix)
}
}
/*static*/ u32 SignatureDB::ComputeCodeChecksum(u32 offsetStart, u32 offsetEnd)
u32 HashSignatureDB::ComputeCodeChecksum(u32 offsetStart, u32 offsetEnd)
{
u32 sum = 0;
for (u32 offset = offsetStart; offset <= offsetEnd; offset += 4)

View File

@ -17,6 +17,52 @@ class SignatureDBFormatHandler;
class SignatureDB
{
public:
enum class HandlerType
{
DSY,
CSV,
MEGA
};
explicit SignatureDB(HandlerType handler);
explicit SignatureDB(const std::string& file_path);
static HandlerType GetHandlerType(const std::string& file_path);
void Clear();
// Does not clear. Remember to clear first if that's what you want.
bool Load(const std::string& file_path);
bool Save(const std::string& file_path) const;
void List() const;
void Populate(const PPCSymbolDB* func_db, const std::string& filter = "");
void Apply(PPCSymbolDB* func_db) const;
bool Add(u32 start_addr, u32 size, const std::string& name);
private:
std::unique_ptr<SignatureDBFormatHandler> CreateFormatHandler(HandlerType handler) const;
std::unique_ptr<SignatureDBFormatHandler> m_handler;
};
class SignatureDBFormatHandler
{
public:
virtual ~SignatureDBFormatHandler();
virtual void Clear() = 0;
virtual bool Load(const std::string& file_path) = 0;
virtual bool Save(const std::string& file_path) const = 0;
virtual void List() const = 0;
virtual void Populate(const PPCSymbolDB* func_db, const std::string& filter = "") = 0;
virtual void Apply(PPCSymbolDB* func_db) const = 0;
virtual bool Add(u32 startAddr, u32 size, const std::string& name) = 0;
};
class HashSignatureDB : public SignatureDBFormatHandler
{
public:
struct DBFunc
{
@ -28,31 +74,18 @@ public:
};
using FuncDB = std::map<u32, DBFunc>;
// Returns the hash.
u32 Add(u32 startAddr, u32 size, const std::string& name);
// Does not clear. Remember to clear first if that's what you want.
bool Load(const std::string& file_path);
bool Save(const std::string& file_path);
void Clear();
void List();
void Initialize(PPCSymbolDB* func_db, const std::string& prefix = "");
void Apply(PPCSymbolDB* func_db);
static u32 ComputeCodeChecksum(u32 offsetStart, u32 offsetEnd);
private:
std::unique_ptr<SignatureDBFormatHandler> CreateFormatHandler(const std::string& file_path);
virtual void Clear() override;
virtual void List() const override;
virtual void Populate(const PPCSymbolDB* func_db, const std::string& filter = "") override;
virtual void Apply(PPCSymbolDB* func_db) const override;
virtual bool Add(u32 startAddr, u32 size, const std::string& name) override;
protected:
// Map from signature to function. We store the DB in this map because it optimizes the
// most common operation - lookup. We don't care about ordering anyway.
FuncDB m_database;
};
class SignatureDBFormatHandler
{
public:
virtual ~SignatureDBFormatHandler();
virtual bool Load(const std::string& file_path, SignatureDB::FuncDB& database) const = 0;
virtual bool Save(const std::string& file_path, const SignatureDB::FuncDB& database) const = 0;
};

View File

@ -166,7 +166,8 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
{
static const wxString signature_selector = _("Dolphin Signature File (*.dsy)") + "|*.dsy|" +
_("Dolphin Signature CSV File (*.csv)") + "|*.csv|" +
wxGetTranslation(wxALL_FILES);
_("WiiTools Signature MEGA File (*.mega)") +
"|*.mega|" + wxGetTranslation(wxALL_FILES);
Parent->ClearStatusBar();
if (!Core::IsRunning())
@ -190,7 +191,7 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
case IDM_SCAN_SIGNATURES:
{
PPCAnalyst::FindFunctions(0x80000000, 0x81800000, &g_symbolDB);
SignatureDB db;
SignatureDB db(SignatureDB::HandlerType::DSY);
if (db.Load(File::GetSysDirectory() + TOTALDB))
{
db.Apply(&g_symbolDB);
@ -239,7 +240,7 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
{
g_symbolDB.Clear();
PPCAnalyst::FindFunctions(0x81300000, 0x81800000, &g_symbolDB);
SignatureDB db;
SignatureDB db(SignatureDB::HandlerType::DSY);
if (db.Load(File::GetSysDirectory() + TOTALDB))
db.Apply(&g_symbolDB);
Parent->StatusBarMessage("'%s' not found, scanning for common functions instead",
@ -352,9 +353,10 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
wxFD_SAVE | wxFD_OVERWRITE_PROMPT, this);
if (!path.IsEmpty())
{
SignatureDB db;
db.Initialize(&g_symbolDB, prefix);
db.Save(WxStrToStr(path));
std::string save_path = WxStrToStr(path);
SignatureDB db(save_path);
db.Populate(&g_symbolDB, prefix);
db.Save(save_path);
db.List();
}
}
@ -375,11 +377,12 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
wxEmptyString, signature_selector, wxFD_SAVE, this);
if (!path.IsEmpty())
{
SignatureDB db;
db.Initialize(&g_symbolDB, prefix);
std::string signature_path = WxStrToStr(path);
SignatureDB db(signature_path);
db.Populate(&g_symbolDB, prefix);
db.List();
db.Load(WxStrToStr(path));
db.Save(WxStrToStr(path));
db.Load(signature_path);
db.Save(signature_path);
db.List();
}
}
@ -392,8 +395,9 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
wxEmptyString, signature_selector, wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path.IsEmpty())
{
SignatureDB db;
db.Load(WxStrToStr(path));
std::string load_path = WxStrToStr(path);
SignatureDB db(load_path);
db.Load(load_path);
db.Apply(&g_symbolDB);
db.List();
NotifyMapLoaded();
@ -407,14 +411,15 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
wxEmptyString, signature_selector, wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path1.IsEmpty())
{
SignatureDB db;
std::string load_path1 = WxStrToStr(path1);
SignatureDB db(load_path1);
wxString path2 =
wxFileSelector(_("Choose secondary input file"), File::GetSysDirectory(), wxEmptyString,
wxEmptyString, signature_selector, wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path2.IsEmpty())
{
db.Load(load_path1);
db.Load(WxStrToStr(path2));
db.Load(WxStrToStr(path1));
path2 = wxFileSelector(_("Save combined output file as"), File::GetSysDirectory(),
wxEmptyString, ".dsy", signature_selector,
@ -425,22 +430,6 @@ void CCodeWindow::OnSymbolsMenu(wxCommandEvent& event)
}
}
break;
case IDM_USE_MEGA_SIGNATURE_FILE:
{
wxString path = wxFileSelector(
_("Apply MEGA signature file"), File::GetSysDirectory(), wxEmptyString, wxEmptyString,
_("MEGA Signature File (*.mega)") + "|*.mega|" + wxGetTranslation(wxALL_FILES),
wxFD_OPEN | wxFD_FILE_MUST_EXIST, this);
if (!path.IsEmpty())
{
MEGASignatureDB db;
db.Load(WxStrToStr(path));
db.Apply(&g_symbolDB);
db.List();
NotifyMapLoaded();
}
}
break;
case IDM_PATCH_HLE_FUNCTIONS:
HLE::PatchFunctions();
Repopulate();

View File

@ -248,7 +248,6 @@ void CFrame::BindDebuggerMenuBarUpdateEvents()
Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_COMBINE_SIGNATURE_FILES);
Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_RENAME_SYMBOLS);
Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_USE_SIGNATURE_FILE);
Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_USE_MEGA_SIGNATURE_FILE);
Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreInitialized, IDM_PATCH_HLE_FUNCTIONS);
Bind(wxEVT_UPDATE_UI, &WxEventUtils::OnEnableIfCoreUninitialized, IDM_JIT_NO_BLOCK_CACHE);

View File

@ -233,7 +233,6 @@ enum
IDM_COMBINE_SIGNATURE_FILES,
IDM_RENAME_SYMBOLS,
IDM_USE_SIGNATURE_FILE,
IDM_USE_MEGA_SIGNATURE_FILE,
IDM_PATCH_HLE_FUNCTIONS,
// JIT

View File

@ -471,11 +471,7 @@ wxMenu* MainMenuBar::CreateSymbolsMenu() const
symbols_menu->Append(
IDM_USE_SIGNATURE_FILE, _("Apply Signat&ure File..."),
_("Must use Generate Symbols first! Recognise names of any standard library functions "
"used in multiple games, by loading them from a .dsy file."));
symbols_menu->Append(
IDM_USE_MEGA_SIGNATURE_FILE, _("Apply &MEGA Signature File..."),
_("Must use Generate Symbols first! Recognise names of any standard library functions "
"used in multiple games, by loading them from a .mega file."));
"used in multiple games, by loading them from a .dsy, .csv, or .mega file."));
symbols_menu->AppendSeparator();
symbols_menu->Append(IDM_PATCH_HLE_FUNCTIONS, _("&Patch HLE Functions"));
symbols_menu->Append(IDM_RENAME_SYMBOLS, _("&Rename Symbols from File..."));