From 00ec82019820af48df2431516fd6e4bc54bcc6a3 Mon Sep 17 00:00:00 2001 From: "Jake.Stine" Date: Mon, 28 Jun 2010 03:05:10 +0000 Subject: [PATCH] GameDB: Fix long loading times of the database, by using a very efficient block-based allocation system, and reserving enough memory for about 9500 titles. Also added a thread at startup to load the gamedb in the background while PCSX2 is opening windows and printing to the console. git-svn-id: http://pcsx2.googlecode.com/svn/trunk@3330 96395faa-99c1-11dd-bbfe-3dabce05a288 --- pcsx2/GameDatabase.cpp | 70 ++++++++++++++++++++++++++++------- pcsx2/GameDatabase.h | 47 +++++++++++------------ pcsx2/gui/AppGameDatabase.cpp | 64 +++++++++++++++++++++++--------- pcsx2/gui/AppInit.cpp | 33 ++++++++++++++++- pcsx2/gui/AppMain.cpp | 18 --------- pcsx2/gui/ExecutorThread.cpp | 2 +- 6 files changed, 161 insertions(+), 73 deletions(-) diff --git a/pcsx2/GameDatabase.cpp b/pcsx2/GameDatabase.cpp index e08672710d..d69ab7e5d1 100644 --- a/pcsx2/GameDatabase.cpp +++ b/pcsx2/GameDatabase.cpp @@ -16,6 +16,35 @@ #include "PrecompiledHeader.h" #include "GameDatabase.h" +BaseGameDatabaseImpl::BaseGameDatabaseImpl() + : gHash( 9400 ) + , m_baseKey( L"Serial" ) +{ + m_BlockTable.reserve(48); + + m_CurBlockWritePos = 0; + m_BlockTableWritePos = 0; + + m_GamesPerBlock = 256; + + m_BlockTable.push_back(NULL); +} + +BaseGameDatabaseImpl::~BaseGameDatabaseImpl() throw() +{ + for(uint blockidx=0; blockidx<=m_BlockTableWritePos; ++blockidx) + { + if( !m_BlockTable[blockidx] ) continue; + + const uint endidx = (blockidx == m_BlockTableWritePos) ? m_CurBlockWritePos : m_GamesPerBlock; + + for( uint gameidx=0; gameidxsecond]; + dest = *iter->second; return true; } -void BaseGameDatabaseImpl::addNewGame(const Game_Data& game) +Game_Data* BaseGameDatabaseImpl::createNewGame( const wxString& id ) { - gList.push_back(game); - gHash[game.id] = gList.size()-1; + if(!m_BlockTable[m_BlockTableWritePos]) + m_BlockTable[m_BlockTableWritePos] = (Game_Data*)malloc(m_GamesPerBlock * sizeof(Game_Data)); + + Game_Data* block = m_BlockTable[m_BlockTableWritePos]; + Game_Data* retval = &block[m_CurBlockWritePos]; + + gHash[id] = retval; + + new (retval) Game_Data(id); + + if( ++m_CurBlockWritePos >= m_GamesPerBlock ) + { + ++m_BlockTableWritePos; + m_CurBlockWritePos = 0; + m_BlockTable.push_back(NULL); + } + + return retval; } void BaseGameDatabaseImpl::updateGame(const Game_Data& game) @@ -40,13 +85,12 @@ void BaseGameDatabaseImpl::updateGame(const Game_Data& game) GameDataHash::const_iterator iter( gHash.find(game.id) ); if( iter == gHash.end() ) { - gList.push_back(game); - gHash[game.id] = gList.size()-1; + *(createNewGame( game.id )) = game; } else { // Re-assign existing vector/array entry! - gList[gHash[game.id]] = game; + *gHash[game.id] = game; } } @@ -54,7 +98,7 @@ void BaseGameDatabaseImpl::updateGame(const Game_Data& game) bool Game_Data::keyExists(const wxChar* key) const { KeyPairArray::const_iterator it( kList.begin() ); for ( ; it != kList.end(); ++it) { - if (it[0].CompareKey(key)) { + if (it->CompareKey(key)) { return true; } } @@ -65,7 +109,7 @@ bool Game_Data::keyExists(const wxChar* key) const { void Game_Data::deleteKey(const wxChar* key) { KeyPairArray::iterator it( kList.begin() ); for ( ; it != kList.end(); ++it) { - if (it[0].CompareKey(key)) { + if (it->CompareKey(key)) { kList.erase(it); return; } @@ -76,8 +120,8 @@ void Game_Data::deleteKey(const wxChar* key) { wxString Game_Data::getString(const wxChar* key) const { KeyPairArray::const_iterator it( kList.begin() ); for ( ; it != kList.end(); ++it) { - if (it[0].CompareKey(key)) { - return it[0].value; + if (it->CompareKey(key)) { + return it->value; } } return wxString(); @@ -86,11 +130,11 @@ wxString Game_Data::getString(const wxChar* key) const { void Game_Data::writeString(const wxString& key, const wxString& value) { KeyPairArray::iterator it( kList.begin() ); for ( ; it != kList.end(); ++it) { - if (it[0].CompareKey(key)) { + if (it->CompareKey(key)) { if( value.IsEmpty() ) kList.erase(it); else - it[0].value = value; + it->value = value; return; } } diff --git a/pcsx2/GameDatabase.h b/pcsx2/GameDatabase.h index 5d40acf319..fc15e2bfb4 100644 --- a/pcsx2/GameDatabase.h +++ b/pcsx2/GameDatabase.h @@ -18,12 +18,25 @@ //#include "Common.h" #include "AppConfig.h" #include "Utilities/HashMap.h" +#include "Utilities/SafeArray.h" #include struct key_pair; struct Game_Data; +class StringHashNoCase +{ +public: + StringHashNoCase() {} + + HashTools::hash_key_t operator()( const wxString& src ) const + { + return HashTools::Hash( (const char *)src.Lower().data(), src.length() * sizeof( wxChar ) ); + } +}; + + typedef std::vector KeyPairArray; struct key_pair { @@ -159,23 +172,11 @@ public: virtual wxString getBaseKey() const=0; virtual bool findGame(Game_Data& dest, const wxString& id)=0; - virtual void addNewGame(const Game_Data& game)=0; + virtual Game_Data* createNewGame( const wxString& id )=0; virtual void updateGame(const Game_Data& game)=0; }; -class StringHashNoCase -{ -public: - StringHashNoCase() {} - - HashTools::hash_key_t operator()( const wxString& src ) const - { - return HashTools::Hash( (const char *)src.Lower().data(), src.length() * sizeof( wxChar ) ); - } -}; - -typedef std::vector GameDataArray; -typedef pxDictionary GameDataHash; +typedef pxDictionary GameDataHash; // -------------------------------------------------------------------------------------- // BaseGameDatabaseImpl @@ -184,24 +185,24 @@ typedef pxDictionary GameDataHash; // faster that way. class BaseGameDatabaseImpl : public IGameDatabase { -public: - GameDataArray gList; // List of all game data +protected: GameDataHash gHash; // hash table of game serials matched to their gList indexes! wxString m_baseKey; -public: - BaseGameDatabaseImpl() - { - m_baseKey = L"Serial"; - } + std::vector m_BlockTable; + uint m_BlockTableWritePos; + int m_CurBlockWritePos; + int m_GamesPerBlock; - virtual ~BaseGameDatabaseImpl() throw() {} +public: + BaseGameDatabaseImpl(); + virtual ~BaseGameDatabaseImpl() throw(); wxString getBaseKey() const { return m_baseKey; } void setBaseKey( const wxString& key ) { m_baseKey = key; } bool findGame(Game_Data& dest, const wxString& id); - void addNewGame(const Game_Data& game); + Game_Data* createNewGame( const wxString& id ); void updateGame(const Game_Data& game); }; diff --git a/pcsx2/gui/AppGameDatabase.cpp b/pcsx2/gui/AppGameDatabase.cpp index a59c2ee417..e20a21ccb7 100644 --- a/pcsx2/gui/AppGameDatabase.cpp +++ b/pcsx2/gui/AppGameDatabase.cpp @@ -118,13 +118,14 @@ wxString DBLoaderHelper::ReadHeader() void DBLoaderHelper::ReadGames() { - Game_Data game; + Game_Data* game = NULL; if (m_keyPair.IsOk()) { - game.writeString(m_keyPair.key, m_keyPair.value); - if( m_keyPair.CompareKey(m_gamedb.getBaseKey()) ) - game.id = m_keyPair.value; + game = m_gamedb.createNewGame(m_keyPair.value); + game->writeString(m_keyPair.key, m_keyPair.value); + //if( m_keyPair.CompareKey(m_gamedb.getBaseKey()) ) + // game.id = m_keyPair.value; } while(!m_reader.Eof()) { // Fill game data, find new game, repeat... @@ -136,13 +137,10 @@ void DBLoaderHelper::ReadGames() if (!extractMultiLine()) extract(); if (!m_keyPair.IsOk()) continue; - if( m_keyPair.CompareKey(m_gamedb.getBaseKey()) ) - { - m_gamedb.addNewGame(game); - game.clear(); - game.id = m_keyPair.value; - } - game.writeString( m_keyPair.key, m_keyPair.value ); + if (m_keyPair.CompareKey(m_gamedb.getBaseKey())) + game = m_gamedb.createNewGame(m_keyPair.value); + + game->writeString( m_keyPair.key, m_keyPair.value ); } } @@ -168,8 +166,13 @@ AppGameDatabase& AppGameDatabase::LoadFromFile(const wxString& file, const wxStr DBLoaderHelper loader( reader, *this ); + u64 qpc_Start = GetCPUTicks(); header = loader.ReadHeader(); loader.ReadGames(); + u64 qpc_end = GetCPUTicks(); + + Console.WriteLn( "(GameDB) %d games on record (loaded in %ums)", + gHash.size(), (u32)(((qpc_end-qpc_Start)*1000) / GetTickFrequency()) ); return *this; } @@ -178,13 +181,40 @@ AppGameDatabase& AppGameDatabase::LoadFromFile(const wxString& file, const wxStr void AppGameDatabase::SaveToFile(const wxString& file) { wxFFileOutputStream writer( file ); pxWriteMultiline(writer, header); - GameDataArray::iterator it( gList.begin() ); - for ( ; it != gList.end(); ++it) { - KeyPairArray::iterator i = it[0].kList.begin(); - for ( ; i != it[0].kList.end(); ++i) { - pxWriteMultiline(writer, i[0].toString() ); + + for(uint blockidx=0; blockidx<=m_BlockTableWritePos; ++blockidx) + { + if( !m_BlockTable[blockidx] ) continue; + + const uint endidx = (blockidx == m_BlockTableWritePos) ? m_CurBlockWritePos : m_GamesPerBlock; + + for( uint gameidx=0; gameidx<=endidx; ++gameidx ) + { + const Game_Data& game( m_BlockTable[blockidx][gameidx] ); + KeyPairArray::const_iterator i(game.kList.begin()); + for ( ; i != game.kList.end(); ++i) { + pxWriteMultiline(writer, i->toString() ); + } + + pxWriteLine(writer, L"---------------------------------------------"); } - pxWriteLine(writer, L"---------------------------------------------"); } } +AppGameDatabase* Pcsx2App::GetGameDatabase() +{ + pxAppResources& res( GetResourceCache() ); + + ScopedLock lock( m_mtx_LoadingGameDB ); + if( !res.GameDB ) + { + res.GameDB = new AppGameDatabase(); + res.GameDB->LoadFromFile(); + } + return res.GameDB; +} + +IGameDatabase* AppHost_GetGameDatabase() +{ + return wxGetApp().GetGameDatabase(); +} diff --git a/pcsx2/gui/AppInit.cpp b/pcsx2/gui/AppInit.cpp index b1541558d0..3b5375d4df 100644 --- a/pcsx2/gui/AppInit.cpp +++ b/pcsx2/gui/AppInit.cpp @@ -475,6 +475,35 @@ bool Pcsx2App::OnCmdLineParsed( wxCmdLineParser& parser ) typedef void (wxEvtHandler::*pxInvokeAppMethodEventFunction)(Pcsx2AppMethodEvent&); typedef void (wxEvtHandler::*pxStuckThreadEventHandler)(pxMessageBoxEvent&); +// -------------------------------------------------------------------------------------- +// CompressThread_gzip +// -------------------------------------------------------------------------------------- +class GameDatabaseLoaderThread : public pxThread +{ + typedef pxThread _parent; + +protected: + gzFile m_gzfp; + +public: + GameDatabaseLoaderThread() : pxThread( L"GameDatabaseLoader" ) {} + virtual ~GameDatabaseLoaderThread() throw() + { + _parent::Cancel(); + } + +protected: + void ExecuteTaskInThread() + { + wxGetApp().GetGameDatabase(); + } + + void OnCleanupInThread() + { + wxGetApp().DeleteThread(this); + } +}; + bool Pcsx2App::OnInit() { EnableAllLogging(); @@ -535,9 +564,11 @@ bool Pcsx2App::OnInit() // ------------------------------------- if( Startup.ForceConsole ) g_Conf->ProgLogBox.Visible = true; OpenProgramLog(); - if( m_UseGUI ) OpenMainFrame(); AllocateCoreStuffs(); + if( m_UseGUI ) OpenMainFrame(); + (new GameDatabaseLoaderThread())->Start(); + if( Startup.SysAutoRun ) { // Notes: Saving/remembering the Iso file is probably fine and desired, so using diff --git a/pcsx2/gui/AppMain.cpp b/pcsx2/gui/AppMain.cpp index 7da8e89300..5a10996d9f 100644 --- a/pcsx2/gui/AppMain.cpp +++ b/pcsx2/gui/AppMain.cpp @@ -1088,21 +1088,3 @@ SysCoreAllocations& GetSysCoreAlloc() { return *wxGetApp().m_CoreAllocs; } - -AppGameDatabase* Pcsx2App::GetGameDatabase() -{ - pxAppResources& res( GetResourceCache() ); - - ScopedLock lock( m_mtx_LoadingGameDB ); - if( !res.GameDB ) - { - res.GameDB = new AppGameDatabase(); - res.GameDB->LoadFromFile(); - } - return res.GameDB; -} - -IGameDatabase* AppHost_GetGameDatabase() -{ - return wxGetApp().GetGameDatabase(); -} diff --git a/pcsx2/gui/ExecutorThread.cpp b/pcsx2/gui/ExecutorThread.cpp index 003bb465ed..4bb0ca1c03 100644 --- a/pcsx2/gui/ExecutorThread.cpp +++ b/pcsx2/gui/ExecutorThread.cpp @@ -190,7 +190,7 @@ void pxEvtHandler::ProcessEvents( pxEvtList& list ) } u64 qpc_end = GetCPUTicks(); - DevCon.WriteLn( L"(pxEvtHandler) Event '%s' completed in %dms", deleteMe->GetEventName().c_str(), ((qpc_end-m_qpc_Start)*1000) / GetTickFrequency() ); + DevCon.WriteLn( L"(pxEvtHandler) Event '%s' completed in %ums", deleteMe->GetEventName().c_str(), (u32)(((qpc_end-m_qpc_Start)*1000) / GetTickFrequency()) ); synclock.Acquire(); m_qpc_Start = 0; // lets the main thread know the message completed.