GameDB: Fix long loading times of the database, by using a very efficient block-based allocation system, and reserving enough memory for about 9500 titles. Also added a thread at startup to load the gamedb in the background while PCSX2 is opening windows and printing to the console.

git-svn-id: http://pcsx2.googlecode.com/svn/trunk@3330 96395faa-99c1-11dd-bbfe-3dabce05a288

pcsx2/GameDatabase.cpp

@@ -16,6 +16,35 @@
 #include "PrecompiledHeader.h"
 #include "GameDatabase.h"
 
+BaseGameDatabaseImpl::BaseGameDatabaseImpl()
+	: gHash( 9400 )
+	, m_baseKey( L"Serial" )
+{
+	m_BlockTable.reserve(48);
+
+	m_CurBlockWritePos		= 0;
+	m_BlockTableWritePos	= 0;
+
+	m_GamesPerBlock			= 256;
+
+	m_BlockTable.push_back(NULL);
+}
+
+BaseGameDatabaseImpl::~BaseGameDatabaseImpl() throw()
+{
+	for(uint blockidx=0; blockidx<=m_BlockTableWritePos; ++blockidx)
+	{
+		if( !m_BlockTable[blockidx] ) continue;
+
+		const uint endidx = (blockidx == m_BlockTableWritePos) ? m_CurBlockWritePos : m_GamesPerBlock;
+
+		for( uint gameidx=0; gameidx<endidx; ++gameidx )
+			m_BlockTable[blockidx][gameidx].~Game_Data();
+
+		safe_free( m_BlockTable[blockidx] );
+	}
+}
+
 // Sets the current game to the one matching the serial id given
 // Returns true if game found, false if not found...
 bool BaseGameDatabaseImpl::findGame(Game_Data& dest, const wxString& id) {
@@ -25,14 +54,30 @@ bool BaseGameDatabaseImpl::findGame(Game_Data& dest, const wxString& id) {
 		dest.clear();
 		return false;
 	}
-	dest = gList[iter->second];
+	dest = *iter->second;
 	return true;
 }
 
-void BaseGameDatabaseImpl::addNewGame(const Game_Data& game)
+Game_Data* BaseGameDatabaseImpl::createNewGame( const wxString& id )
 {
-	gList.push_back(game);
+	if(!m_BlockTable[m_BlockTableWritePos])
-	gHash[game.id] = gList.size()-1;
+		m_BlockTable[m_BlockTableWritePos] = (Game_Data*)malloc(m_GamesPerBlock * sizeof(Game_Data));
+
+	Game_Data* block = m_BlockTable[m_BlockTableWritePos];
+	Game_Data* retval = &block[m_CurBlockWritePos];
+
+	gHash[id] = retval;
+
+	new (retval) Game_Data(id);
+
+	if( ++m_CurBlockWritePos >= m_GamesPerBlock )
+	{
+		++m_BlockTableWritePos;
+		m_CurBlockWritePos = 0;
+		m_BlockTable.push_back(NULL);
+	}
+
+	return retval;
 }
 
 void BaseGameDatabaseImpl::updateGame(const Game_Data& game)
@@ -40,13 +85,12 @@ void BaseGameDatabaseImpl::updateGame(const Game_Data& game)
 	GameDataHash::const_iterator iter( gHash.find(game.id) );
 
 	if( iter == gHash.end() ) {
-		gList.push_back(game);
+		*(createNewGame( game.id )) = game;
-		gHash[game.id] = gList.size()-1;
 	}
 	else
 	{
 		// Re-assign existing vector/array entry!
-		gList[gHash[game.id]] = game;
+		*gHash[game.id] = game;
 	}
 }
 
@@ -54,7 +98,7 @@ void BaseGameDatabaseImpl::updateGame(const Game_Data& game)
 bool Game_Data::keyExists(const wxChar* key) const {
 	KeyPairArray::const_iterator it( kList.begin() );
 	for ( ; it != kList.end(); ++it) {
-		if (it[0].CompareKey(key)) {
+		if (it->CompareKey(key)) {
 			return true;
 		}
 	}
@@ -65,7 +109,7 @@ bool Game_Data::keyExists(const wxChar* key) const {
 void Game_Data::deleteKey(const wxChar* key) {
 	KeyPairArray::iterator it( kList.begin() );
 	for ( ; it != kList.end(); ++it) {
-		if (it[0].CompareKey(key)) {
+		if (it->CompareKey(key)) {
 			kList.erase(it);
 			return;
 		}
@@ -76,8 +120,8 @@ void Game_Data::deleteKey(const wxChar* key) {
 wxString Game_Data::getString(const wxChar* key) const {
 	KeyPairArray::const_iterator it( kList.begin() );
 	for ( ; it != kList.end(); ++it) {
-		if (it[0].CompareKey(key)) {
+		if (it->CompareKey(key)) {
-			return it[0].value;
+			return it->value;
 		}
 	}
 	return wxString();
@@ -86,11 +130,11 @@ wxString Game_Data::getString(const wxChar* key) const {
 void Game_Data::writeString(const wxString& key, const wxString& value) {
 	KeyPairArray::iterator it( kList.begin() );
 	for ( ; it != kList.end(); ++it) {
-		if (it[0].CompareKey(key)) {
+		if (it->CompareKey(key)) {
 			if( value.IsEmpty() )
 				kList.erase(it);
 			else
-				it[0].value = value;
+				it->value = value;
 			return;
 		}
 	}

pcsx2/GameDatabase.h

@@ -18,12 +18,25 @@
 //#include "Common.h"
 #include "AppConfig.h"
 #include "Utilities/HashMap.h"
+#include "Utilities/SafeArray.h"
 
 #include <wx/wfstream.h>
 
 struct	key_pair;
 struct	Game_Data;
 
+class StringHashNoCase
+{
+public:
+	StringHashNoCase() {}
+
+	HashTools::hash_key_t operator()( const wxString& src ) const
+	{
+		return HashTools::Hash( (const char *)src.Lower().data(), src.length() * sizeof( wxChar ) );
+	}
+};
+
+
 typedef std::vector<key_pair>	KeyPairArray;
 
 struct key_pair {
@@ -159,23 +172,11 @@ public:
 
 	virtual wxString getBaseKey() const=0;
 	virtual bool findGame(Game_Data& dest, const wxString& id)=0;
-	virtual void addNewGame(const Game_Data& game)=0;
+	virtual Game_Data* createNewGame( const wxString& id )=0;
 	virtual void updateGame(const Game_Data& game)=0;
 };
 
-class StringHashNoCase
+typedef pxDictionary<Game_Data*,StringHashNoCase>	GameDataHash;
-{
-public:
-	StringHashNoCase() {}
-
-	HashTools::hash_key_t operator()( const wxString& src ) const
-	{
-		return HashTools::Hash( (const char *)src.Lower().data(), src.length() * sizeof( wxChar ) );
-	}
-};
-
-typedef std::vector<Game_Data>				GameDataArray;
-typedef pxDictionary<int,StringHashNoCase>	GameDataHash;
 
 // --------------------------------------------------------------------------------------
 //  BaseGameDatabaseImpl 
@@ -184,24 +185,24 @@ typedef pxDictionary<int,StringHashNoCase>	GameDataHash;
 // faster that way.
 class BaseGameDatabaseImpl : public IGameDatabase
 {
-public:
+protected:
-	GameDataArray	gList;			// List of all game data
 	GameDataHash	gHash;			// hash table of game serials matched to their gList indexes!
 	wxString		m_baseKey;
 
-public:
+	std::vector<Game_Data*>	m_BlockTable;
-	BaseGameDatabaseImpl()
+	uint					m_BlockTableWritePos;
-	{
+	int						m_CurBlockWritePos;
-		m_baseKey = L"Serial";
+	int						m_GamesPerBlock;
-	}
 
-	virtual ~BaseGameDatabaseImpl() throw() {}
+public:
+	BaseGameDatabaseImpl();
+	virtual ~BaseGameDatabaseImpl() throw();
 
 	wxString getBaseKey() const { return m_baseKey; }
 	void setBaseKey( const wxString& key ) { m_baseKey = key; }
 
 	bool findGame(Game_Data& dest, const wxString& id);
-	void addNewGame(const Game_Data& game);
+	Game_Data* createNewGame( const wxString& id );
 	void updateGame(const Game_Data& game);
 };
 

pcsx2/gui/AppGameDatabase.cpp

@@ -118,13 +118,14 @@ wxString DBLoaderHelper::ReadHeader()
 
 void DBLoaderHelper::ReadGames()
 {
-	Game_Data game;
+	Game_Data* game = NULL;
 
 	if (m_keyPair.IsOk())
 	{
-		game.writeString(m_keyPair.key, m_keyPair.value);
+		game = m_gamedb.createNewGame(m_keyPair.value);
-		if( m_keyPair.CompareKey(m_gamedb.getBaseKey()) )
+		game->writeString(m_keyPair.key, m_keyPair.value);
-			game.id = m_keyPair.value;
+		//if( m_keyPair.CompareKey(m_gamedb.getBaseKey()) )
+		//	game.id = m_keyPair.value;
 	}
 
 	while(!m_reader.Eof()) { // Fill game data, find new game, repeat...
@@ -136,13 +137,10 @@ void DBLoaderHelper::ReadGames()
 		if (!extractMultiLine()) extract();
 
 		if (!m_keyPair.IsOk()) continue;
-		if( m_keyPair.CompareKey(m_gamedb.getBaseKey()) )
+		if (m_keyPair.CompareKey(m_gamedb.getBaseKey()))
-		{
+			game = m_gamedb.createNewGame(m_keyPair.value);
-			m_gamedb.addNewGame(game);
+
-			game.clear();
+		game->writeString( m_keyPair.key, m_keyPair.value );
-			game.id = m_keyPair.value;
-		}
-		game.writeString( m_keyPair.key, m_keyPair.value );
 	}
 }
 
@@ -168,8 +166,13 @@ AppGameDatabase& AppGameDatabase::LoadFromFile(const wxString& file, const wxStr
 
 	DBLoaderHelper loader( reader, *this );
 
+	u64 qpc_Start = GetCPUTicks();
 	header = loader.ReadHeader();
 	loader.ReadGames();
+	u64 qpc_end = GetCPUTicks();
+
+	Console.WriteLn( "(GameDB) %d games on record (loaded in %ums)",
+		gHash.size(), (u32)(((qpc_end-qpc_Start)*1000) / GetTickFrequency()) );
 
 	return *this;
 }
@@ -178,13 +181,40 @@ AppGameDatabase& AppGameDatabase::LoadFromFile(const wxString& file, const wxStr
 void AppGameDatabase::SaveToFile(const wxString& file) {
 	wxFFileOutputStream writer( file );
 	pxWriteMultiline(writer, header);
-	GameDataArray::iterator it( gList.begin() );
+
-	for ( ; it != gList.end(); ++it) {
+	for(uint blockidx=0; blockidx<=m_BlockTableWritePos; ++blockidx)
-		KeyPairArray::iterator i = it[0].kList.begin();
+	{
-		for ( ; i != it[0].kList.end(); ++i) {
+		if( !m_BlockTable[blockidx] ) continue;
-			pxWriteMultiline(writer, i[0].toString() );
+
+		const uint endidx = (blockidx == m_BlockTableWritePos) ? m_CurBlockWritePos : m_GamesPerBlock;
+
+		for( uint gameidx=0; gameidx<=endidx; ++gameidx )
+		{
+			const Game_Data& game( m_BlockTable[blockidx][gameidx] );
+			KeyPairArray::const_iterator i(game.kList.begin());
+			for ( ; i != game.kList.end(); ++i) {
+				pxWriteMultiline(writer, i->toString() );
+			}
+
+			pxWriteLine(writer, L"---------------------------------------------");
 		}
-		pxWriteLine(writer, L"---------------------------------------------");
 	}
 }
 
+AppGameDatabase* Pcsx2App::GetGameDatabase()
+{
+	pxAppResources& res( GetResourceCache() );
+
+	ScopedLock lock( m_mtx_LoadingGameDB );
+	if( !res.GameDB )
+	{
+		res.GameDB = new AppGameDatabase();
+		res.GameDB->LoadFromFile();
+	}
+	return res.GameDB;
+}
+
+IGameDatabase* AppHost_GetGameDatabase()
+{
+	return wxGetApp().GetGameDatabase();
+}

pcsx2/gui/AppInit.cpp

@@ -475,6 +475,35 @@ bool Pcsx2App::OnCmdLineParsed( wxCmdLineParser& parser )
 typedef void (wxEvtHandler::*pxInvokeAppMethodEventFunction)(Pcsx2AppMethodEvent&);
 typedef void (wxEvtHandler::*pxStuckThreadEventHandler)(pxMessageBoxEvent&);
 
+// --------------------------------------------------------------------------------------
+//   CompressThread_gzip
+// --------------------------------------------------------------------------------------
+class GameDatabaseLoaderThread : public pxThread
+{
+	typedef pxThread _parent;
+
+protected:
+	gzFile		m_gzfp;
+
+public:
+	GameDatabaseLoaderThread() : pxThread( L"GameDatabaseLoader" ) {}
+	virtual ~GameDatabaseLoaderThread() throw()
+	{
+		_parent::Cancel();
+	}
+
+protected:
+	void ExecuteTaskInThread()
+	{
+		wxGetApp().GetGameDatabase();
+	}
+
+	void OnCleanupInThread()
+	{
+		wxGetApp().DeleteThread(this);
+	}
+};
+
 bool Pcsx2App::OnInit()
 {
 	EnableAllLogging();
@@ -535,9 +564,11 @@ bool Pcsx2App::OnInit()
 		// -------------------------------------
 		if( Startup.ForceConsole ) g_Conf->ProgLogBox.Visible = true;
 		OpenProgramLog();
-		if( m_UseGUI ) OpenMainFrame();
 		AllocateCoreStuffs();
+		if( m_UseGUI ) OpenMainFrame();
 		
+		(new GameDatabaseLoaderThread())->Start();
+
 		if( Startup.SysAutoRun )
 		{
 			// Notes: Saving/remembering the Iso file is probably fine and desired, so using

pcsx2/gui/AppMain.cpp

@@ -1088,21 +1088,3 @@ SysCoreAllocations& GetSysCoreAlloc()
 {
 	return *wxGetApp().m_CoreAllocs;
 }
-
-AppGameDatabase* Pcsx2App::GetGameDatabase()
-{
-	pxAppResources& res( GetResourceCache() );
-
-	ScopedLock lock( m_mtx_LoadingGameDB );
-	if( !res.GameDB )
-	{
-		res.GameDB = new AppGameDatabase();
-		res.GameDB->LoadFromFile();
-	}
-	return res.GameDB;
-}
-
-IGameDatabase* AppHost_GetGameDatabase()
-{
-	return wxGetApp().GetGameDatabase();
-}

pcsx2/gui/ExecutorThread.cpp

@@ -190,7 +190,7 @@ void pxEvtHandler::ProcessEvents( pxEvtList& list )
 			}
 
 			u64 qpc_end = GetCPUTicks();
-			DevCon.WriteLn( L"(pxEvtHandler) Event '%s' completed in %dms", deleteMe->GetEventName().c_str(), ((qpc_end-m_qpc_Start)*1000) / GetTickFrequency() );
+			DevCon.WriteLn( L"(pxEvtHandler) Event '%s' completed in %ums", deleteMe->GetEventName().c_str(), (u32)(((qpc_end-m_qpc_Start)*1000) / GetTickFrequency()) );
 
 			synclock.Acquire();
 			m_qpc_Start = 0;		// lets the main thread know the message completed.