Folder memcards: Add an index file to track timestamps and file order

This resolves folder memory card incompatibilities with games assuming
the order of files in the filesystem - e.g. Grand Theft Auto games.
A newly added index file tracks the order in which files would have
landed in the FAT partition and attempts to preserve this order
upon loading.

pcsx2/gui/MemoryCardFolder.cpp

@@ -22,10 +22,27 @@
 #include "System.h"
 #include "AppConfig.h"
 
+#include "yaml-cpp/yaml.h"
+
 #include "svnrev.h"
 
 bool RemoveDirectory( const wxString& dirname );
 
+// A helper function to parse the YAML file
+static YAML::Node LoadYAMLFromFile( const wxString& fileName ) {
+	YAML::Node index;
+
+	wxFFile indexFile;
+	if ( indexFile.Open( fileName, L"r" ) ) {
+		wxString fileContents;
+		if ( indexFile.ReadAll( &fileContents ) ) {
+			index = YAML::Load( fileContents.mbc_str() );
+		}
+	}
+
+	return index;
+}
+
 FolderMemoryCard::FolderMemoryCard() {
 	m_slot = 0;
 	m_isEnabled = false;
@@ -312,10 +329,7 @@ bool FilterMatches( const wxString& fileName, const wxString& filter ) {
 }
 
 bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, MemoryCardFileMetadataReference* parent, const bool enableFiltering, const wxString& filter ) {
-	wxDir dir( dirPath );
+	if ( wxDir::Exists( dirPath ) ) {
-	if ( dir.IsOpened() ) {
-		wxString fileName;
-		bool hasNext;
 
 		wxString localFilter;
 		if ( enableFiltering ) {
@@ -328,39 +342,31 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
 		}
 
 		int entryNumber = 2; // include . and ..
-		hasNext = dir.GetFirst( &fileName );
+		for ( const auto& file : GetOrderedFiles( dirPath ) ) {
-		while ( hasNext ) {
-			if ( fileName.StartsWith( L"_pcsx2_" ) ) {
-				hasNext = dir.GetNext( &fileName );
-				continue;
-			}
 
-			wxFileName fileInfo( dirPath, fileName );
+			wxFileName fileInfo( dirPath, file.m_fileName );
 			bool isFile = wxFile::Exists( fileInfo.GetFullPath() );
 
 			if ( isFile ) {
 				// don't load files in the root dir if we're filtering; no official software stores files there
 				if ( enableFiltering && parent == nullptr ) {
-					hasNext = dir.GetNext( &fileName );
 					continue;
 				}
-				if ( AddFile( dirEntry, dirPath, fileName, parent ) ) {
+				if ( AddFile( dirEntry, dirPath, file, parent ) ) {
 					++entryNumber;
 				}
 			} else {
 				// if possible filter added directories by game serial
 				// this has the effective result of only files relevant to the current game being loaded into the memory card
 				// which means every game essentially sees the memory card as if no other files exist
-				if ( enableFiltering && !FilterMatches( fileName, localFilter ) ) {
+				if ( enableFiltering && !FilterMatches( file.m_fileName, localFilter ) ) {
-					hasNext = dir.GetNext( &fileName );
 					continue;
 				}
 
 				// make sure we have enough space on the memcard for the directory
-				const u32 newNeededClusters = CalculateRequiredClustersOfDirectory( dirPath + L"/" + fileName ) + ( ( dirEntry->entry.data.length % 2 ) == 0 ? 1 : 0 );
+				const u32 newNeededClusters = CalculateRequiredClustersOfDirectory( dirPath + L"/" + file.m_fileName ) + ( ( dirEntry->entry.data.length % 2 ) == 0 ? 1 : 0 );
 				if ( newNeededClusters > GetAmountFreeDataClusters() ) {
-					Console.Warning( GetCardFullMessage( fileName ) );
+					Console.Warning( GetCardFullMessage( file.m_fileName ) );
-					hasNext = dir.GetNext( &fileName );
 					continue;
 				}
 
@@ -377,19 +383,19 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
 
 				// set metadata
 				wxFileName metaFileName( dirPath, L"_pcsx2_meta_directory" );
-				metaFileName.AppendDir( fileName );
+				metaFileName.AppendDir( file.m_fileName );
 				wxFFile metaFile;
 				if ( metaFileName.FileExists() && metaFile.Open( metaFileName.GetFullPath(), L"rb" ) ) {
 					size_t bytesRead = metaFile.Read( &newDirEntry->entry.raw, sizeof( newDirEntry->entry.raw ) );
 					metaFile.Close();
 					if ( bytesRead < 0x60 ) {
-						strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), fileName.mbc_str() );
+						strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), file.m_fileName.mbc_str() );
 					}
 				} else {
 					newDirEntry->entry.data.mode = MemoryCardFileEntry::DefaultDirMode;
 					newDirEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromWxDateTime( creationTime );
 					newDirEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromWxDateTime( modificationTime );
-					strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), fileName.mbc_str() );
+					strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), file.m_fileName.mbc_str() );
 				}
 
 				// create new cluster for . and .. entries
@@ -416,8 +422,6 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
 				// and add all files in subdir
 				AddFolder( newDirEntry, fileInfo.GetFullPath(), dirRef );
 			}
-
-			hasNext = dir.GetNext( &fileName );
 		}
 
 		return true;
@@ -426,11 +430,11 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
 	return false;
 }
 
-bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const wxString& fileName, MemoryCardFileMetadataReference* parent ) {
+bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const EnumeratedFileEntry& fileEntry, MemoryCardFileMetadataReference* parent ) {
-	wxFileName relativeFilePath( dirPath, fileName );
+	wxFileName relativeFilePath( dirPath, fileEntry.m_fileName );
 	relativeFilePath.MakeRelativeTo( m_folderName.GetPath() );
 
-	wxFileName fileInfo( dirPath, fileName );
+	wxFileName fileInfo( dirPath, fileEntry.m_fileName );
 	wxFFile file( fileInfo.GetFullPath(), L"rb" );
 	if ( file.IsOpened() ) {
 		// make sure we have enough space on the memcard to hold the data
@@ -444,26 +448,24 @@ bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxStr
 		}
 
 		MemoryCardFileEntry* newFileEntry = AppendFileEntryToDir( dirEntry );
-		wxDateTime creationTime, modificationTime;
-		fileInfo.GetTimes( NULL, &modificationTime, &creationTime );
 
 		// set file entry metadata
 		memset( newFileEntry->entry.raw, 0x00, sizeof( newFileEntry->entry.raw ) );
 
-		wxFileName metaFileName( dirPath, fileName );
+		wxFileName metaFileName( dirPath, fileEntry.m_fileName );
 		metaFileName.AppendDir( L"_pcsx2_meta" );
 		wxFFile metaFile;
 		if ( metaFileName.FileExists() && metaFile.Open( metaFileName.GetFullPath(), L"rb" ) ) {
 			size_t bytesRead = metaFile.Read( &newFileEntry->entry.raw, sizeof( newFileEntry->entry.raw ) );
 			metaFile.Close();
 			if ( bytesRead < 0x60 ) {
-				strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileName.mbc_str() );
+				strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileEntry.m_fileName.mbc_str() );
 			}
 		} else {
 			newFileEntry->entry.data.mode = MemoryCardFileEntry::DefaultFileMode;
-			newFileEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromWxDateTime( creationTime );
+			newFileEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromTime( fileEntry.m_timeCreated );
-			newFileEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromWxDateTime( modificationTime );
+			newFileEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromTime( fileEntry.m_timeModified );
-			strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileName.mbc_str() );
+			strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileEntry.m_fileName.mbc_str() );
 		}
 
 		newFileEntry->entry.data.length = filesize;
@@ -507,6 +509,7 @@ u32 FolderMemoryCard::CalculateRequiredClustersOfDirectory( const wxString& dirP
 	u32 requiredFileEntryPages = 2;
 	u32 requiredClusters = 0;
 
+	// No need to read the index file as we are only counting space required; order of files is irrelevant.
 	wxDir dir( dirPath );
 	wxString fileName;
 	bool hasNext = dir.GetFirst( &fileName );
@@ -1073,6 +1076,8 @@ void FolderMemoryCard::FlushDeletedFilesAndRemoveUnchangedDataFromCache( const s
 					RemoveDirectory( newFilePath );
 				}
 				wxRenameFile( filePath, newFilePath );
+				DeleteFromIndex( m_folderName.GetFullPath() + dirPath, fileName );
+
 			} else if ( entry->IsDir() ) {
 				// still exists and is a directory, recursive call for subdir
 				char cleanName[sizeof( entry->entry.data.name )];
@@ -1323,6 +1328,80 @@ void FolderMemoryCard::SetTimeLastWrittenToNow() {
 	m_framesUntilFlush = FramesAfterWriteUntilFlush;
 }
 
+std::vector<FolderMemoryCard::EnumeratedFileEntry> FolderMemoryCard::GetOrderedFiles( const wxString& dirPath ) const
+{
+	std::vector<EnumeratedFileEntry> result;
+
+	wxDir dir( dirPath );
+	if ( dir.IsOpened() ) {
+
+		YAML::Node index = LoadYAMLFromFile( wxFileName( dirPath, "_pcsx2_index" ).GetFullPath() );
+
+		// We must be able to support legacy folder memcards without the index file, so for those
+		// track an order variable and make it negative - this way new files get their order preserved
+		// and old files are listed first.
+		// In the YAML File order is stored as an unsigned int, so use a signed int64_t to accommodate for
+		// all possible values without cutting them off
+		std::map<int64_t, EnumeratedFileEntry> sortContainer;
+		int64_t orderForLegacyFiles = -1;
+
+		wxString fileName;
+		bool hasNext = dir.GetFirst( &fileName );
+		while ( hasNext ) {
+			if ( fileName.StartsWith( L"_pcsx2_" ) ) {
+				hasNext = dir.GetNext( &fileName );
+				continue;
+			}
+
+			const wxFileName fileInfo( dirPath, fileName );
+			try {
+				if ( wxFile::Exists( fileInfo.GetFullPath() ) ) {
+					const YAML::Node& node = index[ fileName.ToStdString() ];
+
+					EnumeratedFileEntry entry { fileName, node["timeCreated"].as<time_t>(), node["timeModified"].as<time_t>() };
+					sortContainer.try_emplace( node["order"].as<unsigned int>(), std::move(entry) );
+				}
+				else {
+					// TODO: Implement directories, for now force it to use the fallback implementation
+					throw YAML::InvalidNode( fileName.ToStdString() );
+				}
+			}
+			catch ( YAML::Exception& /*e*/ ) {
+				// File doesn't exist in index or it's corrupted - fall back to file-based timestamps and a custom order
+				wxDateTime creationTime, modificationTime;
+				fileInfo.GetTimes( nullptr, &modificationTime, &creationTime );
+
+				EnumeratedFileEntry entry { fileName, creationTime.GetTicks(), modificationTime.GetTicks() };
+				sortContainer.try_emplace( orderForLegacyFiles--, std::move(entry) );
+			}
+
+			hasNext = dir.GetNext( &fileName );
+		}
+
+		// Move items from the intermediate map to a final vector
+		result.reserve( sortContainer.size() );
+		for ( auto& e : sortContainer ) {
+			result.push_back( std::move(e.second) );
+		}
+	}
+
+	return result;
+}
+
+void FolderMemoryCard::DeleteFromIndex( const wxString& filePath, const wxString& entry ) const
+{
+	const wxString indexName = wxFileName( filePath, "_pcsx2_index" ).GetFullPath();
+
+	YAML::Node index = LoadYAMLFromFile( indexName );
+	index.remove( entry.ToStdString() );
+
+	// Write out the changes
+	wxFFile indexFile;
+	if ( indexFile.Open( indexName, L"w" ) ) {
+		indexFile.Write( YAML::Dump( index ) );
+	}
+}
+
 // from http://www.oocities.org/siliconvalley/station/8269/sma02/sma02.html#ECC
 void FolderMemoryCard::CalculateECC( u8* ecc, const u8* data ) {
 	static const u8 Table[] = {
@@ -1413,6 +1492,7 @@ wxFFile* FileAccessHelper::Open( const wxFileName& folderName, MemoryCardFileMet
 
 	if ( writeMetadata ) {
 		WriteMetadata( folderName, fileRef );
+		WriteIndex( folderName, fileRef );
 	}
 
 	return file;
@@ -1448,6 +1528,38 @@ void FileAccessHelper::WriteMetadata( wxFileName folderName, const MemoryCardFil
 	}
 }
 
+void FileAccessHelper::WriteIndex( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef )
+{
+	fileRef->GetPath( &folderName );
+
+	const std::string fileName = folderName.GetName();
+	folderName.SetName( L"_pcsx2_index" );
+
+	YAML::Node index = LoadYAMLFromFile( folderName.GetFullPath() );
+	YAML::Node entryNode = index[fileName];
+
+	if ( !entryNode.IsDefined() ) {
+		// Newly added file - figure out the sort order as the entry should be added to the end of the list
+		unsigned int order = 0;
+		for ( const auto& node : index ) {
+			order = std::max( order, node.second["order"].as<unsigned int>() );
+		}
+
+		entryNode["order"] = order + 1;
+	}
+
+	// Update timestamps basing on internal data
+	const auto* entry = &fileRef->entry->entry.data;
+	entryNode["timeCreated"] = entry->timeCreated.ToTime();
+	entryNode["timeModified"] = entry->timeModified.ToTime();
+
+	// Write out the changes
+	wxFFile indexFile;
+	if ( indexFile.Open( folderName.GetFullPath(), L"w" ) ) {
+		indexFile.Write( YAML::Dump( index ) );
+	}
+}
+
 wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileMetadataReference* fileRef, bool writeMetadata ) {
 	std::string internalPath;
 	fileRef->GetInternalPath( &internalPath );
@@ -1459,6 +1571,7 @@ wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileM
 		if ( writeMetadata ) {
 			if ( m_lastWrittenFileRef != fileRef ) {
 				WriteMetadata( folderName, fileRef );
+				WriteIndex( folderName, fileRef );
 				m_lastWrittenFileRef = fileRef;
 			}
 		} else {
@@ -1479,13 +1592,6 @@ wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileM
 void FileAccessHelper::CloseFileHandle( wxFFile* file, const MemoryCardFileEntry* entry ) {
 	file->Close();
 
-	if ( entry != nullptr ) {
-		wxFileName fn( file->GetName() );
-		wxDateTime modified = entry->entry.data.timeModified.ToWxDateTime();
-		wxDateTime created = entry->entry.data.timeCreated.ToWxDateTime();
-		fn.SetTimes( nullptr, &modified, &created );
-	}
-
 	delete file;
 }
 

pcsx2/gui/MemoryCardFolder.h

@@ -87,6 +87,11 @@ struct MemoryCardFileEntryDateTime {
 		return t;
 	}
 
+	static MemoryCardFileEntryDateTime FromTime( time_t time ) {
+		// TODO: When wx is gone, this will have to be handled differently; for now, rely on wx
+		return FromWxDateTime( wxDateTime(time) );
+	}
+
 	wxDateTime ToWxDateTime() const {
 		wxDateTime::Tm tm;
 		tm.sec = this->second;
@@ -100,6 +105,11 @@ struct MemoryCardFileEntryDateTime {
 		return time.FromTimezone( wxDateTime::GMT9 );
 	}
 
+	time_t ToTime() const {
+		// TODO: When wx is gone, this will have to be handled differently; for now, rely on wx
+		return ToWxDateTime().GetTicks();
+	}
+
 	bool operator==( const MemoryCardFileEntryDateTime& other ) const {
 		return unused == other.unused && second == other.second && minute == other.minute && hour == other.hour
 		    && day == other.day && month == other.month && year == other.year;
@@ -247,6 +257,7 @@ private:
 	void CloseFileHandle( wxFFile* file, const MemoryCardFileEntry* entry = nullptr );
 
 	void WriteMetadata( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef );
+	void WriteIndex( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef );
 };
 
 // --------------------------------------------------------------------------------------
@@ -375,6 +386,12 @@ public:
 	void WriteToFile( const wxString& filename );
 
 protected:
+	struct EnumeratedFileEntry {
+		wxString m_fileName; // TODO: Replace with std::string
+		time_t m_timeCreated;
+		time_t m_timeModified;
+	};
+
 	// initializes memory card data, as if it was fresh from the factory
 	void InitializeInternalData();
 
@@ -458,7 +475,7 @@ protected:
 	// - dirPath: the full path to the directory containing the file in the host file system
 	// - fileName: the name of the file, without path
 	// - parent: pointer to the parent dir's quick-access reference element
-	bool AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const wxString& fileName, MemoryCardFileMetadataReference* parent = nullptr );
+	bool AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const EnumeratedFileEntry& fileEntry, MemoryCardFileMetadataReference* parent = nullptr );
 
 	// calculates the amount of clusters a directory would use up if put into a memory card
 	u32 CalculateRequiredClustersOfDirectory( const wxString& dirPath ) const;
@@ -537,6 +554,12 @@ protected:
 	wxString GetCardFullMessage( const wxString& filePath ) const {
 		return wxsFormat( pxE( L"(FolderMcd) Memory Card is full, could not add: %s" ), WX_STR( filePath ) );
 	}
+
+	// get the list of files (and their timestamps) in directory ordered as specified by the index file
+	// for legacy entries without an entry in the index file, order is unspecified and should not be relied on
+	std::vector<EnumeratedFileEntry> GetOrderedFiles( const wxString& dirPath ) const;
+
+	void DeleteFromIndex( const wxString& filePath, const wxString& entry ) const;
 };
 
 // --------------------------------------------------------------------------------------