From a6cec6a04ecc64156c636ce0288c8dc6c3a6fe62 Mon Sep 17 00:00:00 2001 From: Silent Date: Sat, 21 Nov 2020 19:38:08 +0100 Subject: [PATCH] Folder memcards: Add an index file to track timestamps and file order This resolves folder memory card incompatibilities with games assuming the order of files in the filesystem - e.g. Grand Theft Auto games. A newly added index file tracks the order in which files would have landed in the FAT partition and attempts to preserve this order upon loading. --- pcsx2/gui/MemoryCardFolder.cpp | 186 ++++++++++++++++++++++++++------- pcsx2/gui/MemoryCardFolder.h | 25 ++++- 2 files changed, 170 insertions(+), 41 deletions(-) diff --git a/pcsx2/gui/MemoryCardFolder.cpp b/pcsx2/gui/MemoryCardFolder.cpp index 338c69da12..64c30b70da 100644 --- a/pcsx2/gui/MemoryCardFolder.cpp +++ b/pcsx2/gui/MemoryCardFolder.cpp @@ -22,10 +22,27 @@ #include "System.h" #include "AppConfig.h" +#include "yaml-cpp/yaml.h" + #include "svnrev.h" bool RemoveDirectory( const wxString& dirname ); +// A helper function to parse the YAML file +static YAML::Node LoadYAMLFromFile( const wxString& fileName ) { + YAML::Node index; + + wxFFile indexFile; + if ( indexFile.Open( fileName, L"r" ) ) { + wxString fileContents; + if ( indexFile.ReadAll( &fileContents ) ) { + index = YAML::Load( fileContents.mbc_str() ); + } + } + + return index; +} + FolderMemoryCard::FolderMemoryCard() { m_slot = 0; m_isEnabled = false; @@ -312,10 +329,7 @@ bool FilterMatches( const wxString& fileName, const wxString& filter ) { } bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, MemoryCardFileMetadataReference* parent, const bool enableFiltering, const wxString& filter ) { - wxDir dir( dirPath ); - if ( dir.IsOpened() ) { - wxString fileName; - bool hasNext; + if ( wxDir::Exists( dirPath ) ) { wxString localFilter; if ( enableFiltering ) { @@ -328,39 +342,31 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS } int entryNumber = 2; // include . and .. - hasNext = dir.GetFirst( &fileName ); - while ( hasNext ) { - if ( fileName.StartsWith( L"_pcsx2_" ) ) { - hasNext = dir.GetNext( &fileName ); - continue; - } + for ( const auto& file : GetOrderedFiles( dirPath ) ) { - wxFileName fileInfo( dirPath, fileName ); + wxFileName fileInfo( dirPath, file.m_fileName ); bool isFile = wxFile::Exists( fileInfo.GetFullPath() ); if ( isFile ) { // don't load files in the root dir if we're filtering; no official software stores files there if ( enableFiltering && parent == nullptr ) { - hasNext = dir.GetNext( &fileName ); continue; } - if ( AddFile( dirEntry, dirPath, fileName, parent ) ) { + if ( AddFile( dirEntry, dirPath, file, parent ) ) { ++entryNumber; } } else { // if possible filter added directories by game serial // this has the effective result of only files relevant to the current game being loaded into the memory card // which means every game essentially sees the memory card as if no other files exist - if ( enableFiltering && !FilterMatches( fileName, localFilter ) ) { - hasNext = dir.GetNext( &fileName ); + if ( enableFiltering && !FilterMatches( file.m_fileName, localFilter ) ) { continue; } // make sure we have enough space on the memcard for the directory - const u32 newNeededClusters = CalculateRequiredClustersOfDirectory( dirPath + L"/" + fileName ) + ( ( dirEntry->entry.data.length % 2 ) == 0 ? 1 : 0 ); + const u32 newNeededClusters = CalculateRequiredClustersOfDirectory( dirPath + L"/" + file.m_fileName ) + ( ( dirEntry->entry.data.length % 2 ) == 0 ? 1 : 0 ); if ( newNeededClusters > GetAmountFreeDataClusters() ) { - Console.Warning( GetCardFullMessage( fileName ) ); - hasNext = dir.GetNext( &fileName ); + Console.Warning( GetCardFullMessage( file.m_fileName ) ); continue; } @@ -377,19 +383,19 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS // set metadata wxFileName metaFileName( dirPath, L"_pcsx2_meta_directory" ); - metaFileName.AppendDir( fileName ); + metaFileName.AppendDir( file.m_fileName ); wxFFile metaFile; if ( metaFileName.FileExists() && metaFile.Open( metaFileName.GetFullPath(), L"rb" ) ) { size_t bytesRead = metaFile.Read( &newDirEntry->entry.raw, sizeof( newDirEntry->entry.raw ) ); metaFile.Close(); if ( bytesRead < 0x60 ) { - strcpy( reinterpret_cast(newDirEntry->entry.data.name), fileName.mbc_str() ); + strcpy( reinterpret_cast(newDirEntry->entry.data.name), file.m_fileName.mbc_str() ); } } else { newDirEntry->entry.data.mode = MemoryCardFileEntry::DefaultDirMode; newDirEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromWxDateTime( creationTime ); newDirEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromWxDateTime( modificationTime ); - strcpy( reinterpret_cast(newDirEntry->entry.data.name), fileName.mbc_str() ); + strcpy( reinterpret_cast(newDirEntry->entry.data.name), file.m_fileName.mbc_str() ); } // create new cluster for . and .. entries @@ -416,8 +422,6 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS // and add all files in subdir AddFolder( newDirEntry, fileInfo.GetFullPath(), dirRef ); } - - hasNext = dir.GetNext( &fileName ); } return true; @@ -426,11 +430,11 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS return false; } -bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const wxString& fileName, MemoryCardFileMetadataReference* parent ) { - wxFileName relativeFilePath( dirPath, fileName ); +bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const EnumeratedFileEntry& fileEntry, MemoryCardFileMetadataReference* parent ) { + wxFileName relativeFilePath( dirPath, fileEntry.m_fileName ); relativeFilePath.MakeRelativeTo( m_folderName.GetPath() ); - wxFileName fileInfo( dirPath, fileName ); + wxFileName fileInfo( dirPath, fileEntry.m_fileName ); wxFFile file( fileInfo.GetFullPath(), L"rb" ); if ( file.IsOpened() ) { // make sure we have enough space on the memcard to hold the data @@ -444,26 +448,24 @@ bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxStr } MemoryCardFileEntry* newFileEntry = AppendFileEntryToDir( dirEntry ); - wxDateTime creationTime, modificationTime; - fileInfo.GetTimes( NULL, &modificationTime, &creationTime ); // set file entry metadata memset( newFileEntry->entry.raw, 0x00, sizeof( newFileEntry->entry.raw ) ); - wxFileName metaFileName( dirPath, fileName ); + wxFileName metaFileName( dirPath, fileEntry.m_fileName ); metaFileName.AppendDir( L"_pcsx2_meta" ); wxFFile metaFile; if ( metaFileName.FileExists() && metaFile.Open( metaFileName.GetFullPath(), L"rb" ) ) { size_t bytesRead = metaFile.Read( &newFileEntry->entry.raw, sizeof( newFileEntry->entry.raw ) ); metaFile.Close(); if ( bytesRead < 0x60 ) { - strcpy( reinterpret_cast(newFileEntry->entry.data.name), fileName.mbc_str() ); + strcpy( reinterpret_cast(newFileEntry->entry.data.name), fileEntry.m_fileName.mbc_str() ); } } else { newFileEntry->entry.data.mode = MemoryCardFileEntry::DefaultFileMode; - newFileEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromWxDateTime( creationTime ); - newFileEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromWxDateTime( modificationTime ); - strcpy( reinterpret_cast(newFileEntry->entry.data.name), fileName.mbc_str() ); + newFileEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromTime( fileEntry.m_timeCreated ); + newFileEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromTime( fileEntry.m_timeModified ); + strcpy( reinterpret_cast(newFileEntry->entry.data.name), fileEntry.m_fileName.mbc_str() ); } newFileEntry->entry.data.length = filesize; @@ -507,6 +509,7 @@ u32 FolderMemoryCard::CalculateRequiredClustersOfDirectory( const wxString& dirP u32 requiredFileEntryPages = 2; u32 requiredClusters = 0; + // No need to read the index file as we are only counting space required; order of files is irrelevant. wxDir dir( dirPath ); wxString fileName; bool hasNext = dir.GetFirst( &fileName ); @@ -1073,6 +1076,8 @@ void FolderMemoryCard::FlushDeletedFilesAndRemoveUnchangedDataFromCache( const s RemoveDirectory( newFilePath ); } wxRenameFile( filePath, newFilePath ); + DeleteFromIndex( m_folderName.GetFullPath() + dirPath, fileName ); + } else if ( entry->IsDir() ) { // still exists and is a directory, recursive call for subdir char cleanName[sizeof( entry->entry.data.name )]; @@ -1323,6 +1328,80 @@ void FolderMemoryCard::SetTimeLastWrittenToNow() { m_framesUntilFlush = FramesAfterWriteUntilFlush; } +std::vector FolderMemoryCard::GetOrderedFiles( const wxString& dirPath ) const +{ + std::vector result; + + wxDir dir( dirPath ); + if ( dir.IsOpened() ) { + + YAML::Node index = LoadYAMLFromFile( wxFileName( dirPath, "_pcsx2_index" ).GetFullPath() ); + + // We must be able to support legacy folder memcards without the index file, so for those + // track an order variable and make it negative - this way new files get their order preserved + // and old files are listed first. + // In the YAML File order is stored as an unsigned int, so use a signed int64_t to accommodate for + // all possible values without cutting them off + std::map sortContainer; + int64_t orderForLegacyFiles = -1; + + wxString fileName; + bool hasNext = dir.GetFirst( &fileName ); + while ( hasNext ) { + if ( fileName.StartsWith( L"_pcsx2_" ) ) { + hasNext = dir.GetNext( &fileName ); + continue; + } + + const wxFileName fileInfo( dirPath, fileName ); + try { + if ( wxFile::Exists( fileInfo.GetFullPath() ) ) { + const YAML::Node& node = index[ fileName.ToStdString() ]; + + EnumeratedFileEntry entry { fileName, node["timeCreated"].as(), node["timeModified"].as() }; + sortContainer.try_emplace( node["order"].as(), std::move(entry) ); + } + else { + // TODO: Implement directories, for now force it to use the fallback implementation + throw YAML::InvalidNode( fileName.ToStdString() ); + } + } + catch ( YAML::Exception& /*e*/ ) { + // File doesn't exist in index or it's corrupted - fall back to file-based timestamps and a custom order + wxDateTime creationTime, modificationTime; + fileInfo.GetTimes( nullptr, &modificationTime, &creationTime ); + + EnumeratedFileEntry entry { fileName, creationTime.GetTicks(), modificationTime.GetTicks() }; + sortContainer.try_emplace( orderForLegacyFiles--, std::move(entry) ); + } + + hasNext = dir.GetNext( &fileName ); + } + + // Move items from the intermediate map to a final vector + result.reserve( sortContainer.size() ); + for ( auto& e : sortContainer ) { + result.push_back( std::move(e.second) ); + } + } + + return result; +} + +void FolderMemoryCard::DeleteFromIndex( const wxString& filePath, const wxString& entry ) const +{ + const wxString indexName = wxFileName( filePath, "_pcsx2_index" ).GetFullPath(); + + YAML::Node index = LoadYAMLFromFile( indexName ); + index.remove( entry.ToStdString() ); + + // Write out the changes + wxFFile indexFile; + if ( indexFile.Open( indexName, L"w" ) ) { + indexFile.Write( YAML::Dump( index ) ); + } +} + // from http://www.oocities.org/siliconvalley/station/8269/sma02/sma02.html#ECC void FolderMemoryCard::CalculateECC( u8* ecc, const u8* data ) { static const u8 Table[] = { @@ -1413,6 +1492,7 @@ wxFFile* FileAccessHelper::Open( const wxFileName& folderName, MemoryCardFileMet if ( writeMetadata ) { WriteMetadata( folderName, fileRef ); + WriteIndex( folderName, fileRef ); } return file; @@ -1448,6 +1528,38 @@ void FileAccessHelper::WriteMetadata( wxFileName folderName, const MemoryCardFil } } +void FileAccessHelper::WriteIndex( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef ) +{ + fileRef->GetPath( &folderName ); + + const std::string fileName = folderName.GetName(); + folderName.SetName( L"_pcsx2_index" ); + + YAML::Node index = LoadYAMLFromFile( folderName.GetFullPath() ); + YAML::Node entryNode = index[fileName]; + + if ( !entryNode.IsDefined() ) { + // Newly added file - figure out the sort order as the entry should be added to the end of the list + unsigned int order = 0; + for ( const auto& node : index ) { + order = std::max( order, node.second["order"].as() ); + } + + entryNode["order"] = order + 1; + } + + // Update timestamps basing on internal data + const auto* entry = &fileRef->entry->entry.data; + entryNode["timeCreated"] = entry->timeCreated.ToTime(); + entryNode["timeModified"] = entry->timeModified.ToTime(); + + // Write out the changes + wxFFile indexFile; + if ( indexFile.Open( folderName.GetFullPath(), L"w" ) ) { + indexFile.Write( YAML::Dump( index ) ); + } +} + wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileMetadataReference* fileRef, bool writeMetadata ) { std::string internalPath; fileRef->GetInternalPath( &internalPath ); @@ -1459,6 +1571,7 @@ wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileM if ( writeMetadata ) { if ( m_lastWrittenFileRef != fileRef ) { WriteMetadata( folderName, fileRef ); + WriteIndex( folderName, fileRef ); m_lastWrittenFileRef = fileRef; } } else { @@ -1479,13 +1592,6 @@ wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileM void FileAccessHelper::CloseFileHandle( wxFFile* file, const MemoryCardFileEntry* entry ) { file->Close(); - if ( entry != nullptr ) { - wxFileName fn( file->GetName() ); - wxDateTime modified = entry->entry.data.timeModified.ToWxDateTime(); - wxDateTime created = entry->entry.data.timeCreated.ToWxDateTime(); - fn.SetTimes( nullptr, &modified, &created ); - } - delete file; } diff --git a/pcsx2/gui/MemoryCardFolder.h b/pcsx2/gui/MemoryCardFolder.h index 91c9bcc146..d1edc2f71d 100644 --- a/pcsx2/gui/MemoryCardFolder.h +++ b/pcsx2/gui/MemoryCardFolder.h @@ -87,6 +87,11 @@ struct MemoryCardFileEntryDateTime { return t; } + static MemoryCardFileEntryDateTime FromTime( time_t time ) { + // TODO: When wx is gone, this will have to be handled differently; for now, rely on wx + return FromWxDateTime( wxDateTime(time) ); + } + wxDateTime ToWxDateTime() const { wxDateTime::Tm tm; tm.sec = this->second; @@ -100,6 +105,11 @@ struct MemoryCardFileEntryDateTime { return time.FromTimezone( wxDateTime::GMT9 ); } + time_t ToTime() const { + // TODO: When wx is gone, this will have to be handled differently; for now, rely on wx + return ToWxDateTime().GetTicks(); + } + bool operator==( const MemoryCardFileEntryDateTime& other ) const { return unused == other.unused && second == other.second && minute == other.minute && hour == other.hour && day == other.day && month == other.month && year == other.year; @@ -247,6 +257,7 @@ private: void CloseFileHandle( wxFFile* file, const MemoryCardFileEntry* entry = nullptr ); void WriteMetadata( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef ); + void WriteIndex( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef ); }; // -------------------------------------------------------------------------------------- @@ -375,6 +386,12 @@ public: void WriteToFile( const wxString& filename ); protected: + struct EnumeratedFileEntry { + wxString m_fileName; // TODO: Replace with std::string + time_t m_timeCreated; + time_t m_timeModified; + }; + // initializes memory card data, as if it was fresh from the factory void InitializeInternalData(); @@ -458,7 +475,7 @@ protected: // - dirPath: the full path to the directory containing the file in the host file system // - fileName: the name of the file, without path // - parent: pointer to the parent dir's quick-access reference element - bool AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const wxString& fileName, MemoryCardFileMetadataReference* parent = nullptr ); + bool AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const EnumeratedFileEntry& fileEntry, MemoryCardFileMetadataReference* parent = nullptr ); // calculates the amount of clusters a directory would use up if put into a memory card u32 CalculateRequiredClustersOfDirectory( const wxString& dirPath ) const; @@ -537,6 +554,12 @@ protected: wxString GetCardFullMessage( const wxString& filePath ) const { return wxsFormat( pxE( L"(FolderMcd) Memory Card is full, could not add: %s" ), WX_STR( filePath ) ); } + + // get the list of files (and their timestamps) in directory ordered as specified by the index file + // for legacy entries without an entry in the index file, order is unspecified and should not be relied on + std::vector GetOrderedFiles( const wxString& dirPath ) const; + + void DeleteFromIndex( const wxString& filePath, const wxString& entry ) const; }; // --------------------------------------------------------------------------------------