Folder memcards: Add an index file to track timestamps and file order

This resolves folder memory card incompatibilities with games assuming
the order of files in the filesystem - e.g. Grand Theft Auto games.
A newly added index file tracks the order in which files would have
landed in the FAT partition and attempts to preserve this order
upon loading.
This commit is contained in:
Silent 2020-11-21 19:38:08 +01:00 committed by lightningterror
parent 8dee1a3e9f
commit a6cec6a04e
2 changed files with 170 additions and 41 deletions

View File

@ -22,10 +22,27 @@
#include "System.h"
#include "AppConfig.h"
#include "yaml-cpp/yaml.h"
#include "svnrev.h"
bool RemoveDirectory( const wxString& dirname );
// A helper function to parse the YAML file
static YAML::Node LoadYAMLFromFile( const wxString& fileName ) {
YAML::Node index;
wxFFile indexFile;
if ( indexFile.Open( fileName, L"r" ) ) {
wxString fileContents;
if ( indexFile.ReadAll( &fileContents ) ) {
index = YAML::Load( fileContents.mbc_str() );
}
}
return index;
}
FolderMemoryCard::FolderMemoryCard() {
m_slot = 0;
m_isEnabled = false;
@ -312,10 +329,7 @@ bool FilterMatches( const wxString& fileName, const wxString& filter ) {
}
bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, MemoryCardFileMetadataReference* parent, const bool enableFiltering, const wxString& filter ) {
wxDir dir( dirPath );
if ( dir.IsOpened() ) {
wxString fileName;
bool hasNext;
if ( wxDir::Exists( dirPath ) ) {
wxString localFilter;
if ( enableFiltering ) {
@ -328,39 +342,31 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
}
int entryNumber = 2; // include . and ..
hasNext = dir.GetFirst( &fileName );
while ( hasNext ) {
if ( fileName.StartsWith( L"_pcsx2_" ) ) {
hasNext = dir.GetNext( &fileName );
continue;
}
for ( const auto& file : GetOrderedFiles( dirPath ) ) {
wxFileName fileInfo( dirPath, fileName );
wxFileName fileInfo( dirPath, file.m_fileName );
bool isFile = wxFile::Exists( fileInfo.GetFullPath() );
if ( isFile ) {
// don't load files in the root dir if we're filtering; no official software stores files there
if ( enableFiltering && parent == nullptr ) {
hasNext = dir.GetNext( &fileName );
continue;
}
if ( AddFile( dirEntry, dirPath, fileName, parent ) ) {
if ( AddFile( dirEntry, dirPath, file, parent ) ) {
++entryNumber;
}
} else {
// if possible filter added directories by game serial
// this has the effective result of only files relevant to the current game being loaded into the memory card
// which means every game essentially sees the memory card as if no other files exist
if ( enableFiltering && !FilterMatches( fileName, localFilter ) ) {
hasNext = dir.GetNext( &fileName );
if ( enableFiltering && !FilterMatches( file.m_fileName, localFilter ) ) {
continue;
}
// make sure we have enough space on the memcard for the directory
const u32 newNeededClusters = CalculateRequiredClustersOfDirectory( dirPath + L"/" + fileName ) + ( ( dirEntry->entry.data.length % 2 ) == 0 ? 1 : 0 );
const u32 newNeededClusters = CalculateRequiredClustersOfDirectory( dirPath + L"/" + file.m_fileName ) + ( ( dirEntry->entry.data.length % 2 ) == 0 ? 1 : 0 );
if ( newNeededClusters > GetAmountFreeDataClusters() ) {
Console.Warning( GetCardFullMessage( fileName ) );
hasNext = dir.GetNext( &fileName );
Console.Warning( GetCardFullMessage( file.m_fileName ) );
continue;
}
@ -377,19 +383,19 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
// set metadata
wxFileName metaFileName( dirPath, L"_pcsx2_meta_directory" );
metaFileName.AppendDir( fileName );
metaFileName.AppendDir( file.m_fileName );
wxFFile metaFile;
if ( metaFileName.FileExists() && metaFile.Open( metaFileName.GetFullPath(), L"rb" ) ) {
size_t bytesRead = metaFile.Read( &newDirEntry->entry.raw, sizeof( newDirEntry->entry.raw ) );
metaFile.Close();
if ( bytesRead < 0x60 ) {
strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), fileName.mbc_str() );
strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), file.m_fileName.mbc_str() );
}
} else {
newDirEntry->entry.data.mode = MemoryCardFileEntry::DefaultDirMode;
newDirEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromWxDateTime( creationTime );
newDirEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromWxDateTime( modificationTime );
strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), fileName.mbc_str() );
strcpy( reinterpret_cast<char*>(newDirEntry->entry.data.name), file.m_fileName.mbc_str() );
}
// create new cluster for . and .. entries
@ -416,8 +422,6 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
// and add all files in subdir
AddFolder( newDirEntry, fileInfo.GetFullPath(), dirRef );
}
hasNext = dir.GetNext( &fileName );
}
return true;
@ -426,11 +430,11 @@ bool FolderMemoryCard::AddFolder( MemoryCardFileEntry* const dirEntry, const wxS
return false;
}
bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const wxString& fileName, MemoryCardFileMetadataReference* parent ) {
wxFileName relativeFilePath( dirPath, fileName );
bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const EnumeratedFileEntry& fileEntry, MemoryCardFileMetadataReference* parent ) {
wxFileName relativeFilePath( dirPath, fileEntry.m_fileName );
relativeFilePath.MakeRelativeTo( m_folderName.GetPath() );
wxFileName fileInfo( dirPath, fileName );
wxFileName fileInfo( dirPath, fileEntry.m_fileName );
wxFFile file( fileInfo.GetFullPath(), L"rb" );
if ( file.IsOpened() ) {
// make sure we have enough space on the memcard to hold the data
@ -444,26 +448,24 @@ bool FolderMemoryCard::AddFile( MemoryCardFileEntry* const dirEntry, const wxStr
}
MemoryCardFileEntry* newFileEntry = AppendFileEntryToDir( dirEntry );
wxDateTime creationTime, modificationTime;
fileInfo.GetTimes( NULL, &modificationTime, &creationTime );
// set file entry metadata
memset( newFileEntry->entry.raw, 0x00, sizeof( newFileEntry->entry.raw ) );
wxFileName metaFileName( dirPath, fileName );
wxFileName metaFileName( dirPath, fileEntry.m_fileName );
metaFileName.AppendDir( L"_pcsx2_meta" );
wxFFile metaFile;
if ( metaFileName.FileExists() && metaFile.Open( metaFileName.GetFullPath(), L"rb" ) ) {
size_t bytesRead = metaFile.Read( &newFileEntry->entry.raw, sizeof( newFileEntry->entry.raw ) );
metaFile.Close();
if ( bytesRead < 0x60 ) {
strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileName.mbc_str() );
strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileEntry.m_fileName.mbc_str() );
}
} else {
newFileEntry->entry.data.mode = MemoryCardFileEntry::DefaultFileMode;
newFileEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromWxDateTime( creationTime );
newFileEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromWxDateTime( modificationTime );
strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileName.mbc_str() );
newFileEntry->entry.data.timeCreated = MemoryCardFileEntryDateTime::FromTime( fileEntry.m_timeCreated );
newFileEntry->entry.data.timeModified = MemoryCardFileEntryDateTime::FromTime( fileEntry.m_timeModified );
strcpy( reinterpret_cast<char*>(newFileEntry->entry.data.name), fileEntry.m_fileName.mbc_str() );
}
newFileEntry->entry.data.length = filesize;
@ -507,6 +509,7 @@ u32 FolderMemoryCard::CalculateRequiredClustersOfDirectory( const wxString& dirP
u32 requiredFileEntryPages = 2;
u32 requiredClusters = 0;
// No need to read the index file as we are only counting space required; order of files is irrelevant.
wxDir dir( dirPath );
wxString fileName;
bool hasNext = dir.GetFirst( &fileName );
@ -1073,6 +1076,8 @@ void FolderMemoryCard::FlushDeletedFilesAndRemoveUnchangedDataFromCache( const s
RemoveDirectory( newFilePath );
}
wxRenameFile( filePath, newFilePath );
DeleteFromIndex( m_folderName.GetFullPath() + dirPath, fileName );
} else if ( entry->IsDir() ) {
// still exists and is a directory, recursive call for subdir
char cleanName[sizeof( entry->entry.data.name )];
@ -1323,6 +1328,80 @@ void FolderMemoryCard::SetTimeLastWrittenToNow() {
m_framesUntilFlush = FramesAfterWriteUntilFlush;
}
std::vector<FolderMemoryCard::EnumeratedFileEntry> FolderMemoryCard::GetOrderedFiles( const wxString& dirPath ) const
{
std::vector<EnumeratedFileEntry> result;
wxDir dir( dirPath );
if ( dir.IsOpened() ) {
YAML::Node index = LoadYAMLFromFile( wxFileName( dirPath, "_pcsx2_index" ).GetFullPath() );
// We must be able to support legacy folder memcards without the index file, so for those
// track an order variable and make it negative - this way new files get their order preserved
// and old files are listed first.
// In the YAML File order is stored as an unsigned int, so use a signed int64_t to accommodate for
// all possible values without cutting them off
std::map<int64_t, EnumeratedFileEntry> sortContainer;
int64_t orderForLegacyFiles = -1;
wxString fileName;
bool hasNext = dir.GetFirst( &fileName );
while ( hasNext ) {
if ( fileName.StartsWith( L"_pcsx2_" ) ) {
hasNext = dir.GetNext( &fileName );
continue;
}
const wxFileName fileInfo( dirPath, fileName );
try {
if ( wxFile::Exists( fileInfo.GetFullPath() ) ) {
const YAML::Node& node = index[ fileName.ToStdString() ];
EnumeratedFileEntry entry { fileName, node["timeCreated"].as<time_t>(), node["timeModified"].as<time_t>() };
sortContainer.try_emplace( node["order"].as<unsigned int>(), std::move(entry) );
}
else {
// TODO: Implement directories, for now force it to use the fallback implementation
throw YAML::InvalidNode( fileName.ToStdString() );
}
}
catch ( YAML::Exception& /*e*/ ) {
// File doesn't exist in index or it's corrupted - fall back to file-based timestamps and a custom order
wxDateTime creationTime, modificationTime;
fileInfo.GetTimes( nullptr, &modificationTime, &creationTime );
EnumeratedFileEntry entry { fileName, creationTime.GetTicks(), modificationTime.GetTicks() };
sortContainer.try_emplace( orderForLegacyFiles--, std::move(entry) );
}
hasNext = dir.GetNext( &fileName );
}
// Move items from the intermediate map to a final vector
result.reserve( sortContainer.size() );
for ( auto& e : sortContainer ) {
result.push_back( std::move(e.second) );
}
}
return result;
}
void FolderMemoryCard::DeleteFromIndex( const wxString& filePath, const wxString& entry ) const
{
const wxString indexName = wxFileName( filePath, "_pcsx2_index" ).GetFullPath();
YAML::Node index = LoadYAMLFromFile( indexName );
index.remove( entry.ToStdString() );
// Write out the changes
wxFFile indexFile;
if ( indexFile.Open( indexName, L"w" ) ) {
indexFile.Write( YAML::Dump( index ) );
}
}
// from http://www.oocities.org/siliconvalley/station/8269/sma02/sma02.html#ECC
void FolderMemoryCard::CalculateECC( u8* ecc, const u8* data ) {
static const u8 Table[] = {
@ -1413,6 +1492,7 @@ wxFFile* FileAccessHelper::Open( const wxFileName& folderName, MemoryCardFileMet
if ( writeMetadata ) {
WriteMetadata( folderName, fileRef );
WriteIndex( folderName, fileRef );
}
return file;
@ -1448,6 +1528,38 @@ void FileAccessHelper::WriteMetadata( wxFileName folderName, const MemoryCardFil
}
}
void FileAccessHelper::WriteIndex( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef )
{
fileRef->GetPath( &folderName );
const std::string fileName = folderName.GetName();
folderName.SetName( L"_pcsx2_index" );
YAML::Node index = LoadYAMLFromFile( folderName.GetFullPath() );
YAML::Node entryNode = index[fileName];
if ( !entryNode.IsDefined() ) {
// Newly added file - figure out the sort order as the entry should be added to the end of the list
unsigned int order = 0;
for ( const auto& node : index ) {
order = std::max( order, node.second["order"].as<unsigned int>() );
}
entryNode["order"] = order + 1;
}
// Update timestamps basing on internal data
const auto* entry = &fileRef->entry->entry.data;
entryNode["timeCreated"] = entry->timeCreated.ToTime();
entryNode["timeModified"] = entry->timeModified.ToTime();
// Write out the changes
wxFFile indexFile;
if ( indexFile.Open( folderName.GetFullPath(), L"w" ) ) {
indexFile.Write( YAML::Dump( index ) );
}
}
wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileMetadataReference* fileRef, bool writeMetadata ) {
std::string internalPath;
fileRef->GetInternalPath( &internalPath );
@ -1459,6 +1571,7 @@ wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileM
if ( writeMetadata ) {
if ( m_lastWrittenFileRef != fileRef ) {
WriteMetadata( folderName, fileRef );
WriteIndex( folderName, fileRef );
m_lastWrittenFileRef = fileRef;
}
} else {
@ -1479,13 +1592,6 @@ wxFFile* FileAccessHelper::ReOpen( const wxFileName& folderName, MemoryCardFileM
void FileAccessHelper::CloseFileHandle( wxFFile* file, const MemoryCardFileEntry* entry ) {
file->Close();
if ( entry != nullptr ) {
wxFileName fn( file->GetName() );
wxDateTime modified = entry->entry.data.timeModified.ToWxDateTime();
wxDateTime created = entry->entry.data.timeCreated.ToWxDateTime();
fn.SetTimes( nullptr, &modified, &created );
}
delete file;
}

View File

@ -87,6 +87,11 @@ struct MemoryCardFileEntryDateTime {
return t;
}
static MemoryCardFileEntryDateTime FromTime( time_t time ) {
// TODO: When wx is gone, this will have to be handled differently; for now, rely on wx
return FromWxDateTime( wxDateTime(time) );
}
wxDateTime ToWxDateTime() const {
wxDateTime::Tm tm;
tm.sec = this->second;
@ -100,6 +105,11 @@ struct MemoryCardFileEntryDateTime {
return time.FromTimezone( wxDateTime::GMT9 );
}
time_t ToTime() const {
// TODO: When wx is gone, this will have to be handled differently; for now, rely on wx
return ToWxDateTime().GetTicks();
}
bool operator==( const MemoryCardFileEntryDateTime& other ) const {
return unused == other.unused && second == other.second && minute == other.minute && hour == other.hour
&& day == other.day && month == other.month && year == other.year;
@ -247,6 +257,7 @@ private:
void CloseFileHandle( wxFFile* file, const MemoryCardFileEntry* entry = nullptr );
void WriteMetadata( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef );
void WriteIndex( wxFileName folderName, const MemoryCardFileMetadataReference* fileRef );
};
// --------------------------------------------------------------------------------------
@ -375,6 +386,12 @@ public:
void WriteToFile( const wxString& filename );
protected:
struct EnumeratedFileEntry {
wxString m_fileName; // TODO: Replace with std::string
time_t m_timeCreated;
time_t m_timeModified;
};
// initializes memory card data, as if it was fresh from the factory
void InitializeInternalData();
@ -458,7 +475,7 @@ protected:
// - dirPath: the full path to the directory containing the file in the host file system
// - fileName: the name of the file, without path
// - parent: pointer to the parent dir's quick-access reference element
bool AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const wxString& fileName, MemoryCardFileMetadataReference* parent = nullptr );
bool AddFile( MemoryCardFileEntry* const dirEntry, const wxString& dirPath, const EnumeratedFileEntry& fileEntry, MemoryCardFileMetadataReference* parent = nullptr );
// calculates the amount of clusters a directory would use up if put into a memory card
u32 CalculateRequiredClustersOfDirectory( const wxString& dirPath ) const;
@ -537,6 +554,12 @@ protected:
wxString GetCardFullMessage( const wxString& filePath ) const {
return wxsFormat( pxE( L"(FolderMcd) Memory Card is full, could not add: %s" ), WX_STR( filePath ) );
}
// get the list of files (and their timestamps) in directory ordered as specified by the index file
// for legacy entries without an entry in the index file, order is unspecified and should not be relied on
std::vector<EnumeratedFileEntry> GetOrderedFiles( const wxString& dirPath ) const;
void DeleteFromIndex( const wxString& filePath, const wxString& entry ) const;
};
// --------------------------------------------------------------------------------------