Merge pull request #8104 from AdmiralCurtiss/memcard-folder-small

GCMemcardDirectory: Improve logic for initial load of GCI files.

Source/Core/Core/CMakeLists.txt

@@ -107,6 +107,7 @@ add_library(core
   HW/EXI/EXI_DeviceMic.cpp
   HW/GCKeyboard.cpp
   HW/GCKeyboardEmu.cpp
+  HW/GCMemcard/GCIFile.cpp
   HW/GCMemcard/GCMemcard.cpp
   HW/GCMemcard/GCMemcardDirectory.cpp
   HW/GCMemcard/GCMemcardRaw.cpp

Source/Core/Core/Core.vcxproj

@@ -146,6 +146,7 @@
     <ClCompile Include="HW\EXI\EXI_DeviceMic.cpp" />
     <ClCompile Include="HW\GCKeyboard.cpp" />
     <ClCompile Include="HW\GCKeyboardEmu.cpp" />
+    <ClCompile Include="HW\GCMemcard\GCIFile.cpp" />
     <ClCompile Include="HW\GCMemcard\GCMemcard.cpp" />
     <ClCompile Include="HW\GCMemcard\GCMemcardDirectory.cpp" />
     <ClCompile Include="HW\GCMemcard\GCMemcardRaw.cpp" />
@@ -414,6 +415,7 @@
     <ClInclude Include="HW\EXI\EXI_DeviceMic.h" />
     <ClInclude Include="HW\GCKeyboard.h" />
     <ClInclude Include="HW\GCKeyboardEmu.h" />
+    <ClInclude Include="HW\GCMemcard\GCIFile.h" />
     <ClInclude Include="HW\GCMemcard\GCMemcard.h" />
     <ClInclude Include="HW\GCMemcard\GCMemcardDirectory.h" />
     <ClInclude Include="HW\GCMemcard\GCMemcardRaw.h" />

Source/Core/Core/Core.vcxproj.filters

@@ -462,6 +462,9 @@
     <ClCompile Include="HW\Sram.cpp">
       <Filter>HW %28Flipper/Hollywood%29\EXI - Expansion Interface</Filter>
     </ClCompile>
+    <ClCompile Include="HW\GCMemcard\GCIFile.cpp">
+      <Filter>HW %28Flipper/Hollywood%29\GCMemcard</Filter>
+    </ClCompile>
     <ClCompile Include="HW\GCMemcard\GCMemcard.cpp">
       <Filter>HW %28Flipper/Hollywood%29\GCMemcard</Filter>
     </ClCompile>
@@ -1173,6 +1176,9 @@
     <ClInclude Include="HW\Sram.h">
       <Filter>HW %28Flipper/Hollywood%29\EXI - Expansion Interface</Filter>
     </ClInclude>
+    <ClInclude Include="HW\GCMemcard\GCIFile.h">
+      <Filter>HW %28Flipper/Hollywood%29\GCMemcard</Filter>
+    </ClInclude>
     <ClInclude Include="HW\GCMemcard\GCMemcard.h">
       <Filter>HW %28Flipper/Hollywood%29\GCMemcard</Filter>
     </ClInclude>

Source/Core/Core/HW/GCMemcard/GCIFile.cpp

@@ -0,0 +1,102 @@
+// Copyright 2019 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#include "Core/HW/GCMemcard/GCIFile.h"
+
+#include <cinttypes>
+
+#include "Common/ChunkFile.h"
+#include "Common/CommonTypes.h"
+#include "Common/File.h"
+#include "Common/Logging/Log.h"
+
+bool GCIFile::LoadHeader()
+{
+  if (m_filename.empty())
+    return false;
+
+  File::IOFile save_file(m_filename, "rb");
+  if (!save_file)
+    return false;
+
+  INFO_LOG(EXPANSIONINTERFACE, "Reading header from disk for %s", m_filename.c_str());
+  if (!save_file.ReadBytes(&m_gci_header, sizeof(m_gci_header)))
+  {
+    ERROR_LOG(EXPANSIONINTERFACE, "Failed to read header for %s", m_filename.c_str());
+    return false;
+  }
+
+  return true;
+}
+
+bool GCIFile::LoadSaveBlocks()
+{
+  if (m_save_data.empty())
+  {
+    if (m_filename.empty())
+      return false;
+
+    File::IOFile save_file(m_filename, "rb");
+    if (!save_file)
+      return false;
+
+    INFO_LOG(EXPANSIONINTERFACE, "Reading savedata from disk for %s", m_filename.c_str());
+    u16 num_blocks = m_gci_header.m_block_count;
+
+    const u32 size = num_blocks * BLOCK_SIZE;
+    u64 file_size = save_file.GetSize();
+    if (file_size != size + DENTRY_SIZE)
+    {
+      ERROR_LOG(EXPANSIONINTERFACE,
+                "%s\nwas not loaded because it is an invalid GCI.\n File size (0x%" PRIx64
+                ") does not match the size recorded in the header (0x%x)",
+                m_filename.c_str(), file_size, size + DENTRY_SIZE);
+      return false;
+    }
+
+    m_save_data.resize(num_blocks);
+    save_file.Seek(DENTRY_SIZE, SEEK_SET);
+    if (!save_file.ReadBytes(m_save_data.data(), size))
+    {
+      ERROR_LOG(EXPANSIONINTERFACE, "Failed to read data from GCI file %s", m_filename.c_str());
+      m_save_data.clear();
+      return false;
+    }
+  }
+  return true;
+}
+
+bool GCIFile::HasCopyProtection() const
+{
+  if ((strcmp(reinterpret_cast<const char*>(m_gci_header.m_filename.data()), "PSO_SYSTEM") == 0) ||
+      (strcmp(reinterpret_cast<const char*>(m_gci_header.m_filename.data()), "PSO3_SYSTEM") == 0) ||
+      (strcmp(reinterpret_cast<const char*>(m_gci_header.m_filename.data()), "f_zero.dat") == 0))
+    return true;
+  return false;
+}
+
+int GCIFile::UsesBlock(u16 block_num)
+{
+  for (u16 i = 0; i < m_used_blocks.size(); ++i)
+  {
+    if (m_used_blocks[i] == block_num)
+      return i;
+  }
+  return -1;
+}
+
+void GCIFile::DoState(PointerWrap& p)
+{
+  p.DoPOD<DEntry>(m_gci_header);
+  p.Do(m_dirty);
+  p.Do(m_filename);
+  int num_blocks = (int)m_save_data.size();
+  p.Do(num_blocks);
+  m_save_data.resize(num_blocks);
+  for (auto itr = m_save_data.begin(); itr != m_save_data.end(); ++itr)
+  {
+    p.DoPOD<GCMBlock>(*itr);
+  }
+  p.Do(m_used_blocks);
+}

Source/Core/Core/HW/GCMemcard/GCIFile.h

@@ -0,0 +1,29 @@
+// Copyright 2019 Dolphin Emulator Project
+// Licensed under GPLv2+
+// Refer to the license.txt file included.
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "Common/CommonTypes.h"
+#include "Core/HW/GCMemcard/GCMemcard.h"
+
+class PointerWrap;
+
+class GCIFile
+{
+public:
+  bool LoadHeader();
+  bool LoadSaveBlocks();
+  bool HasCopyProtection() const;
+  void DoState(PointerWrap& p);
+  int UsesBlock(u16 blocknum);
+
+  DEntry m_gci_header;
+  std::vector<GCMBlock> m_save_data;
+  std::vector<u16> m_used_blocks;
+  bool m_dirty;
+  std::string m_filename;
+};

Source/Core/Core/HW/GCMemcard/GCMemcard.h

@@ -337,30 +337,6 @@ struct BlockAlloc
 static_assert(sizeof(BlockAlloc) == BLOCK_SIZE);
 #pragma pack(pop)
 
-class GCIFile
-{
-public:
-  bool LoadSaveBlocks();
-  bool HasCopyProtection() const
-  {
-    if ((strcmp(reinterpret_cast<const char*>(m_gci_header.m_filename.data()), "PSO_SYSTEM") ==
-         0) ||
-        (strcmp(reinterpret_cast<const char*>(m_gci_header.m_filename.data()), "PSO3_SYSTEM") ==
-         0) ||
-        (strcmp(reinterpret_cast<const char*>(m_gci_header.m_filename.data()), "f_zero.dat") == 0))
-      return true;
-    return false;
-  }
-
-  void DoState(PointerWrap& p);
-  DEntry m_gci_header;
-  std::vector<GCMBlock> m_save_data;
-  std::vector<u16> m_used_blocks;
-  int UsesBlock(u16 blocknum);
-  bool m_dirty;
-  std::string m_filename;
-};
-
 class GCMemcard
 {
 private:

Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.cpp

@@ -32,96 +32,64 @@
 const int NO_INDEX = -1;
 static const char* MC_HDR = "MC_SYSTEM_AREA";
 
-int GCMemcardDirectory::LoadGCI(const std::string& file_name, bool current_game_only)
+bool GCMemcardDirectory::LoadGCI(GCIFile gci)
 {
-  File::IOFile gci_file(file_name, "rb");
+  // check if any already loaded file has the same internal name as the new file
-  if (gci_file)
+  for (const GCIFile& already_loaded_gci : m_saves)
   {
-    GCIFile gci;
+    if (gci.m_gci_header.GCI_FileName() == already_loaded_gci.m_gci_header.GCI_FileName())
-    gci.m_filename = file_name;
-    gci.m_dirty = false;
-    if (!gci_file.ReadBytes(&(gci.m_gci_header), DENTRY_SIZE))
     {
-      ERROR_LOG(EXPANSIONINTERFACE, "%s failed to read header", file_name.c_str());
+      ERROR_LOG(EXPANSIONINTERFACE,
-      return NO_INDEX;
+                "%s\nwas not loaded because it has the same internal filename as previously "
+                "loaded save\n%s",
+                gci.m_filename.c_str(), already_loaded_gci.m_filename.c_str());
+      return false;
     }
-
-    std::string gci_filename = gci.m_gci_header.GCI_FileName();
-    for (u16 i = 0; i < m_loaded_saves.size(); ++i)
-    {
-      if (m_loaded_saves[i] == gci_filename)
-      {
-        PanicAlertT("%s\nwas not loaded because it has the same internal filename as previously "
-                    "loaded save\n%s",
-                    gci.m_filename.c_str(), m_saves[i].m_filename.c_str());
-        return NO_INDEX;
-      }
-    }
-
-    u16 num_blocks = gci.m_gci_header.m_block_count;
-    // largest number of free blocks on a memory card
-    // in reality, there are not likely any valid gci files > 251 blocks
-    if (num_blocks > 2043)
-    {
-      PanicAlertT(
-          "%s\nwas not loaded because it is an invalid GCI.\n Number of blocks claimed to be %u",
-          gci.m_filename.c_str(), num_blocks);
-      return NO_INDEX;
-    }
-
-    u32 size = num_blocks * BLOCK_SIZE;
-    u64 file_size = gci_file.GetSize();
-    if (file_size != size + DENTRY_SIZE)
-    {
-      PanicAlertT("%s\nwas not loaded because it is an invalid GCI.\n File size (0x%" PRIx64
-                  ") does not match the size recorded in the header (0x%x)",
-                  gci.m_filename.c_str(), file_size, size + DENTRY_SIZE);
-      return NO_INDEX;
-    }
-
-    if (m_game_id == BE32(gci.m_gci_header.m_gamecode.data()))
-    {
-      gci.LoadSaveBlocks();
-    }
-    else
-    {
-      if (current_game_only)
-      {
-        return NO_INDEX;
-      }
-      int total_blocks = m_hdr.m_size_mb * MBIT_TO_BLOCKS - MC_FST_BLOCKS;
-      int free_blocks = m_bat1.m_free_blocks;
-      if (total_blocks > free_blocks * 10)
-      {
-        PanicAlertT("%s\nwas not loaded because there is less than 10%% free blocks available on "
-                    "the memory card\n"
-                    "Total Blocks: %d; Free Blocks: %d",
-                    gci.m_filename.c_str(), total_blocks, free_blocks);
-        return NO_INDEX;
-      }
-    }
-    u16 first_block = m_bat1.AssignBlocksContiguous(num_blocks);
-    if (first_block == 0xFFFF)
-    {
-      PanicAlertT(
-          "%s\nwas not loaded because there are not enough free blocks on the virtual memory card",
-          file_name.c_str());
-      return NO_INDEX;
-    }
-    gci.m_gci_header.m_first_block = first_block;
-    if (gci.HasCopyProtection() && gci.LoadSaveBlocks())
-    {
-      GCMemcard::PSO_MakeSaveGameValid(m_hdr, gci.m_gci_header, gci.m_save_data);
-      GCMemcard::FZEROGX_MakeSaveGameValid(m_hdr, gci.m_gci_header, gci.m_save_data);
-    }
-    int idx = (int)m_saves.size();
-    m_dir1.Replace(gci.m_gci_header, idx);
-    m_saves.push_back(std::move(gci));
-    SetUsedBlocks(idx);
-
-    return idx;
   }
-  return NO_INDEX;
+
+  // check if this file has a valid block size
+  // 2043 is the largest number of free blocks on a memory card
+  // in reality, there are not likely any valid gci files > 251 blocks
+  const u16 num_blocks = gci.m_gci_header.m_block_count;
+  if (num_blocks > 2043)
+  {
+    ERROR_LOG(EXPANSIONINTERFACE,
+              "%s\nwas not loaded because it is an invalid GCI.\nNumber of blocks claimed to be %u",
+              gci.m_filename.c_str(), num_blocks);
+    return false;
+  }
+
+  if (!gci.LoadSaveBlocks())
+  {
+    ERROR_LOG(EXPANSIONINTERFACE, "Failed to load data of %s", gci.m_filename.c_str());
+    return false;
+  }
+
+  // reserve storage for the save file in the BAT
+  u16 first_block = m_bat1.AssignBlocksContiguous(num_blocks);
+  if (first_block == 0xFFFF)
+  {
+    ERROR_LOG(
+        EXPANSIONINTERFACE,
+        "%s\nwas not loaded because there are not enough free blocks on the virtual memory card",
+        gci.m_filename.c_str());
+    return false;
+  }
+  gci.m_gci_header.m_first_block = first_block;
+
+  if (gci.HasCopyProtection())
+  {
+    GCMemcard::PSO_MakeSaveGameValid(m_hdr, gci.m_gci_header, gci.m_save_data);
+    GCMemcard::FZEROGX_MakeSaveGameValid(m_hdr, gci.m_gci_header, gci.m_save_data);
+  }
+
+  // actually load save file into memory card
+  int idx = (int)m_saves.size();
+  m_dir1.Replace(gci.m_gci_header, idx);
+  m_saves.push_back(std::move(gci));
+  SetUsedBlocks(idx);
+
+  return true;
 }
 
 // This is only used by NetPlay but it made sense to put it here to keep the relevant code together
@@ -187,34 +155,69 @@ GCMemcardDirectory::GCMemcardDirectory(const std::string& directory, int slot, u
     File::IOFile((m_save_directory + MC_HDR), "rb").ReadBytes(&m_hdr, BLOCK_SIZE);
   }
 
+  const bool current_game_only = Config::Get(Config::MAIN_GCI_FOLDER_CURRENT_GAME_ONLY);
   std::vector<std::string> filenames = Common::DoFileSearch({m_save_directory}, {".gci"});
 
-  if (filenames.size() > 112)
+  // split up into files for current games we should definitely load,
+  // and files for other games that we don't care too much about
+  std::vector<GCIFile> gci_current_game;
+  std::vector<GCIFile> gci_other_games;
+  for (const std::string& filename : filenames)
   {
-    Core::DisplayMessage("Warning: There are more than 112 save files on this memory card.\n"
+    GCIFile gci;
-                         " Only loading the first 112 in the folder, unless the game ID is the "
+    gci.m_filename = filename;
-                         "same as the current game's ID",
+    gci.m_dirty = false;
-                         4000);
+    if (!gci.LoadHeader())
+    {
+      ERROR_LOG(EXPANSIONINTERFACE, "Failed to load header of %s", filename.c_str());
+      continue;
+    }
+
+    if (m_game_id == BE32(gci.m_gci_header.m_gamecode.data()))
+      gci_current_game.emplace_back(std::move(gci));
+    else if (!current_game_only)
+      gci_other_games.emplace_back(std::move(gci));
   }
 
-  for (const std::string& gci_file : filenames)
+  m_saves.reserve(DIRLEN);
+
+  // load files for current game
+  size_t failed_loads_current_game = 0;
+  for (GCIFile& gci : gci_current_game)
   {
-    if (m_saves.size() == DIRLEN)
+    if (!LoadGCI(std::move(gci)))
     {
-      PanicAlertT(
+      // keep track of how many files failed to load for the current game so we can display a
-          "There are too many GCI files in the folder\n%s.\nOnly the first 127 will be available",
+      // message to the user informing them why some of their saves may not be loaded
-          m_save_directory.c_str());
+      ++failed_loads_current_game;
+    }
+  }
+
+  // leave about 10% of free space on the card if possible
+  const int total_blocks = m_hdr.m_size_mb * MBIT_TO_BLOCKS - MC_FST_BLOCKS;
+  const int reserved_blocks = total_blocks / 10;
+
+  // load files for other games
+  for (GCIFile& gci : gci_other_games)
+  {
+    // leave some free file entries for new saves that might be created
+    if (m_saves.size() > 112)
       break;
-    }
+
-    int index = LoadGCI(gci_file, m_saves.size() > 112 ||
+    // leave some free blocks for new saves that might be created
-                                      Config::Get(Config::MAIN_GCI_FOLDER_CURRENT_GAME_ONLY));
+    const int free_blocks = m_bat1.m_free_blocks;
-    if (index != NO_INDEX)
+    const int gci_blocks = gci.m_gci_header.m_block_count;
-    {
+    if (free_blocks - gci_blocks < reserved_blocks)
-      m_loaded_saves.push_back(m_saves.at(index).m_gci_header.GCI_FileName());
+      continue;
-    }
+
+    LoadGCI(std::move(gci));
+  }
+
+  if (failed_loads_current_game > 0)
+  {
+    Core::DisplayMessage("Warning: Save file(s) of the current game failed to load.", 10000);
   }
 
-  m_loaded_saves.clear();
   m_dir1.FixChecksums();
   m_dir2 = m_dir1;
   m_bat2 = m_bat1;
@@ -693,56 +696,6 @@ void GCMemcardDirectory::DoState(PointerWrap& p)
   }
 }
 
-bool GCIFile::LoadSaveBlocks()
-{
-  if (m_save_data.empty())
-  {
-    if (m_filename.empty())
-      return false;
-
-    File::IOFile save_file(m_filename, "rb");
-    if (!save_file)
-      return false;
-
-    INFO_LOG(EXPANSIONINTERFACE, "Reading savedata from disk for %s", m_filename.c_str());
-    save_file.Seek(DENTRY_SIZE, SEEK_SET);
-    u16 num_blocks = m_gci_header.m_block_count;
-    m_save_data.resize(num_blocks);
-    if (!save_file.ReadBytes(m_save_data.data(), num_blocks * BLOCK_SIZE))
-    {
-      PanicAlertT("Failed to read data from GCI file %s", m_filename.c_str());
-      m_save_data.clear();
-      return false;
-    }
-  }
-  return true;
-}
-
-int GCIFile::UsesBlock(u16 block_num)
-{
-  for (u16 i = 0; i < m_used_blocks.size(); ++i)
-  {
-    if (m_used_blocks[i] == block_num)
-      return i;
-  }
-  return -1;
-}
-
-void GCIFile::DoState(PointerWrap& p)
-{
-  p.DoPOD<DEntry>(m_gci_header);
-  p.Do(m_dirty);
-  p.Do(m_filename);
-  int num_blocks = (int)m_save_data.size();
-  p.Do(num_blocks);
-  m_save_data.resize(num_blocks);
-  for (auto itr = m_save_data.begin(); itr != m_save_data.end(); ++itr)
-  {
-    p.DoPOD<GCMBlock>(*itr);
-  }
-  p.Do(m_used_blocks);
-}
-
 void MigrateFromMemcardFile(const std::string& directory_name, int card_index)
 {
   File::CreateFullPath(directory_name);

Source/Core/Core/HW/GCMemcard/GCMemcardDirectory.h

@@ -10,6 +10,7 @@
 #include <vector>
 
 #include "Common/Event.h"
+#include "Core/HW/GCMemcard/GCIFile.h"
 #include "Core/HW/GCMemcard/GCMemcard.h"
 
 // Uncomment this to write the system data of the memorycard from directory to disc
@@ -39,7 +40,7 @@ public:
   void DoState(PointerWrap& p) override;
 
 private:
-  int LoadGCI(const std::string& file_name, bool current_game_only);
+  bool LoadGCI(GCIFile gci);
   inline s32 SaveAreaRW(u32 block, bool writing = false);
   // s32 DirectoryRead(u32 offset, u32 length, u8* dest_address);
   s32 DirectoryWrite(u32 dest_address, u32 length, const u8* src_address);
@@ -55,7 +56,6 @@ private:
   BlockAlloc m_bat1, m_bat2;
   std::vector<GCIFile> m_saves;
 
-  std::vector<std::string> m_loaded_saves;
   std::string m_save_directory;
   Common::Event m_flush_trigger;
   std::mutex m_write_mutex;