Merge branch 'master' into d3d12

.gitmodules

@@ -18,7 +18,7 @@
 	url = https://github.com/xenia-project/SPIRV-Tools.git
 [submodule "third_party/catch"]
 	path = third_party/catch
-	url = https://github.com/philsquared/Catch.git
+	url = https://github.com/catchorg/Catch2.git
 [submodule "third_party/gflags"]
 	path = third_party/gflags
 	url = https://github.com/benvanik/gflags.git

README.md

@@ -1,6 +1,6 @@
 <p align="center">
     <a href="https://github.com/xenia-project/xenia/tree/master/assets/icon">
-        <img height="370px" src="https://raw.githubusercontent.com/xenia-project/xenia/master/assets/icon/1024.png" />
+        <img height="120px" src="https://raw.githubusercontent.com/xenia-project/xenia/master/assets/icon/128.png" />
     </a>
 </p>
 

src/xenia/app/xenia_main.cc

@@ -169,7 +169,7 @@ int xenia_main(const std::vector<std::wstring>& args) {
 #if defined(XE_PLATFORM_WIN32)
       content_root = xe::join_paths(content_root, L"Xenia");
 #elif defined(XE_PLATFORM_LINUX)
-      content_root = xe::join_paths(content_root, L".xenia");
+      content_root = xe::join_paths(content_root, L"Xenia");
 #else
 #warning Unhandled platform for content root.
       content_root = xe::join_paths(content_root, L"Xenia");

src/xenia/base/filesystem.h

@@ -116,6 +116,7 @@ struct FileInfo {
   };
   Type type;
   std::wstring name;
+  std::wstring path;
   size_t total_size;
   uint64_t create_timestamp;
   uint64_t access_timestamp;

src/xenia/base/filesystem_posix.cc

@@ -12,29 +12,58 @@
 #include "xenia/base/logging.h"
 #include "xenia/base/string.h"
 
+#include <assert.h>
 #include <dirent.h>
 #include <fcntl.h>
 #include <ftw.h>
+#include <libgen.h>
+#include <pwd.h>
+#include <stdio.h>
+#include <string.h>
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <iostream>
 
 namespace xe {
 namespace filesystem {
 
 std::wstring GetExecutablePath() {
-  assert_always();  // IMPLEMENT ME.
+  char buff[FILENAME_MAX] = "";
-  return std::wstring();
+  readlink("/proc/self/exe", buff, FILENAME_MAX);
+  std::string s(buff);
+  return to_wstring(s);
 }
 
 std::wstring GetExecutableFolder() {
-  assert_always();  // IMPLEMENT ME.
+  auto path = GetExecutablePath();
-  return std::wstring();
+  return xe::find_base_path(path);
 }
 
 std::wstring GetUserFolder() {
-  assert_always();  // IMPLEMENT ME.
+  // get preferred data home
-  return std::wstring();
+  char* dataHome = std::getenv("XDG_DATA_HOME");
+
+  // if XDG_DATA_HOME not set, fallback to HOME directory
+  if (dataHome == NULL) {
+    dataHome = std::getenv("HOME");
+  } else {
+    std::string home(dataHome);
+    return to_wstring(home);
+  }
+
+  // if HOME not set, fall back to this
+  if (dataHome == NULL) {
+    struct passwd pw1;
+    struct passwd* pw;
+    char buf[4096];  // could potentionally lower this
+    getpwuid_r(getuid(), &pw1, buf, sizeof(buf), &pw);
+    assert(&pw1 == pw);  // sanity check
+    dataHome = pw->pw_dir;
+  }
+
+  std::string home(dataHome);
+  return to_wstring(home + "/.local/share");
 }
 
 bool PathExists(const std::wstring& path) {

src/xenia/base/filesystem_win.cc

@@ -207,6 +207,7 @@ bool GetInfo(const std::wstring& path, FileInfo* out_info) {
     out_info->total_size =
         (data.nFileSizeHigh * (size_t(MAXDWORD) + 1)) + data.nFileSizeLow;
   }
+  out_info->path = xe::find_base_path(path);
   out_info->name = xe::find_name_from_path(path);
   out_info->create_timestamp = COMBINE_TIME(data.ftCreationTime);
   out_info->access_timestamp = COMBINE_TIME(data.ftLastAccessTime);
@@ -236,6 +237,7 @@ std::vector<FileInfo> ListFiles(const std::wstring& path) {
       info.total_size =
           (ffd.nFileSizeHigh * (size_t(MAXDWORD) + 1)) + ffd.nFileSizeLow;
     }
+    info.path = path;
     info.name = ffd.cFileName;
     info.create_timestamp = COMBINE_TIME(ffd.ftCreationTime);
     info.access_timestamp = COMBINE_TIME(ffd.ftLastAccessTime);

src/xenia/base/string.cc

@@ -9,12 +9,27 @@
 
 #include "xenia/base/string.h"
 
-// TODO(benvanik): when GCC finally gets codecvt, use that.
+// codecvt existence check
-#if XE_PLATFORM_LINUX
+#ifdef __clang__
+// using clang
+#if (__clang_major__ < 4)  // 3.3 has it but I think we need at least 4 anyway
+// insufficient clang version
 #define NO_CODECVT 1
 #else
 #include <codecvt>
-#endif  // XE_PLATFORM_LINUX
+#endif
+#elif defined(__GNUC__) || defined(__GNUG__)
+// using gcc
+#if (__GNUC__ < 5)
+// insufficient clang version
+#define NO_CODECVT 1
+#else
+#include <codecvt>
+#endif
+// since the windows 10 sdk is required, this shouldn't be an issue
+#elif defined(_MSC_VER)
+#include <codecvt>
+#endif
 
 #include <cctype>
 #include <cstring>

src/xenia/vfs/devices/stfs_container_device.cc

@@ -10,6 +10,7 @@
 #include "xenia/vfs/devices/stfs_container_device.h"
 
 #include <algorithm>
+#include <queue>
 #include <vector>
 
 #include "xenia/base/logging.h"
@@ -59,87 +60,88 @@ StfsContainerDevice::StfsContainerDevice(const std::string& mount_path,
 StfsContainerDevice::~StfsContainerDevice() = default;
 
 bool StfsContainerDevice::Initialize() {
-  if (filesystem::IsFolder(local_path_)) {
+  // Resolve a valid STFS file if a directory is given.
-    // Was given a folder. Try to find the file in
+  if (filesystem::IsFolder(local_path_) && !ResolveFromFolder(local_path_)) {
-    // local_path\TITLE_ID\000D0000\HASH_OF_42_CHARS
+    XELOGE("Could not resolve an STFS container given path %s",
-    // We take care to not die if there are additional files around.
+           xe::to_string(local_path_).c_str());
-    bool found_alternative = false;
+    return false;
-    auto files = filesystem::ListFiles(local_path_);
-    for (auto& file : files) {
-      if (file.type != filesystem::FileInfo::Type::kDirectory ||
-          file.name.size() != 8) {
-        continue;
-      }
-      auto child_path = xe::join_paths(local_path_, file.name);
-      auto child_files = filesystem::ListFiles(child_path);
-      for (auto& child_file : child_files) {
-        if (child_file.type != filesystem::FileInfo::Type::kDirectory ||
-            child_file.name != L"000D0000") {
-          continue;
-        }
-        auto stfs_path = xe::join_paths(child_path, child_file.name);
-        auto stfs_files = filesystem::ListFiles(stfs_path);
-        for (auto& stfs_file : stfs_files) {
-          if (stfs_file.type != filesystem::FileInfo::Type::kFile ||
-              stfs_file.name.size() != 42) {
-            continue;
-          }
-          // Probably it!
-          local_path_ = xe::join_paths(stfs_path, stfs_file.name);
-          found_alternative = true;
-          break;
-        }
-        if (found_alternative) {
-          break;
-        }
-      }
-      if (found_alternative) {
-        break;
-      }
-    }
   }
+
   if (!filesystem::PathExists(local_path_)) {
-    XELOGE("STFS container does not exist");
+    XELOGE("Path to STFS container does not exist: %s",
+           xe::to_string(local_path_).c_str());
     return false;
   }
 
-  mmap_ = MappedMemory::Open(local_path_, MappedMemory::Mode::kRead);
+  // Map the data file(s)
-  if (!mmap_) {
+  auto map_result = MapFiles();
-    XELOGE("STFS container could not be mapped");
+  if (map_result != Error::kSuccess) {
-    return false;
+    XELOGE("Failed to map STFS container: %d", map_result);
-  }
-
-  uint8_t* map_ptr = mmap_->data();
-
-  auto result = ReadHeaderAndVerify(map_ptr);
-  if (result != Error::kSuccess) {
-    XELOGE("STFS header read/verification failed: %d", result);
     return false;
   }
 
   switch (header_.descriptor_type) {
     case StfsDescriptorType::kStfs:
-      result = ReadAllEntriesSTFS(map_ptr);
+      return ReadSTFS() == Error::kSuccess;
       break;
     case StfsDescriptorType::kSvod:
-      if (!(header_.svod_volume_descriptor.device_features &
+      return ReadSVOD() == Error::kSuccess;
-            kFeatureHasEnhancedGDFLayout)) {
-        XELOGE("STFS SVOD does not have GDF layout!");
-        return false;
-      }
-
-      result = ReadAllEntriesEGDF(map_ptr);
-      break;
     default:
-      // Shouldn't reach here.
+      XELOGE("Unknown STFS Descriptor Type: %d", header_.descriptor_type);
       return false;
   }
-  if (result != Error::kSuccess) {
+}
-    XELOGE("STFS entry reading failed: %d", result);
+
-    return false;
+StfsContainerDevice::Error StfsContainerDevice::MapFiles() {
+  // Map the file containing the STFS Header and read it.
+  XELOGI("Mapping STFS Header File: %s", xe::to_string(local_path_).c_str());
+  auto header_map = MappedMemory::Open(local_path_, MappedMemory::Mode::kRead);
+
+  auto header_result = ReadHeaderAndVerify(header_map->data());
+  if (header_result != Error::kSuccess) {
+    XELOGE("Error reading STFS Header: %d", header_result);
+    return header_result;
   }
 
-  return true;
+  // If the STFS package is a single file, the header is self contained and
+  // we don't need to map any extra files.
+  // NOTE: data_file_count is 0 for STFS and 1 for SVOD
+  if (header_.data_file_count <= 1) {
+    XELOGI("STFS container is a single file.");
+    mmap_.emplace(std::make_pair(0, std::move(header_map)));
+    return Error::kSuccess;
+  }
+
+  // If the STFS package is multi-file, it is an SVOD system. We need to map
+  // the files in the .data folder and can discard the header.
+  auto data_fragment_path = local_path_ + L".data";
+  if (!filesystem::PathExists(data_fragment_path)) {
+    XELOGE("STFS container is multi-file, but path %s does not exist.",
+           xe::to_string(data_fragment_path).c_str());
+    return Error::kErrorFileMismatch;
+  }
+
+  // Ensure data fragment files are sorted
+  auto fragment_files = filesystem::ListFiles(data_fragment_path);
+  std::sort(fragment_files.begin(), fragment_files.end(),
+            [](filesystem::FileInfo& left, filesystem::FileInfo& right) {
+              return left.name < right.name;
+            });
+
+  if (fragment_files.size() != header_.data_file_count) {
+    XELOGE("SVOD expecting %d data fragments, but %d are present.",
+           header_.data_file_count, fragment_files.size());
+    return Error::kErrorFileMismatch;
+  }
+
+  for (size_t i = 0; i < fragment_files.size(); i++) {
+    auto file = fragment_files.at(i);
+    auto path = xe::join_paths(file.path, file.name);
+    auto data = MappedMemory::Open(path, MappedMemory::Mode::kRead);
+    mmap_.emplace(std::make_pair(i, std::move(data)));
+  }
+  XELOGI("SVOD successfully mapped %d files.", fragment_files.size());
+  return Error::kSuccess;
 }
 
 void StfsContainerDevice::Dump(StringBuffer* string_buffer) {
@@ -196,87 +198,189 @@ StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify(
   return Error::kSuccess;
 }
 
-StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesEGDF(
+StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() {
-    const uint8_t* map_ptr) {
+  // SVOD Systems can have different layouts. The root block is
-  // Verify (and scan) the GDF magic first.
+  // denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in
-  const uint8_t* p = map_ptr + BlockToOffsetSTFS(0);
+  // the first "actual" data fragment of the system.
-  if (std::memcmp(p, "MICROSOFT*XBOX*MEDIA", 20) != 0) {
+  auto data = mmap_.at(0)->data();
-    return Error::kErrorDamagedFile;
+  const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA";
+
+  // Check for EDGF layout
+  auto layout = &header_.svod_volume_descriptor.layout_type;
+  auto features = header_.svod_volume_descriptor.device_features;
+  bool has_egdf_layout = features & kFeatureHasEnhancedGDFLayout;
+
+  if (has_egdf_layout) {
+    // The STFS header has specified that this SVOD system uses the EGDF layout.
+    // We can expect the magic block to be located immediately after the hash
+    // blocks. We also offset block address calculation by 0x1000 by shifting
+    // block indices by +0x2.
+    if (memcmp(data + 0x2000, MEDIA_MAGIC, 20) == 0) {
+      base_offset_ = 0x0000;
+      magic_offset_ = 0x2000;
+      *layout = kEnhancedGDFLayout;
+      XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000.");
+    } else {
+      XELOGE("SVOD uses an EGDF layout, but the magic block was not found.");
+      return Error::kErrorFileMismatch;
+    }
+  } else if (memcmp(data + 0x12000, MEDIA_MAGIC, 20) == 0) {
+    // If the SVOD's magic block is at 0x12000, it is likely using an XSF
+    // layout. This is usually due to converting the game using a third-party
+    // tool, as most of them use a nulled XSF as a template.
+
+    base_offset_ = 0x10000;
+    magic_offset_ = 0x12000;
+
+    // Check for XSF Header
+    const char* XSF_MAGIC = "XSF";
+    if (memcmp(data + 0x2000, XSF_MAGIC, 3) == 0) {
+      *layout = kXSFLayout;
+      XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000.");
+      XELOGI("Game was likely converted using a third-party tool.");
+    } else {
+      *layout = kUnknownLayout;
+      XELOGI("SVOD appears to use an XSF layout, but no header is present.");
+      XELOGI("SVOD magic block found at 0x12000");
+    }
+  } else if (memcmp(data + 0xD000, MEDIA_MAGIC, 20) == 0) {
+    // If the SVOD's magic block is at 0xD000, it most likely means that it is
+    // a single-file system. The STFS Header is 0xB000 bytes , and the remaining
+    // 0x2000 is from hash tables. In most cases, these will be STFS, not SVOD.
+
+    base_offset_ = 0xB000;
+    magic_offset_ = 0xD000;
+
+    // Check for single file system
+    if (header_.data_file_count == 1) {
+      *layout = kSingleFileLayout;
+      XELOGI("SVOD is a single file. Magic block present at 0xD000.");
+    } else {
+      *layout = kUnknownLayout;
+      XELOGE(
+          "SVOD is not a single file, but the magic block was found at "
+          "0xD000.");
+    }
+  } else {
+    XELOGE("Could not locate SVOD magic block.");
+    return Error::kErrorReadError;
   }
 
-  uint32_t root_sector = xe::load<uint32_t>(p + 0x14);
+  // Parse the root directory
-  uint32_t root_size = xe::load<uint32_t>(p + 0x18);
+  uint8_t* magic_block = data + magic_offset_;
+  uint32_t root_block = xe::load<uint32_t>(magic_block + 0x14);
+  uint32_t root_size = xe::load<uint32_t>(magic_block + 0x18);
+  uint32_t root_creation_date = xe::load<uint32_t>(magic_block + 0x1C);
+  uint32_t root_creation_time = xe::load<uint32_t>(magic_block + 0x20);
+  uint64_t root_creation_timestamp =
+      decode_fat_timestamp(root_creation_date, root_creation_time);
 
-  auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get());
+  auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_);
   root_entry->attributes_ = kFileAttributeDirectory;
+  root_entry->access_timestamp_ = root_creation_timestamp;
+  root_entry->create_timestamp_ = root_creation_timestamp;
+  root_entry->write_timestamp_ = root_creation_timestamp;
   root_entry_ = std::unique_ptr<Entry>(root_entry);
 
-  const uint8_t* buffer = map_ptr + BlockToOffsetEGDF(root_sector);
+  // Traverse all child entries
-  return ReadEntryEGDF(buffer, 0, root_entry) ? Error::kSuccess
+  return ReadEntrySVOD(root_block, 0, root_entry);
-                                              : Error::kErrorDamagedFile;
 }
 
-bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer,
+StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD(
-                                        uint16_t entry_ordinal,
+    uint32_t block, uint32_t ordinal, StfsContainerEntry* parent) {
-                                        StfsContainerEntry* parent) {
+  // For games with a large amount of files, the ordinal offset can overrun
-  const uint8_t* p = buffer + (entry_ordinal * 4);
+  // the current block and potentially hit a hash block.
+  size_t ordinal_offset = ordinal * 0x4;
+  size_t block_offset = ordinal_offset / 0x800;
+  size_t true_ordinal_offset = ordinal_offset % 0x800;
 
-  uint16_t node_l = xe::load<uint16_t>(p + 0);
+  // Calculate the file & address of the block
-  uint16_t node_r = xe::load<uint16_t>(p + 2);
+  size_t entry_address, entry_file;
-  uint32_t sector = xe::load<uint32_t>(p + 4);
+  BlockToOffsetSVOD(block + block_offset, &entry_address, &entry_file);
-  uint32_t length = xe::load<uint32_t>(p + 8);
+  entry_address += true_ordinal_offset;
-  uint8_t attributes = xe::load<uint8_t>(p + 12);
-  uint8_t name_length = xe::load<uint8_t>(p + 13);
-  auto name = reinterpret_cast<const char*>(p + 14);
 
-  if (node_l && !ReadEntryEGDF(buffer, node_l, parent)) {
+  // Read block's descriptor
-    return false;
+  auto data = mmap_.at(entry_file)->data() + entry_address;
+
+  uint16_t node_l = xe::load<uint16_t>(data + 0x00);
+  uint16_t node_r = xe::load<uint16_t>(data + 0x02);
+  uint32_t data_block = xe::load<uint32_t>(data + 0x04);
+  uint32_t length = xe::load<uint32_t>(data + 0x08);
+  uint8_t attributes = xe::load<uint8_t>(data + 0x0C);
+  uint8_t name_length = xe::load<uint8_t>(data + 0x0D);
+  auto name = reinterpret_cast<const char*>(data + 0x0E);
+  auto name_str = std::string(name, name_length);
+
+  // Read the left node
+  if (node_l) {
+    auto node_result = ReadEntrySVOD(block, node_l, parent);
+    if (node_result != Error::kSuccess) {
+      return node_result;
+    }
   }
 
-  auto entry = StfsContainerEntry::Create(
+  // Read file & address of block's data
-      this, parent, std::string(name, name_length), mmap_.get());
+  size_t data_address, data_file;
+  BlockToOffsetSVOD(data_block, &data_address, &data_file);
+
+  // Create the entry
+  // NOTE: SVOD entries don't have timestamps for individual files, which can
+  //       cause issues when decrypting games. Using the root entry's timestamp
+  //       solves this issues.
+  auto entry = StfsContainerEntry::Create(this, parent, name_str, &mmap_);
   if (attributes & kFileAttributeDirectory) {
-    // Folder.
+    // Entry is a directory
     entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
     entry->data_offset_ = 0;
     entry->data_size_ = 0;
+    entry->block_ = block;
+    entry->access_timestamp_ = root_entry_->create_timestamp();
+    entry->create_timestamp_ = root_entry_->create_timestamp();
+    entry->write_timestamp_ = root_entry_->create_timestamp();
 
     if (length) {
-      // Not a leaf - read in children.
+      // If length is greater than 0, traverse the directory's children
-      uint8_t* folder_ptr = mmap_->data() + BlockToOffsetEGDF(sector);
+      auto directory_result = ReadEntrySVOD(data_block, 0, entry.get());
-      if (!ReadEntryEGDF(folder_ptr, 0, entry.get())) {
+      if (directory_result != Error::kSuccess) {
-        return false;
+        return directory_result;
       }
     }
   } else {
-    // Regular file.
+    // Entry is a file
     entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
     entry->size_ = length;
     entry->allocation_size_ = xe::round_up(length, bytes_per_sector());
-    entry->data_offset_ = BlockToOffsetEGDF(sector);
+    entry->data_offset_ = data_address;
     entry->data_size_ = length;
+    entry->block_ = data_block;
+    entry->access_timestamp_ = root_entry_->create_timestamp();
+    entry->create_timestamp_ = root_entry_->create_timestamp();
+    entry->write_timestamp_ = root_entry_->create_timestamp();
 
     // Fill in all block records, sector by sector.
     if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
-      uint32_t sector_index = sector;
+      uint32_t block_index = data_block;
       size_t remaining_size = xe::round_up(length, 0x800);
 
       size_t last_record = -1;
       size_t last_offset = -1;
       while (remaining_size) {
-        size_t block_size = 0x800;
+        const size_t BLOCK_SIZE = 0x800;
-        size_t offset = BlockToOffsetEGDF(sector_index);
+
-        sector_index++;
+        size_t offset, file_index;
-        remaining_size -= block_size;
+        BlockToOffsetSVOD(block_index, &offset, &file_index);
+
+        block_index++;
+        remaining_size -= BLOCK_SIZE;
 
         if (offset - last_offset == 0x800) {
           // Consecutive, so append to last entry.
-          entry->block_list_[last_record].length += block_size;
+          entry->block_list_[last_record].length += BLOCK_SIZE;
           last_offset = offset;
           continue;
         }
 
-        entry->block_list_.push_back({offset, block_size});
+        entry->block_list_.push_back({file_index, offset, BLOCK_SIZE});
         last_record = entry->block_list_.size() - 1;
         last_offset = offset;
       }
@@ -285,17 +389,82 @@ bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer,
 
   parent->children_.emplace_back(std::move(entry));
 
-  // Read next file in the list.
+  // Read the right node.
-  if (node_r && !ReadEntryEGDF(buffer, node_r, parent)) {
+  if (node_r) {
-    return false;
+    auto node_result = ReadEntrySVOD(block, node_r, parent);
+    if (node_result != Error::kSuccess) {
+      return node_result;
+    }
   }
 
-  return true;
+  return Error::kSuccess;
 }
 
-StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
+void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address,
-    const uint8_t* map_ptr) {
+                                            size_t* out_file_index) {
-  auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get());
+  // SVOD Systems use hash blocks for integrity checks. These hash blocks
+  // cause blocks to be discontinuous in memory, and must be accounted for.
+  //  - Each data block is 0x800 bytes in length
+  //  - Every group of 0x198 data blocks is preceded a Level0 hash table.
+  //    Level0 tables contain 0xCC hashes, each representing two data blocks.
+  //    The total size of each Level0 hash table is 0x1000 bytes in length.
+  //  - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table.
+  //    Level1 tables contain 0xCB hashes, each representing two Level0 hashes.
+  //    The total size of each Level1 hash table is 0x1000 bytes in length.
+  //  - Files are split into fragments of 0xA290000 bytes in length,
+  //    consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1
+  //    Level1 hash table.
+
+  const size_t BLOCK_SIZE = 0x800;
+  const size_t HASH_BLOCK_SIZE = 0x1000;
+  const size_t BLOCKS_PER_L0_HASH = 0x198;
+  const size_t HASHES_PER_L1_HASH = 0xA1C4;
+  const size_t BLOCKS_PER_FILE = 0x14388;
+  const size_t MAX_FILE_SIZE = 0xA290000;
+  const size_t BLOCK_OFFSET = header_.svod_volume_descriptor.data_block_offset;
+  const SvodLayoutType LAYOUT = header_.svod_volume_descriptor.layout_type;
+
+  // Resolve the true block address and file index
+  size_t true_block = block - (BLOCK_OFFSET * 2);
+  if (LAYOUT == kEnhancedGDFLayout) {
+    // EGDF has an 0x1000 byte offset, which is two blocks
+    true_block += 0x2;
+  }
+
+  size_t file_block = true_block % BLOCKS_PER_FILE;
+  size_t file_index = true_block / BLOCKS_PER_FILE;
+  size_t offset = 0;
+
+  // Calculate offset caused by Level0 Hash Tables
+  size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1;
+  offset += level0_table_count * HASH_BLOCK_SIZE;
+
+  // Calculate offset caused by Level1 Hash Tables
+  size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1;
+  offset += level1_table_count * HASH_BLOCK_SIZE;
+
+  // For single-file SVOD layouts, include the size of the header in the offset.
+  if (LAYOUT == kSingleFileLayout) {
+    offset += base_offset_;
+  }
+
+  size_t block_address = (file_block * BLOCK_SIZE) + offset;
+
+  // If the offset causes the block address to overrun the file, round it.
+  if (block_address >= MAX_FILE_SIZE) {
+    file_index += 1;
+    block_address %= MAX_FILE_SIZE;
+    block_address += 0x2000;
+  }
+
+  *out_address = block_address;
+  *out_file_index = file_index;
+}
+
+StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
+  auto data = mmap_.at(0)->data();
+
+  auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_);
   root_entry->attributes_ = kFileAttributeDirectory;
   root_entry_ = std::unique_ptr<Entry>(root_entry);
 
@@ -305,7 +474,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
   auto& volume_descriptor = header_.stfs_volume_descriptor;
   uint32_t table_block_index = volume_descriptor.file_table_block_number;
   for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) {
-    const uint8_t* p = map_ptr + BlockToOffsetSTFS(table_block_index);
+    const uint8_t* p = data + BlockToOffsetSTFS(table_block_index);
     for (size_t m = 0; m < 0x1000 / 0x40; m++) {
       const uint8_t* filename = p;  // 0x28b
       if (filename[0] == 0) {
@@ -333,11 +502,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
         parent_entry = all_entries[path_indicator];
       }
 
-      auto entry = StfsContainerEntry::Create(
+      std::string name_str(reinterpret_cast<const char*>(filename),
-          this, parent_entry,
+                           filename_length_flags & 0x3F);
-          std::string(reinterpret_cast<const char*>(filename),
+      auto entry =
-                      filename_length_flags & 0x3F),
+          StfsContainerEntry::Create(this, parent_entry, name_str, &mmap_);
-          mmap_.get());
+
       // bit 0x40 = consecutive blocks (not fragmented?)
       if (filename_length_flags & 0x80) {
         entry->attributes_ = kFileAttributeDirectory;
@@ -367,11 +536,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
           size_t block_size =
               std::min(static_cast<size_t>(0x1000), remaining_size);
           size_t offset = BlockToOffsetSTFS(block_index);
-          entry->block_list_.push_back({offset, block_size});
+          entry->block_list_.push_back({0, offset, block_size});
           remaining_size -= block_size;
-          auto block_hash = GetBlockHash(map_ptr, block_index, 0);
+          auto block_hash = GetBlockHash(data, block_index, 0);
           if (table_size_shift_ && block_hash.info < 0x80) {
-            block_hash = GetBlockHash(map_ptr, block_index, 1);
+            block_hash = GetBlockHash(data, block_index, 1);
           }
           block_index = block_hash.next_block_index;
           info = block_hash.info;
@@ -381,9 +550,9 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
       parent_entry->children_.emplace_back(std::move(entry));
     }
 
-    auto block_hash = GetBlockHash(map_ptr, table_block_index, 0);
+    auto block_hash = GetBlockHash(data, table_block_index, 0);
     if (table_size_shift_ && block_hash.info < 0x80) {
-      block_hash = GetBlockHash(map_ptr, table_block_index, 1);
+      block_hash = GetBlockHash(data, table_block_index, 1);
     }
     table_block_index = block_hash.next_block_index;
     if (table_block_index == 0xFFFFFF) {
@@ -402,41 +571,25 @@ size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) {
     block_shift = package_type_ == StfsPackageType::kCon ? 1 : 0;
   }
 
-  if (header_.descriptor_type == StfsDescriptorType::kStfs) {
+  // For every level there is a hash table
-    // For every level there is a hash table
+  // Level 0: hash table of next 170 blocks
-    // Level 0: hash table of next 170 blocks
+  // Level 1: hash table of next 170 hash tables
-    // Level 1: hash table of next 170 hash tables
+  // Level 2: hash table of next 170 level 1 hash tables
-    // Level 2: hash table of next 170 level 1 hash tables
+  // And so on...
-    // And so on...
+  uint64_t base = kSTFSHashSpacing;
-    uint64_t base = kSTFSHashSpacing;
+  block = block_index;
-    block = block_index;
+  for (uint32_t i = 0; i < 3; i++) {
-    for (uint32_t i = 0; i < 3; i++) {
+    block += (block_index + (base << block_shift)) / (base << block_shift);
-      block += (block_index + (base << block_shift)) / (base << block_shift);
+    if (block_index < base) {
-      if (block_index < base) {
+      break;
-        break;
-      }
-
-      base *= kSTFSHashSpacing;
     }
-  } else if (header_.descriptor_type == StfsDescriptorType::kSvod) {
+
-    // Level 0: Hashes for the next 204 blocks
+    base *= kSTFSHashSpacing;
-    // Level 1: Hashes for the next 203 hash blocks + 1 for the next level 0
-    // 10......[204 blocks].....0.....[204 blocks].....0
-    // There are 0xA1C4 (41412) blocks for every level 1 hash table.
-    block = block_index;
-    block += (block_index + 204) / 204;                // Level 0
-    block += (block_index + 204 * 203) / (204 * 203);  // Level 1
   }
 
   return xe::round_up(header_.header_size, 0x1000) + (block << 12);
 }
 
-size_t StfsContainerDevice::BlockToOffsetEGDF(uint64_t sector) {
-  size_t offset = BlockToOffsetSTFS(
-      (sector / 2) - header_.svod_volume_descriptor.data_block_offset + 1);
-  return offset + ((sector & 0x1) << 11);  // Sectors are 0x800 bytes.
-}
-
 StfsContainerDevice::BlockHash StfsContainerDevice::GetBlockHash(
     const uint8_t* map_ptr, uint32_t block_index, uint32_t table_offset) {
   uint32_t record = block_index % 0xAA;
@@ -496,11 +649,6 @@ bool StfsHeader::Read(const uint8_t* p) {
   header_size = xe::load_and_swap<uint32_t>(p + 0x340);
   content_type = (StfsContentType)xe::load_and_swap<uint32_t>(p + 0x344);
   metadata_version = xe::load_and_swap<uint32_t>(p + 0x348);
-  if (metadata_version > 1) {
-    // This is a variant of thumbnail data/etc.
-    // Can just ignore it for now (until we parse thumbnails).
-    XELOGW("STFSContainer doesn't support version %d yet", metadata_version);
-  }
   content_size = xe::load_and_swap<uint32_t>(p + 0x34C);
   media_id = xe::load_and_swap<uint32_t>(p + 0x354);
   version = xe::load_and_swap<uint32_t>(p + 0x358);
@@ -541,8 +689,70 @@ bool StfsHeader::Read(const uint8_t* p) {
   title_thumbnail_image_size = xe::load_and_swap<uint32_t>(p + 0x1716);
   std::memcpy(thumbnail_image, p + 0x171A, 0x4000);
   std::memcpy(title_thumbnail_image, p + 0x571A, 0x4000);
+
+  // Metadata v2 Fields
+  if (metadata_version == 2) {
+    std::memcpy(series_id, p + 0x3B1, 0x10);
+    std::memcpy(season_id, p + 0x3C1, 0x10);
+    season_number = xe::load_and_swap<uint16_t>(p + 0x3D1);
+    episode_number = xe::load_and_swap<uint16_t>(p + 0x3D5);
+
+    for (size_t n = 0; n < 0x300 / 2; n++) {
+      additonal_display_names[n] =
+          xe::load_and_swap<uint16_t>(p + 0x541A + n * 2);
+      additional_display_descriptions[n] =
+          xe::load_and_swap<uint16_t>(p + 0x941A + n * 2);
+    }
+  }
+
+  return true;
+}
+
+const char* StfsContainerDevice::ReadMagic(const std::wstring& path) {
+  auto map = MappedMemory::Open(path, MappedMemory::Mode::kRead, 0, 4);
+  auto magic_data = xe::load<uint32_t>(map->data());
+  auto magic_bytes = static_cast<char*>(static_cast<void*>(&magic_data));
+  return std::move(magic_bytes);
+}
+
+bool StfsContainerDevice::ResolveFromFolder(const std::wstring& path) {
+  // Scan through folders until a file with magic is found
+  std::queue<filesystem::FileInfo> queue;
+
+  filesystem::FileInfo folder;
+  filesystem::GetInfo(local_path_, &folder);
+  queue.push(folder);
+
+  while (!queue.empty()) {
+    auto current_file = queue.front();
+    queue.pop();
+
+    if (current_file.type == filesystem::FileInfo::Type::kDirectory) {
+      auto path = xe::join_paths(current_file.path, current_file.name);
+      auto child_files = filesystem::ListFiles(path);
+      for (auto file : child_files) {
+        queue.push(file);
+      }
+    } else {
+      // Try to read the file's magic
+      auto path = xe::join_paths(current_file.path, current_file.name);
+      auto magic = ReadMagic(path);
+
+      if (memcmp(magic, "LIVE", 4) == 0 || memcmp(magic, "PIRS", 4) == 0 ||
+          memcmp(magic, "CON ", 4) == 0) {
+        local_path_ = xe::join_paths(current_file.path, current_file.name);
+        XELOGI("STFS Package found: %s", xe::to_string(local_path_).c_str());
+        return true;
+      }
+    }
+  }
+
+  if (local_path_ == path) {
+    // Could not find a suitable container file
+    return false;
+  }
   return true;
 }
 
 }  // namespace vfs
-}  // namespace xe
+}  // namespace xe

src/xenia/vfs/devices/stfs_container_device.h

@@ -10,6 +10,7 @@
 #ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_
 #define XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_
 
+#include <map>
 #include <memory>
 #include <string>
 
@@ -91,6 +92,13 @@ enum SvodDeviceFeatures {
   kFeatureHasEnhancedGDFLayout = 0x40,
 };
 
+enum SvodLayoutType {
+  kUnknownLayout = 0x0,
+  kEnhancedGDFLayout = 0x1,
+  kXSFLayout = 0x2,
+  kSingleFileLayout = 0x4,
+};
+
 struct SvodVolumeDescriptor {
   bool Read(const uint8_t* p);
 
@@ -103,6 +111,8 @@ struct SvodVolumeDescriptor {
   uint32_t data_block_count;
   uint32_t data_block_offset;
   // 0x5 padding bytes...
+
+  SvodLayoutType layout_type;
 };
 
 class StfsHeader {
@@ -143,6 +153,14 @@ class StfsHeader {
   uint32_t title_thumbnail_image_size;
   uint8_t thumbnail_image[0x4000];
   uint8_t title_thumbnail_image[0x4000];
+
+  // Metadata v2 Fields
+  uint8_t series_id[0x10];
+  uint8_t season_id[0x10];
+  int16_t season_number;
+  int16_t episode_number;
+  wchar_t additonal_display_names[0x300 / 2];
+  wchar_t additional_display_descriptions[0x300 / 2];
 };
 
 class StfsContainerDevice : public Device {
@@ -156,7 +174,7 @@ class StfsContainerDevice : public Device {
   Entry* ResolvePath(std::string path) override;
 
   uint32_t total_allocation_units() const override {
-    return uint32_t(mmap_->size() / sectors_per_allocation_unit() /
+    return uint32_t(mmap_total_size_ / sectors_per_allocation_unit() /
                     bytes_per_sector());
   }
   uint32_t available_allocation_units() const override { return 0; }
@@ -178,23 +196,30 @@ class StfsContainerDevice : public Device {
   };
 
   const uint32_t kSTFSHashSpacing = 170;
-  const uint32_t kSVODHashSpacing = 204;
 
+  const char* ReadMagic(const std::wstring& path);
+  bool ResolveFromFolder(const std::wstring& path);
+
+  Error MapFiles();
   Error ReadHeaderAndVerify(const uint8_t* map_ptr);
-  Error ReadAllEntriesEGDF(const uint8_t* map_ptr);
-  bool ReadEntryEGDF(const uint8_t* buffer, uint16_t entry_ordinal,
-                     StfsContainerEntry* parent);
 
-  Error ReadAllEntriesSTFS(const uint8_t* map_ptr);
+  Error ReadSVOD();
+  Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal,
+                      StfsContainerEntry* parent);
+  void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index);
+
+  Error ReadSTFS();
   size_t BlockToOffsetSTFS(uint64_t block);
-  size_t BlockToOffsetEGDF(uint64_t block);
 
   BlockHash GetBlockHash(const uint8_t* map_ptr, uint32_t block_index,
                          uint32_t table_offset);
 
   std::wstring local_path_;
-  std::unique_ptr<MappedMemory> mmap_;
+  std::map<size_t, std::unique_ptr<MappedMemory>> mmap_;
+  size_t mmap_total_size_;
 
+  size_t base_offset_;
+  size_t magic_offset_;
   std::unique_ptr<Entry> root_entry_;
   StfsPackageType package_type_;
   StfsHeader header_;

src/xenia/vfs/devices/stfs_container_entry.cc

@@ -8,15 +8,17 @@
  */
 
 #include "xenia/vfs/devices/stfs_container_entry.h"
-
 #include "xenia/base/math.h"
 #include "xenia/vfs/devices/stfs_container_file.h"
 
+#include <map>
+
 namespace xe {
 namespace vfs {
 
 StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
-                                       std::string path, MappedMemory* mmap)
+                                       std::string path,
+                                       MultifileMemoryMap* mmap)
     : Entry(device, parent, path),
       mmap_(mmap),
       data_offset_(0),
@@ -25,7 +27,7 @@ StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
 StfsContainerEntry::~StfsContainerEntry() = default;
 
 std::unique_ptr<StfsContainerEntry> StfsContainerEntry::Create(
-    Device* device, Entry* parent, std::string name, MappedMemory* mmap) {
+    Device* device, Entry* parent, std::string name, MultifileMemoryMap* mmap) {
   auto path = xe::join_paths(parent->path(), name);
   auto entry = std::make_unique<StfsContainerEntry>(device, parent, path, mmap);
 
@@ -38,4 +40,4 @@ X_STATUS StfsContainerEntry::Open(uint32_t desired_access, File** out_file) {
 }
 
 }  // namespace vfs
-}  // namespace xe
+}  // namespace xe

src/xenia/vfs/devices/stfs_container_entry.h

@@ -10,6 +10,7 @@
 #ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
 #define XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
 
+#include <map>
 #include <string>
 #include <vector>
 
@@ -20,27 +21,30 @@
 
 namespace xe {
 namespace vfs {
+typedef std::map<size_t, std::unique_ptr<MappedMemory>> MultifileMemoryMap;
 
 class StfsContainerDevice;
 
 class StfsContainerEntry : public Entry {
  public:
   StfsContainerEntry(Device* device, Entry* parent, std::string path,
-                     MappedMemory* mmap);
+                     MultifileMemoryMap* mmap);
   ~StfsContainerEntry() override;
 
   static std::unique_ptr<StfsContainerEntry> Create(Device* device,
                                                     Entry* parent,
                                                     std::string name,
-                                                    MappedMemory* mmap);
+                                                    MultifileMemoryMap* mmap);
 
-  MappedMemory* mmap() const { return mmap_; }
+  MultifileMemoryMap* mmap() const { return mmap_; }
   size_t data_offset() const { return data_offset_; }
   size_t data_size() const { return data_size_; }
+  size_t block() const { return block_; }
 
   X_STATUS Open(uint32_t desired_access, File** out_file) override;
 
   struct BlockRecord {
+    size_t file;
     size_t offset;
     size_t length;
   };
@@ -49,13 +53,14 @@ class StfsContainerEntry : public Entry {
  private:
   friend class StfsContainerDevice;
 
-  MappedMemory* mmap_;
+  MultifileMemoryMap* mmap_;
   size_t data_offset_;
   size_t data_size_;
+  size_t block_;
   std::vector<BlockRecord> block_list_;
 };
 
 }  // namespace vfs
 }  // namespace xe
 
-#endif  // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
+#endif  // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_

src/xenia/vfs/devices/stfs_container_file.cc

@@ -34,7 +34,6 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
   }
 
   size_t src_offset = 0;
-  uint8_t* src = entry_->mmap()->data();
   uint8_t* p = reinterpret_cast<uint8_t*>(buffer);
   size_t remaining_length =
       std::min(buffer_length, entry_->size() - byte_offset);
@@ -48,6 +47,8 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
       continue;
     }
 
+    uint8_t* src = entry_->mmap()->at(record.file)->data();
+
     size_t read_offset =
         (byte_offset > src_offset) ? byte_offset - src_offset : 0;
     size_t read_length =
@@ -66,4 +67,4 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
 }
 
 }  // namespace vfs
-}  // namespace xe
+}  // namespace xe

third_party/catch

@@ -1 +1 @@
-Subproject commit a0ada2e935324db5c951571c52c9b07307ea422f
+Subproject commit 6860c8def0ba7559bf077515b7a7ff63ad3444f8