Fixes to allow games to make use of cache partitions

Happy new year! Here's my first commit of the 2020s :)

With these fixes, Halo 3 Epsilon will now write cached map data & other things (autosaves/datamine...) to the cache0/cache1 partitions, (as long as mount_cache cvar is set)
(Halo 3 retail will also write some things to cache with this, but oddly doesn't cache map stuff... which is strange because Epsilon was built only a day or two after the retail build, so I'm not sure why it'd work differently...
Maybe retail needs a TU applied for it to work or something like that)
Other games should hopefully work with cache now too (AFAIK the problem was in SDK library code, that a lot of games probably share)
No idea if this will actually improve anything though, but at least things will work closer to what games expect :)

The way this works is by tricking the cache-partition code (staticly linked into the game exe) into thinking that the Partition0/Cache0/Cache1 devices are valid.
To do that I made another kind of VFS device, the NullDevice, which just takes in a list of paths to handle.
Whenever an IO request is made to one of these paths, the NullDevice can simply pretend to the game that everything was successful, which satisfies the requirements needed for caching.

It also makes use of another trick: setting TitleInsecureCacheDrive XEX permission, which seems to skip a huge chunk of cache-init code (STFC filesystem device registration & stuff like that)
I'm not sure if this would work with every single revision of the STFC/cache code though...
At least in Halo 3 the retail code will handle the TitleInsecureCacheDrive case for us fine, but maybe older/more recent versions don't include functionality for it, need to look into it some more.
(I did try an impl. without needing this permission months ago, got pretty far with it but got caught on one tiny issue that I couldn't figure out... too bad I didn't find out about this permission earlier!)

src/xenia/app/xenia_main.cc

@@ -19,6 +19,7 @@
 #include "xenia/debug/ui/debug_window.h"
 #include "xenia/emulator.h"
 #include "xenia/ui/file_picker.h"
+#include "xenia/vfs/virtual_file_system.h"
 #include "xenia/vfs/devices/host_path_device.h"
 
 // Available audio systems:
@@ -68,7 +69,6 @@ DEFINE_path(
     "Storage");
 
 DEFINE_bool(mount_scratch, false, "Enable scratch mount", "Storage");
-DEFINE_bool(mount_cache, false, "Enable cache mount", "Storage");
 
 DEFINE_transient_path(target, "",
                       "Specifies the target .xex or .iso to execute.",

src/xenia/emulator.cc

@@ -13,6 +13,7 @@
 
 #include "config.h"
 #include "third_party/fmt/include/fmt/format.h"
+#include "xenia/app/emulator_window.h"
 #include "xenia/apu/audio_system.h"
 #include "xenia/base/assert.h"
 #include "xenia/base/byte_stream.h"
@@ -39,10 +40,11 @@
 #include "xenia/kernel/xbdm/xbdm_module.h"
 #include "xenia/kernel/xboxkrnl/xboxkrnl_module.h"
 #include "xenia/memory.h"
-#include "xenia/ui/imgui_dialog.h"
 #include "xenia/ui/file_picker.h"
+#include "xenia/ui/imgui_dialog.h"
 #include "xenia/vfs/devices/disc_image_device.h"
 #include "xenia/vfs/devices/host_path_device.h"
+#include "xenia/vfs/devices/null_device.h"
 #include "xenia/vfs/devices/stfs_container_device.h"
 #include "xenia/vfs/virtual_file_system.h"
 
@@ -292,8 +294,8 @@ X_STATUS Emulator::LaunchXexFile(const std::filesystem::path& path,
   // Register the local directory in the virtual filesystem.
   auto parent_path = path.parent_path();
   for (auto mount_path : mount_paths) {
-    auto device =
-        std::make_unique<vfs::HostPathDevice>(mount_path.u8string(), parent_path, true);
+    auto device = std::make_unique<vfs::HostPathDevice>(mount_path.u8string(),
+                                                        parent_path, true);
     if (!device->Initialize()) {
       XELOGE("Unable to scan host path");
       return X_STATUS_NO_SUCH_FILE;
@@ -702,6 +704,20 @@ std::string Emulator::FindLaunchModule() {
 
 X_STATUS Emulator::CompleteLaunch(const std::filesystem::path& path,
                                   const std::string_view module_path) {
+  if (cvars::mount_cache) {
+    // Below are accessed directly by STFC/cache code baked into the game
+    // By using a NullDevice that just returns success to all IO requests, the
+    // cache code should hopefully progress without erroring out
+    std::initializer_list<std::string_view> null_files = {
+        "\\Partition0", "\\Cache0", "\\Cache1"};
+
+    auto null_device =
+        std::make_unique<vfs::NullDevice>("\\Device\\Harddisk0", null_files);
+    if (null_device->Initialize()) {
+      file_system_->RegisterDevice(std::move(null_device));
+    }
+  }
+
   // Reset state.
   title_id_ = 0;
   game_title_ = "";

src/xenia/kernel/xboxkrnl/xboxkrnl_io.cc

@@ -541,6 +541,24 @@ dword_result_t FscSetCacheElementCount(dword_t unk_0, dword_t unk_1) {
 }
 DECLARE_XBOXKRNL_EXPORT1(FscSetCacheElementCount, kFileSystem, kStub);
 
+dword_result_t NtDeviceIoControlFile(
+    dword_t handle, dword_t event_handle, dword_t apc_routine,
+    dword_t apc_context, dword_t io_status_block, dword_t io_control_code,
+    lpvoid_t input_buffer, dword_t input_buffer_len, lpvoid_t output_buffer,
+    dword_t output_buffer_len) {
+  // Called by STFS/cache code, seems to check the sanity of returned values -
+  // the values below appear to pass this check
+  if (io_control_code == 0x74004) {
+    xe::store_and_swap<uint64_t>(output_buffer, 0);
+    xe::store_and_swap<uint64_t>(output_buffer + 8, 0xFF000);
+  } else if (io_control_code == 0x70000) {
+    xe::store_and_swap<uint32_t>(output_buffer, 0xFF000 / 512);
+    xe::store_and_swap<uint32_t>(output_buffer + 4, 512);
+  }
+  return X_STATUS_SUCCESS;
+}
+DECLARE_XBOXKRNL_EXPORT1(NtDeviceIoControlFile, kFileSystem, kStub);
+
 void RegisterIoExports(xe::cpu::ExportResolver* export_resolver,
                        KernelState* kernel_state) {}
 

src/xenia/kernel/xboxkrnl/xboxkrnl_modules.cc

@@ -32,6 +32,22 @@ dword_result_t XexCheckExecutablePrivilege(dword_t privilege) {
     return 0;
   }
 
+  if (privilege == 0xB) {  // TitleInsecureUtilityDrive
+    // If this privilege is set, the cache-partition code baked into most
+    // games skips a huge chunk of device-init code (registering a custom
+    // STFC filesystem handler with the kernel, etc), and just symlinks the
+    // cache partition to the existing device directly (I guess on 360 this
+    // would probably make it FATX, explaining the 'Insecure' part of it)
+
+    // Thanks to this skip we can easily take control of the cache partition
+    // ourselves, just by symlinking it before the game does!
+
+    // TODO: check if this skip-code is actually available on every game that
+    // uses cache - it's possible that early/later SDKs might not have it, and
+    // we won't be able to rely on using this cheat for everything...
+    return 1;
+  }
+
   uint32_t flags = 0;
   module->GetOptHeader<uint32_t>(XEX_HEADER_SYSTEM_FLAGS, &flags);
 

src/xenia/vfs/devices/null_device.cc

@@ -0,0 +1,62 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "xenia/vfs/devices/null_device.h"
+
+#include "xenia/base/filesystem.h"
+#include "xenia/base/logging.h"
+#include "xenia/base/math.h"
+#include "xenia/kernel/xfile.h"
+#include "xenia/vfs/devices/null_entry.h"
+
+namespace xe {
+namespace vfs {
+
+NullDevice::NullDevice(const std::string_view mount_path,
+                       const std::initializer_list<std::string_view> null_paths)
+    : Device(mount_path), null_paths_(null_paths), name_("NULL") {}
+
+NullDevice::~NullDevice() = default;
+
+bool NullDevice::Initialize() {
+  auto root_entry = new NullEntry(this, nullptr, mount_path_);
+  root_entry->attributes_ = kFileAttributeDirectory;
+  root_entry_ = std::unique_ptr<Entry>(root_entry);
+
+  for (auto path : null_paths_) {
+    auto child = NullEntry::Create(this, root_entry, path);
+    root_entry->children_.push_back(std::unique_ptr<Entry>(child));
+  }
+  return true;
+}
+
+void NullDevice::Dump(StringBuffer* string_buffer) {
+  auto global_lock = global_critical_region_.Acquire();
+  root_entry_->Dump(string_buffer, 0);
+}
+
+Entry* NullDevice::ResolvePath(const std::string_view path) {
+  XELOGFS("NullDevice::ResolvePath({})", path);
+
+  auto root = root_entry_.get();
+  if (path.empty()) {
+    return root_entry_.get();
+  }
+
+  for (auto& child : root->children()) {
+    if (!path.compare(child->path())) {
+      return child.get();
+    }
+  }
+
+  return nullptr;
+}
+
+}  // namespace vfs
+}  // namespace xe

src/xenia/vfs/devices/null_device.h

@@ -0,0 +1,54 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_VFS_DEVICES_NULL_DEVICE_H_
+#define XENIA_VFS_DEVICES_NULL_DEVICE_H_
+
+#include <string>
+
+#include "xenia/vfs/device.h"
+
+namespace xe {
+namespace vfs {
+
+class NullEntry;
+
+class NullDevice : public Device {
+ public:
+  NullDevice(const std::string_view mount_path,
+             const std::initializer_list<std::string_view> null_paths);
+  ~NullDevice() override;
+
+  bool Initialize() override;
+  void Dump(StringBuffer* string_buffer) override;
+  Entry* ResolvePath(const std::string_view path) override;
+
+  bool is_read_only() const override { return false; }
+
+  const std::string& name() const override { return name_; }
+  uint32_t attributes() const override { return 0; }
+  uint32_t component_name_max_length() const override { return 40; }
+
+  uint32_t total_allocation_units() const override { return 128 * 1024; }
+  uint32_t available_allocation_units() const override { return 128 * 1024; }
+
+  // STFC/cache code seems to require the product of these two to equal 0x10000!
+  uint32_t sectors_per_allocation_unit() const override { return 1; }
+  uint32_t bytes_per_sector() const override { return 0x10000; }
+
+ private:
+  std::string name_;
+  std::unique_ptr<Entry> root_entry_;
+  std::vector<std::string_view> null_paths_;
+};
+
+}  // namespace vfs
+}  // namespace xe
+
+#endif  // XENIA_VFS_DEVICES_NULL_DEVICE_H_

src/xenia/vfs/devices/null_entry.cc

@@ -0,0 +1,55 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "xenia/vfs/devices/null_entry.h"
+
+#include "xenia/base/filesystem.h"
+#include "xenia/base/logging.h"
+#include "xenia/base/mapped_memory.h"
+#include "xenia/base/math.h"
+#include "xenia/base/string.h"
+#include "xenia/vfs/device.h"
+#include "xenia/vfs/devices/null_file.h"
+
+namespace xe {
+namespace vfs {
+
+NullEntry::NullEntry(Device* device, Entry* parent, const std::string_view path)
+    : Entry(device, parent, path) {}
+
+NullEntry::~NullEntry() = default;
+
+NullEntry* NullEntry::Create(Device* device, Entry* parent,
+                             const std::filesystem::path& full_path) {
+  auto entry = new NullEntry(device, parent, full_path.u8string());
+
+  entry->create_timestamp_ = 0;
+  entry->access_timestamp_ = 0;
+  entry->write_timestamp_ = 0;
+
+  entry->attributes_ = kFileAttributeNormal;
+
+  entry->size_ = 0;
+  entry->allocation_size_ = 0;
+  return entry;
+}
+
+X_STATUS NullEntry::Open(uint32_t desired_access, File** out_file) {
+  if (is_read_only() && (desired_access & (FileAccess::kFileWriteData |
+                                           FileAccess::kFileAppendData))) {
+    XELOGE("Attempting to open file for write access on read-only device");
+    return X_STATUS_ACCESS_DENIED;
+  }
+
+  *out_file = new NullFile(desired_access, this);
+  return X_STATUS_SUCCESS;
+}
+
+}  // namespace vfs
+}  // namespace xe

src/xenia/vfs/devices/null_entry.h

@@ -0,0 +1,42 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_VFS_DEVICES_NULL_ENTRY_H_
+#define XENIA_VFS_DEVICES_NULL_ENTRY_H_
+
+#include <string>
+
+#include "xenia/base/filesystem.h"
+#include "xenia/vfs/entry.h"
+
+namespace xe {
+namespace vfs {
+
+class NullDevice;
+
+class NullEntry : public Entry {
+ public:
+  NullEntry(Device* device, Entry* parent, const std::string_view path);
+  ~NullEntry() override;
+
+  static NullEntry* Create(Device* device, Entry* parent,
+                           const std::filesystem::path& full_path);
+
+  X_STATUS Open(uint32_t desired_access, File** out_file) override;
+
+  bool can_map() const override { return false; }
+
+ private:
+  friend class NullDevice;
+};
+
+}  // namespace vfs
+}  // namespace xe
+
+#endif  // XENIA_VFS_DEVICES_NULL_ENTRY_H_

src/xenia/vfs/devices/null_file.cc

@@ -0,0 +1,52 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "xenia/vfs/devices/null_file.h"
+
+#include "xenia/vfs/devices/null_entry.h"
+
+namespace xe {
+namespace vfs {
+
+NullFile::NullFile(uint32_t file_access, NullEntry* entry)
+    : File(file_access, entry) {}
+
+NullFile::~NullFile() = default;
+
+void NullFile::Destroy() { delete this; }
+
+X_STATUS NullFile::ReadSync(void* buffer, size_t buffer_length,
+                            size_t byte_offset, size_t* out_bytes_read) {
+  if (!(file_access_ & FileAccess::kFileReadData)) {
+    return X_STATUS_ACCESS_DENIED;
+  }
+
+  return X_STATUS_SUCCESS;
+}
+
+X_STATUS NullFile::WriteSync(const void* buffer, size_t buffer_length,
+                             size_t byte_offset, size_t* out_bytes_written) {
+  if (!(file_access_ &
+        (FileAccess::kFileWriteData | FileAccess::kFileAppendData))) {
+    return X_STATUS_ACCESS_DENIED;
+  }
+
+  return X_STATUS_SUCCESS;
+}
+
+X_STATUS NullFile::SetLength(size_t length) {
+  if (!(file_access_ & FileAccess::kFileWriteData)) {
+    return X_STATUS_ACCESS_DENIED;
+  }
+
+  return X_STATUS_SUCCESS;
+}
+
+}  // namespace vfs
+}  // namespace xe

src/xenia/vfs/devices/null_file.h

@@ -0,0 +1,40 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_VFS_DEVICES_NULL_FILE_H_
+#define XENIA_VFS_DEVICES_NULL_FILE_H_
+
+#include <string>
+
+#include "xenia/base/filesystem.h"
+#include "xenia/vfs/file.h"
+
+namespace xe {
+namespace vfs {
+
+class NullEntry;
+
+class NullFile : public File {
+ public:
+  NullFile(uint32_t file_access, NullEntry* entry);
+  ~NullFile() override;
+
+  void Destroy() override;
+
+  X_STATUS ReadSync(void* buffer, size_t buffer_length, size_t byte_offset,
+                    size_t* out_bytes_read) override;
+  X_STATUS WriteSync(const void* buffer, size_t buffer_length,
+                     size_t byte_offset, size_t* out_bytes_written) override;
+  X_STATUS SetLength(size_t length) override;
+};
+
+}  // namespace vfs
+}  // namespace xe
+
+#endif  // XENIA_VFS_DEVICES_NULL_FILE_H_

src/xenia/vfs/virtual_file_system.cc

@@ -13,6 +13,8 @@
 #include "xenia/base/string.h"
 #include "xenia/kernel/xfile.h"
 
+DEFINE_bool(mount_cache, false, "Enable cache mount", "Storage");
+
 namespace xe {
 namespace vfs {
 

src/xenia/vfs/virtual_file_system.h

@@ -15,11 +15,14 @@
 #include <unordered_map>
 #include <vector>
 
+#include "xenia/base/cvar.h"
 #include "xenia/base/mutex.h"
 #include "xenia/vfs/device.h"
 #include "xenia/vfs/entry.h"
 #include "xenia/vfs/file.h"
 
+DECLARE_bool(mount_cache);
+
 namespace xe {
 namespace vfs {