[VFS] Add support for loading ZArchive files

.gitmodules

@@ -85,3 +85,9 @@
 [submodule "third_party/VulkanMemoryAllocator"]
 	path = third_party/VulkanMemoryAllocator
 	url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git
+[submodule "third_party/zstd"]
+	path = third_party/zstd
+	url = https://github.com/facebook/zstd.git
+[submodule "third_party/zarchive"]
+	path = third_party/zarchive
+	url = https://github.com/exzap/ZArchive.git

premake5.lua

@@ -260,6 +260,8 @@ workspace("xenia")
   include("third_party/mspack.lua")
   include("third_party/snappy.lua")
   include("third_party/xxhash.lua")
+  include("third_party/zarchive.lua")
+  include("third_party/zstd.lua")
 
   if not os.istarget("android") then
     -- SDL2 requires sdl2-config, and as of November 2020 isn't high-quality on

src/xenia/app/emulator_window.cc

@@ -881,8 +881,9 @@ void EmulatorWindow::FileOpen() {
   file_picker->set_multi_selection(false);
   file_picker->set_title("Select Content Package");
   file_picker->set_extensions({
-      {"Supported Files", "*.iso;*.xex;*.*"},
+      {"Supported Files", "*.iso;*.xex;*.zar;*.*"},
       {"Disc Image (*.iso)", "*.iso"},
+      {"Disc Archive (*.zar)", "*.zar"},
       {"Xbox Executable (*.xex)", "*.xex"},
       //{"Content Package (*.xcp)", "*.xcp" },
       {"All Files (*.*)", "*.*"},

src/xenia/emulator.cc

@@ -50,6 +50,7 @@
 #include "xenia/ui/windowed_app_context.h"
 #include "xenia/vfs/device.h"
 #include "xenia/vfs/devices/disc_image_device.h"
+#include "xenia/vfs/devices/disc_zarchive_device.h"
 #include "xenia/vfs/devices/host_path_device.h"
 #include "xenia/vfs/devices/null_device.h"
 #include "xenia/vfs/virtual_file_system.h"
@@ -420,6 +421,9 @@ X_STATUS Emulator::LaunchPath(const std::filesystem::path& path) {
     // Treat as a naked xex file.
     MountPath(path, "\\Device\\Harddisk0\\Partition1");
     return LaunchXexFile(path);
+  } else if (extension == ".zar") {
+    // Assume a disc image.
+    return LaunchDiscArchive(path);
   } else {
     // Assume a disc image.
     MountPath(path, "\\Device\\Cdrom0");
@@ -447,6 +451,29 @@ X_STATUS Emulator::LaunchDiscImage(const std::filesystem::path& path) {
   return CompleteLaunch(path, module_path);
 }
 
+X_STATUS Emulator::LaunchDiscArchive(const std::filesystem::path& path) {
+  auto mount_path = "\\Device\\Cdrom0";
+
+  // Register the disc image in the virtual filesystem.
+  auto device = std::make_unique<vfs::DiscZarchiveDevice>(mount_path, path);
+  if (!device->Initialize()) {
+    xe::FatalError("Unable to mount disc image; file not found or corrupt.");
+    return X_STATUS_NO_SUCH_FILE;
+  }
+  if (!file_system_->RegisterDevice(std::move(device))) {
+    xe::FatalError("Unable to register disc image.");
+    return X_STATUS_NO_SUCH_FILE;
+  }
+
+  // Create symlinks to the device.
+  file_system_->RegisterSymbolicLink("game:", mount_path);
+  file_system_->RegisterSymbolicLink("d:", mount_path);
+
+  // Launch the game.
+  auto module_path(FindLaunchModule());
+  return CompleteLaunch(path, module_path);
+}
+
 X_STATUS Emulator::LaunchStfsContainer(const std::filesystem::path& path) {
   auto module_path(FindLaunchModule());
   return CompleteLaunch(path, module_path);

src/xenia/emulator.h

@@ -196,6 +196,9 @@ class Emulator {
   // Launches a game from a disc image file (.iso, etc).
   X_STATUS LaunchDiscImage(const std::filesystem::path& path);
 
+  // Launches a game from a disc archive file (.zar, etc).
+  X_STATUS LaunchDiscArchive(const std::filesystem::path& path);
+
   // Launches a game from an STFS container file.
   X_STATUS LaunchStfsContainer(const std::filesystem::path& path);
 

src/xenia/vfs/devices/disc_zarchive_device.cc

@@ -0,0 +1,139 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2020 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "xenia/vfs/devices/disc_zarchive_device.h"
+
+#include "xenia/base/literals.h"
+#include "xenia/base/logging.h"
+#include "xenia/base/math.h"
+#include "xenia/vfs/devices/disc_zarchive_entry.h"
+
+#include "third_party/zarchive/include/zarchive/zarchivereader.h"
+
+namespace xe {
+namespace vfs {
+
+using namespace xe::literals;
+
+const size_t kXESectorSize = 2_KiB;
+
+DiscZarchiveDevice::DiscZarchiveDevice(const std::string_view mount_path,
+                                       const std::filesystem::path& host_path)
+    : Device(mount_path),
+      name_("GDFX"),
+      host_path_(host_path),
+      opaque_(nullptr) {}
+
+DiscZarchiveDevice::~DiscZarchiveDevice() {
+  ZArchiveReader* reader = static_cast<ZArchiveReader*>(opaque_);
+  if (reader != nullptr) delete reader;
+};
+
+bool DiscZarchiveDevice::Initialize() {
+  ZArchiveReader* reader = nullptr;
+  reader = ZArchiveReader::OpenFromFile(host_path_);
+
+  if (!reader) {
+    XELOGE("Disc ZArchive could not be opened");
+    return false;
+  }
+
+  opaque_ = static_cast<void*>(reader);
+  bool result = false;
+
+  result = reader->IsFile(reader->LookUp("default.xex", true, false));
+  if (!result) XELOGE("Failed to verify disc ZArchive (no default.xex)");
+
+  const std::string root_path = std::string("/");
+  ZArchiveNodeHandle handle = reader->LookUp(root_path);
+  auto root_entry = new DiscZarchiveEntry(this, nullptr, root_path, reader);
+  root_entry->attributes_ = kFileAttributeDirectory;
+  root_entry->handle_ = static_cast<uint32_t>(handle);
+  root_entry->name_ = root_path;
+  root_entry->absolute_path_ = root_path;
+  root_entry_ = std::unique_ptr<Entry>(root_entry);
+  result = ReadAllEntries(reader, "", root_entry, nullptr);
+
+  return result;
+}
+
+void DiscZarchiveDevice::Dump(StringBuffer* string_buffer) {
+  auto global_lock = global_critical_region_.Acquire();
+  root_entry_->Dump(string_buffer, 0);
+}
+
+Entry* DiscZarchiveDevice::ResolvePath(const std::string_view path) {
+  // The filesystem will have stripped our prefix off already, so the path will
+  // be in the form:
+  // some\PATH.foo
+  XELOGFS("DiscZarchiveDevice::ResolvePath({})", path);
+
+  ZArchiveReader* reader = static_cast<ZArchiveReader*>(opaque_);
+  if (!reader) return nullptr;
+
+  ZArchiveNodeHandle handle = reader->LookUp(path);
+  bool result = (handle != ZARCHIVE_INVALID_NODE);
+
+  if (!result) return nullptr;
+
+  return root_entry_->ResolvePath(path);
+}
+
+bool DiscZarchiveDevice::ReadAllEntries(void* opaque, const std::string& path,
+                                        DiscZarchiveEntry* node,
+                                        DiscZarchiveEntry* parent) {
+  ZArchiveReader* reader = static_cast<ZArchiveReader*>(opaque);
+  ZArchiveNodeHandle handle = node->handle_;
+  if (handle == ZARCHIVE_INVALID_NODE) return false;
+  if (reader->IsDirectory(handle)) {
+    uint32_t count = reader->GetDirEntryCount(handle);
+    for (uint32_t i = 0; i < count; i++) {
+      ZArchiveReader::DirEntry dirEntry;
+      if (!reader->GetDirEntry(handle, i, dirEntry)) return false;
+      std::string full_path = path + std::string(dirEntry.name);
+      ZArchiveNodeHandle fileHandle = reader->LookUp(full_path);
+      if (handle == ZARCHIVE_INVALID_NODE) return false;
+      auto entry = new DiscZarchiveEntry(this, parent, full_path, opaque);
+      entry->handle_ = static_cast<uint32_t>(fileHandle);
+      entry->data_offset_ = 0;
+      // Set to January 1, 1970 (UTC) in 100-nanosecond intervals
+      entry->create_timestamp_ = 10000 * 11644473600000LL;
+      entry->access_timestamp_ = 10000 * 11644473600000LL;
+      entry->write_timestamp_ = 10000 * 11644473600000LL;
+      entry->parent_ = node;
+      if (dirEntry.isDirectory) {
+        entry->data_size_ = 0;
+        entry->size_ = dirEntry.size;
+        entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
+        node->children_.push_back(std::unique_ptr<Entry>(entry));
+        if (!ReadAllEntries(reader, full_path + "\\", entry, node))
+          return false;
+      } else if (dirEntry.isFile) {
+        entry->data_size_ = entry->size_ = reader->GetFileSize(fileHandle);
+        entry->attributes_ = kFileAttributeReadOnly;
+        entry->allocation_size_ =
+            xe::round_up(entry->size_, bytes_per_sector());
+        node->children_.push_back(std::unique_ptr<Entry>(entry));
+      }
+    }
+    return true;
+  } else if (reader->IsFile(handle)) {
+    auto entry = new DiscZarchiveEntry(this, parent, path, opaque);
+    entry->attributes_ = kFileAttributeReadOnly;
+    entry->handle_ = static_cast<uint32_t>(handle);
+    entry->parent_ = parent;
+    entry->children_.push_back(std::unique_ptr<Entry>(entry));
+    return true;
+  }
+
+  return false;
+}
+
+}  // namespace vfs
+}  // namespace xe

src/xenia/vfs/devices/disc_zarchive_device.h

@@ -0,0 +1,56 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2020 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_VFS_DEVICES_DISC_ZARCHIVE_DEVICE_H_
+#define XENIA_VFS_DEVICES_DISC_ZARCHIVE_DEVICE_H_
+
+#include <memory>
+#include <string>
+
+#include "xenia/base/mapped_memory.h"
+#include "xenia/vfs/device.h"
+
+namespace xe {
+namespace vfs {
+
+class DiscZarchiveEntry;
+
+class DiscZarchiveDevice : public Device {
+ public:
+  DiscZarchiveDevice(const std::string_view mount_path,
+                     const std::filesystem::path& host_path);
+  ~DiscZarchiveDevice() override;
+
+  bool Initialize() override;
+  void Dump(StringBuffer* string_buffer) override;
+  Entry* ResolvePath(const std::string_view path) override;
+
+  const std::string& name() const override { return name_; }
+  uint32_t attributes() const override { return 0; }
+  uint32_t component_name_max_length() const override { return 255; }
+
+  uint32_t total_allocation_units() const override { return 128 * 1024; }
+  uint32_t available_allocation_units() const override { return 0; }
+  uint32_t sectors_per_allocation_unit() const override { return 1; }
+  uint32_t bytes_per_sector() const override { return 0x200; }
+
+ private:
+  bool ReadAllEntries(void* opaque, const std::string& path,
+                      DiscZarchiveEntry* node, DiscZarchiveEntry* parent);
+
+  std::string name_;
+  std::filesystem::path host_path_;
+  std::unique_ptr<Entry> root_entry_;
+  void* opaque_;
+};
+
+}  // namespace vfs
+}  // namespace xe
+
+#endif  // XENIA_VFS_DEVICES_DISC_ZARCHIVE_DEVICE_H_

src/xenia/vfs/devices/disc_zarchive_entry.cc

@@ -0,0 +1,50 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2020 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "xenia/vfs/devices/disc_zarchive_entry.h"
+
+#include <algorithm>
+
+#include "xenia/base/math.h"
+#include "xenia/vfs/devices/disc_zarchive_file.h"
+
+#include "third_party/zarchive/include/zarchive/zarchivereader.h"
+namespace xe {
+namespace vfs {
+
+DiscZarchiveEntry::DiscZarchiveEntry(Device* device, Entry* parent,
+                                     const std::string_view path, void* opaque)
+    : Entry(device, parent, path),
+      opaque_(opaque),
+      data_offset_(0),
+      data_size_(0),
+      handle_(ZARCHIVE_INVALID_NODE) {}
+
+DiscZarchiveEntry::~DiscZarchiveEntry() = default;
+
+std::unique_ptr<DiscZarchiveEntry> DiscZarchiveEntry::Create(
+    Device* device, Entry* parent, const std::string_view name, void* opaque) {
+  auto path = name;  // xe::utf8::join_guest_paths(parent->path(), name);
+  auto entry =
+      std::make_unique<DiscZarchiveEntry>(device, parent, path, opaque);
+  return std::move(entry);
+}
+
+X_STATUS DiscZarchiveEntry::Open(uint32_t desired_access, File** out_file) {
+  *out_file = new DiscZarchiveFile(desired_access, this);
+  return X_STATUS_SUCCESS;
+}
+
+std::unique_ptr<MappedMemory> DiscZarchiveEntry::OpenMapped(
+    MappedMemory::Mode mode, size_t offset, size_t length) {
+  return nullptr;
+}
+
+}  // namespace vfs
+}  // namespace xe

src/xenia/vfs/devices/disc_zarchive_entry.h

@@ -0,0 +1,59 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2020 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_VFS_DEVICES_DISC_ZARCHIVE_ENTRY_H_
+#define XENIA_VFS_DEVICES_DISC_ZARCHIVE_ENTRY_H_
+
+#include <string>
+#include <vector>
+
+#include "xenia/base/mapped_memory.h"
+#include "xenia/vfs/entry.h"
+
+namespace xe {
+namespace vfs {
+
+class DiscZarchiveDevice;
+
+class DiscZarchiveEntry : public Entry {
+ public:
+  DiscZarchiveEntry(Device* device, Entry* parent, const std::string_view path,
+                    void* opaque);
+  ~DiscZarchiveEntry() override;
+
+  static std::unique_ptr<DiscZarchiveEntry> Create(Device* device,
+                                                   Entry* parent,
+                                                   const std::string_view name,
+                                                   void* opaque);
+
+  MappedMemory* mmap() const { return nullptr; }
+  size_t data_offset() const { return data_offset_; }
+  size_t data_size() const { return data_size_; }
+
+  X_STATUS Open(uint32_t desired_access, File** out_file) override;
+
+  bool can_map() const override { return false; }
+  std::unique_ptr<MappedMemory> OpenMapped(MappedMemory::Mode mode,
+                                           size_t offset,
+                                           size_t length) override;
+
+ private:
+  friend class DiscZarchiveDevice;
+  friend class DiscZarchiveFile;
+
+  void* opaque_;
+  uint32_t handle_;
+  size_t data_offset_;
+  size_t data_size_;
+};
+
+}  // namespace vfs
+}  // namespace xe
+
+#endif  // XENIA_VFS_DEVICES_DISC_ZARCHIVE_ENTRY_H_

src/xenia/vfs/devices/disc_zarchive_file.cc

@@ -0,0 +1,46 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#include "xenia/vfs/devices/disc_zarchive_file.h"
+
+#include <algorithm>
+
+#include "xenia/vfs/devices/disc_zarchive_entry.h"
+
+#include "third_party/zarchive/include/zarchive/zarchivereader.h"
+
+namespace xe {
+namespace vfs {
+
+DiscZarchiveFile::DiscZarchiveFile(uint32_t file_access,
+                                   DiscZarchiveEntry* entry)
+    : File(file_access, entry), entry_(entry) {}
+
+DiscZarchiveFile::~DiscZarchiveFile() = default;
+
+void DiscZarchiveFile::Destroy() { delete this; }
+
+X_STATUS DiscZarchiveFile::ReadSync(void* buffer, size_t buffer_length,
+                                    size_t byte_offset,
+                                    size_t* out_bytes_read) {
+  if (byte_offset >= entry_->size()) {
+    return X_STATUS_END_OF_FILE;
+  }
+  ZArchiveReader* reader = static_cast<ZArchiveReader*>(entry_->opaque_);
+  uint64_t bytes_read =
+      reader->ReadFromFile(entry_->handle_, byte_offset, buffer_length, buffer);
+  size_t real_offset = entry_->data_offset() + byte_offset;
+  size_t real_length =
+      std::min(buffer_length, entry_->data_size() - byte_offset);
+  *out_bytes_read = real_length;
+  return X_STATUS_SUCCESS;
+}
+
+}  // namespace vfs
+}  // namespace xe

src/xenia/vfs/devices/disc_zarchive_file.h

@@ -0,0 +1,42 @@
+/**
+ ******************************************************************************
+ * Xenia : Xbox 360 Emulator Research Project                                 *
+ ******************************************************************************
+ * Copyright 2013 Ben Vanik. All rights reserved.                             *
+ * Released under the BSD license - see LICENSE in the root for more details. *
+ ******************************************************************************
+ */
+
+#ifndef XENIA_VFS_DEVICES_DISC_ZARCHIVE_FILE_H_
+#define XENIA_VFS_DEVICES_DISC_ZARCHIVE_FILE_H_
+
+#include "xenia/vfs/file.h"
+
+namespace xe {
+namespace vfs {
+
+class DiscZarchiveEntry;
+
+class DiscZarchiveFile : public File {
+ public:
+  DiscZarchiveFile(uint32_t file_access, DiscZarchiveEntry* entry);
+  ~DiscZarchiveFile() override;
+
+  void Destroy() override;
+
+  X_STATUS ReadSync(void* buffer, size_t buffer_length, size_t byte_offset,
+                    size_t* out_bytes_read) override;
+  X_STATUS WriteSync(const void* buffer, size_t buffer_length,
+                     size_t byte_offset, size_t* out_bytes_written) override {
+    return X_STATUS_ACCESS_DENIED;
+  }
+  X_STATUS SetLength(size_t length) override { return X_STATUS_ACCESS_DENIED; }
+
+ private:
+  DiscZarchiveEntry* entry_;
+};
+
+}  // namespace vfs
+}  // namespace xe
+
+#endif  // XENIA_VFS_DEVICES_DISC_ZARCHIVE_FILE_H_

src/xenia/vfs/premake5.lua

@@ -7,7 +7,7 @@ project("xenia-vfs")
   kind("StaticLib")
   language("C++")
   links({
-    "xenia-base",
+    "xenia-base", "zstd", "zarchive"
   })
   defines({
   })

third_party/zarchive.lua

@@ -0,0 +1,21 @@
+group("third_party")
+project("zarchive")
+  uuid("d32f03aa-f0c9-11ed-a05b-0242ac120003")
+  kind("StaticLib")
+  language("C++")
+  links({
+  })
+  defines({
+    "_LIB",
+  })
+  includedirs({
+    "zarchive/include",
+    "zstd/lib",
+  })
+  files({
+    "zarchive/include/zarchive/zarchivecommon.h",
+    "zarchive/include/zarchive/zarchivereader.h",
+    "zarchive/include/zarchive/zarchivewriter.h",
+    "zarchive/src/zarchivereader.cpp",
+    "zarchive/src/zarchivewriter.cpp",
+  })

third_party/zstd.lua

@@ -0,0 +1,87 @@
+-- This GENie/premake file copies the behavior of the Makefile in the lib folder.
+-- Basic usage: project_zstd(ZSTD_DIR)
+
+function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy)
+	if compression == nil then compression = true end
+	if decompression == nil then decompression = true end
+	if deprecated == nil then deprecated = false end
+	if dictbuilder == nil then dictbuilder = false end
+
+	if legacy == nil then legacy = 0 end
+
+	if not compression then
+		dictbuilder = false
+		deprecated = false
+	end
+
+	if not decompression then
+		legacy = 0
+		deprecated = false
+	end
+
+	project 'zstd'
+		kind 'StaticLib'
+		language 'C'
+
+		files {
+			dir .. 'zstd.h',
+			dir .. 'common/**.c',
+			dir .. 'common/**.h'
+		}
+
+		if compression then
+			files {
+				dir .. 'compress/**.c',
+				dir .. 'compress/**.h'
+			}
+		end
+
+		if decompression then
+			files {
+				dir .. 'decompress/**.c',
+				dir .. 'decompress/**.h'
+			}
+		end
+
+		if dictbuilder then
+			files {
+				dir .. 'dictBuilder/**.c',
+				dir .. 'dictBuilder/**.h'
+			}
+		end
+
+		if deprecated then
+			files {
+				dir .. 'deprecated/**.c',
+				dir .. 'deprecated/**.h'
+			}
+		end
+
+		if legacy ~= 0 then
+			if legacy >= 8 then
+				files {
+					dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*'
+				}
+			end
+			includedirs {
+				dir .. 'legacy'
+			}
+		end
+
+		includedirs {
+			dir,
+			dir .. 'common'
+		}
+
+		defines {
+			'XXH_NAMESPACE=ZSTD_',
+			'ZSTD_LEGACY_SUPPORT=' .. legacy
+		}
+end
+
+
+group("third_party")
+project("zstd")
+  uuid("df336aac-f0c8-11ed-a05b-0242ac120003")
+  project_zstd('./zstd/lib/')
+