From 2cd2d1d620508dac3c1ec94ef5714b1bfd4396ff Mon Sep 17 00:00:00 2001 From: Romain Tisserand Date: Sun, 21 May 2023 21:33:10 +0200 Subject: [PATCH] [VFS] Add support for loading ZArchive files --- .gitmodules | 6 + premake5.lua | 2 + src/xenia/app/emulator_window.cc | 3 +- src/xenia/emulator.cc | 27 ++++ src/xenia/emulator.h | 3 + src/xenia/vfs/devices/disc_zarchive_device.cc | 139 ++++++++++++++++++ src/xenia/vfs/devices/disc_zarchive_device.h | 56 +++++++ src/xenia/vfs/devices/disc_zarchive_entry.cc | 50 +++++++ src/xenia/vfs/devices/disc_zarchive_entry.h | 59 ++++++++ src/xenia/vfs/devices/disc_zarchive_file.cc | 46 ++++++ src/xenia/vfs/devices/disc_zarchive_file.h | 42 ++++++ src/xenia/vfs/premake5.lua | 2 +- third_party/zarchive.lua | 21 +++ third_party/zstd.lua | 87 +++++++++++ 14 files changed, 541 insertions(+), 2 deletions(-) create mode 100644 src/xenia/vfs/devices/disc_zarchive_device.cc create mode 100644 src/xenia/vfs/devices/disc_zarchive_device.h create mode 100644 src/xenia/vfs/devices/disc_zarchive_entry.cc create mode 100644 src/xenia/vfs/devices/disc_zarchive_entry.h create mode 100644 src/xenia/vfs/devices/disc_zarchive_file.cc create mode 100644 src/xenia/vfs/devices/disc_zarchive_file.h create mode 100644 third_party/zarchive.lua create mode 100644 third_party/zstd.lua diff --git a/.gitmodules b/.gitmodules index d4d8f6cca..23d7fa2bb 100644 --- a/.gitmodules +++ b/.gitmodules @@ -85,3 +85,9 @@ [submodule "third_party/VulkanMemoryAllocator"] path = third_party/VulkanMemoryAllocator url = https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator.git +[submodule "third_party/zstd"] + path = third_party/zstd + url = https://github.com/facebook/zstd.git +[submodule "third_party/zarchive"] + path = third_party/zarchive + url = https://github.com/exzap/ZArchive.git diff --git a/premake5.lua b/premake5.lua index 71efb4a85..1faedd90c 100644 --- a/premake5.lua +++ b/premake5.lua @@ -260,6 +260,8 @@ workspace("xenia") include("third_party/mspack.lua") include("third_party/snappy.lua") include("third_party/xxhash.lua") + include("third_party/zarchive.lua") + include("third_party/zstd.lua") if not os.istarget("android") then -- SDL2 requires sdl2-config, and as of November 2020 isn't high-quality on diff --git a/src/xenia/app/emulator_window.cc b/src/xenia/app/emulator_window.cc index 26214c91b..7f2dffcfb 100644 --- a/src/xenia/app/emulator_window.cc +++ b/src/xenia/app/emulator_window.cc @@ -881,8 +881,9 @@ void EmulatorWindow::FileOpen() { file_picker->set_multi_selection(false); file_picker->set_title("Select Content Package"); file_picker->set_extensions({ - {"Supported Files", "*.iso;*.xex;*.*"}, + {"Supported Files", "*.iso;*.xex;*.zar;*.*"}, {"Disc Image (*.iso)", "*.iso"}, + {"Disc Archive (*.zar)", "*.zar"}, {"Xbox Executable (*.xex)", "*.xex"}, //{"Content Package (*.xcp)", "*.xcp" }, {"All Files (*.*)", "*.*"}, diff --git a/src/xenia/emulator.cc b/src/xenia/emulator.cc index bddf899a0..bacdefeb5 100644 --- a/src/xenia/emulator.cc +++ b/src/xenia/emulator.cc @@ -50,6 +50,7 @@ #include "xenia/ui/windowed_app_context.h" #include "xenia/vfs/device.h" #include "xenia/vfs/devices/disc_image_device.h" +#include "xenia/vfs/devices/disc_zarchive_device.h" #include "xenia/vfs/devices/host_path_device.h" #include "xenia/vfs/devices/null_device.h" #include "xenia/vfs/virtual_file_system.h" @@ -420,6 +421,9 @@ X_STATUS Emulator::LaunchPath(const std::filesystem::path& path) { // Treat as a naked xex file. MountPath(path, "\\Device\\Harddisk0\\Partition1"); return LaunchXexFile(path); + } else if (extension == ".zar") { + // Assume a disc image. + return LaunchDiscArchive(path); } else { // Assume a disc image. MountPath(path, "\\Device\\Cdrom0"); @@ -447,6 +451,29 @@ X_STATUS Emulator::LaunchDiscImage(const std::filesystem::path& path) { return CompleteLaunch(path, module_path); } +X_STATUS Emulator::LaunchDiscArchive(const std::filesystem::path& path) { + auto mount_path = "\\Device\\Cdrom0"; + + // Register the disc image in the virtual filesystem. + auto device = std::make_unique(mount_path, path); + if (!device->Initialize()) { + xe::FatalError("Unable to mount disc image; file not found or corrupt."); + return X_STATUS_NO_SUCH_FILE; + } + if (!file_system_->RegisterDevice(std::move(device))) { + xe::FatalError("Unable to register disc image."); + return X_STATUS_NO_SUCH_FILE; + } + + // Create symlinks to the device. + file_system_->RegisterSymbolicLink("game:", mount_path); + file_system_->RegisterSymbolicLink("d:", mount_path); + + // Launch the game. + auto module_path(FindLaunchModule()); + return CompleteLaunch(path, module_path); +} + X_STATUS Emulator::LaunchStfsContainer(const std::filesystem::path& path) { auto module_path(FindLaunchModule()); return CompleteLaunch(path, module_path); diff --git a/src/xenia/emulator.h b/src/xenia/emulator.h index 0e0893fb8..32070d8a6 100644 --- a/src/xenia/emulator.h +++ b/src/xenia/emulator.h @@ -196,6 +196,9 @@ class Emulator { // Launches a game from a disc image file (.iso, etc). X_STATUS LaunchDiscImage(const std::filesystem::path& path); + // Launches a game from a disc archive file (.zar, etc). + X_STATUS LaunchDiscArchive(const std::filesystem::path& path); + // Launches a game from an STFS container file. X_STATUS LaunchStfsContainer(const std::filesystem::path& path); diff --git a/src/xenia/vfs/devices/disc_zarchive_device.cc b/src/xenia/vfs/devices/disc_zarchive_device.cc new file mode 100644 index 000000000..ff31d660e --- /dev/null +++ b/src/xenia/vfs/devices/disc_zarchive_device.cc @@ -0,0 +1,139 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/vfs/devices/disc_zarchive_device.h" + +#include "xenia/base/literals.h" +#include "xenia/base/logging.h" +#include "xenia/base/math.h" +#include "xenia/vfs/devices/disc_zarchive_entry.h" + +#include "third_party/zarchive/include/zarchive/zarchivereader.h" + +namespace xe { +namespace vfs { + +using namespace xe::literals; + +const size_t kXESectorSize = 2_KiB; + +DiscZarchiveDevice::DiscZarchiveDevice(const std::string_view mount_path, + const std::filesystem::path& host_path) + : Device(mount_path), + name_("GDFX"), + host_path_(host_path), + opaque_(nullptr) {} + +DiscZarchiveDevice::~DiscZarchiveDevice() { + ZArchiveReader* reader = static_cast(opaque_); + if (reader != nullptr) delete reader; +}; + +bool DiscZarchiveDevice::Initialize() { + ZArchiveReader* reader = nullptr; + reader = ZArchiveReader::OpenFromFile(host_path_); + + if (!reader) { + XELOGE("Disc ZArchive could not be opened"); + return false; + } + + opaque_ = static_cast(reader); + bool result = false; + + result = reader->IsFile(reader->LookUp("default.xex", true, false)); + if (!result) XELOGE("Failed to verify disc ZArchive (no default.xex)"); + + const std::string root_path = std::string("/"); + ZArchiveNodeHandle handle = reader->LookUp(root_path); + auto root_entry = new DiscZarchiveEntry(this, nullptr, root_path, reader); + root_entry->attributes_ = kFileAttributeDirectory; + root_entry->handle_ = static_cast(handle); + root_entry->name_ = root_path; + root_entry->absolute_path_ = root_path; + root_entry_ = std::unique_ptr(root_entry); + result = ReadAllEntries(reader, "", root_entry, nullptr); + + return result; +} + +void DiscZarchiveDevice::Dump(StringBuffer* string_buffer) { + auto global_lock = global_critical_region_.Acquire(); + root_entry_->Dump(string_buffer, 0); +} + +Entry* DiscZarchiveDevice::ResolvePath(const std::string_view path) { + // The filesystem will have stripped our prefix off already, so the path will + // be in the form: + // some\PATH.foo + XELOGFS("DiscZarchiveDevice::ResolvePath({})", path); + + ZArchiveReader* reader = static_cast(opaque_); + if (!reader) return nullptr; + + ZArchiveNodeHandle handle = reader->LookUp(path); + bool result = (handle != ZARCHIVE_INVALID_NODE); + + if (!result) return nullptr; + + return root_entry_->ResolvePath(path); +} + +bool DiscZarchiveDevice::ReadAllEntries(void* opaque, const std::string& path, + DiscZarchiveEntry* node, + DiscZarchiveEntry* parent) { + ZArchiveReader* reader = static_cast(opaque); + ZArchiveNodeHandle handle = node->handle_; + if (handle == ZARCHIVE_INVALID_NODE) return false; + if (reader->IsDirectory(handle)) { + uint32_t count = reader->GetDirEntryCount(handle); + for (uint32_t i = 0; i < count; i++) { + ZArchiveReader::DirEntry dirEntry; + if (!reader->GetDirEntry(handle, i, dirEntry)) return false; + std::string full_path = path + std::string(dirEntry.name); + ZArchiveNodeHandle fileHandle = reader->LookUp(full_path); + if (handle == ZARCHIVE_INVALID_NODE) return false; + auto entry = new DiscZarchiveEntry(this, parent, full_path, opaque); + entry->handle_ = static_cast(fileHandle); + entry->data_offset_ = 0; + // Set to January 1, 1970 (UTC) in 100-nanosecond intervals + entry->create_timestamp_ = 10000 * 11644473600000LL; + entry->access_timestamp_ = 10000 * 11644473600000LL; + entry->write_timestamp_ = 10000 * 11644473600000LL; + entry->parent_ = node; + if (dirEntry.isDirectory) { + entry->data_size_ = 0; + entry->size_ = dirEntry.size; + entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly; + node->children_.push_back(std::unique_ptr(entry)); + if (!ReadAllEntries(reader, full_path + "\\", entry, node)) + return false; + } else if (dirEntry.isFile) { + entry->data_size_ = entry->size_ = reader->GetFileSize(fileHandle); + entry->attributes_ = kFileAttributeReadOnly; + entry->allocation_size_ = + xe::round_up(entry->size_, bytes_per_sector()); + node->children_.push_back(std::unique_ptr(entry)); + } + } + return true; + } else if (reader->IsFile(handle)) { + auto entry = new DiscZarchiveEntry(this, parent, path, opaque); + entry->attributes_ = kFileAttributeReadOnly; + entry->handle_ = static_cast(handle); + entry->parent_ = parent; + entry->children_.push_back(std::unique_ptr(entry)); + return true; + } + + return false; +} + +} // namespace vfs +} // namespace xe diff --git a/src/xenia/vfs/devices/disc_zarchive_device.h b/src/xenia/vfs/devices/disc_zarchive_device.h new file mode 100644 index 000000000..22edbb362 --- /dev/null +++ b/src/xenia/vfs/devices/disc_zarchive_device.h @@ -0,0 +1,56 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_VFS_DEVICES_DISC_ZARCHIVE_DEVICE_H_ +#define XENIA_VFS_DEVICES_DISC_ZARCHIVE_DEVICE_H_ + +#include +#include + +#include "xenia/base/mapped_memory.h" +#include "xenia/vfs/device.h" + +namespace xe { +namespace vfs { + +class DiscZarchiveEntry; + +class DiscZarchiveDevice : public Device { + public: + DiscZarchiveDevice(const std::string_view mount_path, + const std::filesystem::path& host_path); + ~DiscZarchiveDevice() override; + + bool Initialize() override; + void Dump(StringBuffer* string_buffer) override; + Entry* ResolvePath(const std::string_view path) override; + + const std::string& name() const override { return name_; } + uint32_t attributes() const override { return 0; } + uint32_t component_name_max_length() const override { return 255; } + + uint32_t total_allocation_units() const override { return 128 * 1024; } + uint32_t available_allocation_units() const override { return 0; } + uint32_t sectors_per_allocation_unit() const override { return 1; } + uint32_t bytes_per_sector() const override { return 0x200; } + + private: + bool ReadAllEntries(void* opaque, const std::string& path, + DiscZarchiveEntry* node, DiscZarchiveEntry* parent); + + std::string name_; + std::filesystem::path host_path_; + std::unique_ptr root_entry_; + void* opaque_; +}; + +} // namespace vfs +} // namespace xe + +#endif // XENIA_VFS_DEVICES_DISC_ZARCHIVE_DEVICE_H_ diff --git a/src/xenia/vfs/devices/disc_zarchive_entry.cc b/src/xenia/vfs/devices/disc_zarchive_entry.cc new file mode 100644 index 000000000..fe3c1919a --- /dev/null +++ b/src/xenia/vfs/devices/disc_zarchive_entry.cc @@ -0,0 +1,50 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/vfs/devices/disc_zarchive_entry.h" + +#include + +#include "xenia/base/math.h" +#include "xenia/vfs/devices/disc_zarchive_file.h" + +#include "third_party/zarchive/include/zarchive/zarchivereader.h" +namespace xe { +namespace vfs { + +DiscZarchiveEntry::DiscZarchiveEntry(Device* device, Entry* parent, + const std::string_view path, void* opaque) + : Entry(device, parent, path), + opaque_(opaque), + data_offset_(0), + data_size_(0), + handle_(ZARCHIVE_INVALID_NODE) {} + +DiscZarchiveEntry::~DiscZarchiveEntry() = default; + +std::unique_ptr DiscZarchiveEntry::Create( + Device* device, Entry* parent, const std::string_view name, void* opaque) { + auto path = name; // xe::utf8::join_guest_paths(parent->path(), name); + auto entry = + std::make_unique(device, parent, path, opaque); + return std::move(entry); +} + +X_STATUS DiscZarchiveEntry::Open(uint32_t desired_access, File** out_file) { + *out_file = new DiscZarchiveFile(desired_access, this); + return X_STATUS_SUCCESS; +} + +std::unique_ptr DiscZarchiveEntry::OpenMapped( + MappedMemory::Mode mode, size_t offset, size_t length) { + return nullptr; +} + +} // namespace vfs +} // namespace xe diff --git a/src/xenia/vfs/devices/disc_zarchive_entry.h b/src/xenia/vfs/devices/disc_zarchive_entry.h new file mode 100644 index 000000000..f0595045b --- /dev/null +++ b/src/xenia/vfs/devices/disc_zarchive_entry.h @@ -0,0 +1,59 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2020 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_VFS_DEVICES_DISC_ZARCHIVE_ENTRY_H_ +#define XENIA_VFS_DEVICES_DISC_ZARCHIVE_ENTRY_H_ + +#include +#include + +#include "xenia/base/mapped_memory.h" +#include "xenia/vfs/entry.h" + +namespace xe { +namespace vfs { + +class DiscZarchiveDevice; + +class DiscZarchiveEntry : public Entry { + public: + DiscZarchiveEntry(Device* device, Entry* parent, const std::string_view path, + void* opaque); + ~DiscZarchiveEntry() override; + + static std::unique_ptr Create(Device* device, + Entry* parent, + const std::string_view name, + void* opaque); + + MappedMemory* mmap() const { return nullptr; } + size_t data_offset() const { return data_offset_; } + size_t data_size() const { return data_size_; } + + X_STATUS Open(uint32_t desired_access, File** out_file) override; + + bool can_map() const override { return false; } + std::unique_ptr OpenMapped(MappedMemory::Mode mode, + size_t offset, + size_t length) override; + + private: + friend class DiscZarchiveDevice; + friend class DiscZarchiveFile; + + void* opaque_; + uint32_t handle_; + size_t data_offset_; + size_t data_size_; +}; + +} // namespace vfs +} // namespace xe + +#endif // XENIA_VFS_DEVICES_DISC_ZARCHIVE_ENTRY_H_ diff --git a/src/xenia/vfs/devices/disc_zarchive_file.cc b/src/xenia/vfs/devices/disc_zarchive_file.cc new file mode 100644 index 000000000..39cc745f1 --- /dev/null +++ b/src/xenia/vfs/devices/disc_zarchive_file.cc @@ -0,0 +1,46 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2013 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#include "xenia/vfs/devices/disc_zarchive_file.h" + +#include + +#include "xenia/vfs/devices/disc_zarchive_entry.h" + +#include "third_party/zarchive/include/zarchive/zarchivereader.h" + +namespace xe { +namespace vfs { + +DiscZarchiveFile::DiscZarchiveFile(uint32_t file_access, + DiscZarchiveEntry* entry) + : File(file_access, entry), entry_(entry) {} + +DiscZarchiveFile::~DiscZarchiveFile() = default; + +void DiscZarchiveFile::Destroy() { delete this; } + +X_STATUS DiscZarchiveFile::ReadSync(void* buffer, size_t buffer_length, + size_t byte_offset, + size_t* out_bytes_read) { + if (byte_offset >= entry_->size()) { + return X_STATUS_END_OF_FILE; + } + ZArchiveReader* reader = static_cast(entry_->opaque_); + uint64_t bytes_read = + reader->ReadFromFile(entry_->handle_, byte_offset, buffer_length, buffer); + size_t real_offset = entry_->data_offset() + byte_offset; + size_t real_length = + std::min(buffer_length, entry_->data_size() - byte_offset); + *out_bytes_read = real_length; + return X_STATUS_SUCCESS; +} + +} // namespace vfs +} // namespace xe diff --git a/src/xenia/vfs/devices/disc_zarchive_file.h b/src/xenia/vfs/devices/disc_zarchive_file.h new file mode 100644 index 000000000..eda3ec25c --- /dev/null +++ b/src/xenia/vfs/devices/disc_zarchive_file.h @@ -0,0 +1,42 @@ +/** + ****************************************************************************** + * Xenia : Xbox 360 Emulator Research Project * + ****************************************************************************** + * Copyright 2013 Ben Vanik. All rights reserved. * + * Released under the BSD license - see LICENSE in the root for more details. * + ****************************************************************************** + */ + +#ifndef XENIA_VFS_DEVICES_DISC_ZARCHIVE_FILE_H_ +#define XENIA_VFS_DEVICES_DISC_ZARCHIVE_FILE_H_ + +#include "xenia/vfs/file.h" + +namespace xe { +namespace vfs { + +class DiscZarchiveEntry; + +class DiscZarchiveFile : public File { + public: + DiscZarchiveFile(uint32_t file_access, DiscZarchiveEntry* entry); + ~DiscZarchiveFile() override; + + void Destroy() override; + + X_STATUS ReadSync(void* buffer, size_t buffer_length, size_t byte_offset, + size_t* out_bytes_read) override; + X_STATUS WriteSync(const void* buffer, size_t buffer_length, + size_t byte_offset, size_t* out_bytes_written) override { + return X_STATUS_ACCESS_DENIED; + } + X_STATUS SetLength(size_t length) override { return X_STATUS_ACCESS_DENIED; } + + private: + DiscZarchiveEntry* entry_; +}; + +} // namespace vfs +} // namespace xe + +#endif // XENIA_VFS_DEVICES_DISC_ZARCHIVE_FILE_H_ diff --git a/src/xenia/vfs/premake5.lua b/src/xenia/vfs/premake5.lua index 4fc5cf6e9..e33df18f6 100644 --- a/src/xenia/vfs/premake5.lua +++ b/src/xenia/vfs/premake5.lua @@ -7,7 +7,7 @@ project("xenia-vfs") kind("StaticLib") language("C++") links({ - "xenia-base", + "xenia-base", "zstd", "zarchive" }) defines({ }) diff --git a/third_party/zarchive.lua b/third_party/zarchive.lua new file mode 100644 index 000000000..e2a7e803a --- /dev/null +++ b/third_party/zarchive.lua @@ -0,0 +1,21 @@ +group("third_party") +project("zarchive") + uuid("d32f03aa-f0c9-11ed-a05b-0242ac120003") + kind("StaticLib") + language("C++") + links({ + }) + defines({ + "_LIB", + }) + includedirs({ + "zarchive/include", + "zstd/lib", + }) + files({ + "zarchive/include/zarchive/zarchivecommon.h", + "zarchive/include/zarchive/zarchivereader.h", + "zarchive/include/zarchive/zarchivewriter.h", + "zarchive/src/zarchivereader.cpp", + "zarchive/src/zarchivewriter.cpp", + }) diff --git a/third_party/zstd.lua b/third_party/zstd.lua new file mode 100644 index 000000000..f273a6748 --- /dev/null +++ b/third_party/zstd.lua @@ -0,0 +1,87 @@ +-- This GENie/premake file copies the behavior of the Makefile in the lib folder. +-- Basic usage: project_zstd(ZSTD_DIR) + +function project_zstd(dir, compression, decompression, deprecated, dictbuilder, legacy) + if compression == nil then compression = true end + if decompression == nil then decompression = true end + if deprecated == nil then deprecated = false end + if dictbuilder == nil then dictbuilder = false end + + if legacy == nil then legacy = 0 end + + if not compression then + dictbuilder = false + deprecated = false + end + + if not decompression then + legacy = 0 + deprecated = false + end + + project 'zstd' + kind 'StaticLib' + language 'C' + + files { + dir .. 'zstd.h', + dir .. 'common/**.c', + dir .. 'common/**.h' + } + + if compression then + files { + dir .. 'compress/**.c', + dir .. 'compress/**.h' + } + end + + if decompression then + files { + dir .. 'decompress/**.c', + dir .. 'decompress/**.h' + } + end + + if dictbuilder then + files { + dir .. 'dictBuilder/**.c', + dir .. 'dictBuilder/**.h' + } + end + + if deprecated then + files { + dir .. 'deprecated/**.c', + dir .. 'deprecated/**.h' + } + end + + if legacy ~= 0 then + if legacy >= 8 then + files { + dir .. 'legacy/zstd_v0' .. (legacy - 7) .. '.*' + } + end + includedirs { + dir .. 'legacy' + } + end + + includedirs { + dir, + dir .. 'common' + } + + defines { + 'XXH_NAMESPACE=ZSTD_', + 'ZSTD_LEGACY_SUPPORT=' .. legacy + } +end + + +group("third_party") +project("zstd") + uuid("df336aac-f0c8-11ed-a05b-0242ac120003") + project_zstd('./zstd/lib/') +