ShuriZma
/
xenia-canary
mirror of https://github.com/xenia-canary/xenia-canary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Separation of STFS, SVOD into different entities

This commit is contained in:
Gliniak 2022-06-21 11:05:32 +02:00
parent c1bd30eb7f
commit e191f2d8d0
16 changed files with 1294 additions and 1085 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
src/xenia/emulator.cc
View File

@ -2,7 +2,7 @@
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2021 Ben Vanik. All rights reserved. *
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
@ -48,10 +48,12 @@
#include "xenia/ui/imgui_drawer.h"
#include "xenia/ui/window.h"
#include "xenia/ui/windowed_app_context.h"
#include "xenia/vfs/device.h"
#include "xenia/vfs/devices/disc_image_device.h"
#include "xenia/vfs/devices/host_path_device.h"
#include "xenia/vfs/devices/null_device.h"
#include "xenia/vfs/devices/stfs_container_device.h"
#include "xenia/vfs/virtual_file_system.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#if XE_ARCH_AMD64
#include "xenia/cpu/backend/x64/x64_backend.h"
@ -320,7 +322,7 @@ std::string Emulator::CanonicalizeFileExtension(
const std::unique_ptr<vfs::Device> Emulator::CreateVfsDeviceBasedOnPath(
const std::filesystem::path& path, const std::string_view mount_path) {
if (!path.has_extension()) {
return std::make_unique<vfs::StfsContainerDevice>(mount_path, path);
return vfs::XContentContainerDevice::CreateContentDevice(mount_path, path);
}
auto extension = CanonicalizeFileExtension(path);
if (extension == ".xex" || extension == ".elf" || extension == ".exe") {
@ -451,20 +453,23 @@ X_STATUS Emulator::LaunchStfsContainer(const std::filesystem::path& path) {
}
X_STATUS Emulator::InstallContentPackage(const std::filesystem::path& path) {
std::unique_ptr<vfs::StfsContainerDevice> device =
std::make_unique<vfs::StfsContainerDevice>("", path);
std::unique_ptr<vfs::Device> device =
vfs::XContentContainerDevice::CreateContentDevice("", path);
if (!device->Initialize()) {
XELOGE("Failed to initialize device");
return X_STATUS_INVALID_PARAMETER;
}
const vfs::XContentContainerDevice* dev =
(vfs::XContentContainerDevice*)device.get();
std::filesystem::path installation_path =
content_root() / fmt::format("{:08X}", device->title_id()) /
fmt::format("{:08X}", device->content_type()) / path.filename();
content_root() / fmt::format("{:08X}", dev->title_id()) /
fmt::format("{:08X}", dev->content_type()) / path.filename();
std::filesystem::path header_path =
content_root() / fmt::format("{:08X}", device->title_id()) / "Headers" /
fmt::format("{:08X}", device->content_type()) / path.filename();
content_root() / fmt::format("{:08X}", dev->title_id()) / "Headers" /
fmt::format("{:08X}", dev->content_type()) / path.filename();
if (std::filesystem::exists(installation_path)) {
// TODO(Gliniak): Popup

862
src/xenia/vfs/devices/stfs_container_device.cc
View File

@ -1,862 +0,0 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2020 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/vfs/devices/stfs_container_device.h"
#include <algorithm>
#include <queue>
#include <vector>
#include "xenia/base/logging.h"
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_entry.h"
namespace xe {
namespace vfs {
StfsContainerDevice::StfsContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path)
: Device(mount_path),
name_("STFS"),
host_path_(host_path),
files_total_size_(),
component_name_max_length_(40),
svod_base_offset_(),
header_(),
svod_layout_(),
blocks_per_hash_table_(1),
block_step{0, 0} {}
StfsContainerDevice::~StfsContainerDevice() { CloseFiles(); }
bool StfsContainerDevice::Initialize() {
// Resolve a valid STFS file if a directory is given.
if (std::filesystem::is_directory(host_path_) &&
!ResolveFromFolder(host_path_)) {
XELOGE("Could not resolve an STFS container given path {}",
xe::path_to_utf8(host_path_));
return false;
}
if (!std::filesystem::exists(host_path_)) {
XELOGE("Path to STFS container does not exist: {}",
xe::path_to_utf8(host_path_));
return false;
}
// Open the data file(s)
auto open_result = OpenFiles();
if (open_result != Error::kSuccess) {
XELOGE("Failed to open STFS container: {}", open_result);
return false;
}
switch (header_.metadata.volume_type) {
case XContentVolumeType::kStfs:
return ReadSTFS() == Error::kSuccess;
break;
case XContentVolumeType::kSvod:
component_name_max_length_ = 255;
return ReadSVOD() == Error::kSuccess;
default:
XELOGE("Unknown XContent volume type: {}",
xe::byte_swap(uint32_t(header_.metadata.volume_type.value)));
return false;
}
}
StfsContainerDevice::Error StfsContainerDevice::OpenFiles() {
// Map the file containing the STFS Header and read it.
XELOGI("Loading STFS header file: {}", xe::path_to_utf8(host_path_));
auto header_file = xe::filesystem::OpenFile(host_path_, "rb");
if (!header_file) {
XELOGE("Error opening STFS header file.");
return Error::kErrorReadError;
}
auto header_result = ReadHeaderAndVerify(header_file);
if (header_result != Error::kSuccess) {
XELOGE("Error reading STFS header: {}", header_result);
fclose(header_file);
files_total_size_ = 0;
return header_result;
}
// If the STFS package is a single file, the header is self contained and
// we don't need to map any extra files.
// NOTE: data_file_count is 0 for STFS and 1 for SVOD
if (header_.metadata.data_file_count <= 1) {
XELOGI("STFS container is a single file.");
files_.emplace(std::make_pair(0, header_file));
return Error::kSuccess;
}
// If the STFS package is multi-file, it is an SVOD system. We need to map
// the files in the .data folder and can discard the header.
auto data_fragment_path = host_path_;
data_fragment_path += ".data";
if (!std::filesystem::exists(data_fragment_path)) {
XELOGE("STFS container is multi-file, but path {} does not exist.",
xe::path_to_utf8(data_fragment_path));
return Error::kErrorFileMismatch;
}
// Ensure data fragment files are sorted
auto fragment_files = filesystem::ListFiles(data_fragment_path);
std::sort(fragment_files.begin(), fragment_files.end(),
[](filesystem::FileInfo& left, filesystem::FileInfo& right) {
return left.name < right.name;
});
if (fragment_files.size() != header_.metadata.data_file_count) {
XELOGE("SVOD expecting {} data fragments, but {} are present.",
header_.metadata.data_file_count, fragment_files.size());
return Error::kErrorFileMismatch;
}
for (size_t i = 0; i < fragment_files.size(); i++) {
auto& fragment = fragment_files.at(i);
auto path = fragment.path / fragment.name;
auto file = xe::filesystem::OpenFile(path, "rb");
if (!file) {
XELOGI("Failed to map SVOD file {}.", xe::path_to_utf8(path));
CloseFiles();
return Error::kErrorReadError;
}
xe::filesystem::Seek(file, 0L, SEEK_END);
files_total_size_ += xe::filesystem::Tell(file);
// no need to seek back, any reads from this file will seek first anyway
files_.emplace(std::make_pair(i, file));
}
XELOGI("SVOD successfully mapped {} files.", fragment_files.size());
return Error::kSuccess;
}
void StfsContainerDevice::CloseFiles() {
for (auto& file : files_) {
fclose(file.second);
}
files_.clear();
files_total_size_ = 0;
}
void StfsContainerDevice::Dump(StringBuffer* string_buffer) {
auto global_lock = global_critical_region_.Acquire();
root_entry_->Dump(string_buffer, 0);
}
Entry* StfsContainerDevice::ResolvePath(const std::string_view path) {
// The filesystem will have stripped our prefix off already, so the path will
// be in the form:
// some\PATH.foo
XELOGFS("StfsContainerDevice::ResolvePath({})", path);
return root_entry_->ResolvePath(path);
}
StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify(
FILE* header_file) {
// Check size of the file is enough to store an STFS header
xe::filesystem::Seek(header_file, 0L, SEEK_END);
files_total_size_ = xe::filesystem::Tell(header_file);
xe::filesystem::Seek(header_file, 0L, SEEK_SET);
if (sizeof(StfsHeader) > files_total_size_) {
return Error::kErrorTooSmall;
}
// Read header & check signature
if (fread(&header_, sizeof(StfsHeader), 1, header_file) != 1) {
return Error::kErrorReadError;
}
if (!header_.header.is_magic_valid()) {
// Unexpected format.
return Error::kErrorFileMismatch;
}
// Pre-calculate some values used in block number calculations
if (header_.metadata.volume_type == XContentVolumeType::kStfs) {
blocks_per_hash_table_ =
header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format ? 1
: 2;
block_step[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_;
block_step[1] = kBlocksPerHashLevel[1] +
((kBlocksPerHashLevel[0] + 1) * blocks_per_hash_table_);
}
return Error::kSuccess;
}
StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() {
// SVOD Systems can have different layouts. The root block is
// denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in
// the first "actual" data fragment of the system.
auto& svod_header = files_.at(0);
const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA";
uint8_t magic_buf[20];
size_t magic_offset;
// Check for EDGF layout
if (header_.metadata.volume_descriptor.svod.features.bits
.enhanced_gdf_layout) {
// The STFS header has specified that this SVOD system uses the EGDF layout.
// We can expect the magic block to be located immediately after the hash
// blocks. We also offset block address calculation by 0x1000 by shifting
// block indices by +0x2.
xe::filesystem::Seek(svod_header, 0x2000, SEEK_SET);
if (fread(magic_buf, 1, countof(magic_buf), svod_header) !=
countof(magic_buf)) {
XELOGE("ReadSVOD failed to read SVOD magic at 0x2000");
return Error::kErrorReadError;
}
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) {
svod_base_offset_ = 0x0000;
magic_offset = 0x2000;
svod_layout_ = SvodLayoutType::kEnhancedGDF;
XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000.");
} else {
XELOGE("SVOD uses an EGDF layout, but the magic block was not found.");
return Error::kErrorFileMismatch;
}
} else {
xe::filesystem::Seek(svod_header, 0x12000, SEEK_SET);
if (fread(magic_buf, 1, countof(magic_buf), svod_header) !=
countof(magic_buf)) {
XELOGE("ReadSVOD failed to read SVOD magic at 0x12000");
return Error::kErrorReadError;
}
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0x12000, it is likely using an XSF
// layout. This is usually due to converting the game using a third-party
// tool, as most of them use a nulled XSF as a template.
svod_base_offset_ = 0x10000;
magic_offset = 0x12000;
// Check for XSF Header
const char* XSF_MAGIC = "XSF";
xe::filesystem::Seek(svod_header, 0x2000, SEEK_SET);
if (fread(magic_buf, 1, 3, svod_header) != 3) {
XELOGE("ReadSVOD failed to read SVOD XSF magic at 0x2000");
return Error::kErrorReadError;
}
if (std::memcmp(magic_buf, XSF_MAGIC, 3) == 0) {
svod_layout_ = SvodLayoutType::kXSF;
XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000.");
XELOGI("Game was likely converted using a third-party tool.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGI("SVOD appears to use an XSF layout, but no header is present.");
XELOGI("SVOD magic block found at 0x12000");
}
} else {
xe::filesystem::Seek(svod_header, 0xD000, SEEK_SET);
if (fread(magic_buf, 1, countof(magic_buf), svod_header) !=
countof(magic_buf)) {
XELOGE("ReadSVOD failed to read SVOD magic at 0xD000");
return Error::kErrorReadError;
}
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0xD000, it most likely means that it
// is a single-file system. The STFS Header is 0xB000 bytes , and the
// remaining 0x2000 is from hash tables. In most cases, these will be
// STFS, not SVOD.
svod_base_offset_ = 0xB000;
magic_offset = 0xD000;
// Check for single file system
if (header_.metadata.data_file_count == 1) {
svod_layout_ = SvodLayoutType::kSingleFile;
XELOGI("SVOD is a single file. Magic block present at 0xD000.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGE(
"SVOD is not a single file, but the magic block was found at "
"0xD000.");
}
} else {
XELOGE("Could not locate SVOD magic block.");
return Error::kErrorReadError;
}
}
}
// Parse the root directory
xe::filesystem::Seek(svod_header, magic_offset + 0x14, SEEK_SET);
struct {
uint32_t block;
uint32_t size;
uint32_t creation_date;
uint32_t creation_time;
} root_data;
static_assert_size(root_data, 0x10);
if (fread(&root_data, sizeof(root_data), 1, svod_header) != 1) {
XELOGE("ReadSVOD failed to read root block data at 0x{X}",
magic_offset + 0x14);
return Error::kErrorReadError;
}
uint64_t root_creation_timestamp =
decode_fat_timestamp(root_data.creation_date, root_data.creation_time);
auto root_entry = new StfsContainerEntry(this, nullptr, "", &files_);
root_entry->attributes_ = kFileAttributeDirectory;
root_entry->access_timestamp_ = root_creation_timestamp;
root_entry->create_timestamp_ = root_creation_timestamp;
root_entry->write_timestamp_ = root_creation_timestamp;
root_entry_ = std::unique_ptr<Entry>(root_entry);
// Traverse all child entries
return ReadEntrySVOD(root_data.block, 0, root_entry);
}
StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD(
uint32_t block, uint32_t ordinal, StfsContainerEntry* parent) {
// For games with a large amount of files, the ordinal offset can overrun
// the current block and potentially hit a hash block.
size_t ordinal_offset = ordinal * 0x4;
size_t block_offset = ordinal_offset / 0x800;
size_t true_ordinal_offset = ordinal_offset % 0x800;
// Calculate the file & address of the block
size_t entry_address, entry_file;
BlockToOffsetSVOD(block + block_offset, &entry_address, &entry_file);
entry_address += true_ordinal_offset;
// Read directory entry
auto& file = files_.at(entry_file);
xe::filesystem::Seek(file, entry_address, SEEK_SET);
#pragma pack(push, 1)
struct {
uint16_t node_l;
uint16_t node_r;
uint32_t data_block;
uint32_t length;
uint8_t attributes;
uint8_t name_length;
} dir_entry;
static_assert_size(dir_entry, 0xE);
#pragma pack(pop)
if (fread(&dir_entry, sizeof(dir_entry), 1, file) != 1) {
XELOGE("ReadEntrySVOD failed to read directory entry at 0x{X}",
entry_address);
return Error::kErrorReadError;
}
auto name_buffer = std::make_unique<char[]>(dir_entry.name_length);
if (fread(name_buffer.get(), 1, dir_entry.name_length, file) !=
dir_entry.name_length) {
XELOGE("ReadEntrySVOD failed to read directory entry name at 0x{X}",
entry_address);
return Error::kErrorReadError;
}
auto name = std::string(name_buffer.get(), dir_entry.name_length);
// Read the left node
if (dir_entry.node_l) {
auto node_result = ReadEntrySVOD(block, dir_entry.node_l, parent);
if (node_result != Error::kSuccess) {
return node_result;
}
}
// Read file & address of block's data
size_t data_address, data_file;
BlockToOffsetSVOD(dir_entry.data_block, &data_address, &data_file);
// Create the entry
// NOTE: SVOD entries don't have timestamps for individual files, which can
// cause issues when decrypting games. Using the root entry's timestamp
// solves this issues.
auto entry = StfsContainerEntry::Create(this, parent, name, &files_);
if (dir_entry.attributes & kFileAttributeDirectory) {
// Entry is a directory
entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
entry->data_offset_ = 0;
entry->data_size_ = 0;
entry->block_ = block;
entry->access_timestamp_ = root_entry_->create_timestamp();
entry->create_timestamp_ = root_entry_->create_timestamp();
entry->write_timestamp_ = root_entry_->create_timestamp();
if (dir_entry.length) {
// If length is greater than 0, traverse the directory's children
auto directory_result =
ReadEntrySVOD(dir_entry.data_block, 0, entry.get());
if (directory_result != Error::kSuccess) {
return directory_result;
}
}
} else {
// Entry is a file
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->size_ = dir_entry.length;
entry->allocation_size_ = xe::round_up(dir_entry.length, kBlockSize);
entry->data_offset_ = data_address;
entry->data_size_ = dir_entry.length;
entry->block_ = dir_entry.data_block;
entry->access_timestamp_ = root_entry_->create_timestamp();
entry->create_timestamp_ = root_entry_->create_timestamp();
entry->write_timestamp_ = root_entry_->create_timestamp();
// Fill in all block records, sector by sector.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t block_index = dir_entry.data_block;
size_t remaining_size = xe::round_up(dir_entry.length, 0x800);
size_t last_record = -1;
size_t last_offset = -1;
while (remaining_size) {
const size_t BLOCK_SIZE = 0x800;
size_t offset, file_index;
BlockToOffsetSVOD(block_index, &offset, &file_index);
block_index++;
remaining_size -= BLOCK_SIZE;
if (offset - last_offset == 0x800) {
// Consecutive, so append to last entry.
entry->block_list_[last_record].length += BLOCK_SIZE;
last_offset = offset;
continue;
}
entry->block_list_.push_back({file_index, offset, BLOCK_SIZE});
last_record = entry->block_list_.size() - 1;
last_offset = offset;
}
}
}
parent->children_.emplace_back(std::move(entry));
// Read the right node.
if (dir_entry.node_r) {
auto node_result = ReadEntrySVOD(block, dir_entry.node_r, parent);
if (node_result != Error::kSuccess) {
return node_result;
}
}
return Error::kSuccess;
}
void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address,
size_t* out_file_index) {
// SVOD Systems use hash blocks for integrity checks. These hash blocks
// cause blocks to be discontinuous in memory, and must be accounted for.
// - Each data block is 0x800 bytes in length
// - Every group of 0x198 data blocks is preceded a Level0 hash table.
// Level0 tables contain 0xCC hashes, each representing two data blocks.
// The total size of each Level0 hash table is 0x1000 bytes in length.
// - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table.
// Level1 tables contain 0xCB hashes, each representing two Level0 hashes.
// The total size of each Level1 hash table is 0x1000 bytes in length.
// - Files are split into fragments of 0xA290000 bytes in length,
// consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1
// Level1 hash table.
const size_t BLOCK_SIZE = 0x800;
const size_t HASH_BLOCK_SIZE = 0x1000;
const size_t BLOCKS_PER_L0_HASH = 0x198;
const size_t HASHES_PER_L1_HASH = 0xA1C4;
const size_t BLOCKS_PER_FILE = 0x14388;
const size_t MAX_FILE_SIZE = 0xA290000;
const size_t BLOCK_OFFSET =
header_.metadata.volume_descriptor.svod.start_data_block();
// Resolve the true block address and file index
size_t true_block = block - (BLOCK_OFFSET * 2);
if (svod_layout_ == SvodLayoutType::kEnhancedGDF) {
// EGDF has an 0x1000 byte offset, which is two blocks
true_block += 0x2;
}
size_t file_block = true_block % BLOCKS_PER_FILE;
size_t file_index = true_block / BLOCKS_PER_FILE;
size_t offset = 0;
// Calculate offset caused by Level0 Hash Tables
size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1;
offset += level0_table_count * HASH_BLOCK_SIZE;
// Calculate offset caused by Level1 Hash Tables
size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1;
offset += level1_table_count * HASH_BLOCK_SIZE;
// For single-file SVOD layouts, include the size of the header in the offset.
if (svod_layout_ == SvodLayoutType::kSingleFile) {
offset += svod_base_offset_;
}
size_t block_address = (file_block * BLOCK_SIZE) + offset;
// If the offset causes the block address to overrun the file, round it.
if (block_address >= MAX_FILE_SIZE) {
file_index += 1;
block_address %= MAX_FILE_SIZE;
block_address += 0x2000;
}
*out_address = block_address;
*out_file_index = file_index;
}
StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
auto& file = files_.at(0);
auto root_entry = new StfsContainerEntry(this, nullptr, "", &files_);
root_entry->attributes_ = kFileAttributeDirectory;
root_entry_ = std::unique_ptr<Entry>(root_entry);
std::vector<StfsContainerEntry*> all_entries;
// Load all listings.
StfsDirectoryBlock directory;
auto& descriptor = header_.metadata.volume_descriptor.stfs;
uint32_t table_block_index = descriptor.file_table_block_number();
size_t n = 0;
for (n = 0; n < descriptor.file_table_block_count; n++) {
auto offset = BlockToOffsetSTFS(table_block_index);
xe::filesystem::Seek(file, offset, SEEK_SET);
if (fread(&directory, sizeof(StfsDirectoryBlock), 1, file) != 1) {
XELOGE("ReadSTFS failed to read directory block at 0x{X}", offset);
return Error::kErrorReadError;
}
for (size_t m = 0; m < kEntriesPerDirectoryBlock; m++) {
auto& dir_entry = directory.entries[m];
if (dir_entry.name[0] == 0) {
// Done.
break;
}
StfsContainerEntry* parent_entry = nullptr;
if (dir_entry.directory_index == 0xFFFF) {
parent_entry = root_entry;
} else {
parent_entry = all_entries[dir_entry.directory_index];
}
std::string name(reinterpret_cast<const char*>(dir_entry.name),
dir_entry.flags.name_length & 0x3F);
auto entry =
StfsContainerEntry::Create(this, parent_entry, name, &files_);
if (dir_entry.flags.directory) {
entry->attributes_ = kFileAttributeDirectory;
} else {
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->data_offset_ = BlockToOffsetSTFS(dir_entry.start_block_number());
entry->data_size_ = dir_entry.length;
}
entry->size_ = dir_entry.length;
entry->allocation_size_ = xe::round_up(dir_entry.length, kBlockSize);
entry->create_timestamp_ =
decode_fat_timestamp(dir_entry.create_date, dir_entry.create_time);
entry->write_timestamp_ = decode_fat_timestamp(dir_entry.modified_date,
dir_entry.modified_time);
entry->access_timestamp_ = entry->write_timestamp_;
all_entries.push_back(entry.get());
// Fill in all block records.
// It's easier to do this now and just look them up later, at the cost
// of some memory. Nasty chain walk.
// TODO(benvanik): optimize if flags.contiguous is set.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t block_index = dir_entry.start_block_number();
size_t remaining_size = dir_entry.length;
while (remaining_size && block_index != kEndOfChain) {
size_t block_size =
std::min(static_cast<size_t>(kBlockSize), remaining_size);
size_t offset = BlockToOffsetSTFS(block_index);
entry->block_list_.push_back({0, offset, block_size});
remaining_size -= block_size;
auto block_hash = GetBlockHash(block_index);
block_index = block_hash->level0_next_block();
}
if (remaining_size) {
// Loop above must have exited prematurely, bad hash tables?
XELOGW(
"STFS file {} only found {} bytes for file, expected {} ({} "
"bytes missing)",
name, dir_entry.length - remaining_size, dir_entry.length,
remaining_size);
assert_always();
}
// Check that the number of blocks retrieved from hash entries matches
// the block count read from the file entry
if (entry->block_list_.size() != dir_entry.allocated_data_blocks()) {
XELOGW(
"STFS failed to read correct block-chain for entry {}, read {} "
"blocks, expected {}",
entry->name_, entry->block_list_.size(),
dir_entry.allocated_data_blocks());
assert_always();
}
}
parent_entry->children_.emplace_back(std::move(entry));
}
auto block_hash = GetBlockHash(table_block_index);
table_block_index = block_hash->level0_next_block();
if (table_block_index == kEndOfChain) {
break;
}
}
if (n + 1 != descriptor.file_table_block_count) {
XELOGW("STFS read {} file table blocks, but STFS headers expected {}!",
n + 1, descriptor.file_table_block_count);
assert_always();
}
return Error::kSuccess;
}
size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) const {
// For every level there is a hash table
// Level 0: hash table of next 170 blocks
// Level 1: hash table of next 170 hash tables
// Level 2: hash table of next 170 level 1 hash tables
// And so on...
uint64_t base = kBlocksPerHashLevel[0];
uint64_t block = block_index;
for (uint32_t i = 0; i < 3; i++) {
block += ((block_index + base) / base) * blocks_per_hash_table_;
if (block_index < base) {
break;
}
base *= kBlocksPerHashLevel[0];
}
return xe::round_up(header_.header.header_size, kBlockSize) + (block << 12);
}
uint32_t StfsContainerDevice::BlockToHashBlockNumberSTFS(
uint32_t block_index, uint32_t hash_level) const {
uint32_t block = 0;
if (hash_level == 0) {
if (block_index < kBlocksPerHashLevel[0]) {
return 0;
}
block = (block_index / kBlocksPerHashLevel[0]) * block_step[0];
block +=
((block_index / kBlocksPerHashLevel[1]) + 1) * blocks_per_hash_table_;
if (block_index < kBlocksPerHashLevel[1]) {
return block;
}
return block + blocks_per_hash_table_;
}
if (hash_level == 1) {
if (block_index < kBlocksPerHashLevel[1]) {
return block_step[0];
}
block = (block_index / kBlocksPerHashLevel[1]) * block_step[1];
return block + blocks_per_hash_table_;
}
// Level 2 is always at blockStep1
return block_step[1];
}
size_t StfsContainerDevice::BlockToHashBlockOffsetSTFS(
uint32_t block_index, uint32_t hash_level) const {
uint64_t block = BlockToHashBlockNumberSTFS(block_index, hash_level);
return xe::round_up(header_.header.header_size, kBlockSize) + (block << 12);
}
const StfsHashEntry* StfsContainerDevice::GetBlockHash(uint32_t block_index) {
auto& file = files_.at(0);
auto& descriptor = header_.metadata.volume_descriptor.stfs;
// Offset for selecting the secondary hash block, in packages that have them
uint32_t secondary_table_offset =
descriptor.flags.bits.root_active_index ? kBlockSize : 0;
auto hash_offset_lv0 = BlockToHashBlockOffsetSTFS(block_index, 0);
if (!cached_hash_tables_.count(hash_offset_lv0)) {
// If this is read_only_format then it doesn't contain secondary blocks, no
// need to check upper hash levels
if (descriptor.flags.bits.read_only_format) {
secondary_table_offset = 0;
} else {
// Not a read-only package, need to check each levels active index flag to
// see if we need to use secondary block or not
// Check level1 table if package has it
if (descriptor.total_block_count > kBlocksPerHashLevel[0]) {
auto hash_offset_lv1 = BlockToHashBlockOffsetSTFS(block_index, 1);
if (!cached_hash_tables_.count(hash_offset_lv1)) {
// Check level2 table if package has it
if (descriptor.total_block_count > kBlocksPerHashLevel[1]) {
auto hash_offset_lv2 = BlockToHashBlockOffsetSTFS(block_index, 2);
if (!cached_hash_tables_.count(hash_offset_lv2)) {
xe::filesystem::Seek(
file, hash_offset_lv2 + secondary_table_offset, SEEK_SET);
StfsHashTable table_lv2;
if (fread(&table_lv2, sizeof(StfsHashTable), 1, file) != 1) {
XELOGE("GetBlockHash failed to read level2 hash table at 0x{X}",
hash_offset_lv2 + secondary_table_offset);
return nullptr;
}
cached_hash_tables_[hash_offset_lv2] = table_lv2;
}
auto record =
(block_index / kBlocksPerHashLevel[1]) % kBlocksPerHashLevel[0];
auto record_data =
&cached_hash_tables_[hash_offset_lv2].entries[record];
secondary_table_offset =
record_data->levelN_active_index() ? kBlockSize : 0;
}
xe::filesystem::Seek(file, hash_offset_lv1 + secondary_table_offset,
SEEK_SET);
StfsHashTable table_lv1;
if (fread(&table_lv1, sizeof(StfsHashTable), 1, file) != 1) {
XELOGE("GetBlockHash failed to read level1 hash table at 0x{X}",
hash_offset_lv1 + secondary_table_offset);
return nullptr;
}
cached_hash_tables_[hash_offset_lv1] = table_lv1;
}
auto record =
(block_index / kBlocksPerHashLevel[0]) % kBlocksPerHashLevel[0];
auto record_data =
&cached_hash_tables_[hash_offset_lv1].entries[record];
secondary_table_offset =
record_data->levelN_active_index() ? kBlockSize : 0;
}
}
xe::filesystem::Seek(file, hash_offset_lv0 + secondary_table_offset,
SEEK_SET);
StfsHashTable table_lv0;
if (fread(&table_lv0, sizeof(StfsHashTable), 1, file) != 1) {
XELOGE("GetBlockHash failed to read level0 hash table at 0x{X}",
hash_offset_lv0 + secondary_table_offset);
return nullptr;
}
cached_hash_tables_[hash_offset_lv0] = table_lv0;
}
auto record = block_index % kBlocksPerHashLevel[0];
auto record_data = &cached_hash_tables_[hash_offset_lv0].entries[record];
return record_data;
}
XContentPackageType StfsContainerDevice::ReadMagic(
const std::filesystem::path& path) {
auto map = MappedMemory::Open(path, MappedMemory::Mode::kRead, 0, 4);
return XContentPackageType(xe::load_and_swap<uint32_t>(map->data()));
}
bool StfsContainerDevice::ResolveFromFolder(const std::filesystem::path& path) {
// Scan through folders until a file with magic is found
std::queue<filesystem::FileInfo> queue;
filesystem::FileInfo folder;
filesystem::GetInfo(host_path_, &folder);
queue.push(folder);
while (!queue.empty()) {
auto current_file = queue.front();
queue.pop();
if (current_file.type == filesystem::FileInfo::Type::kDirectory) {
auto path = current_file.path / current_file.name;
auto child_files = filesystem::ListFiles(path);
for (auto file : child_files) {
queue.push(file);
}
} else {
// Try to read the file's magic
auto path = current_file.path / current_file.name;
auto magic = ReadMagic(path);
if (magic == XContentPackageType::kCon ||
magic == XContentPackageType::kLive ||
magic == XContentPackageType::kPirs) {
host_path_ = current_file.path / current_file.name;
XELOGI("STFS Package found: {}", xe::path_to_utf8(host_path_));
return true;
}
}
}
if (host_path_ == path) {
// Could not find a suitable container file
return false;
}
return true;
}
kernel::xam::XCONTENT_AGGREGATE_DATA StfsContainerDevice::content_header() const {
kernel::xam::XCONTENT_AGGREGATE_DATA data;
std::memset(&data, 0, sizeof(kernel::xam::XCONTENT_AGGREGATE_DATA));
data.device_id = 1;
data.title_id = header_.metadata.execution_info.title_id;
data.content_type = header_.metadata.content_type;
auto name = header_.metadata.display_name(XLanguage::kEnglish);
if (name.empty()) {
// Find first filled language and use it. It might be incorrect, but meh
// until stfs support is done.
for (uint8_t i = 0; i < header_.metadata.kNumLanguagesV2; i++) {
name = header_.metadata.display_name((XLanguage)i);
if (!name.empty()) {
break;
}
}
}
data.set_display_name(name);
return data;
}
} // namespace vfs
} // namespace xe

157
src/xenia/vfs/devices/stfs_container_device.h
View File

@ -1,157 +0,0 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2020 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_
#define XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include "xenia/base/math.h"
#include "xenia/base/string_util.h"
#include "xenia/kernel/util/xex2_info.h"
#include "xenia/kernel/xam/content_manager.h"
#include "xenia/vfs/device.h"
#include "xenia/vfs/devices/stfs_xbox.h"
namespace xe {
namespace vfs {
// https://free60project.github.io/wiki/STFS.html
class StfsContainerEntry;
class StfsContainerDevice : public Device {
public:
const static uint32_t kBlockSize = 0x1000;
StfsContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path);
~StfsContainerDevice() override;
bool Initialize() override;
bool is_read_only() const override {
return header_.metadata.volume_type != XContentVolumeType::kStfs ||
header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format;
}
void Dump(StringBuffer* string_buffer) override;
Entry* ResolvePath(const std::string_view path) override;
const std::string& name() const override { return name_; }
uint32_t attributes() const override { return 0; }
uint32_t component_name_max_length() const override {
return component_name_max_length_;
}
uint32_t total_allocation_units() const override {
if (header_.metadata.volume_type == XContentVolumeType::kStfs) {
return header_.metadata.volume_descriptor.stfs.total_block_count;
}
return uint32_t(data_size() / sectors_per_allocation_unit() /
bytes_per_sector());
}
uint32_t available_allocation_units() const override {
if (!is_read_only()) {
auto& descriptor = header_.metadata.volume_descriptor.stfs;
return kBlocksPerHashLevel[2] -
(descriptor.total_block_count - descriptor.free_block_count);
}
return 0;
}
uint32_t sectors_per_allocation_unit() const override { return 8; }
uint32_t bytes_per_sector() const override { return 0x200; }
size_t data_size() const {
if (header_.header.header_size) {
if (header_.metadata.volume_type == XContentVolumeType::kStfs) {
return header_.metadata.volume_descriptor.stfs.total_block_count *
kBlockSize;
}
return files_total_size_ -
xe::round_up(header_.header.header_size, kBlockSize);
}
return files_total_size_ - sizeof(StfsHeader);
}
uint32_t title_id() const { return header_.metadata.execution_info.title_id; }
XContentType content_type() const { return header_.metadata.content_type; }
kernel::xam::XCONTENT_AGGREGATE_DATA content_header() const;
private:
const uint32_t kBlocksPerHashLevel[3] = {170, 28900, 4913000};
const uint32_t kEndOfChain = 0xFFFFFF;
const uint32_t kEntriesPerDirectoryBlock =
kBlockSize / sizeof(StfsDirectoryEntry);
enum class Error {
kSuccess = 0,
kErrorOutOfMemory = -1,
kErrorReadError = -10,
kErrorFileMismatch = -30,
kErrorDamagedFile = -31,
kErrorTooSmall = -32,
};
enum class SvodLayoutType {
kUnknown = 0x0,
kEnhancedGDF = 0x1,
kXSF = 0x2,
kSingleFile = 0x4,
};
XContentPackageType ReadMagic(const std::filesystem::path& path);
bool ResolveFromFolder(const std::filesystem::path& path);
Error OpenFiles();
void CloseFiles();
Error ReadHeaderAndVerify(FILE* header_file);
Error ReadSVOD();
Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal,
StfsContainerEntry* parent);
void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index);
Error ReadSTFS();
size_t BlockToOffsetSTFS(uint64_t block_index) const;
uint32_t BlockToHashBlockNumberSTFS(uint32_t block_index,
uint32_t hash_level) const;
size_t BlockToHashBlockOffsetSTFS(uint32_t block_index,
uint32_t hash_level) const;
const StfsHashEntry* GetBlockHash(uint32_t block_index);
std::string name_;
std::filesystem::path host_path_;
std::map<size_t, FILE*> files_;
size_t files_total_size_;
size_t svod_base_offset_;
uint32_t component_name_max_length_;
std::unique_ptr<Entry> root_entry_;
StfsHeader header_;
SvodLayoutType svod_layout_;
uint32_t blocks_per_hash_table_;
uint32_t block_step[2];
std::unordered_map<size_t, StfsHashTable> cached_hash_tables_;
};
} // namespace vfs
} // namespace xe
#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_

21
src/xenia/vfs/devices/stfs_xbox.h
View File

@ -2,7 +2,7 @@
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2021 Ben Vanik. All rights reserved. *
* Copyright 2022 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
@ -10,14 +10,15 @@
#ifndef XENIA_VFS_DEVICES_STFS_XBOX_H_
#define XENIA_VFS_DEVICES_STFS_XBOX_H_
#include "xenia/xbox.h"
#include "xenia/base/string_util.h"
#include "xenia/kernel/util/xex2_info.h"
#include "xenia/xbox.h"
namespace xe {
namespace vfs {
// Convert FAT timestamp to 100-nanosecond intervals since January 1, 1601 (UTC)
inline uint64_t decode_fat_timestamp(const uint32_t date, const uint32_t time) {
struct tm tm = {0};
// 80 is the difference between 1980 (FAT) and 1900 (tm);
@ -481,13 +482,21 @@ struct XContentHeader {
static_assert_size(XContentHeader, 0x344);
#pragma pack(pop)
struct StfsHeader {
XContentHeader header;
XContentMetadata metadata;
struct XContentContainerHeader {
XContentHeader content_header;
XContentMetadata content_metadata;
// TODO: title/system updates contain more data after XContentMetadata, seems
// to affect header.header_size
bool is_package_readonly() const {
if (content_metadata.volume_type == vfs::XContentVolumeType::kSvod) {
return true;
}
return content_metadata.volume_descriptor.stfs.flags.bits.read_only_format;
}
};
static_assert_size(StfsHeader, 0x971A);
static_assert_size(XContentContainerHeader, 0x971A);
} // namespace vfs
} // namespace xe

192
src/xenia/vfs/devices/xcontent_container_device.cc Normal file
View File

@ -0,0 +1,192 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/base/logging.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#include "xenia/vfs/devices/xcontent_devices/stfs_container_device.h"
#include "xenia/vfs/devices/xcontent_devices/svod_container_device.h"
namespace xe {
namespace vfs {
std::unique_ptr<Device> XContentContainerDevice::CreateContentDevice(
const std::string_view mount_path, const std::filesystem::path& host_path) {
if (!std::filesystem::exists(host_path)) {
XELOGE("Path to XContent container does not exist: {}",
xe::path_to_utf8(host_path));
return nullptr;
}
if (std::filesystem::is_directory(host_path)) {
return nullptr;
}
FILE* host_file = xe::filesystem::OpenFile(host_path, "rb");
if (!host_file) {
XELOGE("Error opening XContent file.");
return nullptr;
}
const uint64_t package_size = std::filesystem::file_size(host_path);
if (package_size < sizeof(XContentContainerHeader)) {
return nullptr;
}
const auto header = XContentContainerDevice::ReadContainerHeader(host_file);
if (header == nullptr) {
return nullptr;
}
fclose(host_file);
if (!header->content_header.is_magic_valid()) {
return nullptr;
}
switch (header->content_metadata.volume_type) {
case XContentVolumeType::kStfs:
return std::make_unique<StfsContainerDevice>(mount_path, host_path);
break;
case XContentVolumeType::kSvod:
return std::make_unique<SvodContainerDevice>(mount_path, host_path);
break;
default:
break;
}
return nullptr;
}
XContentContainerDevice::XContentContainerDevice(
const std::string_view mount_path, const std::filesystem::path& host_path)
: Device(mount_path),
name_("XContent"),
host_path_(host_path),
files_total_size_(0),
header_(std::make_unique<XContentContainerHeader>()) {}
XContentContainerDevice::~XContentContainerDevice() {}
bool XContentContainerDevice::Initialize() {
if (!std::filesystem::exists(host_path_)) {
XELOGE("Path to XContent container does not exist: {}",
xe::path_to_utf8(host_path_));
return false;
}
if (std::filesystem::is_directory(host_path_)) {
return false;
}
XELOGI("Loading XContent header file: {}", xe::path_to_utf8(host_path_));
auto header_file = xe::filesystem::OpenFile(host_path_, "rb");
if (!header_file) {
XELOGE("Error opening XContent header file.");
return false;
}
auto header_loading_result = ReadHeaderAndVerify(header_file);
if (header_loading_result != Result::kSuccess) {
XELOGE("Error reading XContent header: {}", header_loading_result);
fclose(header_file);
return false;
}
SetupContainer();
if (LoadHostFiles(header_file) != Result::kSuccess) {
XELOGE("Error loading XContent host files.");
return false;
}
return Read() == Result::kSuccess;
}
XContentContainerHeader* XContentContainerDevice::ReadContainerHeader(
FILE* host_file) {
XContentContainerHeader* header = new XContentContainerHeader();
// Read header & check signature
if (fread(header, sizeof(XContentContainerHeader), 1, host_file) != 1) {
return nullptr;
}
return header;
}
Entry* XContentContainerDevice::ResolvePath(const std::string_view path) {
// The filesystem will have stripped our prefix off already, so the path will
// be in the form:
// some\PATH.foo
XELOGFS("StfsContainerDevice::ResolvePath({})", path);
return root_entry_->ResolvePath(path);
}
void XContentContainerDevice::Dump(StringBuffer* string_buffer) {
auto global_lock = global_critical_region_.Acquire();
root_entry_->Dump(string_buffer, 0);
}
void XContentContainerDevice::CloseFiles() {
for (auto& file : files_) {
fclose(file.second);
}
files_.clear();
files_total_size_ = 0;
}
kernel::xam::XCONTENT_AGGREGATE_DATA XContentContainerDevice::content_header()
const {
kernel::xam::XCONTENT_AGGREGATE_DATA data;
std::memset(&data, 0, sizeof(kernel::xam::XCONTENT_AGGREGATE_DATA));
data.device_id = 1;
data.title_id = header_->content_metadata.execution_info.title_id;
data.content_type = header_->content_metadata.content_type;
auto name = header_->content_metadata.display_name(XLanguage::kEnglish);
if (name.empty()) {
// Find first filled language and use it. It might be incorrect, but meh
// until stfs support is done.
for (uint8_t i = 0; i < header_->content_metadata.kNumLanguagesV2; i++) {
name = header_->content_metadata.display_name((XLanguage)i);
if (!name.empty()) {
break;
}
}
}
data.set_display_name(name);
return data;
}
XContentContainerDevice::Result XContentContainerDevice::ReadHeaderAndVerify(
FILE* header_file) {
files_total_size_ = std::filesystem::file_size(host_path_);
if (files_total_size_ < sizeof(XContentContainerHeader)) {
return Result::kTooSmall;
}
const XContentContainerHeader* header = ReadContainerHeader(header_file);
if (header == nullptr) {
return Result::kReadError;
}
std::memcpy(header_.get(), header, sizeof(XContentContainerHeader));
if (!header_->content_header.is_magic_valid()) {
// Unexpected format.
return Result::kFileMismatch;
}
return Result::kSuccess;
}
} // namespace vfs
} // namespace xe

109
src/xenia/vfs/devices/xcontent_container_device.h Normal file
View File

@ -0,0 +1,109 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_VFS_DEVICES_XCONTENT_CONTAINER_DEVICE_H_
#define XENIA_VFS_DEVICES_XCONTENT_CONTAINER_DEVICE_H_
#include <filesystem>
#include <map>
#include <string_view>
#include "xenia/base/math.h"
#include "xenia/kernel/util/xex2_info.h"
#include "xenia/vfs/device.h"
#include "xenia/vfs/devices/stfs_xbox.h"
#include "xenia/kernel/xam/content_manager.h"
namespace xe {
namespace vfs {
class XContentContainerDevice : public Device {
public:
const static uint32_t kBlockSize = 0x1000;
static std::unique_ptr<Device> CreateContentDevice(
const std::string_view mount_path,
const std::filesystem::path& host_path);
~XContentContainerDevice() override;
bool Initialize();
const std::string& name() const override { return name_; }
uint32_t attributes() const override { return 0; }
uint32_t sectors_per_allocation_unit() const override { return 8; }
uint32_t bytes_per_sector() const override { return 0x200; }
size_t data_size() const {
if (header_->content_header.header_size) {
return files_total_size_ -
xe::round_up(header_->content_header.header_size, kBlockSize);
}
return files_total_size_ - sizeof(XContentContainerHeader);
}
uint32_t title_id() const {
return header_->content_metadata.execution_info.title_id;
}
uint32_t content_type() const {
return (uint32_t)header_->content_metadata.content_type.get();
}
kernel::xam::XCONTENT_AGGREGATE_DATA content_header() const;
protected:
XContentContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path);
enum class Result {
kSuccess = 0,
kOutOfMemory = -1,
kReadError = -10,
kFileMismatch = -30,
kDamagedFile = -31,
kTooSmall = -32,
};
virtual Result Read() = 0;
// Load all host files. Usually STFS is only 1 file, meanwhile SVOD is usually multiple file.
virtual Result LoadHostFiles(FILE* header_file) = 0;
// Initialize any container specific fields.
virtual void SetupContainer() { };
Entry* ResolvePath(const std::string_view path);
void CloseFiles();
void Dump(StringBuffer* string_buffer);
Result ReadHeaderAndVerify(FILE* header_file);
void SetName(std::string name) { name_ = name; }
const std::string& GetName() const { return name_; }
void SetFilesSize(uint64_t files_size) { files_total_size_ = files_size; }
const uint64_t GetFilesSize() const { return files_total_size_; }
const std::filesystem::path& GetHostPath() const { return host_path_; }
const XContentContainerHeader* GetContainerHeader() const { return header_.get(); }
std::string name_;
std::filesystem::path host_path_;
std::map<size_t, FILE*> files_;
size_t files_total_size_;
std::unique_ptr<Entry> root_entry_;
std::unique_ptr<XContentContainerHeader> header_;
private:
static XContentContainerHeader* ReadContainerHeader(FILE* host_file);
};
} // namespace vfs
} // namespace xe
#endif

20
src/xenia/vfs/devices/stfs_container_entry.cc → src/xenia/vfs/devices/xcontent_container_entry.cc
View File

@ -2,21 +2,20 @@
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2020 Ben Vanik. All rights reserved. *
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/vfs/devices/stfs_container_entry.h"
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_file.h"
#include "xenia/vfs/devices/xcontent_container_entry.h"
#include "xenia/vfs/devices/xcontent_container_file.h"
#include <map>
namespace xe {
namespace vfs {
StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
XContentContainerEntry::XContentContainerEntry(Device* device, Entry* parent,
const std::string_view path,
MultiFileHandles* files)
: Entry(device, parent, path),
@ -25,20 +24,21 @@ StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
data_size_(0),
block_(0) {}
StfsContainerEntry::~StfsContainerEntry() = default;
XContentContainerEntry::~XContentContainerEntry() = default;
std::unique_ptr<StfsContainerEntry> StfsContainerEntry::Create(
std::unique_ptr<XContentContainerEntry> XContentContainerEntry::Create(
Device* device, Entry* parent, const std::string_view name,
MultiFileHandles* files) {
auto path = xe::utf8::join_guest_paths(parent->path(), name);
auto entry =
std::make_unique<StfsContainerEntry>(device, parent, path, files);
std::make_unique<XContentContainerEntry>(device, parent, path, files);
return std::move(entry);
}
X_STATUS StfsContainerEntry::Open(uint32_t desired_access, File** out_file) {
*out_file = new StfsContainerFile(desired_access, this);
X_STATUS XContentContainerEntry::Open(uint32_t desired_access,
File** out_file) {
*out_file = new XContentContainerFile(desired_access, this);
return X_STATUS_SUCCESS;
}

24
src/xenia/vfs/devices/stfs_container_entry.h → src/xenia/vfs/devices/xcontent_container_entry.h
View File

@ -2,13 +2,13 @@
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2020 Ben Vanik. All rights reserved. *
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
#define XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
#ifndef XENIA_VFS_DEVICES_XCONTENT_CONTAINER_ENTRY_H_
#define XENIA_VFS_DEVICES_XCONTENT_CONTAINER_ENTRY_H_
#include <map>
#include <string>
@ -21,17 +21,16 @@ namespace xe {
namespace vfs {
typedef std::map<size_t, FILE*> MultiFileHandles;
class StfsContainerDevice;
class XContentContainerDevice;
class StfsContainerEntry : public Entry {
class XContentContainerEntry : public Entry {
public:
StfsContainerEntry(Device* device, Entry* parent, const std::string_view path,
MultiFileHandles* files);
~StfsContainerEntry() override;
XContentContainerEntry(Device* device, Entry* parent,
const std::string_view path, MultiFileHandles* files);
~XContentContainerEntry() override;
static std::unique_ptr<StfsContainerEntry> Create(Device* device,
Entry* parent,
const std::string_view name,
static std::unique_ptr<XContentContainerEntry> Create(
Device* device, Entry* parent, const std::string_view name,
MultiFileHandles* files);
MultiFileHandles* files() const { return files_; }
@ -50,6 +49,7 @@ class StfsContainerEntry : public Entry {
private:
friend class StfsContainerDevice;
friend class SvodContainerDevice;
MultiFileHandles* files_;
size_t data_offset_;
@ -61,4 +61,4 @@ class StfsContainerEntry : public Entry {
} // namespace vfs
} // namespace xe
#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
#endif // XENIA_VFS_DEVICES_XCONTENT_CONTAINER_ENTRY_H_

17
src/xenia/vfs/devices/stfs_container_file.cc → src/xenia/vfs/devices/xcontent_container_file.cc
View File

@ -2,31 +2,30 @@
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2014 Ben Vanik. All rights reserved. *
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include "xenia/vfs/devices/stfs_container_file.h"
#include <algorithm>
#include <cmath>
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_entry.h"
#include "xenia/vfs/devices/xcontent_container_entry.h"
#include "xenia/vfs/devices/xcontent_container_file.h"
namespace xe {
namespace vfs {
StfsContainerFile::StfsContainerFile(uint32_t file_access,
StfsContainerEntry* entry)
XContentContainerFile::XContentContainerFile(uint32_t file_access,
XContentContainerEntry* entry)
: File(file_access, entry), entry_(entry) {}
StfsContainerFile::~StfsContainerFile() = default;
XContentContainerFile::~XContentContainerFile() = default;
void StfsContainerFile::Destroy() { delete this; }
void XContentContainerFile::Destroy() { delete this; }
X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
X_STATUS XContentContainerFile::ReadSync(void* buffer, size_t buffer_length,
size_t byte_offset,
size_t* out_bytes_read) {
if (byte_offset >= entry_->size()) {

18
src/xenia/vfs/devices/stfs_container_file.h → src/xenia/vfs/devices/xcontent_container_file.h
View File

@ -2,13 +2,13 @@
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2014 Ben Vanik. All rights reserved. *
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_FILE_H_
#define XENIA_VFS_DEVICES_STFS_CONTAINER_FILE_H_
#ifndef XENIA_VFS_DEVICES_XCONTENT_CONTAINER_FILE_H_
#define XENIA_VFS_DEVICES_XCONTENT_CONTAINER_FILE_H_
#include "xenia/vfs/file.h"
@ -17,12 +17,12 @@
namespace xe {
namespace vfs {
class StfsContainerEntry;
class XContentContainerEntry;
class StfsContainerFile : public File {
class XContentContainerFile : public File {
public:
StfsContainerFile(uint32_t file_access, StfsContainerEntry* entry);
~StfsContainerFile() override;
XContentContainerFile(uint32_t file_access, XContentContainerEntry* entry);
~XContentContainerFile() override;
void Destroy() override;
@ -35,10 +35,10 @@ class StfsContainerFile : public File {
X_STATUS SetLength(size_t length) override { return X_STATUS_ACCESS_DENIED; }
private:
StfsContainerEntry* entry_;
XContentContainerEntry* entry_;
};
} // namespace vfs
} // namespace xe
#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_FILE_H_
#endif // XENIA_VFS_DEVICES_XCONTENT_CONTAINER_FILE_H_

324
src/xenia/vfs/devices/xcontent_devices/stfs_container_device.cc Normal file
View File

@ -0,0 +1,324 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include <algorithm>
#include <vector>
#include "xenia/base/logging.h"
#include "xenia/kernel/xam/content_manager.h"
#include "xenia/vfs/devices/xcontent_container_entry.h"
#include "xenia/vfs/devices/xcontent_devices/stfs_container_device.h"
namespace xe {
namespace vfs {
StfsContainerDevice::StfsContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path)
: XContentContainerDevice(mount_path, host_path),
blocks_per_hash_table_(1),
block_step_{0, 0} {
SetName("STFS");
}
StfsContainerDevice::~StfsContainerDevice() { CloseFiles(); }
void StfsContainerDevice::SetupContainer() {
// Additional part specific to STFS container.
const XContentContainerHeader* header = GetContainerHeader();
blocks_per_hash_table_ = header->is_package_readonly() ? 1 : 2;
block_step_[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_;
block_step_[1] = kBlocksPerHashLevel[1] +
((kBlocksPerHashLevel[0] + 1) * blocks_per_hash_table_);
}
XContentContainerDevice::Result StfsContainerDevice::LoadHostFiles(
FILE* header_file) {
const XContentContainerHeader* header = GetContainerHeader();
if (header->content_metadata.data_file_count > 0) {
XELOGW("STFS container is not a single file. Loading might fail!");
}
files_.emplace(std::make_pair(0, header_file));
return Result::kSuccess;
}
StfsContainerDevice::Result StfsContainerDevice::Read() {
auto& file = files_.at(0);
auto root_entry = new XContentContainerEntry(this, nullptr, "", &files_);
root_entry->attributes_ = kFileAttributeDirectory;
root_entry_ = std::unique_ptr<Entry>(root_entry);
std::vector<XContentContainerEntry*> all_entries;
// Load all listings.
StfsDirectoryBlock directory;
auto& descriptor =
GetContainerHeader()->content_metadata.volume_descriptor.stfs;
uint32_t table_block_index = descriptor.file_table_block_number();
size_t n = 0;
for (n = 0; n < descriptor.file_table_block_count; n++) {
const size_t offset = BlockToOffset(table_block_index);
xe::filesystem::Seek(file, offset, SEEK_SET);
if (fread(&directory, sizeof(StfsDirectoryBlock), 1, file) != 1) {
XELOGE("ReadSTFS failed to read directory block at 0x{X}", offset);
return Result::kReadError;
}
for (size_t m = 0; m < kEntriesPerDirectoryBlock; m++) {
const StfsDirectoryEntry& dir_entry = directory.entries[m];
if (dir_entry.name[0] == 0) {
// Done.
break;
}
XContentContainerEntry* parent_entry =
dir_entry.directory_index == 0xFFFF
? root_entry
: all_entries[dir_entry.directory_index];
std::unique_ptr<XContentContainerEntry> entry =
ReadEntry(parent_entry, &files_, &dir_entry);
all_entries.push_back(entry.get());
parent_entry->children_.emplace_back(std::move(entry));
}
const StfsHashEntry* block_hash = GetBlockHash(table_block_index);
table_block_index = block_hash->level0_next_block();
if (table_block_index == kEndOfChain) {
break;
}
}
if (n + 1 != descriptor.file_table_block_count) {
XELOGW("STFS read {} file table blocks, but STFS headers expected {}!",
n + 1, descriptor.file_table_block_count);
assert_always();
}
return Result::kSuccess;
}
std::unique_ptr<XContentContainerEntry> StfsContainerDevice::ReadEntry(
Entry* parent, MultiFileHandles* files,
const StfsDirectoryEntry* dir_entry) {
std::string name(reinterpret_cast<const char*>(dir_entry->name),
dir_entry->flags.name_length & 0x3F);
auto entry = XContentContainerEntry::Create(this, parent, name, &files_);
if (dir_entry->flags.directory) {
entry->attributes_ = kFileAttributeDirectory;
} else {
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->data_offset_ = BlockToOffset(dir_entry->start_block_number());
entry->data_size_ = dir_entry->length;
}
entry->size_ = dir_entry->length;
entry->allocation_size_ = xe::round_up(dir_entry->length, kBlockSize);
entry->create_timestamp_ =
decode_fat_timestamp(dir_entry->create_date, dir_entry->create_time);
entry->write_timestamp_ =
decode_fat_timestamp(dir_entry->modified_date, dir_entry->modified_time);
entry->access_timestamp_ = entry->write_timestamp_;
// Fill in all block records.
// It's easier to do this now and just look them up later, at the cost
// of some memory. Nasty chain walk.
// TODO(benvanik): optimize if flags.contiguous is set.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t block_index = dir_entry->start_block_number();
size_t remaining_size = dir_entry->length;
while (remaining_size && block_index != kEndOfChain) {
size_t block_size =
std::min(static_cast<size_t>(kBlockSize), remaining_size);
size_t offset = BlockToOffset(block_index);
entry->block_list_.push_back({0, offset, block_size});
remaining_size -= block_size;
auto block_hash = GetBlockHash(block_index);
block_index = block_hash->level0_next_block();
}
if (remaining_size) {
// Loop above must have exited prematurely, bad hash tables?
XELOGW(
"STFS file {} only found {} bytes for file, expected {} ({} "
"bytes missing)",
name, dir_entry->length - remaining_size, dir_entry->length,
remaining_size);
assert_always();
}
// Check that the number of blocks retrieved from hash entries matches
// the block count read from the file entry
if (entry->block_list_.size() != dir_entry->allocated_data_blocks()) {
XELOGW(
"STFS failed to read correct block-chain for entry {}, read {} "
"blocks, expected {}",
entry->name_, entry->block_list_.size(),
dir_entry->allocated_data_blocks());
assert_always();
}
}
return entry;
}
size_t StfsContainerDevice::BlockToOffset(uint64_t block_index) const {
// For every level there is a hash table
// Level 0: hash table of next 170 blocks
// Level 1: hash table of next 170 hash tables
// Level 2: hash table of next 170 level 1 hash tables
// And so on...
uint64_t block = block_index;
for (uint32_t i = 0; i < kBlocksHashLevelAmount; i++) {
const uint32_t level_base = kBlocksPerHashLevel[i];
block += ((block_index + level_base) / level_base) * blocks_per_hash_table_;
if (block_index < level_base) {
break;
}
}
return xe::round_up(GetContainerHeader()->content_header.header_size,
kBlockSize) +
(block << 12);
}
uint32_t StfsContainerDevice::BlockToHashBlockNumber(
uint32_t block_index, uint32_t hash_level) const {
if (hash_level == 2) {
return block_step_[1];
}
if (block_index < kBlocksPerHashLevel[hash_level]) {
return hash_level == 0 ? 0 : block_step_[hash_level - 1];
}
uint32_t block =
(block_index / kBlocksPerHashLevel[hash_level]) * block_step_[hash_level];
if (hash_level == 0) {
block +=
((block_index / kBlocksPerHashLevel[1]) + 1) * blocks_per_hash_table_;
if (block_index < kBlocksPerHashLevel[1]) {
return block;
}
}
return block + blocks_per_hash_table_;
}
size_t StfsContainerDevice::BlockToHashBlockOffset(uint32_t block_index,
uint32_t hash_level) const {
const uint64_t block = BlockToHashBlockNumber(block_index, hash_level);
return xe::round_up(header_->content_header.header_size, kBlockSize) +
(block << 12);
}
const uint8_t StfsContainerDevice::GetAmountOfHashLevelsToCheck(
uint32_t total_block_count) const {
for (uint8_t level = 0; level < kBlocksHashLevelAmount; level++) {
if (total_block_count < kBlocksPerHashLevel[level]) {
return level;
}
}
XELOGE("GetAmountOfHashLevelsToCheck - Invalid total_block_count: {}",
total_block_count);
return 0;
}
void StfsContainerDevice::UpdateCachedHashTable(
uint32_t block_index, uint8_t hash_level,
uint32_t& secondary_table_offset) {
const size_t hash_offset = BlockToHashBlockOffset(block_index, hash_level);
// Do nothing. It's already there.
if (!cached_hash_tables_.count(hash_offset)) {
auto& file = files_.at(0);
xe::filesystem::Seek(file, hash_offset + secondary_table_offset, SEEK_SET);
StfsHashTable table;
if (fread(&table, sizeof(StfsHashTable), 1, file) != 1) {
XELOGE("GetBlockHash failed to read level{} hash table at 0x{X}",
hash_level, hash_offset + secondary_table_offset);
return;
}
cached_hash_tables_[hash_offset] = table;
}
uint32_t record = block_index % kBlocksPerHashLevel[0];
if (hash_level >= 1) {
record = (block_index / kBlocksPerHashLevel[hash_level - 1]) %
kBlocksPerHashLevel[0];
}
const StfsHashEntry* record_data =
&cached_hash_tables_[hash_offset].entries[record];
secondary_table_offset = record_data->levelN_active_index() ? kBlockSize : 0;
}
void StfsContainerDevice::UpdateCachedHashTables(
uint32_t block_index, uint8_t highest_hash_level_to_update,
uint32_t& secondary_table_offset) {
for (int8_t level = highest_hash_level_to_update; level >= 0; level--) {
UpdateCachedHashTable(block_index, level, secondary_table_offset);
}
}
const StfsHashEntry* StfsContainerDevice::GetBlockHash(uint32_t block_index) {
auto& file = files_.at(0);
const StfsVolumeDescriptor& descriptor =
header_->content_metadata.volume_descriptor.stfs;
// Offset for selecting the secondary hash block, in packages that have them
uint32_t secondary_table_offset =
descriptor.flags.bits.root_active_index ? kBlockSize : 0;
uint8_t hash_levels_to_process =
GetAmountOfHashLevelsToCheck(descriptor.total_block_count);
if (header_->is_package_readonly()) {
secondary_table_offset = 0;
// Because we have read only package we only need to check first hash level.
hash_levels_to_process = 0;
}
UpdateCachedHashTables(block_index, hash_levels_to_process,
secondary_table_offset);
const size_t hash_offset = BlockToHashBlockOffset(block_index, 0);
const uint32_t record = block_index % kBlocksPerHashLevel[0];
return &cached_hash_tables_[hash_offset].entries[record];
}
const uint8_t StfsContainerDevice::GetBlocksPerHashTableFromContainerHeader()
const {
const XContentContainerHeader* header = GetContainerHeader();
if (!header) {
XELOGE(
"VFS: SetBlocksPerHashTableBasedOnContainerHeader - Missing "
"Container "
"Header!");
return 0;
}
if (header->is_package_readonly()) {
return 1;
}
return 2;
}
} // namespace vfs
} // namespace xe

95
src/xenia/vfs/devices/xcontent_devices/stfs_container_device.h Normal file
View File

@ -0,0 +1,95 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_VFS_DEVICES_XCONTENT_DEVICES_STFS_CONTAINER_DEVICE_H_
#define XENIA_VFS_DEVICES_XCONTENT_DEVICES_STFS_CONTAINER_DEVICE_H_
#include <string>
#include <unordered_map>
#include "xenia/base/string_util.h"
#include "xenia/kernel/util/xex2_info.h"
#include "xenia/vfs/device.h"
#include "xenia/vfs/devices/stfs_xbox.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#include "xenia/vfs/devices/xcontent_container_entry.h"
namespace xe {
namespace vfs {
// https://free60project.github.io/wiki/STFS.html
class StfsContainerDevice : public XContentContainerDevice {
public:
StfsContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path);
~StfsContainerDevice() override;
bool is_read_only() const override {
return GetContainerHeader()
->content_metadata.volume_descriptor.stfs.flags.bits.read_only_format;
}
uint32_t component_name_max_length() const override { return 40; }
uint32_t total_allocation_units() const override {
return GetContainerHeader()
->content_metadata.volume_descriptor.stfs.total_block_count;
}
uint32_t available_allocation_units() const override {
if (!is_read_only()) {
auto& descriptor =
GetContainerHeader()->content_metadata.volume_descriptor.stfs;
return kBlocksPerHashLevel[2] -
(descriptor.total_block_count - descriptor.free_block_count);
}
return 0;
}
private:
static const uint8_t kBlocksHashLevelAmount = 3;
const uint32_t kBlocksPerHashLevel[kBlocksHashLevelAmount] = {170, 28900,
4913000};
const uint32_t kEndOfChain = 0xFFFFFF;
const uint32_t kEntriesPerDirectoryBlock =
kBlockSize / sizeof(StfsDirectoryEntry);
void SetupContainer() override;
Result LoadHostFiles(FILE* header_file) override;
Result Read() override;
std::unique_ptr<XContentContainerEntry> ReadEntry(
Entry* parent, MultiFileHandles* files,
const StfsDirectoryEntry* dir_entry);
size_t BlockToOffset(uint64_t block_index) const;
uint32_t BlockToHashBlockNumber(uint32_t block_index,
uint32_t hash_level) const;
size_t BlockToHashBlockOffset(uint32_t block_index,
uint32_t hash_level) const;
const uint8_t GetAmountOfHashLevelsToCheck(uint32_t total_block_count) const;
const StfsHashEntry* GetBlockHash(uint32_t block_index);
void UpdateCachedHashTable(uint32_t block_index, uint8_t hash_level,
uint32_t& secondary_table_offset);
void UpdateCachedHashTables(uint32_t block_index,
uint8_t highest_hash_level_to_update,
uint32_t& secondary_table_offset);
const uint8_t GetBlocksPerHashTableFromContainerHeader() const;
uint8_t blocks_per_hash_table_;
uint32_t block_step_[2];
std::unordered_map<size_t, StfsHashTable> cached_hash_tables_;
};
} // namespace vfs
} // namespace xe
#endif // XENIA_VFS_DEVICES_XCONTENT_DEVICES_STFS_CONTAINER_DEVICE_H_

417
src/xenia/vfs/devices/xcontent_devices/svod_container_device.cc Normal file
View File

@ -0,0 +1,417 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#include <algorithm>
#include <vector>
#include "xenia/base/logging.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#include "xenia/vfs/devices/xcontent_container_entry.h"
#include "xenia/vfs/devices/xcontent_devices/svod_container_device.h"
namespace xe {
namespace vfs {
SvodContainerDevice::SvodContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path)
: XContentContainerDevice(mount_path, host_path),
svod_base_offset_(),
svod_layout_() {
SetName("FATX");
}
SvodContainerDevice::~SvodContainerDevice() { CloseFiles(); }
SvodContainerDevice::Result SvodContainerDevice::LoadHostFiles(
FILE* header_file) {
std::filesystem::path data_fragment_path = host_path_;
data_fragment_path += ".data";
if (!std::filesystem::exists(data_fragment_path)) {
XELOGE("STFS container is multi-file, but path {} does not exist.",
xe::path_to_utf8(data_fragment_path));
return Result::kFileMismatch;
}
// Ensure data fragment files are sorted
auto fragment_files = filesystem::ListFiles(data_fragment_path);
std::sort(fragment_files.begin(), fragment_files.end(),
[](filesystem::FileInfo& left, filesystem::FileInfo& right) {
return left.name < right.name;
});
if (fragment_files.size() != header_->content_metadata.data_file_count) {
XELOGE("SVOD expecting {} data fragments, but {} are present.",
header_->content_metadata.data_file_count, fragment_files.size());
return Result::kFileMismatch;
}
for (size_t i = 0; i < fragment_files.size(); i++) {
auto& fragment = fragment_files.at(i);
auto path = fragment.path / fragment.name;
auto file = xe::filesystem::OpenFile(path, "rb");
if (!file) {
XELOGI("Failed to map SVOD file {}.", xe::path_to_utf8(path));
CloseFiles();
return Result::kReadError;
}
xe::filesystem::Seek(file, 0L, SEEK_END);
files_total_size_ += xe::filesystem::Tell(file);
// no need to seek back, any reads from this file will seek first anyway
files_.emplace(std::make_pair(i, file));
}
XELOGI("SVOD successfully mapped {} files.", fragment_files.size());
return Result::kSuccess;
}
XContentContainerDevice::Result SvodContainerDevice::Read() {
// SVOD Systems can have different layouts. The root block is
// denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in
// the first "actual" data fragment of the system.
auto& svod_header = files_.at(0);
size_t magic_offset;
SetLayout(svod_header, magic_offset);
// Parse the root directory
xe::filesystem::Seek(svod_header, magic_offset + 0x14, SEEK_SET);
struct {
uint32_t block;
uint32_t size;
uint32_t creation_date;
uint32_t creation_time;
} root_data;
static_assert_size(root_data, 0x10);
if (fread(&root_data, sizeof(root_data), 1, svod_header) != 1) {
XELOGE("ReadSVOD failed to read root block data at 0x{X}",
magic_offset + 0x14);
return Result::kReadError;
}
const uint64_t root_creation_timestamp =
decode_fat_timestamp(root_data.creation_date, root_data.creation_time);
auto root_entry = new XContentContainerEntry(this, nullptr, "", &files_);
root_entry->attributes_ = kFileAttributeDirectory;
root_entry->access_timestamp_ = root_creation_timestamp;
root_entry->create_timestamp_ = root_creation_timestamp;
root_entry->write_timestamp_ = root_creation_timestamp;
root_entry_ = std::unique_ptr<Entry>(root_entry);
// Traverse all child entries
return ReadEntry(root_data.block, 0, root_entry);
}
SvodContainerDevice::Result SvodContainerDevice::ReadEntry(
uint32_t block, uint32_t ordinal, XContentContainerEntry* parent) {
// For games with a large amount of files, the ordinal offset can overrun
// the current block and potentially hit a hash block.
size_t ordinal_offset = ordinal * 0x4;
size_t block_offset = ordinal_offset / 0x800;
size_t true_ordinal_offset = ordinal_offset % 0x800;
// Calculate the file & address of the block
size_t entry_address, entry_file;
BlockToOffset(block + block_offset, &entry_address, &entry_file);
entry_address += true_ordinal_offset;
// Read directory entry
auto& file = files_.at(entry_file);
xe::filesystem::Seek(file, entry_address, SEEK_SET);
#pragma pack(push, 1)
struct {
uint16_t node_l;
uint16_t node_r;
uint32_t data_block;
uint32_t length;
uint8_t attributes;
uint8_t name_length;
} dir_entry;
static_assert_size(dir_entry, 0xE);
#pragma pack(pop)
if (fread(&dir_entry, sizeof(dir_entry), 1, file) != 1) {
XELOGE("ReadEntrySVOD failed to read directory entry at 0x{X}",
entry_address);
return Result::kReadError;
}
auto name_buffer = std::make_unique<char[]>(dir_entry.name_length);
if (fread(name_buffer.get(), 1, dir_entry.name_length, file) !=
dir_entry.name_length) {
XELOGE("ReadEntrySVOD failed to read directory entry name at 0x{X}",
entry_address);
return Result::kReadError;
}
auto name = std::string(name_buffer.get(), dir_entry.name_length);
// Read the left node
if (dir_entry.node_l) {
auto node_result = ReadEntry(block, dir_entry.node_l, parent);
if (node_result != Result::kSuccess) {
return node_result;
}
}
// Read file & address of block's data
size_t data_address, data_file;
BlockToOffset(dir_entry.data_block, &data_address, &data_file);
// Create the entry
// NOTE: SVOD entries don't have timestamps for individual files, which can
// cause issues when decrypting games. Using the root entry's timestamp
// solves this issues.
auto entry = XContentContainerEntry::Create(this, parent, name, &files_);
if (dir_entry.attributes & kFileAttributeDirectory) {
// Entry is a directory
entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
entry->data_offset_ = 0;
entry->data_size_ = 0;
entry->block_ = block;
entry->access_timestamp_ = root_entry_->create_timestamp();
entry->create_timestamp_ = root_entry_->create_timestamp();
entry->write_timestamp_ = root_entry_->create_timestamp();
if (dir_entry.length) {
// If length is greater than 0, traverse the directory's children
auto directory_result = ReadEntry(dir_entry.data_block, 0, entry.get());
if (directory_result != Result::kSuccess) {
return directory_result;
}
}
} else {
// Entry is a file
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->size_ = dir_entry.length;
entry->allocation_size_ = xe::round_up(dir_entry.length, kBlockSize);
entry->data_offset_ = data_address;
entry->data_size_ = dir_entry.length;
entry->block_ = dir_entry.data_block;
entry->access_timestamp_ = root_entry_->create_timestamp();
entry->create_timestamp_ = root_entry_->create_timestamp();
entry->write_timestamp_ = root_entry_->create_timestamp();
// Fill in all block records, sector by sector.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t block_index = dir_entry.data_block;
size_t remaining_size = xe::round_up(dir_entry.length, 0x800);
size_t last_record = -1;
size_t last_offset = -1;
while (remaining_size) {
const size_t BLOCK_SIZE = 0x800;
size_t offset, file_index;
BlockToOffset(block_index, &offset, &file_index);
block_index++;
remaining_size -= BLOCK_SIZE;
if (offset - last_offset == BLOCK_SIZE) {
// Consecutive, so append to last entry.
entry->block_list_[last_record].length += BLOCK_SIZE;
last_offset = offset;
continue;
}
entry->block_list_.push_back({file_index, offset, BLOCK_SIZE});
last_record = entry->block_list_.size() - 1;
last_offset = offset;
}
}
}
parent->children_.emplace_back(std::move(entry));
// Read the right node.
if (dir_entry.node_r) {
auto node_result = ReadEntry(block, dir_entry.node_r, parent);
if (node_result != Result::kSuccess) {
return node_result;
}
}
return Result::kSuccess;
}
XContentContainerDevice::Result SvodContainerDevice::SetLayout(
FILE* header, size_t& magic_offset) {
if (IsEDGFLayout()) {
return SetEDGFLayout(header, magic_offset);
}
if (IsXSFLayout(header)) {
return SetXSFLayout(header, magic_offset);
}
return SetNormalLayout(header, magic_offset);
}
XContentContainerDevice::Result SvodContainerDevice::SetEDGFLayout(
FILE* header, size_t& magic_offset) {
uint8_t magic_buf[20];
xe::filesystem::Seek(header, 0x2000, SEEK_SET);
if (fread(magic_buf, 1, countof(magic_buf), header) != countof(magic_buf)) {
XELOGE("ReadSVOD failed to read SVOD magic at 0x2000");
return Result::kReadError;
}
if (std::memcmp(magic_buf, MEDIA_MAGIC, countof(magic_buf)) != 0) {
XELOGE("SVOD uses an EGDF layout, but the magic block was not found.");
return Result::kFileMismatch;
}
svod_base_offset_ = 0x0000;
magic_offset = 0x2000;
svod_layout_ = SvodLayoutType::kEnhancedGDF;
XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000.");
return Result::kSuccess;
}
const bool SvodContainerDevice::IsXSFLayout(FILE* header) const {
uint8_t magic_buf[20];
xe::filesystem::Seek(header, 0x12000, SEEK_SET);
if (fread(magic_buf, 1, countof(magic_buf), header) != countof(magic_buf)) {
XELOGE("ReadSVOD failed to read SVOD magic at 0x12000");
return false;
}
return std::memcmp(magic_buf, MEDIA_MAGIC, countof(magic_buf)) == 0;
}
XContentContainerDevice::Result SvodContainerDevice::SetXSFLayout(
FILE* header, size_t& magic_offset) {
uint8_t magic_buf[20];
const char* XSF_MAGIC = "XSF";
xe::filesystem::Seek(header, 0x2000, SEEK_SET);
if (fread(magic_buf, 1, 3, header) != 3) {
XELOGE("ReadSVOD failed to read SVOD XSF magic at 0x2000");
return Result::kReadError;
}
svod_base_offset_ = 0x10000;
magic_offset = 0x12000;
if (std::memcmp(magic_buf, XSF_MAGIC, 3) != 0) {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGI("SVOD appears to use an XSF layout, but no header is present.");
XELOGI("SVOD magic block found at 0x12000");
return Result::kSuccess;
}
svod_layout_ = SvodLayoutType::kXSF;
XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000.");
XELOGI("Game was likely converted using a third-party tool.");
return Result::kSuccess;
}
XContentContainerDevice::Result SvodContainerDevice::SetNormalLayout(
FILE* header, size_t& magic_offset) {
uint8_t magic_buf[20];
xe::filesystem::Seek(header, 0xD000, SEEK_SET);
if (fread(magic_buf, 1, countof(magic_buf), header) != countof(magic_buf)) {
XELOGE("ReadSVOD failed to read SVOD magic at 0xD000");
return Result::kReadError;
}
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) != 0) {
XELOGE("Could not locate SVOD magic block.");
return Result::kReadError;
}
// If the SVOD's magic block is at 0xD000, it most likely means that it
// is a single-file system. The STFS Header is 0xB000 bytes and the
// remaining 0x2000 is from hash tables. In most cases, these will be
// STFS, not SVOD.
svod_base_offset_ = 0xB000;
magic_offset = 0xD000;
// Check for single file system
if (header_->content_metadata.data_file_count == 1) {
svod_layout_ = SvodLayoutType::kSingleFile;
XELOGI("SVOD is a single file. Magic block present at 0xD000.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGE(
"SVOD is not a single file, but the magic block was found at "
"0xD000.");
}
return Result::kSuccess;
}
void SvodContainerDevice::BlockToOffset(size_t block, size_t* out_address,
size_t* out_file_index) const {
// SVOD Systems use hash blocks for integrity checks. These hash blocks
// cause blocks to be discontinuous in memory, and must be accounted for.
// - Each data block is 0x800 bytes in length
// - Every group of 0x198 data blocks is preceded a Level0 hash table.
// Level0 tables contain 0xCC hashes, each representing two data blocks.
// The total size of each Level0 hash table is 0x1000 bytes in length.
// - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table.
// Level1 tables contain 0xCB hashes, each representing two Level0 hashes.
// The total size of each Level1 hash table is 0x1000 bytes in length.
// - Files are split into fragments of 0xA290000 bytes in length,
// consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1
// Level1 hash table.
const size_t BLOCK_SIZE = 0x800;
const size_t HASH_BLOCK_SIZE = 0x1000;
const size_t BLOCKS_PER_L0_HASH = 0x198;
const size_t HASHES_PER_L1_HASH = 0xA1C4;
const size_t BLOCKS_PER_FILE = 0x14388;
const size_t MAX_FILE_SIZE = 0xA290000;
const size_t BLOCK_OFFSET =
header_->content_metadata.volume_descriptor.svod.start_data_block();
// Resolve the true block address and file index
size_t true_block = block - (BLOCK_OFFSET * 2);
if (svod_layout_ == SvodLayoutType::kEnhancedGDF) {
// EGDF has an 0x1000 byte offset, which is two blocks
true_block += 0x2;
}
size_t file_block = true_block % BLOCKS_PER_FILE;
size_t file_index = true_block / BLOCKS_PER_FILE;
size_t offset = 0;
// Calculate offset caused by Level0 Hash Tables
size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1;
offset += level0_table_count * HASH_BLOCK_SIZE;
// Calculate offset caused by Level1 Hash Tables
size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1;
offset += level1_table_count * HASH_BLOCK_SIZE;
// For single-file SVOD layouts, include the size of the header in the offset.
if (svod_layout_ == SvodLayoutType::kSingleFile) {
offset += svod_base_offset_;
}
size_t block_address = (file_block * BLOCK_SIZE) + offset;
// If the offset causes the block address to overrun the file, round it.
if (block_address >= MAX_FILE_SIZE) {
file_index += 1;
block_address %= MAX_FILE_SIZE;
block_address += 0x2000;
}
*out_address = block_address;
*out_file_index = file_index;
}
} // namespace vfs
} // namespace xe

77
src/xenia/vfs/devices/xcontent_devices/svod_container_device.h Normal file
View File

@ -0,0 +1,77 @@
/**
******************************************************************************
* Xenia : Xbox 360 Emulator Research Project *
******************************************************************************
* Copyright 2023 Ben Vanik. All rights reserved. *
* Released under the BSD license - see LICENSE in the root for more details. *
******************************************************************************
*/
#ifndef XENIA_VFS_DEVICES_XCONTENT_DEVICES_SVOD_CONTAINER_DEVICE_H_
#define XENIA_VFS_DEVICES_XCONTENT_DEVICES_SVOD_CONTAINER_DEVICE_H_
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include "xenia/base/string_util.h"
#include "xenia/kernel/util/xex2_info.h"
#include "xenia/vfs/device.h"
#include "xenia/vfs/devices/stfs_xbox.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#include "xenia/vfs/devices/xcontent_container_entry.h"
namespace xe {
namespace vfs {
class SvodContainerDevice : public XContentContainerDevice {
public:
SvodContainerDevice(const std::string_view mount_path,
const std::filesystem::path& host_path);
~SvodContainerDevice() override;
bool is_read_only() const override { return true; }
uint32_t component_name_max_length() const override { return 255; }
uint32_t total_allocation_units() const override {
return uint32_t(data_size() / sectors_per_allocation_unit() /
bytes_per_sector());
}
uint32_t available_allocation_units() const override { return 0; }
private:
enum class SvodLayoutType {
kUnknown = 0x0,
kEnhancedGDF = 0x1,
kXSF = 0x2,
kSingleFile = 0x4,
};
const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA";
Result LoadHostFiles(FILE* header_file) override;
Result Read() override;
Result ReadEntry(uint32_t sector, uint32_t ordinal,
XContentContainerEntry* parent);
void BlockToOffset(size_t sector, size_t* address, size_t* file_index) const;
Result SetLayout(FILE* header, size_t& magic_offset);
Result SetEDGFLayout(FILE* header, size_t& magic_offset);
Result SetXSFLayout(FILE* header, size_t& magic_offset);
Result SetNormalLayout(FILE* header, size_t& magic_offset);
const bool IsEDGFLayout() const {
return header_->content_metadata.volume_descriptor.svod.features.bits
.enhanced_gdf_layout;
}
const bool IsXSFLayout(FILE* header) const;
size_t svod_base_offset_;
SvodLayoutType svod_layout_;
};
} // namespace vfs
} // namespace xe
#endif // XENIA_VFS_DEVICES_XCONTENT_DEVICES_SVOD_CONTAINER_DEVICE_H_

7
src/xenia/vfs/vfs_dump.cc
View File

@ -17,7 +17,7 @@
#include "xenia/base/logging.h"
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_device.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#include "xenia/vfs/file.h"
#include "xenia/vfs/virtual_file_system.h"
@ -37,10 +37,9 @@ int vfs_dump_main(const std::vector<std::string>& args) {
}
std::filesystem::path base_path = cvars::dump_path;
std::unique_ptr<vfs::Device> device;
std::unique_ptr<vfs::Device> device =
vfs::XContentContainerDevice::CreateContentDevice("", cvars::source);
// TODO: Flags specifying the type of device.
device = std::make_unique<vfs::StfsContainerDevice>("", cvars::source);
if (!device->Initialize()) {
XELOGE("Failed to initialize device");
return 1;

8
src/xenia/vfs/virtual_file_system.cc
View File

@ -9,7 +9,7 @@
#include "xenia/vfs/virtual_file_system.h"
#include "xenia/kernel/xam/content_manager.h"
#include "xenia/vfs/devices/stfs_container_device.h"
#include "xenia/vfs/devices/xcontent_container_device.h"
#include "devices/host_path_entry.h"
#include "xenia/base/literals.h"
@ -404,7 +404,8 @@ X_STATUS VirtualFileSystem::ExtractContentFiles(
void VirtualFileSystem::ExtractContentHeader(Device* device,
std::filesystem::path base_path) {
auto stfs_device = ((StfsContainerDevice*)device);
const XContentContainerDevice* xcontent_device =
((XContentContainerDevice*)device);
if (!std::filesystem::exists(base_path.parent_path())) {
if (!std::filesystem::create_directories(base_path.parent_path())) {
@ -417,7 +418,8 @@ void VirtualFileSystem::ExtractContentHeader(Device* device,
if (std::filesystem::exists(header_path)) {
auto file = xe::filesystem::OpenFile(header_path, "wb");
kernel::xam::XCONTENT_AGGREGATE_DATA data = stfs_device->content_header();
kernel::xam::XCONTENT_AGGREGATE_DATA data =
xcontent_device->content_header();
data.set_file_name(base_path.filename().string());
fwrite(&data, 1, sizeof(kernel::xam::XCONTENT_AGGREGATE_DATA), file);
fclose(file);