ShuriZma
/
xenia
mirror of https://github.com/xenia-project/xenia.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[VFS/STFS] Use xe::filesystem::OpenFile instead of memory-mapping 

Adds StfsDirectoryEntry / StfsDirectoryBlock to make it easier to fread()
This commit is contained in:
emoose 2021-05-11 03:26:39 +01:00 committed by Rick Gibbed
parent 2659b70c90
commit 478bb90922
6 changed files with 312 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

379
src/xenia/vfs/devices/stfs_container_device.cc
View File

@ -49,13 +49,15 @@ StfsContainerDevice::StfsContainerDevice(const std::string_view mount_path,
: Device(mount_path), : Device(mount_path),
name_("STFS"), name_("STFS"),
host_path_(host_path), host_path_(host_path),
mmap_total_size_(), files_total_size_(),
base_offset_(), base_offset_(),
magic_offset_(), magic_offset_(),
header_(), header_(),
svod_layout_() {} svod_layout_(),
blocks_per_hash_table_(1),
block_step{0, 0} {}
StfsContainerDevice::~StfsContainerDevice() = default; StfsContainerDevice::~StfsContainerDevice() { CloseFiles(); }
bool StfsContainerDevice::Initialize() { bool StfsContainerDevice::Initialize() {
// Resolve a valid STFS file if a directory is given. // Resolve a valid STFS file if a directory is given.
@ -72,10 +74,10 @@ bool StfsContainerDevice::Initialize() {
return false; return false;
} }
// Map the data file(s) // Open the data file(s)
auto map_result = MapFiles(); auto open_result = OpenFiles();
if (map_result != Error::kSuccess) { if (open_result != Error::kSuccess) {
XELOGE("Failed to map STFS container: {}", map_result); XELOGE("Failed to open STFS container: {}", open_result);
return false; return false;
} }
@ -86,36 +88,36 @@ bool StfsContainerDevice::Initialize() {
case XContentVolumeType::kSvod: case XContentVolumeType::kSvod:
return ReadSVOD() == Error::kSuccess; return ReadSVOD() == Error::kSuccess;
default: default:
XELOGE("Unknown STFS Descriptor Type: {}", XELOGE("Unknown XContent volume type: {}",
xe::byte_swap(uint32_t(header_.metadata.volume_type.value))); xe::byte_swap(uint32_t(header_.metadata.volume_type.value)));
return false; return false;
} }
} }
StfsContainerDevice::Error StfsContainerDevice::MapFiles() { StfsContainerDevice::Error StfsContainerDevice::OpenFiles() {
// Map the file containing the STFS Header and read it. // Map the file containing the STFS Header and read it.
XELOGI("Mapping STFS Header file: {}", xe::path_to_utf8(host_path_)); XELOGI("Loading STFS header file: {}", xe::path_to_utf8(host_path_));
auto header_map = MappedMemory::Open(host_path_, MappedMemory::Mode::kRead);
if (!header_map) { auto header_file = xe::filesystem::OpenFile(host_path_, "rb");
XELOGE("Error mapping STFS Header file."); if (!header_file) {
XELOGE("Error opening STFS header file.");
return Error::kErrorReadError; return Error::kErrorReadError;
} }
auto header_result = auto header_result = ReadHeaderAndVerify(header_file);
ReadHeaderAndVerify(header_map->data(), header_map->size());
if (header_result != Error::kSuccess) { if (header_result != Error::kSuccess) {
XELOGE("Error reading STFS Header: {}", header_result); XELOGE("Error reading STFS header: {}", header_result);
fclose(header_file);
files_total_size_ = 0;
return header_result; return header_result;
} }
mmap_total_size_ = header_map->size();
// If the STFS package is a single file, the header is self contained and // If the STFS package is a single file, the header is self contained and
// we don't need to map any extra files. // we don't need to map any extra files.
// NOTE: data_file_count is 0 for STFS and 1 for SVOD // NOTE: data_file_count is 0 for STFS and 1 for SVOD
if (header_.metadata.data_file_count <= 1) { if (header_.metadata.data_file_count <= 1) {
XELOGI("STFS container is a single file."); XELOGI("STFS container is a single file.");
mmap_.emplace(std::make_pair(0, std::move(header_map))); files_.emplace(std::make_pair(0, header_file));
return Error::kSuccess; return Error::kSuccess;
} }
@ -143,22 +145,32 @@ StfsContainerDevice::Error StfsContainerDevice::MapFiles() {
} }
for (size_t i = 0; i < fragment_files.size(); i++) { for (size_t i = 0; i < fragment_files.size(); i++) {
auto file = fragment_files.at(i); auto& fragment = fragment_files.at(i);
auto path = file.path / file.name; auto path = fragment.path / fragment.name;
auto data = MappedMemory::Open(path, MappedMemory::Mode::kRead); auto file = xe::filesystem::OpenFile(path, "rb");
if (!data) { if (!file) {
XELOGI("Failed to map SVOD file {}.", xe::path_to_utf8(path)); XELOGI("Failed to map SVOD file {}.", xe::path_to_utf8(path));
mmap_.clear(); CloseFiles();
mmap_total_size_ = 0;
return Error::kErrorReadError; return Error::kErrorReadError;
} }
mmap_total_size_ += data->size();
mmap_.emplace(std::make_pair(i, std::move(data))); xe::filesystem::Seek(file, 0L, SEEK_END);
files_total_size_ += xe::filesystem::Tell(file);
// no need to seek back, any reads from this file will seek first anyway
files_.emplace(std::make_pair(i, file));
} }
XELOGI("SVOD successfully mapped {} files.", fragment_files.size()); XELOGI("SVOD successfully mapped {} files.", fragment_files.size());
return Error::kSuccess; return Error::kSuccess;
} }
void StfsContainerDevice::CloseFiles() {
for (auto& file : files_) {
fclose(file.second);
}
files_.clear();
files_total_size_ = 0;
}
void StfsContainerDevice::Dump(StringBuffer* string_buffer) { void StfsContainerDevice::Dump(StringBuffer* string_buffer) {
auto global_lock = global_critical_region_.Acquire(); auto global_lock = global_critical_region_.Acquire();
root_entry_->Dump(string_buffer, 0); root_entry_->Dump(string_buffer, 0);
@ -173,22 +185,34 @@ Entry* StfsContainerDevice::ResolvePath(const std::string_view path) {
} }
StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify( StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify(
const uint8_t* map_ptr, size_t map_size) { FILE* header_file) {
// Copy header & check signature // Check size of the file is enough to store an STFS header
memcpy(&header_, map_ptr, sizeof(StfsHeader)); xe::filesystem::Seek(header_file, 0L, SEEK_END);
files_total_size_ = xe::filesystem::Tell(header_file);
xe::filesystem::Seek(header_file, 0L, SEEK_SET);
if (sizeof(StfsHeader) > files_total_size_) {
return Error::kErrorTooSmall;
}
// Read header & check signature
fread(&header_, sizeof(StfsHeader), 1, header_file);
if (!header_.header.is_magic_valid()) { if (!header_.header.is_magic_valid()) {
// Unexpected format. // Unexpected format.
return Error::kErrorFileMismatch; return Error::kErrorFileMismatch;
} }
// Pre-calculate some values used in block number calculations // Pre-calculate some values used in block number calculations
blocks_per_hash_table_ = if (header_.metadata.volume_type == XContentVolumeType::kStfs) {
header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format ? 1 blocks_per_hash_table_ =
: 2; header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format ? 1
: 2;
block_step[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_; block_step[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_;
block_step[1] = kBlocksPerHashLevel[1] + block_step[1] = kBlocksPerHashLevel[1] +
((kBlocksPerHashLevel[0] + 1) * blocks_per_hash_table_); ((kBlocksPerHashLevel[0] + 1) * blocks_per_hash_table_);
}
return Error::kSuccess; return Error::kSuccess;
} }
@ -197,9 +221,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() {
// SVOD Systems can have different layouts. The root block is // SVOD Systems can have different layouts. The root block is
// denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in // denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in
// the first "actual" data fragment of the system. // the first "actual" data fragment of the system.
auto data = mmap_.at(0)->data(); auto& svod_header = files_.at(0);
const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA"; const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA";
uint8_t magic_buf[20];
// Check for EDGF layout // Check for EDGF layout
if (header_.metadata.volume_descriptor.svod.features.bits if (header_.metadata.volume_descriptor.svod.features.bits
.enhanced_gdf_layout) { .enhanced_gdf_layout) {
@ -207,7 +233,9 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() {
// We can expect the magic block to be located immediately after the hash // We can expect the magic block to be located immediately after the hash
// blocks. We also offset block address calculation by 0x1000 by shifting // blocks. We also offset block address calculation by 0x1000 by shifting
// block indices by +0x2. // block indices by +0x2.
if (memcmp(data + 0x2000, MEDIA_MAGIC, 20) == 0) { xe::filesystem::Seek(svod_header, 0x2000, SEEK_SET);
fread(magic_buf, 1, 20, svod_header);
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) {
base_offset_ = 0x0000; base_offset_ = 0x0000;
magic_offset_ = 0x2000; magic_offset_ = 0x2000;
svod_layout_ = SvodLayoutType::kEnhancedGDF; svod_layout_ = SvodLayoutType::kEnhancedGDF;
@ -216,58 +244,75 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() {
XELOGE("SVOD uses an EGDF layout, but the magic block was not found."); XELOGE("SVOD uses an EGDF layout, but the magic block was not found.");
return Error::kErrorFileMismatch; return Error::kErrorFileMismatch;
} }
} else if (memcmp(data + 0x12000, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0x12000, it is likely using an XSF
// layout. This is usually due to converting the game using a third-party
// tool, as most of them use a nulled XSF as a template.
base_offset_ = 0x10000;
magic_offset_ = 0x12000;
// Check for XSF Header
const char* XSF_MAGIC = "XSF";
if (memcmp(data + 0x2000, XSF_MAGIC, 3) == 0) {
svod_layout_ = SvodLayoutType::kXSF;
XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000.");
XELOGI("Game was likely converted using a third-party tool.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGI("SVOD appears to use an XSF layout, but no header is present.");
XELOGI("SVOD magic block found at 0x12000");
}
} else if (memcmp(data + 0xD000, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0xD000, it most likely means that it is
// a single-file system. The STFS Header is 0xB000 bytes , and the remaining
// 0x2000 is from hash tables. In most cases, these will be STFS, not SVOD.
base_offset_ = 0xB000;
magic_offset_ = 0xD000;
// Check for single file system
if (header_.metadata.data_file_count == 1) {
svod_layout_ = SvodLayoutType::kSingleFile;
XELOGI("SVOD is a single file. Magic block present at 0xD000.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGE(
"SVOD is not a single file, but the magic block was found at "
"0xD000.");
}
} else { } else {
XELOGE("Could not locate SVOD magic block."); xe::filesystem::Seek(svod_header, 0x12000, SEEK_SET);
return Error::kErrorReadError; fread(magic_buf, 1, 20, svod_header);
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0x12000, it is likely using an XSF
// layout. This is usually due to converting the game using a third-party
// tool, as most of them use a nulled XSF as a template.
base_offset_ = 0x10000;
magic_offset_ = 0x12000;
// Check for XSF Header
const char* XSF_MAGIC = "XSF";
xe::filesystem::Seek(svod_header, 0x2000, SEEK_SET);
fread(magic_buf, 1, 3, svod_header);
if (std::memcmp(magic_buf, XSF_MAGIC, 3) == 0) {
svod_layout_ = SvodLayoutType::kXSF;
XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000.");
XELOGI("Game was likely converted using a third-party tool.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGI("SVOD appears to use an XSF layout, but no header is present.");
XELOGI("SVOD magic block found at 0x12000");
}
} else {
xe::filesystem::Seek(svod_header, 0xD000, SEEK_SET);
fread(magic_buf, 1, 20, svod_header);
if (std::memcmp(magic_buf, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0xD000, it most likely means that it
// is a single-file system. The STFS Header is 0xB000 bytes , and the
// remaining 0x2000 is from hash tables. In most cases, these will be
// STFS, not SVOD.
base_offset_ = 0xB000;
magic_offset_ = 0xD000;
// Check for single file system
if (header_.metadata.data_file_count == 1) {
svod_layout_ = SvodLayoutType::kSingleFile;
XELOGI("SVOD is a single file. Magic block present at 0xD000.");
} else {
svod_layout_ = SvodLayoutType::kUnknown;
XELOGE(
"SVOD is not a single file, but the magic block was found at "
"0xD000.");
}
} else {
XELOGE("Could not locate SVOD magic block.");
return Error::kErrorReadError;
}
}
} }
// Parse the root directory // Parse the root directory
uint8_t* magic_block = data + magic_offset_; xe::filesystem::Seek(svod_header, magic_offset_ + 0x14, SEEK_SET);
uint32_t root_block = xe::load<uint32_t>(magic_block + 0x14);
uint32_t root_size = xe::load<uint32_t>(magic_block + 0x18); uint32_t root_block;
uint32_t root_creation_date = xe::load<uint32_t>(magic_block + 0x1C); uint32_t root_size;
uint32_t root_creation_time = xe::load<uint32_t>(magic_block + 0x20); uint32_t root_creation_date;
uint32_t root_creation_time;
fread(&root_block, sizeof(uint32_t), 1, svod_header);
fread(&root_size, sizeof(uint32_t), 1, svod_header);
fread(&root_creation_date, sizeof(uint32_t), 1, svod_header);
fread(&root_creation_time, sizeof(uint32_t), 1, svod_header);
uint64_t root_creation_timestamp = uint64_t root_creation_timestamp =
decode_fat_timestamp(root_creation_date, root_creation_time); decode_fat_timestamp(root_creation_date, root_creation_time);
auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_); auto root_entry = new StfsContainerEntry(this, nullptr, "", &files_);
root_entry->attributes_ = kFileAttributeDirectory; root_entry->attributes_ = kFileAttributeDirectory;
root_entry->access_timestamp_ = root_creation_timestamp; root_entry->access_timestamp_ = root_creation_timestamp;
root_entry->create_timestamp_ = root_creation_timestamp; root_entry->create_timestamp_ = root_creation_timestamp;
@ -292,16 +337,26 @@ StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD(
entry_address += true_ordinal_offset; entry_address += true_ordinal_offset;
// Read block's descriptor // Read block's descriptor
auto data = mmap_.at(entry_file)->data() + entry_address; auto& file = files_.at(entry_file);
xe::filesystem::Seek(file, entry_address, SEEK_SET);
uint16_t node_l = xe::load<uint16_t>(data + 0x00); uint16_t node_l;
uint16_t node_r = xe::load<uint16_t>(data + 0x02); uint16_t node_r;
uint32_t data_block = xe::load<uint32_t>(data + 0x04); uint32_t data_block;
uint32_t length = xe::load<uint32_t>(data + 0x08); uint32_t length;
uint8_t attributes = xe::load<uint8_t>(data + 0x0C); uint8_t attributes;
uint8_t name_length = xe::load<uint8_t>(data + 0x0D); uint8_t name_length;
auto name_buffer = reinterpret_cast<const char*>(data + 0x0E); fread(&node_l, sizeof(uint16_t), 1, file);
auto name = std::string(name_buffer, name_length); fread(&node_r, sizeof(uint16_t), 1, file);
fread(&data_block, sizeof(uint32_t), 1, file);
fread(&length, sizeof(uint32_t), 1, file);
fread(&attributes, sizeof(uint8_t), 1, file);
fread(&name_length, sizeof(uint8_t), 1, file);
auto name_buffer = std::make_unique<char[]>(name_length);
fread(name_buffer.get(), 1, name_length, file);
auto name = std::string(name_buffer.get(), name_length);
// Read the left node // Read the left node
if (node_l) { if (node_l) {
@ -319,7 +374,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD(
// NOTE: SVOD entries don't have timestamps for individual files, which can // NOTE: SVOD entries don't have timestamps for individual files, which can
// cause issues when decrypting games. Using the root entry's timestamp // cause issues when decrypting games. Using the root entry's timestamp
// solves this issues. // solves this issues.
auto entry = StfsContainerEntry::Create(this, parent, name, &mmap_); auto entry = StfsContainerEntry::Create(this, parent, name, &files_);
if (attributes & kFileAttributeDirectory) { if (attributes & kFileAttributeDirectory) {
// Entry is a directory // Entry is a directory
entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly; entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
@ -454,90 +509,98 @@ void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address,
} }
StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() { StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
auto data = mmap_.at(0)->data(); auto& file = files_.at(0);
auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_); auto root_entry = new StfsContainerEntry(this, nullptr, "", &files_);
root_entry->attributes_ = kFileAttributeDirectory; root_entry->attributes_ = kFileAttributeDirectory;
root_entry_ = std::unique_ptr<Entry>(root_entry); root_entry_ = std::unique_ptr<Entry>(root_entry);
std::vector<StfsContainerEntry*> all_entries; std::vector<StfsContainerEntry*> all_entries;
// Load all listings. // Load all listings.
auto& volume_descriptor = header_.metadata.volume_descriptor.stfs; StfsDirectoryBlock directory;
uint32_t table_block_index = volume_descriptor.file_table_block_number();
for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) { auto& descriptor = header_.metadata.volume_descriptor.stfs;
const uint8_t* p = data + BlockToOffsetSTFS(table_block_index); uint32_t table_block_index = descriptor.file_table_block_number();
size_t n = 0;
for (n = 0; n < descriptor.file_table_block_count; n++) {
auto offset = BlockToOffsetSTFS(table_block_index);
xe::filesystem::Seek(file, offset, SEEK_SET);
fread(&directory, sizeof(StfsDirectoryBlock), 1, file);
for (size_t m = 0; m < kSectorSize / 0x40; m++) { for (size_t m = 0; m < kSectorSize / 0x40; m++) {
const uint8_t* name_buffer = p; // 0x28b auto& dir_entry = directory.entries[m];
if (name_buffer[0] == 0) {
if (dir_entry.name[0] == 0) {
// Done. // Done.
break; break;
} }
uint8_t name_length_flags = xe::load_and_swap<uint8_t>(p + 0x28);
uint32_t allocated_block_count = load_uint24_le(p + 0x2C);
uint32_t start_block_index = load_uint24_le(p + 0x2F);
uint16_t path_indicator = xe::load_and_swap<uint16_t>(p + 0x32);
uint32_t file_size = xe::load_and_swap<uint32_t>(p + 0x34);
// both date and time parts of the timestamp are big endian
uint16_t update_date = xe::load_and_swap<uint16_t>(p + 0x38);
uint16_t update_time = xe::load_and_swap<uint16_t>(p + 0x3A);
uint32_t access_date = xe::load_and_swap<uint16_t>(p + 0x3C);
uint32_t access_time = xe::load_and_swap<uint16_t>(p + 0x3E);
p += 0x40;
StfsContainerEntry* parent_entry = nullptr; StfsContainerEntry* parent_entry = nullptr;
if (path_indicator == 0xFFFF) { if (dir_entry.directory_index == 0xFFFF) {
parent_entry = root_entry; parent_entry = root_entry;
} else { } else {
parent_entry = all_entries[path_indicator]; parent_entry = all_entries[dir_entry.directory_index];
} }
std::string name(reinterpret_cast<const char*>(name_buffer), std::string name(reinterpret_cast<const char*>(dir_entry.name),
name_length_flags & 0x3F); dir_entry.flags.name_length & 0x3F);
auto entry = StfsContainerEntry::Create(this, parent_entry, name, &mmap_); auto entry =
StfsContainerEntry::Create(this, parent_entry, name, &files_);
// bit 0x40 = consecutive blocks (not fragmented?) if (dir_entry.flags.directory) {
if (name_length_flags & 0x80) {
entry->attributes_ = kFileAttributeDirectory; entry->attributes_ = kFileAttributeDirectory;
} else { } else {
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->data_offset_ = BlockToOffsetSTFS(start_block_index); entry->data_offset_ = BlockToOffsetSTFS(dir_entry.start_block_number());
entry->data_size_ = file_size; entry->data_size_ = dir_entry.length;
} }
entry->size_ = file_size; entry->size_ = dir_entry.length;
entry->allocation_size_ = xe::round_up(file_size, kSectorSize); entry->allocation_size_ = xe::round_up(dir_entry.length, kSectorSize);
entry->create_timestamp_ = decode_fat_timestamp(update_date, update_time); entry->create_timestamp_ =
entry->access_timestamp_ = decode_fat_timestamp(access_date, access_time); decode_fat_timestamp(dir_entry.create_date, dir_entry.create_time);
entry->write_timestamp_ = entry->create_timestamp_; entry->write_timestamp_ = decode_fat_timestamp(dir_entry.modified_date,
dir_entry.modified_time);
entry->access_timestamp_ = entry->write_timestamp_;
all_entries.push_back(entry.get()); all_entries.push_back(entry.get());
// Fill in all block records. // Fill in all block records.
// It's easier to do this now and just look them up later, at the cost // It's easier to do this now and just look them up later, at the cost
// of some memory. Nasty chain walk. // of some memory. Nasty chain walk.
// TODO(benvanik): optimize if flag 0x40 (consecutive) is set. // TODO(benvanik): optimize if flags.contiguous is set.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) { if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t block_index = start_block_index; uint32_t block_index = dir_entry.start_block_number();
size_t remaining_size = file_size; size_t remaining_size = dir_entry.length;
while (remaining_size && block_index != 0xFFFFFF) { while (remaining_size && block_index != 0xFFFFFF) {
size_t block_size = size_t block_size =
std::min(static_cast<size_t>(kSectorSize), remaining_size); std::min(static_cast<size_t>(kSectorSize), remaining_size);
size_t offset = BlockToOffsetSTFS(block_index); size_t offset = BlockToOffsetSTFS(block_index);
entry->block_list_.push_back({0, offset, block_size}); entry->block_list_.push_back({0, offset, block_size});
remaining_size -= block_size; remaining_size -= block_size;
auto block_hash = GetBlockHash(data, block_index); auto block_hash = GetBlockHash(block_index);
block_index = block_hash->level0_next_block(); block_index = block_hash->level0_next_block();
} }
if (remaining_size) {
// Loop above must have exited prematurely, bad hash tables?
XELOGW(
"STFS file {} only found {} bytes for file, expected {} ({} "
"bytes missing)",
name, dir_entry.length - remaining_size, dir_entry.length,
remaining_size);
assert_always();
}
// Check that the number of blocks retrieved from hash entries matches // Check that the number of blocks retrieved from hash entries matches
// the block count read from the file entry // the block count read from the file entry
if (entry->block_list_.size() != allocated_block_count) { if (entry->block_list_.size() != dir_entry.allocated_data_blocks()) {
XELOGW( XELOGW(
"STFS failed to read correct block-chain for entry {}, read {} " "STFS failed to read correct block-chain for entry {}, read {} "
"blocks, expected {}", "blocks, expected {}",
entry->name_, entry->block_list_.size(), allocated_block_count); entry->name_, entry->block_list_.size(),
dir_entry.allocated_data_blocks());
assert_always(); assert_always();
} }
} }
@ -545,13 +608,19 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
parent_entry->children_.emplace_back(std::move(entry)); parent_entry->children_.emplace_back(std::move(entry));
} }
auto block_hash = GetBlockHash(data, table_block_index); auto block_hash = GetBlockHash(table_block_index);
table_block_index = block_hash->level0_next_block(); table_block_index = block_hash->level0_next_block();
if (table_block_index == 0xFFFFFF) { if (table_block_index == 0xFFFFFF) {
break; break;
} }
} }
if (n + 1 != descriptor.file_table_block_count) {
XELOGW("STFS read {} file table blocks, but STFS headers expected {}!",
n + 1, descriptor.file_table_block_count);
assert_always();
}
return Error::kSuccess; return Error::kSuccess;
} }
@ -613,37 +682,41 @@ size_t StfsContainerDevice::BlockToHashBlockOffsetSTFS(
return xe::round_up(header_.header.header_size, kSectorSize) + (block << 12); return xe::round_up(header_.header.header_size, kSectorSize) + (block << 12);
} }
const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr, const StfsHashEntry* StfsContainerDevice::GetBlockHash(uint32_t block_index) {
uint32_t block_index) { auto& file = files_.at(0);
auto& volume_descriptor = header_.metadata.volume_descriptor.stfs;
auto& descriptor = header_.metadata.volume_descriptor.stfs;
// Offset for selecting the secondary hash block, in packages that have them // Offset for selecting the secondary hash block, in packages that have them
uint32_t secondary_table_offset = uint32_t secondary_table_offset =
volume_descriptor.flags.bits.root_active_index ? kSectorSize : 0; descriptor.flags.bits.root_active_index ? kSectorSize : 0;
auto hash_offset_lv0 = BlockToHashBlockOffsetSTFS(block_index, 0); auto hash_offset_lv0 = BlockToHashBlockOffsetSTFS(block_index, 0);
if (!cached_hash_tables_.count(hash_offset_lv0)) { if (!cached_hash_tables_.count(hash_offset_lv0)) {
// If this is read_only_format then it doesn't contain secondary blocks, no // If this is read_only_format then it doesn't contain secondary blocks, no
// need to check upper hash levels // need to check upper hash levels
if (volume_descriptor.flags.bits.read_only_format) { if (descriptor.flags.bits.read_only_format) {
secondary_table_offset = 0; secondary_table_offset = 0;
} else { } else {
// Not a read-only package, need to check each levels active index flag to // Not a read-only package, need to check each levels active index flag to
// see if we need to use secondary block or not // see if we need to use secondary block or not
// Check level1 table if package has it // Check level1 table if package has it
if (volume_descriptor.total_block_count > kBlocksPerHashLevel[0]) { if (descriptor.total_block_count > kBlocksPerHashLevel[0]) {
auto hash_offset_lv1 = BlockToHashBlockOffsetSTFS(block_index, 1); auto hash_offset_lv1 = BlockToHashBlockOffsetSTFS(block_index, 1);
if (!cached_hash_tables_.count(hash_offset_lv1)) { if (!cached_hash_tables_.count(hash_offset_lv1)) {
// Check level2 table if package has it // Check level2 table if package has it
if (volume_descriptor.total_block_count > kBlocksPerHashLevel[1]) { if (descriptor.total_block_count > kBlocksPerHashLevel[1]) {
auto hash_offset_lv2 = BlockToHashBlockOffsetSTFS(block_index, 2); auto hash_offset_lv2 = BlockToHashBlockOffsetSTFS(block_index, 2);
if (!cached_hash_tables_.count(hash_offset_lv2)) { if (!cached_hash_tables_.count(hash_offset_lv2)) {
cached_hash_tables_[hash_offset_lv2] = xe::filesystem::Seek(
*reinterpret_cast<const StfsHashTable*>( file, hash_offset_lv2 + secondary_table_offset, SEEK_SET);
map_ptr + hash_offset_lv2 + secondary_table_offset);
StfsHashTable table_lv2;
fread(&table_lv2, sizeof(StfsHashTable), 1, file);
cached_hash_tables_[hash_offset_lv2] = table_lv2;
} }
auto record = auto record =
@ -654,9 +727,12 @@ const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr,
record_data->levelN_active_index() ? kSectorSize : 0; record_data->levelN_active_index() ? kSectorSize : 0;
} }
cached_hash_tables_[hash_offset_lv1] = xe::filesystem::Seek(file, hash_offset_lv1 + secondary_table_offset,
*reinterpret_cast<const StfsHashTable*>( SEEK_SET);
map_ptr + hash_offset_lv1 + secondary_table_offset);
StfsHashTable table_lv1;
fread(&table_lv1, sizeof(StfsHashTable), 1, file);
cached_hash_tables_[hash_offset_lv1] = table_lv1;
} }
auto record = auto record =
@ -668,9 +744,12 @@ const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr,
} }
} }
cached_hash_tables_[hash_offset_lv0] = xe::filesystem::Seek(file, hash_offset_lv0 + secondary_table_offset,
*reinterpret_cast<const StfsHashTable*>(map_ptr + hash_offset_lv0 + SEEK_SET);
secondary_table_offset);
StfsHashTable table_lv0;
fread(&table_lv0, sizeof(StfsHashTable), 1, file);
cached_hash_tables_[hash_offset_lv0] = table_lv0;
} }
auto record = block_index % kBlocksPerHashLevel[0]; auto record = block_index % kBlocksPerHashLevel[0];

25
src/xenia/vfs/devices/stfs_container_device.h
View File

@ -15,7 +15,6 @@
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include "xenia/base/mapped_memory.h"
#include "xenia/base/math.h" #include "xenia/base/math.h"
#include "xenia/base/string_util.h" #include "xenia/base/string_util.h"
#include "xenia/kernel/util/xex2_info.h" #include "xenia/kernel/util/xex2_info.h"
@ -55,10 +54,15 @@ class StfsContainerDevice : public Device {
// TODO: use allocated_block_count inside volume-descriptor? // TODO: use allocated_block_count inside volume-descriptor?
size_t data_size() const { size_t data_size() const {
if (header_.header.header_size) { if (header_.header.header_size) {
return mmap_total_size_ - if (header_.metadata.volume_type == XContentVolumeType::kStfs &&
header_.metadata.volume_descriptor.stfs.is_valid()) {
return header_.metadata.volume_descriptor.stfs.total_block_count *
kSectorSize;
}
return files_total_size_ -
xe::round_up(header_.header.header_size, kSectorSize); xe::round_up(header_.header.header_size, kSectorSize);
} }
return mmap_total_size_ - sizeof(StfsHeader); return files_total_size_ - sizeof(StfsHeader);
} }
private: private:
@ -71,6 +75,7 @@ class StfsContainerDevice : public Device {
kErrorReadError = -10, kErrorReadError = -10,
kErrorFileMismatch = -30, kErrorFileMismatch = -30,
kErrorDamagedFile = -31, kErrorDamagedFile = -31,
kErrorTooSmall = -32,
}; };
enum class SvodLayoutType { enum class SvodLayoutType {
@ -83,8 +88,10 @@ class StfsContainerDevice : public Device {
XContentPackageType ReadMagic(const std::filesystem::path& path); XContentPackageType ReadMagic(const std::filesystem::path& path);
bool ResolveFromFolder(const std::filesystem::path& path); bool ResolveFromFolder(const std::filesystem::path& path);
Error MapFiles(); Error OpenFiles();
Error ReadHeaderAndVerify(const uint8_t* map_ptr, size_t map_size); void CloseFiles();
Error ReadHeaderAndVerify(FILE* header_file);
Error ReadSVOD(); Error ReadSVOD();
Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal, Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal,
@ -98,13 +105,13 @@ class StfsContainerDevice : public Device {
size_t BlockToHashBlockOffsetSTFS(uint32_t block_index, size_t BlockToHashBlockOffsetSTFS(uint32_t block_index,
uint32_t hash_level) const; uint32_t hash_level) const;
const StfsHashEntry* GetBlockHash(const uint8_t* map_ptr, const StfsHashEntry* GetBlockHash(uint32_t block_index);
uint32_t block_index);
std::string name_; std::string name_;
std::filesystem::path host_path_; std::filesystem::path host_path_;
std::map<size_t, std::unique_ptr<MappedMemory>> mmap_;
size_t mmap_total_size_; std::map<size_t, FILE*> files_;
size_t files_total_size_;
size_t base_offset_; size_t base_offset_;
size_t magic_offset_; size_t magic_offset_;

9
src/xenia/vfs/devices/stfs_container_entry.cc
View File

@ -18,9 +18,9 @@ namespace vfs {
StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent, StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
const std::string_view path, const std::string_view path,
MultifileMemoryMap* mmap) MultiFileHandles* files)
: Entry(device, parent, path), : Entry(device, parent, path),
mmap_(mmap), files_(files),
data_offset_(0), data_offset_(0),
data_size_(0), data_size_(0),
block_(0) {} block_(0) {}
@ -29,9 +29,10 @@ StfsContainerEntry::~StfsContainerEntry() = default;
std::unique_ptr<StfsContainerEntry> StfsContainerEntry::Create( std::unique_ptr<StfsContainerEntry> StfsContainerEntry::Create(
Device* device, Entry* parent, const std::string_view name, Device* device, Entry* parent, const std::string_view name,
MultifileMemoryMap* mmap) { MultiFileHandles* files) {
auto path = xe::utf8::join_guest_paths(parent->path(), name); auto path = xe::utf8::join_guest_paths(parent->path(), name);
auto entry = std::make_unique<StfsContainerEntry>(device, parent, path, mmap); auto entry =
std::make_unique<StfsContainerEntry>(device, parent, path, files);
return std::move(entry); return std::move(entry);
} }

11
src/xenia/vfs/devices/stfs_container_entry.h
View File

@ -14,28 +14,27 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include "xenia/base/mapped_memory.h"
#include "xenia/vfs/entry.h" #include "xenia/vfs/entry.h"
#include "xenia/vfs/file.h" #include "xenia/vfs/file.h"
namespace xe { namespace xe {
namespace vfs { namespace vfs {
typedef std::map<size_t, std::unique_ptr<MappedMemory>> MultifileMemoryMap; typedef std::map<size_t, FILE*> MultiFileHandles;
class StfsContainerDevice; class StfsContainerDevice;
class StfsContainerEntry : public Entry { class StfsContainerEntry : public Entry {
public: public:
StfsContainerEntry(Device* device, Entry* parent, const std::string_view path, StfsContainerEntry(Device* device, Entry* parent, const std::string_view path,
MultifileMemoryMap* mmap); MultiFileHandles* files);
~StfsContainerEntry() override; ~StfsContainerEntry() override;
static std::unique_ptr<StfsContainerEntry> Create(Device* device, static std::unique_ptr<StfsContainerEntry> Create(Device* device,
Entry* parent, Entry* parent,
const std::string_view name, const std::string_view name,
MultifileMemoryMap* mmap); MultiFileHandles* files);
MultifileMemoryMap* mmap() const { return mmap_; } MultiFileHandles* files() const { return files_; }
size_t data_offset() const { return data_offset_; } size_t data_offset() const { return data_offset_; }
size_t data_size() const { return data_size_; } size_t data_size() const { return data_size_; }
size_t block() const { return block_; } size_t block() const { return block_; }
@ -52,7 +51,7 @@ class StfsContainerEntry : public Entry {
private: private:
friend class StfsContainerDevice; friend class StfsContainerDevice;
MultifileMemoryMap* mmap_; MultiFileHandles* files_;
size_t data_offset_; size_t data_offset_;
size_t data_size_; size_t data_size_;
size_t block_; size_t block_;

7
src/xenia/vfs/devices/stfs_container_file.cc
View File

@ -47,13 +47,14 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
continue; continue;
} }
uint8_t* src = entry_->mmap()->at(record.file)->data();
size_t read_offset = size_t read_offset =
(byte_offset > src_offset) ? byte_offset - src_offset : 0; (byte_offset > src_offset) ? byte_offset - src_offset : 0;
size_t read_length = size_t read_length =
std::min(record.length - read_offset, remaining_length); std::min(record.length - read_offset, remaining_length);
std::memcpy(p, src + record.offset + read_offset, read_length);
auto& file = entry_->files()->at(record.file);
xe::filesystem::Seek(file, record.offset + read_offset, SEEK_SET);
fread(p, 1, read_length, file);
p += read_length; p += read_length;
src_offset += record.length; src_offset += record.length;

53
src/xenia/vfs/devices/stfs_xbox.h
View File

@ -171,6 +171,59 @@ struct StfsHashTable {
}; };
static_assert_size(StfsHashTable, 0x1000); static_assert_size(StfsHashTable, 0x1000);
struct StfsDirectoryEntry {
char name[40];
struct {
uint8_t name_length : 6;
uint8_t contiguous : 1;
uint8_t directory : 1;
} flags;
uint8_t valid_data_blocks_raw[3];
uint8_t allocated_data_blocks_raw[3];
uint8_t start_block_number_raw[3];
be<uint16_t> directory_index;
be<uint32_t> length;
be<uint16_t> create_date;
be<uint16_t> create_time;
be<uint16_t> modified_date;
be<uint16_t> modified_time;
uint32_t valid_data_blocks() const {
return load_uint24_le(valid_data_blocks_raw);
}
void set_valid_data_blocks(uint32_t value) {
store_uint24_le(valid_data_blocks_raw, value);
}
uint32_t allocated_data_blocks() const {
return load_uint24_le(allocated_data_blocks_raw);
}
void set_allocated_data_blocks(uint32_t value) {
store_uint24_le(allocated_data_blocks_raw, value);
}
uint32_t start_block_number() const {
return load_uint24_le(start_block_number_raw);
}
void set_start_block_number(uint32_t value) {
store_uint24_le(start_block_number_raw, value);
}
};
static_assert_size(StfsDirectoryEntry, 0x40);
struct StfsDirectoryBlock {
StfsDirectoryEntry entries[0x40];
};
static_assert_size(StfsDirectoryBlock, 0x1000);
/* SVOD structures */ /* SVOD structures */
struct SvodDeviceDescriptor { struct SvodDeviceDescriptor {
uint8_t descriptor_length; uint8_t descriptor_length;