ShuriZma
/
xenia-canary
mirror of https://github.com/xenia-canary/xenia-canary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[VFS] Some more STFS code improvements 

Reverted to older BlockToOffsetSTFS code (with some improvements, now named STFSDataBlockToBackingBlock)
Split the To*Offset functions into ToBackingBlock & To*Offset funcs, for later use when we need the actual block numbers instead of the offsets.
Store the num blocks per hash table & block step count in the device instance, instead of calculating it each time.
Add read_only_package() & root_table_secondary() getters to the STFS descriptor struct (note: will be replaced with better stuff once stfs-headers is eventually merged)
This commit is contained in:
emoose 2020-01-11 15:57:41 +00:00 committed by illusion
parent dc444a4a2d
commit f82cdfb539
2 changed files with 110 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
src/xenia/vfs/devices/stfs_container_device.cc
View File

@ -220,6 +220,23 @@ StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify(
return Error::kErrorDamagedFile; return Error::kErrorDamagedFile;
} }
// Pre-calculate some values used in block number calculations
blocks_per_hash_table_ = 1;
block_step_[0] = 0xAB;
block_step_[1] = 0x718F;
// TODO: it seems if header_.header_size > 0xA000, this should make sure never
// to follow the branch below? Since the header size would spill over into the
// first hash tables primary block (@0xA000), that must mean it only uses a
// single block for each table, right?
// Need to verify with kernel if it actually bases anything on the header_size
// field.
if (!header_.stfs_volume_descriptor.read_only_package()) {
blocks_per_hash_table_ = 2;
block_step_[0] = 0xAC;
block_step_[1] = 0x723A;
}
return Error::kSuccess; return Error::kSuccess;
} }
@ -499,7 +516,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
auto& volume_descriptor = header_.stfs_volume_descriptor; auto& volume_descriptor = header_.stfs_volume_descriptor;
uint32_t table_block_index = volume_descriptor.file_table_block_number; uint32_t table_block_index = volume_descriptor.file_table_block_number;
for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) { for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) {
const uint8_t* p = data + BlockToOffsetSTFS(table_block_index); const uint8_t* p = data + STFSDataBlockToOffset(table_block_index);
for (size_t m = 0; m < 0x1000 / 0x40; m++) { for (size_t m = 0; m < 0x1000 / 0x40; m++) {
const uint8_t* filename = p; // 0x28b const uint8_t* filename = p; // 0x28b
if (filename[0] == 0) { if (filename[0] == 0) {
@ -537,7 +554,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
entry->attributes_ = kFileAttributeDirectory; entry->attributes_ = kFileAttributeDirectory;
} else { } else {
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->data_offset_ = BlockToOffsetSTFS(start_block_index); entry->data_offset_ = STFSDataBlockToOffset(start_block_index);
entry->data_size_ = file_size; entry->data_size_ = file_size;
} }
entry->size_ = file_size; entry->size_ = file_size;
@ -560,10 +577,10 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
while (remaining_size && block_index) { while (remaining_size && block_index) {
size_t block_size = size_t block_size =
std::min(static_cast<size_t>(0x1000), remaining_size); std::min(static_cast<size_t>(0x1000), remaining_size);
size_t offset = BlockToOffsetSTFS(block_index); size_t offset = STFSDataBlockToOffset(block_index);
entry->block_list_.push_back({0, offset, block_size}); entry->block_list_.push_back({0, offset, block_size});
remaining_size -= block_size; remaining_size -= block_size;
auto block_hash = GetBlockHash(data, block_index); auto block_hash = STFSGetLevel0HashEntry(data, block_index);
block_index = block_hash.next_block_index; block_index = block_hash.next_block_index;
info = block_hash.info; info = block_hash.info;
} }
@ -572,7 +589,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
parent_entry->children_.emplace_back(std::move(entry)); parent_entry->children_.emplace_back(std::move(entry));
} }
auto block_hash = GetBlockHash(data, table_block_index); auto block_hash = STFSGetLevel0HashEntry(data, table_block_index);
table_block_index = block_hash.next_block_index; table_block_index = block_hash.next_block_index;
} }
@ -580,121 +597,86 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
return Error::kSuccess; return Error::kSuccess;
} }
// Failed to load any entries, try with old algo if we haven't already // No entries found... return failure
if (!use_old_algorithm_) {
use_old_algorithm_ = true;
return ReadSTFS();
}
// Tried with old algo and still no entries... return failure
return Error::kErrorReadError; return Error::kErrorReadError;
} }
size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) { uint64_t StfsContainerDevice::STFSDataBlockToBackingBlock(
if (use_old_algorithm_) { uint64_t block_index) {
return BlockToOffsetSTFS_Old(block_index);
}
uint32_t num_tables = 1; // num hashtables per block? or maybe backingblocks?
if (!read_only_package()) {
num_tables++;
}
uint64_t dataBlock =
block_index + num_tables * ((block_index + kSTFSBlocksPerHashTable) /
kSTFSBlocksPerHashTable);
if (block_index >= kSTFSBlocksPerHashTable) {
dataBlock += num_tables * ((block_index + kSTFSBlocksPerL1HashTable) /
kSTFSBlocksPerL1HashTable);
if (block_index >= kSTFSBlocksPerL1HashTable) {
dataBlock += num_tables * ((block_index + kSTFSBlocksPerL2HashTable) /
kSTFSBlocksPerL2HashTable);
}
}
return BackingBlockToOffsetSTFS(dataBlock);
}
size_t StfsContainerDevice::BlockToOffsetSTFS_Old(uint64_t block_index) {
uint64_t block;
uint32_t block_shift = 0;
if (((header_.header_size + 0x0FFF) & 0xB000) == 0xB000 ||
(header_.stfs_volume_descriptor.flags & 0x1) == 0x0) {
block_shift = package_type_ == StfsPackageType::kCon ? 1 : 0;
}
// For every level there is a hash table // For every level there is a hash table
// Level 0: hash table of next 170 blocks // Level 0: hash table of next 170 blocks
// Level 1: hash table of next 170 hash tables // Level 1: hash table of next 170 hash tables
// Level 2: hash table of next 170 level 1 hash tables // Level 2: hash table of next 170 level 1 hash tables
// And so on... // And so on...
uint64_t base = kSTFSBlocksPerHashTable; uint64_t block = block_index;
block = block_index;
for (uint32_t i = 0; i < 3; i++) { for (uint32_t i = 0; i < 3; i++) {
block += (block_index + (base << block_shift)) / (base << block_shift); block += blocks_per_hash_table_ *
if (block_index < base) { ((block_index + kSTFSDataBlocksPerHashLevel[i]) /
kSTFSDataBlocksPerHashLevel[i]);
if (block_index < kSTFSDataBlocksPerHashLevel[i]) {
break; break;
} }
base *= kSTFSBlocksPerHashTable;
} }
return xe::round_up(header_.header_size, 0x1000) + (block << 12); return block;
}
size_t StfsContainerDevice::BackingBlockToOffsetSTFS(uint64_t backing_block) {
return xe::round_up(header_.header_size, 0x1000) + (backing_block * 0x1000);
}
size_t StfsContainerDevice::BlockToHashBlockOffset(uint64_t block,
uint32_t hash_level) {
uint32_t num_tables = 1;
uint32_t block_step0 = 0xAB;
uint32_t block_step1 = 0x718F;
if (!read_only_package()) {
num_tables = 2;
block_step0 = 0xAC;
block_step1 = 0x723A;
} }
uint64_t StfsContainerDevice::STFSDataBlockToBackingHashBlock(uint64_t block,
uint32_t level) {
uint64_t backing_num = 0; uint64_t backing_num = 0;
switch (hash_level) { switch (level) {
case 0: case 0:
backing_num = (block / kSTFSBlocksPerHashTable) * block_step0; backing_num = (block / kSTFSDataBlocksPerHashLevel[0]) * block_step_[0];
if (block / kSTFSBlocksPerHashTable == 0) { if (block / kSTFSDataBlocksPerHashLevel[0] == 0) {
return BackingBlockToOffsetSTFS(backing_num); return backing_num;
} }
backing_num += ((block / kSTFSBlocksPerL1HashTable) + 1) * num_tables; backing_num += ((block / kSTFSDataBlocksPerHashLevel[1]) + 1) *
if (block / kSTFSBlocksPerL1HashTable == 0) { blocks_per_hash_table_;
return BackingBlockToOffsetSTFS(backing_num); if (block / kSTFSDataBlocksPerHashLevel[1] == 0) {
return backing_num;
} }
break; break;
case 1: case 1:
backing_num = (block / kSTFSBlocksPerL1HashTable) * block_step1; backing_num = (block / kSTFSDataBlocksPerHashLevel[1]) * block_step_[1];
if (block / kSTFSBlocksPerL1HashTable == 0) { if (block / kSTFSDataBlocksPerHashLevel[1] == 0) {
return BackingBlockToOffsetSTFS(backing_num + block_step0); return backing_num + block_step_[0];
} }
break; break;
default: default:
return BackingBlockToOffsetSTFS(block_step1); return block_step_[1];
} }
return BackingBlockToOffsetSTFS(backing_num + num_tables); return backing_num + blocks_per_hash_table_;
} }
StfsContainerDevice::BlockHash StfsContainerDevice::GetHashEntry( size_t StfsContainerDevice::STFSBackingBlockToOffset(uint64_t backing_block) {
return xe::round_up(header_.header_size, 0x1000) + (backing_block * 0x1000);
}
size_t StfsContainerDevice::STFSDataBlockToOffset(uint64_t block) {
return STFSBackingBlockToOffset(STFSDataBlockToBackingBlock(block));
}
size_t StfsContainerDevice::STFSDataBlockToBackingHashBlockOffset(
uint64_t block, uint32_t level) {
return STFSBackingBlockToOffset(
STFSDataBlockToBackingHashBlock(block, level));
}
StfsContainerDevice::BlockHash StfsContainerDevice::STFSGetLevelNHashEntry(
const uint8_t* map_ptr, uint32_t block_index, uint32_t level, const uint8_t* map_ptr, uint32_t block_index, uint32_t level,
bool secondary_table) { bool secondary_block) {
uint32_t record = block_index; uint32_t record = block_index;
for (uint32_t i = 0; i < level; i++) { for (uint32_t i = 0; i < level; i++) {
record = record / kSTFSBlocksPerHashTable; record = record / kSTFSDataBlocksPerHashLevel[0];
} }
record = record % kSTFSBlocksPerHashTable; record = record % kSTFSDataBlocksPerHashLevel[0];
size_t hash_offset = BlockToHashBlockOffset(block_index, level); size_t hash_offset =
if (secondary_table) { STFSDataBlockToBackingHashBlockOffset(block_index, level);
hash_offset += bytes_per_sector(); // use alternate hash table? if (secondary_block && !header_.stfs_volume_descriptor.read_only_package()) {
hash_offset += bytes_per_sector(); // read from this tables secondary block
} }
const uint8_t* hash_data = map_ptr + hash_offset; const uint8_t* hash_data = map_ptr + hash_offset;
@ -705,44 +687,44 @@ StfsContainerDevice::BlockHash StfsContainerDevice::GetHashEntry(
return {next_block_index, info}; return {next_block_index, info};
} }
StfsContainerDevice::BlockHash StfsContainerDevice::GetBlockHash( StfsContainerDevice::BlockHash StfsContainerDevice::STFSGetLevel0HashEntry(
const uint8_t* map_ptr, uint32_t block_index) { const uint8_t* map_ptr, uint32_t block_index) {
bool use_secondary_table = false; bool use_secondary_block = false;
// Use secondary table for root table if RootActiveIndex flag is set // Use secondary block for root table if RootActiveIndex flag is set
if (header_.stfs_volume_descriptor.flags & 2) { if (header_.stfs_volume_descriptor.root_table_secondary()) {
use_secondary_table = true; use_secondary_block = true;
} }
// Check upper hash table levels to find which table (primary/secondary) to // Check upper hash table levels to find which table (primary/secondary) to
// use. // use.
// Hopefully we can skip doing this if the package is read-only // We should be able to skip this if it's a read-only package, since the hash
// (and RootActiveIndex isn't set) // tables in those only use one block
if (!read_only_package() || use_secondary_table) { if (!header_.stfs_volume_descriptor.read_only_package()) {
auto num_blocks = auto num_blocks =
header_.stfs_volume_descriptor.total_allocated_block_count; header_.stfs_volume_descriptor.total_allocated_block_count;
if (num_blocks >= kSTFSBlocksPerL1HashTable) { if (num_blocks >= kSTFSDataBlocksPerHashLevel[1]) {
// Get the L2 entry for the block // Get the L2 entry for the block
auto l2_entry = auto l2_entry =
GetHashEntry(map_ptr, block_index, 2, use_secondary_table); STFSGetLevelNHashEntry(map_ptr, block_index, 2, use_secondary_block);
use_secondary_table = false; use_secondary_block = false;
if (l2_entry.info & 0x40) { // ActiveIndex flag if (l2_entry.info & 0x40) { // ActiveIndex flag
use_secondary_table = true; use_secondary_block = true;
} }
} }
if (num_blocks >= kSTFSBlocksPerHashTable) { if (num_blocks >= kSTFSDataBlocksPerHashLevel[0]) {
// Get the L1 entry for this block // Get the L1 entry for this block
auto l1_entry = auto l1_entry =
GetHashEntry(map_ptr, block_index, 1, use_secondary_table); STFSGetLevelNHashEntry(map_ptr, block_index, 1, use_secondary_block);
use_secondary_table = false; use_secondary_block = false;
if (l1_entry.info & 0x40) { // ActiveIndex flag if (l1_entry.info & 0x40) { // ActiveIndex flag
use_secondary_table = true; use_secondary_block = true;
} }
} }
} }
return GetHashEntry(map_ptr, block_index, 0, use_secondary_table); return STFSGetLevelNHashEntry(map_ptr, block_index, 0, use_secondary_block);
} }
bool StfsVolumeDescriptor::Read(const uint8_t* p) { bool StfsVolumeDescriptor::Read(const uint8_t* p) {

41
src/xenia/vfs/devices/stfs_container_device.h
View File

@ -86,6 +86,16 @@ struct StfsVolumeDescriptor {
uint8_t top_hash_table_hash[0x14]; uint8_t top_hash_table_hash[0x14];
uint32_t total_allocated_block_count; uint32_t total_allocated_block_count;
uint32_t total_unallocated_block_count; uint32_t total_unallocated_block_count;
// Whether this is a read-only package - these will only use a single block
// for each hash table, compared to the two blocks used in non-read-only
bool read_only_package() { return (flags & 1) != 0; }
// Whether the root hash table is stored in the hash tables secondary block
// Only valid if read_only_package is false
bool root_table_secondary() {
return !read_only_package() && (flags & 2) != 0;
}
}; };
enum SvodDeviceFeatures { enum SvodDeviceFeatures {
@ -183,10 +193,6 @@ class StfsContainerDevice : public Device {
uint32_t sectors_per_allocation_unit() const override { return 1; } uint32_t sectors_per_allocation_unit() const override { return 1; }
uint32_t bytes_per_sector() const override { return 4 * 1024; } uint32_t bytes_per_sector() const override { return 4 * 1024; }
bool read_only_package() {
return (header_.stfs_volume_descriptor.flags & 1) != 0;
}
StfsHeader& header() { return header_; } StfsHeader& header() { return header_; }
uint32_t ExtractToFolder(const std::wstring& dest_path); uint32_t ExtractToFolder(const std::wstring& dest_path);
@ -205,9 +211,7 @@ class StfsContainerDevice : public Device {
uint32_t info; uint32_t info;
}; };
const uint32_t kSTFSBlocksPerHashTable = 0xAA; const uint32_t kSTFSDataBlocksPerHashLevel[3] = {0xAA, 0x70E4, 0x4AF768};
const uint32_t kSTFSBlocksPerL1HashTable = 0x70E4;
const uint32_t kSTFSBlocksPerL2HashTable = 0x4AF768;
bool ResolveFromFolder(const std::wstring& path); bool ResolveFromFolder(const std::wstring& path);
@ -222,29 +226,34 @@ class StfsContainerDevice : public Device {
void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index); void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index);
Error ReadSTFS(); Error ReadSTFS();
size_t BlockToOffsetSTFS(uint64_t block);
size_t BlockToOffsetSTFS_Old(uint64_t block);
size_t BackingBlockToOffsetSTFS(uint64_t backing_block); uint64_t STFSDataBlockToBackingBlock(uint64_t block);
uint64_t STFSDataBlockToBackingHashBlock(uint64_t block, uint32_t level = 0);
size_t BlockToHashBlockOffset(uint64_t block, uint32_t hash_level = 0); size_t STFSBackingBlockToOffset(uint64_t backing_block);
size_t STFSDataBlockToOffset(uint64_t block);
size_t STFSDataBlockToBackingHashBlockOffset(uint64_t block,
uint32_t level = 0);
BlockHash GetHashEntry(const uint8_t* map_ptr, uint32_t block_index, BlockHash STFSGetLevelNHashEntry(const uint8_t* map_ptr, uint32_t block_index,
uint32_t level, bool secondary_table = false); uint32_t level,
bool secondary_block = false);
BlockHash GetBlockHash(const uint8_t* map_ptr, uint32_t block_index); BlockHash STFSGetLevel0HashEntry(const uint8_t* map_ptr,
uint32_t block_index);
std::wstring local_path_; std::wstring local_path_;
std::map<size_t, std::unique_ptr<MappedMemory>> mmap_; std::map<size_t, std::unique_ptr<MappedMemory>> mmap_;
size_t mmap_total_size_; size_t mmap_total_size_;
uint32_t blocks_per_hash_table_ = 1;
uint32_t block_step_[2] = {0xAB, 0x718F};
size_t base_offset_; size_t base_offset_;
size_t magic_offset_; size_t magic_offset_;
std::unique_ptr<Entry> root_entry_; std::unique_ptr<Entry> root_entry_;
StfsPackageType package_type_; StfsPackageType package_type_;
StfsHeader header_; StfsHeader header_;
bool use_old_algorithm_ = false;
}; };
} // namespace vfs } // namespace vfs