From 2659b70c90b3c4741189cbb01172a625d2b1ccb6 Mon Sep 17 00:00:00 2001 From: emoose Date: Tue, 11 May 2021 03:13:00 +0100 Subject: [PATCH] [VFS/STFS] Improved StfsHashEntry, minor struct improvements Changed XEPACKEDSTRUCT to #pragma pack, added setters for some fields that didn't have them, named some anonymous structs which might help with other compilers. --- .../vfs/devices/stfs_container_device.cc | 40 +-- src/xenia/vfs/devices/stfs_container_device.h | 2 +- src/xenia/vfs/devices/stfs_xbox.h | 233 ++++++++++-------- 3 files changed, 150 insertions(+), 125 deletions(-) diff --git a/src/xenia/vfs/devices/stfs_container_device.cc b/src/xenia/vfs/devices/stfs_container_device.cc index 6ce92c00b..e8f6f72be 100644 --- a/src/xenia/vfs/devices/stfs_container_device.cc +++ b/src/xenia/vfs/devices/stfs_container_device.cc @@ -176,16 +176,15 @@ StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify( const uint8_t* map_ptr, size_t map_size) { // Copy header & check signature memcpy(&header_, map_ptr, sizeof(StfsHeader)); - if (header_.header.magic != XContentPackageType::kPackageTypeCon && - header_.header.magic != XContentPackageType::kPackageTypeLive && - header_.header.magic != XContentPackageType::kPackageTypePirs) { + if (!header_.header.is_magic_valid()) { // Unexpected format. return Error::kErrorFileMismatch; } // Pre-calculate some values used in block number calculations blocks_per_hash_table_ = - header_.metadata.stfs_volume_descriptor.flags.read_only_format ? 1 : 2; + header_.metadata.volume_descriptor.stfs.flags.bits.read_only_format ? 1 + : 2; block_step[0] = kBlocksPerHashLevel[0] + blocks_per_hash_table_; block_step[1] = kBlocksPerHashLevel[1] + @@ -202,7 +201,8 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() { const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA"; // Check for EDGF layout - if (header_.metadata.svod_volume_descriptor.features.enhanced_gdf_layout) { + if (header_.metadata.volume_descriptor.svod.features.bits + .enhanced_gdf_layout) { // The STFS header has specified that this SVOD system uses the EGDF layout. // We can expect the magic block to be located immediately after the hash // blocks. We also offset block address calculation by 0x1000 by shifting @@ -414,7 +414,7 @@ void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address, const size_t BLOCKS_PER_FILE = 0x14388; const size_t MAX_FILE_SIZE = 0xA290000; const size_t BLOCK_OFFSET = - header_.metadata.svod_volume_descriptor.start_data_block(); + header_.metadata.volume_descriptor.svod.start_data_block(); // Resolve the true block address and file index size_t true_block = block - (BLOCK_OFFSET * 2); @@ -463,7 +463,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() { std::vector all_entries; // Load all listings. - auto& volume_descriptor = header_.metadata.stfs_volume_descriptor; + auto& volume_descriptor = header_.metadata.volume_descriptor.stfs; uint32_t table_block_index = volume_descriptor.file_table_block_number(); for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) { const uint8_t* p = data + BlockToOffsetSTFS(table_block_index); @@ -615,30 +615,29 @@ size_t StfsContainerDevice::BlockToHashBlockOffsetSTFS( const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr, uint32_t block_index) { - auto& volume_descriptor = header_.metadata.stfs_volume_descriptor; + auto& volume_descriptor = header_.metadata.volume_descriptor.stfs; // Offset for selecting the secondary hash block, in packages that have them uint32_t secondary_table_offset = - volume_descriptor.flags.root_active_index ? kSectorSize : 0; + volume_descriptor.flags.bits.root_active_index ? kSectorSize : 0; auto hash_offset_lv0 = BlockToHashBlockOffsetSTFS(block_index, 0); if (!cached_hash_tables_.count(hash_offset_lv0)) { // If this is read_only_format then it doesn't contain secondary blocks, no // need to check upper hash levels - if (volume_descriptor.flags.read_only_format) { + if (volume_descriptor.flags.bits.read_only_format) { secondary_table_offset = 0; } else { // Not a read-only package, need to check each levels active index flag to // see if we need to use secondary block or not // Check level1 table if package has it - if (volume_descriptor.allocated_block_count > kBlocksPerHashLevel[0]) { + if (volume_descriptor.total_block_count > kBlocksPerHashLevel[0]) { auto hash_offset_lv1 = BlockToHashBlockOffsetSTFS(block_index, 1); if (!cached_hash_tables_.count(hash_offset_lv1)) { // Check level2 table if package has it - if (volume_descriptor.allocated_block_count > - kBlocksPerHashLevel[1]) { + if (volume_descriptor.total_block_count > kBlocksPerHashLevel[1]) { auto hash_offset_lv2 = BlockToHashBlockOffsetSTFS(block_index, 2); if (!cached_hash_tables_.count(hash_offset_lv2)) { @@ -652,7 +651,7 @@ const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr, auto record_data = &cached_hash_tables_[hash_offset_lv2].entries[record]; secondary_table_offset = - record_data->levelN_activeindex() ? kSectorSize : 0; + record_data->levelN_active_index() ? kSectorSize : 0; } cached_hash_tables_[hash_offset_lv1] = @@ -665,7 +664,7 @@ const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr, auto record_data = &cached_hash_tables_[hash_offset_lv1].entries[record]; secondary_table_offset = - record_data->levelN_activeindex() ? kSectorSize : 0; + record_data->levelN_active_index() ? kSectorSize : 0; } } @@ -680,9 +679,10 @@ const StfsHashEntry* StfsContainerDevice::GetBlockHash(const uint8_t* map_ptr, return record_data; } -uint32_t StfsContainerDevice::ReadMagic(const std::filesystem::path& path) { +XContentPackageType StfsContainerDevice::ReadMagic( + const std::filesystem::path& path) { auto map = MappedMemory::Open(path, MappedMemory::Mode::kRead, 0, 4); - return xe::load_and_swap(map->data()); + return XContentPackageType(xe::load_and_swap(map->data())); } bool StfsContainerDevice::ResolveFromFolder(const std::filesystem::path& path) { @@ -708,9 +708,9 @@ bool StfsContainerDevice::ResolveFromFolder(const std::filesystem::path& path) { auto path = current_file.path / current_file.name; auto magic = ReadMagic(path); - if (magic == XContentPackageType::kPackageTypeCon || - magic == XContentPackageType::kPackageTypeLive || - magic == XContentPackageType::kPackageTypePirs) { + if (magic == XContentPackageType::kCon || + magic == XContentPackageType::kLive || + magic == XContentPackageType::kPirs) { host_path_ = current_file.path / current_file.name; XELOGI("STFS Package found: {}", xe::path_to_utf8(host_path_)); return true; diff --git a/src/xenia/vfs/devices/stfs_container_device.h b/src/xenia/vfs/devices/stfs_container_device.h index 96bd40f86..d7fc36aa3 100644 --- a/src/xenia/vfs/devices/stfs_container_device.h +++ b/src/xenia/vfs/devices/stfs_container_device.h @@ -80,7 +80,7 @@ class StfsContainerDevice : public Device { kSingleFile = 0x4, }; - uint32_t ReadMagic(const std::filesystem::path& path); + XContentPackageType ReadMagic(const std::filesystem::path& path); bool ResolveFromFolder(const std::filesystem::path& path); Error MapFiles(); diff --git a/src/xenia/vfs/devices/stfs_xbox.h b/src/xenia/vfs/devices/stfs_xbox.h index 93fcbaaad..9ff657502 100644 --- a/src/xenia/vfs/devices/stfs_xbox.h +++ b/src/xenia/vfs/devices/stfs_xbox.h @@ -19,18 +19,21 @@ namespace vfs { // Structs used for interchange between Xenia and actual Xbox360 kernel/XAM inline uint32_t load_uint24_be(const uint8_t* p) { - return (static_cast(p[0]) << 16) | - (static_cast(p[1]) << 8) | static_cast(p[2]); + return (uint32_t(p[0]) << 16) | (uint32_t(p[1]) << 8) | uint32_t(p[2]); } inline uint32_t load_uint24_le(const uint8_t* p) { - return (static_cast(p[2]) << 16) | - (static_cast(p[1]) << 8) | static_cast(p[0]); + return (uint32_t(p[2]) << 16) | (uint32_t(p[1]) << 8) | uint32_t(p[0]); +} +inline void store_uint24_le(uint8_t* p, uint32_t value) { + p[2] = uint8_t((value >> 16) & 0xFF); + p[1] = uint8_t((value >> 8) & 0xFF); + p[0] = uint8_t(value & 0xFF); } -enum XContentPackageType : uint32_t { - kPackageTypeCon = 0x434F4E20, - kPackageTypePirs = 0x50495253, - kPackageTypeLive = 0x4C495645, +enum class XContentPackageType : uint32_t { + kCon = 0x434F4E20, + kPirs = 0x50495253, + kLive = 0x4C495645, }; enum XContentType : uint32_t { @@ -76,7 +79,8 @@ enum class XContentVolumeType : uint32_t { }; /* STFS structures */ -XEPACKEDSTRUCT(StfsVolumeDescriptor, { +#pragma pack(push, 1) +struct StfsVolumeDescriptor { uint8_t descriptor_length; uint8_t version; union { @@ -86,52 +90,77 @@ XEPACKEDSTRUCT(StfsVolumeDescriptor, { // otherwise uses two uint8_t root_active_index : 1; // if set, uses secondary backing-block // for the highest-level hash table + uint8_t directory_overallocated : 1; uint8_t directory_index_bounds_valid : 1; - }; + } bits; uint8_t as_byte; } flags; uint16_t file_table_block_count; - uint8_t file_table_block_number_0; - uint8_t file_table_block_number_1; - uint8_t file_table_block_number_2; + uint8_t file_table_block_number_raw[3]; uint8_t top_hash_table_hash[0x14]; - be allocated_block_count; + be total_block_count; be free_block_count; - uint32_t file_table_block_number() { - return uint32_t(file_table_block_number_0) | - (uint32_t(file_table_block_number_1) << 8) | - (uint32_t(file_table_block_number_2) << 16); + uint32_t file_table_block_number() const { + return load_uint24_le(file_table_block_number_raw); } -}); + + void set_file_table_block_number(uint32_t value) { + store_uint24_le(file_table_block_number_raw, value); + } + + bool is_valid() const { + return descriptor_length == sizeof(StfsVolumeDescriptor); + } +}; static_assert_size(StfsVolumeDescriptor, 0x24); +#pragma pack(pop) + +enum class StfsHashState : uint8_t { + kFree = 0, // unallocated but doesn't exist in package (needs to expand)? + kFree2 = 1, // unallocated but exists in package? + kInUse = 2, +}; struct StfsHashEntry { uint8_t sha1[0x14]; - uint8_t info0; // usually contains flags + xe::be info_raw; - uint8_t info1; - uint8_t info2; - uint8_t info3; - - // If this is a level0 entry, this points to the next block in the chain - uint32_t level0_next_block() const { - return uint32_t(info3) | (uint32_t(info2) << 8) | (uint32_t(info1) << 16); + uint32_t level0_next_block() const { return info_raw & 0xFFFFFF; } + void set_level0_next_block(uint32_t value) { + info_raw = (info_raw & ~0xFFFFFF) | (value & 0xFFFFFF); } - void level0_next_block(uint32_t value) { - info3 = uint8_t(value & 0xFF); - info2 = uint8_t((value >> 8) & 0xFF); - info1 = uint8_t((value >> 16) & 0xFF); + StfsHashState level0_allocation_state() const { + return StfsHashState(uint8_t(((info_raw & 0xC0000000) >> 30) & 0xFF)); + } + void set_level0_allocation_state(StfsHashState value) { + info_raw = (info_raw & ~0xC0000000) | (uint32_t(value) << 30); } - // If this is level 1 or 2, this says whether the hash table this entry refers - // to is using the secondary block or not - bool levelN_activeindex() const { return info0 & 0x40; } + uint32_t levelN_num_blocks_free() const { return info_raw & 0x7FFF; } + void set_levelN_num_blocks_free(uint32_t value) { + info_raw = (info_raw & ~0x7FFF) | (value & 0x7FFF); + } - bool levelN_writeable() const { return info0 & 0x80; } + uint32_t levelN_num_blocks_unk() const { + return ((info_raw & 0x3FFF8000) >> 15) & 0x7FFF; + } + void set_levelN_num_blocks_unk(uint32_t value) { + info_raw = (info_raw & ~0x3FFF8000) | ((value & 0x7FFF) << 15); + } + + bool levelN_active_index() const { return (info_raw & 0x40000000) != 0; } + void set_levelN_active_index(bool value) { + info_raw = (info_raw & ~0x40000000) | (value ? 0x40000000 : 0); + } + + bool levelN_writeable() const { return (info_raw & 0x80000000) != 0; } + void set_levelN_writeable(bool value) { + info_raw = (info_raw & ~0x80000000) | (value ? 0x80000000 : 0); + } }; static_assert_size(StfsHashEntry, 0x18); @@ -154,30 +183,27 @@ struct SvodDeviceDescriptor { uint8_t must_be_zero_for_future_usage : 6; uint8_t enhanced_gdf_layout : 1; uint8_t zero_for_downlevel_clients : 1; - }; + } bits; uint8_t as_byte; } features; - uint8_t num_data_blocks2; - uint8_t num_data_blocks1; - uint8_t num_data_blocks0; - uint8_t start_data_block0; - uint8_t start_data_block1; - uint8_t start_data_block2; + uint8_t num_data_blocks_raw[3]; + uint8_t start_data_block_raw[3]; uint8_t reserved[5]; - uint32_t num_data_blocks() { - return uint32_t(num_data_blocks0) | (uint32_t(num_data_blocks1) << 8) | - (uint32_t(num_data_blocks2) << 16); - } + uint32_t num_data_blocks() { return load_uint24_le(num_data_blocks_raw); } - uint32_t start_data_block() { - return uint32_t(start_data_block0) | (uint32_t(start_data_block1) << 8) | - (uint32_t(start_data_block2) << 16); - } + uint32_t start_data_block() { return load_uint24_le(start_data_block_raw); } }; static_assert_size(SvodDeviceDescriptor, 0x24); /* XContent structures */ +struct XContentLicense { + be licensee_id; + be license_bits; + be license_flags; +}; +static_assert_size(XContentLicense, 0x10); + struct XContentMediaData { uint8_t series_id[0x10]; uint8_t season_id[0x10]; @@ -206,7 +232,8 @@ struct XContentAttributes { }; static_assert_size(XContentAttributes, 1); -XEPACKEDSTRUCT(XContentMetadata, { +#pragma pack(push, 1) +struct XContentMetadata { static const uint32_t kThumbLengthV1 = 0x4000; static const uint32_t kThumbLengthV2 = 0x3D00; @@ -221,9 +248,9 @@ XEPACKEDSTRUCT(XContentMetadata, { uint8_t console_id[5]; be profile_id; union { - StfsVolumeDescriptor stfs_volume_descriptor; - SvodDeviceDescriptor svod_volume_descriptor; - }; + StfsVolumeDescriptor stfs; + SvodDeviceDescriptor svod; + } volume_descriptor; be data_file_count; be data_file_size; be volume_type; @@ -233,24 +260,24 @@ XEPACKEDSTRUCT(XContentMetadata, { union { XContentMediaData media_data; XContentAvatarAssetData avatar_asset_data; - }; + } metadata_v2; uint8_t device_id[0x14]; union { - be display_name_raw[kNumLanguagesV1][128]; - char16_t display_name_chars[kNumLanguagesV1][128]; - }; + be uint[kNumLanguagesV1][128]; + char16_t chars[kNumLanguagesV1][128]; + } display_name_raw; union { - be description_raw[kNumLanguagesV1][128]; - char16_t description_chars[kNumLanguagesV1][128]; - }; + be uint[kNumLanguagesV1][128]; + char16_t chars[kNumLanguagesV1][128]; + } description_raw; union { - be publisher_raw[64]; - char16_t publisher_chars[64]; - }; + be uint[64]; + char16_t chars[64]; + } publisher_raw; union { - be title_name_raw[64]; - char16_t title_name_chars[64]; - }; + be uint[64]; + char16_t chars[64]; + } title_name_raw; union { XContentAttributes bits; uint8_t as_byte; @@ -259,14 +286,14 @@ XEPACKEDSTRUCT(XContentMetadata, { be title_thumbnail_size; uint8_t thumbnail[kThumbLengthV2]; union { - be display_name_ex_raw[kNumLanguagesV2 - kNumLanguagesV1][128]; - char16_t display_name_ex_chars[kNumLanguagesV2 - kNumLanguagesV1][128]; - }; + be uint[kNumLanguagesV2 - kNumLanguagesV1][128]; + char16_t chars[kNumLanguagesV2 - kNumLanguagesV1][128]; + } display_name_ex_raw; uint8_t title_thumbnail[kThumbLengthV2]; union { - be description_ex_raw[kNumLanguagesV2 - kNumLanguagesV1][128]; - char16_t description_ex_chars[kNumLanguagesV2 - kNumLanguagesV1][128]; - }; + be uint[kNumLanguagesV2 - kNumLanguagesV1][128]; + char16_t chars[kNumLanguagesV2 - kNumLanguagesV1][128]; + } description_ex_raw; std::u16string display_name(XLanguage language) const { uint32_t lang_id = uint32_t(language) - 1; @@ -279,10 +306,10 @@ XEPACKEDSTRUCT(XContentMetadata, { const be* str = 0; if (lang_id >= 0 && lang_id < kNumLanguagesV1) { - str = display_name_raw[lang_id]; + str = display_name_raw.uint[lang_id]; } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 && metadata_version >= 2) { - str = display_name_ex_raw[lang_id - kNumLanguagesV1]; + str = display_name_ex_raw.uint[lang_id - kNumLanguagesV1]; } if (!str) { @@ -305,10 +332,10 @@ XEPACKEDSTRUCT(XContentMetadata, { const be* str = 0; if (lang_id >= 0 && lang_id < kNumLanguagesV1) { - str = description_raw[lang_id]; + str = description_raw.uint[lang_id]; } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 && metadata_version >= 2) { - str = description_ex_raw[lang_id - kNumLanguagesV1]; + str = description_ex_raw.uint[lang_id - kNumLanguagesV1]; } if (!str) { @@ -321,11 +348,11 @@ XEPACKEDSTRUCT(XContentMetadata, { } std::u16string publisher() const { - return load_and_swap(publisher_raw); + return load_and_swap(publisher_raw.uint); } std::u16string title_name() const { - return load_and_swap(title_name_raw); + return load_and_swap(title_name_raw.uint); } bool set_display_name(XLanguage language, const std::u16string_view value) { @@ -339,10 +366,10 @@ XEPACKEDSTRUCT(XContentMetadata, { char16_t* str = 0; if (lang_id >= 0 && lang_id < kNumLanguagesV1) { - str = display_name_chars[lang_id]; + str = display_name_raw.chars[lang_id]; } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 && metadata_version >= 2) { - str = display_name_ex_chars[lang_id - kNumLanguagesV1]; + str = display_name_ex_raw.chars[lang_id - kNumLanguagesV1]; } if (!str) { @@ -352,7 +379,7 @@ XEPACKEDSTRUCT(XContentMetadata, { } string_util::copy_and_swap_truncating(str, value, - countof(display_name_chars[0])); + countof(display_name_raw.chars[0])); return true; } @@ -367,10 +394,10 @@ XEPACKEDSTRUCT(XContentMetadata, { char16_t* str = 0; if (lang_id >= 0 && lang_id < kNumLanguagesV1) { - str = description_chars[lang_id]; + str = description_raw.chars[lang_id]; } else if (lang_id >= kNumLanguagesV1 && lang_id < kNumLanguagesV2 && metadata_version >= 2) { - str = description_ex_chars[lang_id - kNumLanguagesV1]; + str = description_ex_raw.chars[lang_id - kNumLanguagesV1]; } if (!str) { @@ -380,39 +407,37 @@ XEPACKEDSTRUCT(XContentMetadata, { } string_util::copy_and_swap_truncating(str, value, - countof(description_chars[0])); + countof(description_raw.chars[0])); return true; } - bool set_publisher(const std::u16string_view value) { - string_util::copy_and_swap_truncating(publisher_chars, value, - countof(publisher_chars)); - return true; + void set_publisher(const std::u16string_view value) { + string_util::copy_and_swap_truncating(publisher_raw.chars, value, + countof(publisher_raw.chars)); } - bool set_title_name(const std::u16string_view value) { - string_util::copy_and_swap_truncating(title_name_chars, value, - countof(title_name_chars)); - return true; + void set_title_name(const std::u16string_view value) { + string_util::copy_and_swap_truncating(title_name_raw.chars, value, + countof(title_name_raw.chars)); } -}); +}; static_assert_size(XContentMetadata, 0x93D6); -struct XContentLicense { - be licensee_id; - be license_bits; - be license_flags; -}; -static_assert_size(XContentLicense, 0x10); - -XEPACKEDSTRUCT(XContentHeader, { +struct XContentHeader { be magic; uint8_t signature[0x228]; XContentLicense licenses[0x10]; uint8_t content_id[0x14]; be header_size; -}); + + bool is_magic_valid() const { + return magic == XContentPackageType::kCon || + magic == XContentPackageType::kLive || + magic == XContentPackageType::kPirs; + } +}; static_assert_size(XContentHeader, 0x344); +#pragma pack(pop) struct StfsHeader { XContentHeader header; @@ -425,4 +450,4 @@ static_assert_size(StfsHeader, 0x971A); } // namespace vfs } // namespace xe -#endif // XENIA_VFS_DEVICES_STFS_XBOX_H_ \ No newline at end of file +#endif // XENIA_VFS_DEVICES_STFS_XBOX_H_