diff --git a/src/xenia/base/filesystem.h b/src/xenia/base/filesystem.h index 8cce19719..d627796e7 100644 --- a/src/xenia/base/filesystem.h +++ b/src/xenia/base/filesystem.h @@ -116,6 +116,7 @@ struct FileInfo { }; Type type; std::wstring name; + std::wstring path; size_t total_size; uint64_t create_timestamp; uint64_t access_timestamp; diff --git a/src/xenia/base/filesystem_win.cc b/src/xenia/base/filesystem_win.cc index 25b0c319d..c36197a05 100644 --- a/src/xenia/base/filesystem_win.cc +++ b/src/xenia/base/filesystem_win.cc @@ -207,6 +207,7 @@ bool GetInfo(const std::wstring& path, FileInfo* out_info) { out_info->total_size = (data.nFileSizeHigh * (size_t(MAXDWORD) + 1)) + data.nFileSizeLow; } + out_info->path = xe::find_base_path(path); out_info->name = xe::find_name_from_path(path); out_info->create_timestamp = COMBINE_TIME(data.ftCreationTime); out_info->access_timestamp = COMBINE_TIME(data.ftLastAccessTime); @@ -236,6 +237,7 @@ std::vector ListFiles(const std::wstring& path) { info.total_size = (ffd.nFileSizeHigh * (size_t(MAXDWORD) + 1)) + ffd.nFileSizeLow; } + info.path = path; info.name = ffd.cFileName; info.create_timestamp = COMBINE_TIME(ffd.ftCreationTime); info.access_timestamp = COMBINE_TIME(ffd.ftLastAccessTime); diff --git a/src/xenia/vfs/devices/stfs_container_device.cc b/src/xenia/vfs/devices/stfs_container_device.cc index 977ad9398..b93173319 100644 --- a/src/xenia/vfs/devices/stfs_container_device.cc +++ b/src/xenia/vfs/devices/stfs_container_device.cc @@ -10,6 +10,7 @@ #include "xenia/vfs/devices/stfs_container_device.h" #include +#include #include #include "xenia/base/logging.h" @@ -59,87 +60,88 @@ StfsContainerDevice::StfsContainerDevice(const std::string& mount_path, StfsContainerDevice::~StfsContainerDevice() = default; bool StfsContainerDevice::Initialize() { - if (filesystem::IsFolder(local_path_)) { - // Was given a folder. Try to find the file in - // local_path\TITLE_ID\000D0000\HASH_OF_42_CHARS - // We take care to not die if there are additional files around. - bool found_alternative = false; - auto files = filesystem::ListFiles(local_path_); - for (auto& file : files) { - if (file.type != filesystem::FileInfo::Type::kDirectory || - file.name.size() != 8) { - continue; - } - auto child_path = xe::join_paths(local_path_, file.name); - auto child_files = filesystem::ListFiles(child_path); - for (auto& child_file : child_files) { - if (child_file.type != filesystem::FileInfo::Type::kDirectory || - child_file.name != L"000D0000") { - continue; - } - auto stfs_path = xe::join_paths(child_path, child_file.name); - auto stfs_files = filesystem::ListFiles(stfs_path); - for (auto& stfs_file : stfs_files) { - if (stfs_file.type != filesystem::FileInfo::Type::kFile || - stfs_file.name.size() != 42) { - continue; - } - // Probably it! - local_path_ = xe::join_paths(stfs_path, stfs_file.name); - found_alternative = true; - break; - } - if (found_alternative) { - break; - } - } - if (found_alternative) { - break; - } - } + // Resolve a valid STFS file if a directory is given. + if (filesystem::IsFolder(local_path_) && !ResolveFromFolder(local_path_)) { + XELOGE("Could not resolve an STFS container given path %s", + xe::to_string(local_path_).c_str()); + return false; } + if (!filesystem::PathExists(local_path_)) { - XELOGE("STFS container does not exist"); + XELOGE("Path to STFS container does not exist: %s", + xe::to_string(local_path_).c_str()); return false; } - mmap_ = MappedMemory::Open(local_path_, MappedMemory::Mode::kRead); - if (!mmap_) { - XELOGE("STFS container could not be mapped"); - return false; - } - - uint8_t* map_ptr = mmap_->data(); - - auto result = ReadHeaderAndVerify(map_ptr); - if (result != Error::kSuccess) { - XELOGE("STFS header read/verification failed: %d", result); + // Map the data file(s) + auto map_result = MapFiles(); + if (map_result != Error::kSuccess) { + XELOGE("Failed to map STFS container: %d", map_result); return false; } switch (header_.descriptor_type) { case StfsDescriptorType::kStfs: - result = ReadAllEntriesSTFS(map_ptr); + return ReadSTFS() == Error::kSuccess; break; case StfsDescriptorType::kSvod: - if (!(header_.svod_volume_descriptor.device_features & - kFeatureHasEnhancedGDFLayout)) { - XELOGE("STFS SVOD does not have GDF layout!"); - return false; - } - - result = ReadAllEntriesEGDF(map_ptr); - break; + return ReadSVOD() == Error::kSuccess; default: - // Shouldn't reach here. + XELOGE("Unknown STFS Descriptor Type: %d", header_.descriptor_type); return false; } - if (result != Error::kSuccess) { - XELOGE("STFS entry reading failed: %d", result); - return false; +} + +StfsContainerDevice::Error StfsContainerDevice::MapFiles() { + // Map the file containing the STFS Header and read it. + XELOGI("Mapping STFS Header File: %s", xe::to_string(local_path_).c_str()); + auto header_map = MappedMemory::Open(local_path_, MappedMemory::Mode::kRead); + + auto header_result = ReadHeaderAndVerify(header_map->data()); + if (header_result != Error::kSuccess) { + XELOGE("Error reading STFS Header: %d", header_result); + return header_result; } - return true; + // If the STFS package is a single file, the header is self contained and + // we don't need to map any extra files. + // NOTE: data_file_count is 0 for STFS and 1 for SVOD + if (header_.data_file_count <= 1) { + XELOGI("STFS container is a single file."); + mmap_.emplace(std::make_pair(0, std::move(header_map))); + return Error::kSuccess; + } + + // If the STFS package is multi-file, it is an SVOD system. We need to map + // the files in the .data folder and can discard the header. + auto data_fragment_path = local_path_ + L".data"; + if (!filesystem::PathExists(data_fragment_path)) { + XELOGE("STFS container is multi-file, but path %s does not exist.", + xe::to_string(data_fragment_path).c_str()); + return Error::kErrorFileMismatch; + } + + // Ensure data fragment files are sorted + auto fragment_files = filesystem::ListFiles(data_fragment_path); + std::sort(fragment_files.begin(), fragment_files.end(), + [](filesystem::FileInfo& left, filesystem::FileInfo& right) { + return left.name < right.name; + }); + + if (fragment_files.size() != header_.data_file_count) { + XELOGE("SVOD expecting %d data fragments, but %d are present.", + header_.data_file_count, fragment_files.size()); + return Error::kErrorFileMismatch; + } + + for (size_t i = 0; i < fragment_files.size(); i++) { + auto file = fragment_files.at(i); + auto path = xe::join_paths(file.path, file.name); + auto data = MappedMemory::Open(path, MappedMemory::Mode::kRead); + mmap_.emplace(std::make_pair(i, std::move(data))); + } + XELOGI("SVOD successfully mapped %d files.", fragment_files.size()); + return Error::kSuccess; } void StfsContainerDevice::Dump(StringBuffer* string_buffer) { @@ -196,87 +198,189 @@ StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify( return Error::kSuccess; } -StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesEGDF( - const uint8_t* map_ptr) { - // Verify (and scan) the GDF magic first. - const uint8_t* p = map_ptr + BlockToOffsetSTFS(0); - if (std::memcmp(p, "MICROSOFT*XBOX*MEDIA", 20) != 0) { - return Error::kErrorDamagedFile; +StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() { + // SVOD Systems can have different layouts. The root block is + // denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in + // the first "actual" data fragment of the system. + auto data = mmap_.at(0)->data(); + const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA"; + + // Check for EDGF layout + auto layout = &header_.svod_volume_descriptor.layout_type; + auto features = header_.svod_volume_descriptor.device_features; + bool has_egdf_layout = features & kFeatureHasEnhancedGDFLayout; + + if (has_egdf_layout) { + // The STFS header has specified that this SVOD system uses the EGDF layout. + // We can expect the magic block to be located immediately after the hash + // blocks. We also offset block address calculation by 0x1000 by shifting + // block indices by +0x2. + if (memcmp(data + 0x2000, MEDIA_MAGIC, 20) == 0) { + base_offset_ = 0x0000; + magic_offset_ = 0x2000; + *layout = kEnhancedGDFLayout; + XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000."); + } else { + XELOGE("SVOD uses an EGDF layout, but the magic block was not found."); + return Error::kErrorFileMismatch; + } + } else if (memcmp(data + 0x12000, MEDIA_MAGIC, 20) == 0) { + // If the SVOD's magic block is at 0x12000, it is likely using an XSF + // layout. This is usually due to converting the game using a third-party + // tool, as most of them use a nulled XSF as a template. + + base_offset_ = 0x10000; + magic_offset_ = 0x12000; + + // Check for XSF Header + const char* XSF_MAGIC = "XSF"; + if (memcmp(data + 0x2000, XSF_MAGIC, 3) == 0) { + *layout = kXSFLayout; + XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000."); + XELOGI("Game was likely converted using a third-party tool."); + } else { + *layout = kUnknownLayout; + XELOGI("SVOD appears to use an XSF layout, but no header is present."); + XELOGI("SVOD magic block found at 0x12000"); + } + } else if (memcmp(data + 0xD000, MEDIA_MAGIC, 20) == 0) { + // If the SVOD's magic block is at 0xD000, it most likely means that it is + // a single-file system. The STFS Header is 0xB000 bytes , and the remaining + // 0x2000 is from hash tables. In most cases, these will be STFS, not SVOD. + + base_offset_ = 0xB000; + magic_offset_ = 0xD000; + + // Check for single file system + if (header_.data_file_count == 1) { + *layout = kSingleFileLayout; + XELOGI("SVOD is a single file. Magic block present at 0xD000."); + } else { + *layout = kUnknownLayout; + XELOGE( + "SVOD is not a single file, but the magic block was found at " + "0xD000."); + } + } else { + XELOGE("Could not locate SVOD magic block."); + return Error::kErrorReadError; } - uint32_t root_sector = xe::load(p + 0x14); - uint32_t root_size = xe::load(p + 0x18); + // Parse the root directory + uint8_t* magic_block = data + magic_offset_; + uint32_t root_block = xe::load(magic_block + 0x14); + uint32_t root_size = xe::load(magic_block + 0x18); + uint32_t root_creation_date = xe::load(magic_block + 0x1C); + uint32_t root_creation_time = xe::load(magic_block + 0x20); + uint64_t root_creation_timestamp = + decode_fat_timestamp(root_creation_date, root_creation_time); - auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get()); + auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_); root_entry->attributes_ = kFileAttributeDirectory; + root_entry->access_timestamp_ = root_creation_timestamp; + root_entry->create_timestamp_ = root_creation_timestamp; + root_entry->write_timestamp_ = root_creation_timestamp; root_entry_ = std::unique_ptr(root_entry); - const uint8_t* buffer = map_ptr + BlockToOffsetEGDF(root_sector); - return ReadEntryEGDF(buffer, 0, root_entry) ? Error::kSuccess - : Error::kErrorDamagedFile; + // Traverse all child entries + return ReadEntrySVOD(root_block, 0, root_entry); } -bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer, - uint16_t entry_ordinal, - StfsContainerEntry* parent) { - const uint8_t* p = buffer + (entry_ordinal * 4); +StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD( + uint32_t block, uint32_t ordinal, StfsContainerEntry* parent) { + // For games with a large amount of files, the ordinal offset can overrun + // the current block and potentially hit a hash block. + size_t ordinal_offset = ordinal * 0x4; + size_t block_offset = ordinal_offset / 0x800; + size_t true_ordinal_offset = ordinal_offset % 0x800; - uint16_t node_l = xe::load(p + 0); - uint16_t node_r = xe::load(p + 2); - uint32_t sector = xe::load(p + 4); - uint32_t length = xe::load(p + 8); - uint8_t attributes = xe::load(p + 12); - uint8_t name_length = xe::load(p + 13); - auto name = reinterpret_cast(p + 14); + // Calculate the file & address of the block + size_t entry_address, entry_file; + BlockToOffsetSVOD(block + block_offset, &entry_address, &entry_file); + entry_address += true_ordinal_offset; - if (node_l && !ReadEntryEGDF(buffer, node_l, parent)) { - return false; + // Read block's descriptor + auto data = mmap_.at(entry_file)->data() + entry_address; + + uint16_t node_l = xe::load(data + 0x00); + uint16_t node_r = xe::load(data + 0x02); + uint32_t data_block = xe::load(data + 0x04); + uint32_t length = xe::load(data + 0x08); + uint8_t attributes = xe::load(data + 0x0C); + uint8_t name_length = xe::load(data + 0x0D); + auto name = reinterpret_cast(data + 0x0E); + auto name_str = std::string(name, name_length); + + // Read the left node + if (node_l) { + auto node_result = ReadEntrySVOD(block, node_l, parent); + if (node_result != Error::kSuccess) { + return node_result; + } } - auto entry = StfsContainerEntry::Create( - this, parent, std::string(name, name_length), mmap_.get()); + // Read file & address of block's data + size_t data_address, data_file; + BlockToOffsetSVOD(data_block, &data_address, &data_file); + + // Create the entry + // NOTE: SVOD entries don't have timestamps for individual files, which can + // cause issues when decrypting games. Using the root entry's timestamp + // solves this issues. + auto entry = StfsContainerEntry::Create(this, parent, name_str, &mmap_); if (attributes & kFileAttributeDirectory) { - // Folder. + // Entry is a directory entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly; entry->data_offset_ = 0; entry->data_size_ = 0; + entry->block_ = block; + entry->access_timestamp_ = root_entry_->create_timestamp(); + entry->create_timestamp_ = root_entry_->create_timestamp(); + entry->write_timestamp_ = root_entry_->create_timestamp(); if (length) { - // Not a leaf - read in children. - uint8_t* folder_ptr = mmap_->data() + BlockToOffsetEGDF(sector); - if (!ReadEntryEGDF(folder_ptr, 0, entry.get())) { - return false; + // If length is greater than 0, traverse the directory's children + auto directory_result = ReadEntrySVOD(data_block, 0, entry.get()); + if (directory_result != Error::kSuccess) { + return directory_result; } } } else { - // Regular file. + // Entry is a file entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly; entry->size_ = length; entry->allocation_size_ = xe::round_up(length, bytes_per_sector()); - entry->data_offset_ = BlockToOffsetEGDF(sector); + entry->data_offset_ = data_address; entry->data_size_ = length; + entry->block_ = data_block; + entry->access_timestamp_ = root_entry_->create_timestamp(); + entry->create_timestamp_ = root_entry_->create_timestamp(); + entry->write_timestamp_ = root_entry_->create_timestamp(); // Fill in all block records, sector by sector. if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) { - uint32_t sector_index = sector; + uint32_t block_index = data_block; size_t remaining_size = xe::round_up(length, 0x800); size_t last_record = -1; size_t last_offset = -1; while (remaining_size) { - size_t block_size = 0x800; - size_t offset = BlockToOffsetEGDF(sector_index); - sector_index++; - remaining_size -= block_size; + const size_t BLOCK_SIZE = 0x800; + + size_t offset, file_index; + BlockToOffsetSVOD(block_index, &offset, &file_index); + + block_index++; + remaining_size -= BLOCK_SIZE; if (offset - last_offset == 0x800) { // Consecutive, so append to last entry. - entry->block_list_[last_record].length += block_size; + entry->block_list_[last_record].length += BLOCK_SIZE; last_offset = offset; continue; } - entry->block_list_.push_back({offset, block_size}); + entry->block_list_.push_back({file_index, offset, BLOCK_SIZE}); last_record = entry->block_list_.size() - 1; last_offset = offset; } @@ -285,17 +389,82 @@ bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer, parent->children_.emplace_back(std::move(entry)); - // Read next file in the list. - if (node_r && !ReadEntryEGDF(buffer, node_r, parent)) { - return false; + // Read the right node. + if (node_r) { + auto node_result = ReadEntrySVOD(block, node_r, parent); + if (node_result != Error::kSuccess) { + return node_result; + } } - return true; + return Error::kSuccess; } -StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS( - const uint8_t* map_ptr) { - auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get()); +void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address, + size_t* out_file_index) { + // SVOD Systems use hash blocks for integrity checks. These hash blocks + // cause blocks to be discontinuous in memory, and must be accounted for. + // - Each data block is 0x800 bytes in length + // - Every group of 0x198 data blocks is preceded a Level0 hash table. + // Level0 tables contain 0xCC hashes, each representing two data blocks. + // The total size of each Level0 hash table is 0x1000 bytes in length. + // - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table. + // Level1 tables contain 0xCB hashes, each representing two Level0 hashes. + // The total size of each Level1 hash table is 0x1000 bytes in length. + // - Files are split into fragments of 0xA290000 bytes in length, + // consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1 + // Level1 hash table. + + const size_t BLOCK_SIZE = 0x800; + const size_t HASH_BLOCK_SIZE = 0x1000; + const size_t BLOCKS_PER_L0_HASH = 0x198; + const size_t HASHES_PER_L1_HASH = 0xA1C4; + const size_t BLOCKS_PER_FILE = 0x14388; + const size_t MAX_FILE_SIZE = 0xA290000; + const size_t BLOCK_OFFSET = header_.svod_volume_descriptor.data_block_offset; + const SvodLayoutType LAYOUT = header_.svod_volume_descriptor.layout_type; + + // Resolve the true block address and file index + size_t true_block = block - (BLOCK_OFFSET * 2); + if (LAYOUT == kEnhancedGDFLayout) { + // EGDF has an 0x1000 byte offset, which is two blocks + true_block += 0x2; + } + + size_t file_block = true_block % BLOCKS_PER_FILE; + size_t file_index = true_block / BLOCKS_PER_FILE; + size_t offset = 0; + + // Calculate offset caused by Level0 Hash Tables + size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1; + offset += level0_table_count * HASH_BLOCK_SIZE; + + // Calculate offset caused by Level1 Hash Tables + size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1; + offset += level1_table_count * HASH_BLOCK_SIZE; + + // For single-file SVOD layouts, include the size of the header in the offset. + if (LAYOUT == kSingleFileLayout) { + offset += base_offset_; + } + + size_t block_address = (file_block * BLOCK_SIZE) + offset; + + // If the offset causes the block address to overrun the file, round it. + if (block_address >= MAX_FILE_SIZE) { + file_index += 1; + block_address %= MAX_FILE_SIZE; + block_address += 0x2000; + } + + *out_address = block_address; + *out_file_index = file_index; +} + +StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() { + auto data = mmap_.at(0)->data(); + + auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_); root_entry->attributes_ = kFileAttributeDirectory; root_entry_ = std::unique_ptr(root_entry); @@ -305,7 +474,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS( auto& volume_descriptor = header_.stfs_volume_descriptor; uint32_t table_block_index = volume_descriptor.file_table_block_number; for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) { - const uint8_t* p = map_ptr + BlockToOffsetSTFS(table_block_index); + const uint8_t* p = data + BlockToOffsetSTFS(table_block_index); for (size_t m = 0; m < 0x1000 / 0x40; m++) { const uint8_t* filename = p; // 0x28b if (filename[0] == 0) { @@ -333,11 +502,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS( parent_entry = all_entries[path_indicator]; } - auto entry = StfsContainerEntry::Create( - this, parent_entry, - std::string(reinterpret_cast(filename), - filename_length_flags & 0x3F), - mmap_.get()); + std::string name_str(reinterpret_cast(filename), + filename_length_flags & 0x3F); + auto entry = + StfsContainerEntry::Create(this, parent_entry, name_str, &mmap_); + // bit 0x40 = consecutive blocks (not fragmented?) if (filename_length_flags & 0x80) { entry->attributes_ = kFileAttributeDirectory; @@ -367,11 +536,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS( size_t block_size = std::min(static_cast(0x1000), remaining_size); size_t offset = BlockToOffsetSTFS(block_index); - entry->block_list_.push_back({offset, block_size}); + entry->block_list_.push_back({0, offset, block_size}); remaining_size -= block_size; - auto block_hash = GetBlockHash(map_ptr, block_index, 0); + auto block_hash = GetBlockHash(data, block_index, 0); if (table_size_shift_ && block_hash.info < 0x80) { - block_hash = GetBlockHash(map_ptr, block_index, 1); + block_hash = GetBlockHash(data, block_index, 1); } block_index = block_hash.next_block_index; info = block_hash.info; @@ -381,9 +550,9 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS( parent_entry->children_.emplace_back(std::move(entry)); } - auto block_hash = GetBlockHash(map_ptr, table_block_index, 0); + auto block_hash = GetBlockHash(data, table_block_index, 0); if (table_size_shift_ && block_hash.info < 0x80) { - block_hash = GetBlockHash(map_ptr, table_block_index, 1); + block_hash = GetBlockHash(data, table_block_index, 1); } table_block_index = block_hash.next_block_index; if (table_block_index == 0xFFFFFF) { @@ -402,41 +571,25 @@ size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) { block_shift = package_type_ == StfsPackageType::kCon ? 1 : 0; } - if (header_.descriptor_type == StfsDescriptorType::kStfs) { - // For every level there is a hash table - // Level 0: hash table of next 170 blocks - // Level 1: hash table of next 170 hash tables - // Level 2: hash table of next 170 level 1 hash tables - // And so on... - uint64_t base = kSTFSHashSpacing; - block = block_index; - for (uint32_t i = 0; i < 3; i++) { - block += (block_index + (base << block_shift)) / (base << block_shift); - if (block_index < base) { - break; - } - - base *= kSTFSHashSpacing; + // For every level there is a hash table + // Level 0: hash table of next 170 blocks + // Level 1: hash table of next 170 hash tables + // Level 2: hash table of next 170 level 1 hash tables + // And so on... + uint64_t base = kSTFSHashSpacing; + block = block_index; + for (uint32_t i = 0; i < 3; i++) { + block += (block_index + (base << block_shift)) / (base << block_shift); + if (block_index < base) { + break; } - } else if (header_.descriptor_type == StfsDescriptorType::kSvod) { - // Level 0: Hashes for the next 204 blocks - // Level 1: Hashes for the next 203 hash blocks + 1 for the next level 0 - // 10......[204 blocks].....0.....[204 blocks].....0 - // There are 0xA1C4 (41412) blocks for every level 1 hash table. - block = block_index; - block += (block_index + 204) / 204; // Level 0 - block += (block_index + 204 * 203) / (204 * 203); // Level 1 + + base *= kSTFSHashSpacing; } return xe::round_up(header_.header_size, 0x1000) + (block << 12); } -size_t StfsContainerDevice::BlockToOffsetEGDF(uint64_t sector) { - size_t offset = BlockToOffsetSTFS( - (sector / 2) - header_.svod_volume_descriptor.data_block_offset + 1); - return offset + ((sector & 0x1) << 11); // Sectors are 0x800 bytes. -} - StfsContainerDevice::BlockHash StfsContainerDevice::GetBlockHash( const uint8_t* map_ptr, uint32_t block_index, uint32_t table_offset) { uint32_t record = block_index % 0xAA; @@ -496,11 +649,6 @@ bool StfsHeader::Read(const uint8_t* p) { header_size = xe::load_and_swap(p + 0x340); content_type = (StfsContentType)xe::load_and_swap(p + 0x344); metadata_version = xe::load_and_swap(p + 0x348); - if (metadata_version > 1) { - // This is a variant of thumbnail data/etc. - // Can just ignore it for now (until we parse thumbnails). - XELOGW("STFSContainer doesn't support version %d yet", metadata_version); - } content_size = xe::load_and_swap(p + 0x34C); media_id = xe::load_and_swap(p + 0x354); version = xe::load_and_swap(p + 0x358); @@ -541,8 +689,70 @@ bool StfsHeader::Read(const uint8_t* p) { title_thumbnail_image_size = xe::load_and_swap(p + 0x1716); std::memcpy(thumbnail_image, p + 0x171A, 0x4000); std::memcpy(title_thumbnail_image, p + 0x571A, 0x4000); + + // Metadata v2 Fields + if (metadata_version == 2) { + std::memcpy(series_id, p + 0x3B1, 0x10); + std::memcpy(season_id, p + 0x3C1, 0x10); + season_number = xe::load_and_swap(p + 0x3D1); + episode_number = xe::load_and_swap(p + 0x3D5); + + for (size_t n = 0; n < 0x300 / 2; n++) { + additonal_display_names[n] = + xe::load_and_swap(p + 0x541A + n * 2); + additional_display_descriptions[n] = + xe::load_and_swap(p + 0x941A + n * 2); + } + } + + return true; +} + +const char* StfsContainerDevice::ReadMagic(const std::wstring& path) { + auto map = MappedMemory::Open(path, MappedMemory::Mode::kRead, 0, 4); + auto magic_data = xe::load(map->data()); + auto magic_bytes = static_cast(static_cast(&magic_data)); + return std::move(magic_bytes); +} + +bool StfsContainerDevice::ResolveFromFolder(const std::wstring& path) { + // Scan through folders until a file with magic is found + std::queue queue; + + filesystem::FileInfo folder; + filesystem::GetInfo(local_path_, &folder); + queue.push(folder); + + while (!queue.empty()) { + auto current_file = queue.front(); + queue.pop(); + + if (current_file.type == filesystem::FileInfo::Type::kDirectory) { + auto path = xe::join_paths(current_file.path, current_file.name); + auto child_files = filesystem::ListFiles(path); + for (auto file : child_files) { + queue.push(file); + } + } else { + // Try to read the file's magic + auto path = xe::join_paths(current_file.path, current_file.name); + auto magic = ReadMagic(path); + + if (memcmp(magic, "LIVE", 4) == 0 || memcmp(magic, "PIRS", 4) == 0 || + memcmp(magic, "CON ", 4) == 0) { + local_path_ = xe::join_paths(current_file.path, current_file.name); + XELOGI("STFS Package found: %s", xe::to_string(local_path_).c_str()); + return true; + } + } + } + + if (local_path_ == path) { + // Could not find a suitable container file + return false; + } return true; } } // namespace vfs -} // namespace xe +} // namespace xe \ No newline at end of file diff --git a/src/xenia/vfs/devices/stfs_container_device.h b/src/xenia/vfs/devices/stfs_container_device.h index 16514761f..9819084f0 100644 --- a/src/xenia/vfs/devices/stfs_container_device.h +++ b/src/xenia/vfs/devices/stfs_container_device.h @@ -10,6 +10,7 @@ #ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_ #define XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_ +#include #include #include @@ -91,6 +92,13 @@ enum SvodDeviceFeatures { kFeatureHasEnhancedGDFLayout = 0x40, }; +enum SvodLayoutType { + kUnknownLayout = 0x0, + kEnhancedGDFLayout = 0x1, + kXSFLayout = 0x2, + kSingleFileLayout = 0x4, +}; + struct SvodVolumeDescriptor { bool Read(const uint8_t* p); @@ -103,6 +111,8 @@ struct SvodVolumeDescriptor { uint32_t data_block_count; uint32_t data_block_offset; // 0x5 padding bytes... + + SvodLayoutType layout_type; }; class StfsHeader { @@ -143,6 +153,14 @@ class StfsHeader { uint32_t title_thumbnail_image_size; uint8_t thumbnail_image[0x4000]; uint8_t title_thumbnail_image[0x4000]; + + // Metadata v2 Fields + uint8_t series_id[0x10]; + uint8_t season_id[0x10]; + int16_t season_number; + int16_t episode_number; + wchar_t additonal_display_names[0x300 / 2]; + wchar_t additional_display_descriptions[0x300 / 2]; }; class StfsContainerDevice : public Device { @@ -156,7 +174,7 @@ class StfsContainerDevice : public Device { Entry* ResolvePath(std::string path) override; uint32_t total_allocation_units() const override { - return uint32_t(mmap_->size() / sectors_per_allocation_unit() / + return uint32_t(mmap_total_size_ / sectors_per_allocation_unit() / bytes_per_sector()); } uint32_t available_allocation_units() const override { return 0; } @@ -178,23 +196,30 @@ class StfsContainerDevice : public Device { }; const uint32_t kSTFSHashSpacing = 170; - const uint32_t kSVODHashSpacing = 204; + const char* ReadMagic(const std::wstring& path); + bool ResolveFromFolder(const std::wstring& path); + + Error MapFiles(); Error ReadHeaderAndVerify(const uint8_t* map_ptr); - Error ReadAllEntriesEGDF(const uint8_t* map_ptr); - bool ReadEntryEGDF(const uint8_t* buffer, uint16_t entry_ordinal, - StfsContainerEntry* parent); - Error ReadAllEntriesSTFS(const uint8_t* map_ptr); + Error ReadSVOD(); + Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal, + StfsContainerEntry* parent); + void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index); + + Error ReadSTFS(); size_t BlockToOffsetSTFS(uint64_t block); - size_t BlockToOffsetEGDF(uint64_t block); BlockHash GetBlockHash(const uint8_t* map_ptr, uint32_t block_index, uint32_t table_offset); std::wstring local_path_; - std::unique_ptr mmap_; + std::map> mmap_; + size_t mmap_total_size_; + size_t base_offset_; + size_t magic_offset_; std::unique_ptr root_entry_; StfsPackageType package_type_; StfsHeader header_; diff --git a/src/xenia/vfs/devices/stfs_container_entry.cc b/src/xenia/vfs/devices/stfs_container_entry.cc index 71c2440ba..63c1a4df1 100644 --- a/src/xenia/vfs/devices/stfs_container_entry.cc +++ b/src/xenia/vfs/devices/stfs_container_entry.cc @@ -8,15 +8,17 @@ */ #include "xenia/vfs/devices/stfs_container_entry.h" - #include "xenia/base/math.h" #include "xenia/vfs/devices/stfs_container_file.h" +#include + namespace xe { namespace vfs { StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent, - std::string path, MappedMemory* mmap) + std::string path, + MultifileMemoryMap* mmap) : Entry(device, parent, path), mmap_(mmap), data_offset_(0), @@ -25,7 +27,7 @@ StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent, StfsContainerEntry::~StfsContainerEntry() = default; std::unique_ptr StfsContainerEntry::Create( - Device* device, Entry* parent, std::string name, MappedMemory* mmap) { + Device* device, Entry* parent, std::string name, MultifileMemoryMap* mmap) { auto path = xe::join_paths(parent->path(), name); auto entry = std::make_unique(device, parent, path, mmap); @@ -38,4 +40,4 @@ X_STATUS StfsContainerEntry::Open(uint32_t desired_access, File** out_file) { } } // namespace vfs -} // namespace xe +} // namespace xe \ No newline at end of file diff --git a/src/xenia/vfs/devices/stfs_container_entry.h b/src/xenia/vfs/devices/stfs_container_entry.h index 279c83523..4ffe13423 100644 --- a/src/xenia/vfs/devices/stfs_container_entry.h +++ b/src/xenia/vfs/devices/stfs_container_entry.h @@ -10,6 +10,7 @@ #ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ #define XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ +#include #include #include @@ -20,27 +21,30 @@ namespace xe { namespace vfs { +typedef std::map> MultifileMemoryMap; class StfsContainerDevice; class StfsContainerEntry : public Entry { public: StfsContainerEntry(Device* device, Entry* parent, std::string path, - MappedMemory* mmap); + MultifileMemoryMap* mmap); ~StfsContainerEntry() override; static std::unique_ptr Create(Device* device, Entry* parent, std::string name, - MappedMemory* mmap); + MultifileMemoryMap* mmap); - MappedMemory* mmap() const { return mmap_; } + MultifileMemoryMap* mmap() const { return mmap_; } size_t data_offset() const { return data_offset_; } size_t data_size() const { return data_size_; } + size_t block() const { return block_; } X_STATUS Open(uint32_t desired_access, File** out_file) override; struct BlockRecord { + size_t file; size_t offset; size_t length; }; @@ -49,13 +53,14 @@ class StfsContainerEntry : public Entry { private: friend class StfsContainerDevice; - MappedMemory* mmap_; + MultifileMemoryMap* mmap_; size_t data_offset_; size_t data_size_; + size_t block_; std::vector block_list_; }; } // namespace vfs } // namespace xe -#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ +#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_ \ No newline at end of file diff --git a/src/xenia/vfs/devices/stfs_container_file.cc b/src/xenia/vfs/devices/stfs_container_file.cc index 06e8b52c4..dd6c7e2ad 100644 --- a/src/xenia/vfs/devices/stfs_container_file.cc +++ b/src/xenia/vfs/devices/stfs_container_file.cc @@ -34,7 +34,6 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length, } size_t src_offset = 0; - uint8_t* src = entry_->mmap()->data(); uint8_t* p = reinterpret_cast(buffer); size_t remaining_length = std::min(buffer_length, entry_->size() - byte_offset); @@ -48,6 +47,8 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length, continue; } + uint8_t* src = entry_->mmap()->at(record.file)->data(); + size_t read_offset = (byte_offset > src_offset) ? byte_offset - src_offset : 0; size_t read_length = @@ -66,4 +67,4 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length, } } // namespace vfs -} // namespace xe +} // namespace xe \ No newline at end of file