ShuriZma
/
xenia-canary
mirror of https://github.com/xenia-canary/xenia-canary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[VFS] SVOD (with EGDF) support

This commit is contained in:
Dr. Chat 2018-05-29 14:10:09 -05:00
parent 62bb26878f
commit a0464f54ba
4 changed files with 250 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

239
src/xenia/vfs/devices/stfs_container_device.cc
View File

@ -117,7 +117,23 @@ bool StfsContainerDevice::Initialize() {
return false;
}
result = ReadAllEntries(map_ptr);
switch (header_.descriptor_type) {
case StfsDescriptorType::kStfs:
result = ReadAllEntriesSTFS(map_ptr);
break;
case StfsDescriptorType::kSvod:
if (!(header_.svod_volume_descriptor.device_features &
kFeatureHasEnhancedGDFLayout)) {
XELOGE("STFS SVOD does not have GDF layout!");
return false;
}
result = ReadAllEntriesEGDF(map_ptr);
break;
default:
// Shouldn't reach here.
return false;
}
if (result != Error::kSuccess) {
XELOGE("STFS entry reading failed: %d", result);
return false;
@ -180,21 +196,116 @@ StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify(
return Error::kSuccess;
}
StfsContainerDevice::Error StfsContainerDevice::ReadAllEntries(
StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesEGDF(
const uint8_t* map_ptr) {
// Verify (and scan) the GDF magic first.
const uint8_t* p = map_ptr + BlockToOffsetSTFS(0);
if (std::memcmp(p, "MICROSOFT*XBOX*MEDIA", 20) != 0) {
return Error::kErrorDamagedFile;
}
uint32_t root_sector = xe::load<uint32_t>(p + 0x14);
uint32_t root_size = xe::load<uint32_t>(p + 0x18);
auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get());
root_entry->attributes_ = kFileAttributeDirectory;
root_entry_ = std::unique_ptr<Entry>(root_entry);
const uint8_t* buffer = map_ptr + BlockToOffsetEGDF(root_sector);
return ReadEntryEGDF(buffer, 0, root_entry) ? Error::kSuccess
: Error::kErrorDamagedFile;
}
bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer,
uint16_t entry_ordinal,
StfsContainerEntry* parent) {
const uint8_t* p = buffer + (entry_ordinal * 4);
uint16_t node_l = xe::load<uint16_t>(p + 0);
uint16_t node_r = xe::load<uint16_t>(p + 2);
uint32_t sector = xe::load<uint32_t>(p + 4);
uint32_t length = xe::load<uint32_t>(p + 8);
uint8_t attributes = xe::load<uint8_t>(p + 12);
uint8_t name_length = xe::load<uint8_t>(p + 13);
auto name = reinterpret_cast<const char*>(p + 14);
if (node_l && !ReadEntryEGDF(buffer, node_l, parent)) {
return false;
}
auto entry = StfsContainerEntry::Create(
this, parent, std::string(name, name_length), mmap_.get());
if (attributes & kFileAttributeDirectory) {
// Folder.
entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
entry->data_offset_ = 0;
entry->data_size_ = 0;
if (length) {
// Not a leaf - read in children.
uint8_t* folder_ptr = mmap_->data() + BlockToOffsetEGDF(sector);
if (!ReadEntryEGDF(folder_ptr, 0, entry.get())) {
return false;
}
}
} else {
// Regular file.
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->size_ = length;
entry->allocation_size_ = xe::round_up(length, bytes_per_sector());
entry->data_offset_ = BlockToOffsetEGDF(sector);
entry->data_size_ = length;
// Fill in all block records, sector by sector.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t sector_index = sector;
size_t remaining_size = xe::round_up(length, 0x800);
size_t last_record = -1;
size_t last_offset = -1;
while (remaining_size) {
size_t block_size = 0x800;
size_t offset = BlockToOffsetEGDF(sector_index);
sector_index++;
remaining_size -= block_size;
if (offset - last_offset == 0x800) {
// Consecutive, so append to last entry.
entry->block_list_[last_record].length += block_size;
last_offset = offset;
continue;
}
entry->block_list_.push_back({offset, block_size});
last_record = entry->block_list_.size() - 1;
last_offset = offset;
}
}
}
parent->children_.emplace_back(std::move(entry));
// Read next file in the list.
if (node_r && !ReadEntryEGDF(buffer, node_r, parent)) {
return false;
}
return true;
}
StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
const uint8_t* map_ptr) {
auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get());
root_entry->attributes_ = kFileAttributeDirectory;
root_entry_ = std::unique_ptr<Entry>(root_entry);
std::vector<StfsContainerEntry*> all_entries;
// Load all listings.
auto& volume_descriptor = header_.volume_descriptor;
auto& volume_descriptor = header_.stfs_volume_descriptor;
uint32_t table_block_index = volume_descriptor.file_table_block_number;
for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) {
const uint8_t* p =
map_ptr + BlockToOffset(ComputeBlockNumber(table_block_index));
const uint8_t* p = map_ptr + BlockToOffsetSTFS(table_block_index);
for (size_t m = 0; m < 0x1000 / 0x40; m++) {
const uint8_t* filename = p; // 0x28b
if (filename[0] == 0) {
@ -232,8 +343,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntries(
entry->attributes_ = kFileAttributeDirectory;
} else {
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->data_offset_ =
BlockToOffset(ComputeBlockNumber(start_block_index));
entry->data_offset_ = BlockToOffsetSTFS(start_block_index);
entry->data_size_ = file_size;
}
entry->size_ = file_size;
@ -256,7 +366,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntries(
while (remaining_size && block_index && info >= 0x80) {
size_t block_size =
std::min(static_cast<size_t>(0x1000), remaining_size);
size_t offset = BlockToOffset(ComputeBlockNumber(block_index));
size_t offset = BlockToOffsetSTFS(block_index);
entry->block_list_.push_back({offset, block_size});
remaining_size -= block_size;
auto block_hash = GetBlockHash(map_ptr, block_index, 0);
@ -284,46 +394,47 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntries(
return Error::kSuccess;
}
size_t StfsContainerDevice::BlockToOffset(uint32_t block) {
if (block >= 0xFFFFFF) {
return ~0ull;
} else {
return ((header_.header_size + 0x0FFF) & 0xF000) + (block << 12);
size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) {
uint64_t block;
uint32_t block_shift = 0;
if (((header_.header_size + 0x0FFF) & 0xB000) == 0xB000 ||
(header_.stfs_volume_descriptor.flags & 0x1) == 0x0) {
block_shift = package_type_ == StfsPackageType::kCon ? 1 : 0;
}
if (header_.descriptor_type == StfsDescriptorType::kStfs) {
// For every level there is a hash table
// Level 0: hash table of next 170 blocks
// Level 1: hash table of next 170 hash tables
// Level 2: hash table of next 170 level 1 hash tables
// And so on...
uint64_t base = kSTFSHashSpacing;
block = block_index;
for (uint32_t i = 0; i < 3; i++) {
block += (block_index + (base << block_shift)) / (base << block_shift);
if (block_index < base) {
break;
}
base *= kSTFSHashSpacing;
}
} else if (header_.descriptor_type == StfsDescriptorType::kSvod) {
// Level 0: Hashes for the next 204 blocks
// Level 1: Hashes for the next 203 hash blocks + 1 for the next level 0
// 10......[204 blocks].....0.....[204 blocks].....0
// There are 0xA1C4 (41412) blocks for every level 1 hash table.
block = block_index;
block += (block_index + 204) / 204; // Level 0
block += (block_index + 204 * 203) / (204 * 203); // Level 1
}
return xe::round_up(header_.header_size, 0x1000) + (block << 12);
}
uint32_t StfsContainerDevice::ComputeBlockNumber(uint32_t block_index) {
uint32_t block_shift = 0;
if (((header_.header_size + 0x0FFF) & 0xB000) == 0xB000) {
block_shift = 1;
} else {
if ((header_.volume_descriptor.flags & 0x1) == 0x1) {
block_shift = 0;
} else {
block_shift = 1;
}
}
uint32_t base = (block_index + 0xAA) / 0xAA;
if (package_type_ == StfsPackageType::kCon) {
base <<= block_shift;
}
uint32_t block = base + block_index;
if (block_index >= 0xAA) {
base = (block_index + 0x70E4) / 0x70E4;
if (package_type_ == StfsPackageType::kCon) {
base <<= block_shift;
}
block += base;
if (block_index >= 0x70E4) {
base = (block_index + 0x4AF768) / 0x4AF768;
if (package_type_ == StfsPackageType::kCon) {
base <<= block_shift;
}
block += base;
}
}
return block;
size_t StfsContainerDevice::BlockToOffsetEGDF(uint64_t sector) {
size_t offset = BlockToOffsetSTFS(
(sector / 2) - header_.svod_volume_descriptor.data_block_offset + 1);
return offset + ((sector & 0x1) << 11); // Sectors are 0x800 bytes.
}
StfsContainerDevice::BlockHash StfsContainerDevice::GetBlockHash(
@ -344,7 +455,7 @@ StfsContainerDevice::BlockHash StfsContainerDevice::GetBlockHash(
}
}
// table_index += table_offset - (1 << table_size_shift_);
const uint8_t* hash_data = map_ptr + BlockToOffset(table_index);
const uint8_t* hash_data = map_ptr + BlockToOffsetSTFS(table_index);
const uint8_t* record_data = hash_data + record * 0x18;
uint32_t info = xe::load_and_swap<uint8_t>(record_data + 0x14);
uint32_t next_block_index = load_uint24_be(record_data + 0x15);
@ -368,6 +479,24 @@ bool StfsVolumeDescriptor::Read(const uint8_t* p) {
return true;
}
bool SvodVolumeDescriptor::Read(const uint8_t* p) {
descriptor_size = xe::load<uint8_t>(p + 0x00);
if (descriptor_size != 0x24) {
XELOGE("SVOD volume descriptor size mismatch, expected 0x24 but got 0x%X",
descriptor_size);
return false;
}
block_cache_element_count = xe::load<uint8_t>(p + 0x01);
worker_thread_processor = xe::load<uint8_t>(p + 0x02);
worker_thread_priority = xe::load<uint8_t>(p + 0x03);
std::memcpy(hash, p + 0x04, 0x14);
device_features = xe::load<uint8_t>(p + 0x18);
data_block_count = load_uint24_be(p + 0x19);
data_block_offset = load_uint24_le(p + 0x1C);
return true;
}
bool StfsHeader::Read(const uint8_t* p) {
std::memcpy(license_entries, p + 0x22C, 0x100);
std::memcpy(header_hash, p + 0x32C, 0x14);
@ -394,12 +523,16 @@ bool StfsHeader::Read(const uint8_t* p) {
data_file_count = xe::load_and_swap<uint32_t>(p + 0x39D);
data_file_combined_size = xe::load_and_swap<uint64_t>(p + 0x3A1);
descriptor_type = (StfsDescriptorType)xe::load_and_swap<uint32_t>(p + 0x3A9);
if (descriptor_type != StfsDescriptorType::kStfs) {
XELOGE("STFS descriptor format not supported: %d", descriptor_type);
return false;
}
if (!volume_descriptor.Read(p + 0x379)) {
return false;
switch (descriptor_type) {
case StfsDescriptorType::kStfs:
stfs_volume_descriptor.Read(p + 0x379);
break;
case StfsDescriptorType::kSvod:
svod_volume_descriptor.Read(p + 0x379);
break;
default:
XELOGE("STFS descriptor format not supported: %d", descriptor_type);
return false;
}
memcpy(device_id, p + 0x3FD, 0x14);
for (size_t n = 0; n < 0x900 / 2; n++) {

38
src/xenia/vfs/devices/stfs_container_device.h
View File

@ -21,6 +21,8 @@ namespace vfs {
// http://www.free60.org/wiki/STFS
class StfsContainerEntry;
enum class StfsPackageType {
kCon,
kPirs,
@ -85,6 +87,24 @@ struct StfsVolumeDescriptor {
uint32_t total_unallocated_block_count;
};
enum SvodDeviceFeatures {
kFeatureHasEnhancedGDFLayout = 0x40,
};
struct SvodVolumeDescriptor {
bool Read(const uint8_t* p);
uint8_t descriptor_size;
uint8_t block_cache_element_count;
uint8_t worker_thread_processor;
uint8_t worker_thread_priority;
uint8_t hash[0x14];
uint8_t device_features;
uint32_t data_block_count;
uint32_t data_block_offset;
// 0x5 padding bytes...
};
class StfsHeader {
public:
bool Read(const uint8_t* p);
@ -106,7 +126,10 @@ class StfsHeader {
uint32_t save_game_id;
uint8_t console_id[0x5];
uint8_t profile_id[0x8];
StfsVolumeDescriptor volume_descriptor;
union {
StfsVolumeDescriptor stfs_volume_descriptor;
SvodVolumeDescriptor svod_volume_descriptor;
};
uint32_t data_file_count;
uint64_t data_file_combined_size;
StfsDescriptorType descriptor_type;
@ -154,10 +177,17 @@ class StfsContainerDevice : public Device {
uint32_t info;
};
const uint32_t kSTFSHashSpacing = 170;
const uint32_t kSVODHashSpacing = 204;
Error ReadHeaderAndVerify(const uint8_t* map_ptr);
Error ReadAllEntries(const uint8_t* map_ptr);
size_t BlockToOffset(uint32_t block);
uint32_t ComputeBlockNumber(uint32_t block_index);
Error ReadAllEntriesEGDF(const uint8_t* map_ptr);
bool ReadEntryEGDF(const uint8_t* buffer, uint16_t entry_ordinal,
StfsContainerEntry* parent);
Error ReadAllEntriesSTFS(const uint8_t* map_ptr);
size_t BlockToOffsetSTFS(uint64_t block);
size_t BlockToOffsetEGDF(uint64_t block);
BlockHash GetBlockHash(const uint8_t* map_ptr, uint32_t block_index,
uint32_t table_offset);

47
src/xenia/vfs/devices/stfs_container_file.cc
View File

@ -12,6 +12,7 @@
#include <algorithm>
#include <cmath>
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_entry.h"
namespace xe {
@ -32,29 +33,35 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
return X_STATUS_END_OF_FILE;
}
// Each block is 4096.
// Blocks may not be sequential, so we need to read by blocks and handle the
// offsets.
size_t real_length = std::min(buffer_length, entry_->size() - byte_offset);
size_t start_block = byte_offset / 4096;
size_t end_block =
std::min(entry_->block_list().size(),
(size_t)ceil((byte_offset + real_length) / 4096.0));
uint8_t* dest_ptr = reinterpret_cast<uint8_t*>(buffer);
size_t remaining_length = real_length;
for (size_t n = start_block; n < end_block; n++) {
auto& record = entry_->block_list()[n];
size_t offset = record.offset;
size_t read_length = std::min(remaining_length, record.length);
if (n == start_block) {
offset += byte_offset % 4096;
read_length = std::min(read_length, record.length - (byte_offset % 4096));
size_t src_offset = 0;
uint8_t* src = entry_->mmap()->data();
uint8_t* p = reinterpret_cast<uint8_t*>(buffer);
size_t remaining_length =
std::min(buffer_length, entry_->size() - byte_offset);
*out_bytes_read = remaining_length;
for (size_t i = 0; i < entry_->block_list().size(); i++) {
auto& record = entry_->block_list()[i];
if (src_offset + record.length <= byte_offset) {
// Doesn't begin in this region. Skip it.
src_offset += record.length;
continue;
}
memcpy(dest_ptr, entry_->mmap()->data() + offset, read_length);
dest_ptr += read_length;
size_t read_offset =
(byte_offset > src_offset) ? byte_offset - src_offset : 0;
size_t read_length =
std::min(record.length - read_offset, remaining_length);
std::memcpy(p, src + record.offset + read_offset, read_length);
p += read_length;
src_offset += record.length;
remaining_length -= read_length;
if (remaining_length == 0) {
break;
}
}
*out_bytes_read = real_length;
return X_STATUS_SUCCESS;
}

3
src/xenia/vfs/entry.h
View File

@ -98,6 +98,9 @@ class Entry {
Entry* GetChild(std::string name);
const std::vector<std::unique_ptr<Entry>>& children() const {
return children_;
}
size_t child_count() const { return children_.size(); }
Entry* IterateChildren(const xe::filesystem::WildcardEngine& engine,
size_t* current_index);