ShuriZma
/
xenia-canary
mirror of https://github.com/xenia-canary/xenia-canary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into d3d12

This commit is contained in:
gibbed 2019-05-04 13:09:08 -05:00
parent fac37e7c50 00e061c4e4
commit f54dddee90
13 changed files with 480 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitmodules vendored
View File

@ -18,7 +18,7 @@
url = https://github.com/xenia-project/SPIRV-Tools.git
[submodule "third_party/catch"]
path = third_party/catch
url = https://github.com/philsquared/Catch.git
url = https://github.com/catchorg/Catch2.git
[submodule "third_party/gflags"]
path = third_party/gflags
url = https://github.com/benvanik/gflags.git

2
README.md
View File

@ -1,6 +1,6 @@
<p align="center">
<a href="https://github.com/xenia-project/xenia/tree/master/assets/icon">
<img height="370px" src="https://raw.githubusercontent.com/xenia-project/xenia/master/assets/icon/1024.png" />
<img height="120px" src="https://raw.githubusercontent.com/xenia-project/xenia/master/assets/icon/128.png" />
</a>
</p>

2
src/xenia/app/xenia_main.cc
View File

@ -169,7 +169,7 @@ int xenia_main(const std::vector<std::wstring>& args) {
#if defined(XE_PLATFORM_WIN32)
content_root = xe::join_paths(content_root, L"Xenia");
#elif defined(XE_PLATFORM_LINUX)
content_root = xe::join_paths(content_root, L".xenia");
content_root = xe::join_paths(content_root, L"Xenia");
#else
#warning Unhandled platform for content root.
content_root = xe::join_paths(content_root, L"Xenia");

1
src/xenia/base/filesystem.h
View File

@ -116,6 +116,7 @@ struct FileInfo {
};
Type type;
std::wstring name;
std::wstring path;
size_t total_size;
uint64_t create_timestamp;
uint64_t access_timestamp;

41
src/xenia/base/filesystem_posix.cc
View File

@ -12,29 +12,58 @@
#include "xenia/base/logging.h"
#include "xenia/base/string.h"
#include <assert.h>
#include <dirent.h>
#include <fcntl.h>
#include <ftw.h>
#include <libgen.h>
#include <pwd.h>
#include <stdio.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
namespace xe {
namespace filesystem {
std::wstring GetExecutablePath() {
assert_always(); // IMPLEMENT ME.
return std::wstring();
char buff[FILENAME_MAX] = "";
readlink("/proc/self/exe", buff, FILENAME_MAX);
std::string s(buff);
return to_wstring(s);
}
std::wstring GetExecutableFolder() {
assert_always(); // IMPLEMENT ME.
return std::wstring();
auto path = GetExecutablePath();
return xe::find_base_path(path);
}
std::wstring GetUserFolder() {
assert_always(); // IMPLEMENT ME.
return std::wstring();
// get preferred data home
char* dataHome = std::getenv("XDG_DATA_HOME");
// if XDG_DATA_HOME not set, fallback to HOME directory
if (dataHome == NULL) {
dataHome = std::getenv("HOME");
} else {
std::string home(dataHome);
return to_wstring(home);
}
// if HOME not set, fall back to this
if (dataHome == NULL) {
struct passwd pw1;
struct passwd* pw;
char buf[4096]; // could potentionally lower this
getpwuid_r(getuid(), &pw1, buf, sizeof(buf), &pw);
assert(&pw1 == pw); // sanity check
dataHome = pw->pw_dir;
}
std::string home(dataHome);
return to_wstring(home + "/.local/share");
}
bool PathExists(const std::wstring& path) {

2
src/xenia/base/filesystem_win.cc
View File

@ -207,6 +207,7 @@ bool GetInfo(const std::wstring& path, FileInfo* out_info) {
out_info->total_size =
(data.nFileSizeHigh * (size_t(MAXDWORD) + 1)) + data.nFileSizeLow;
}
out_info->path = xe::find_base_path(path);
out_info->name = xe::find_name_from_path(path);
out_info->create_timestamp = COMBINE_TIME(data.ftCreationTime);
out_info->access_timestamp = COMBINE_TIME(data.ftLastAccessTime);
@ -236,6 +237,7 @@ std::vector<FileInfo> ListFiles(const std::wstring& path) {
info.total_size =
(ffd.nFileSizeHigh * (size_t(MAXDWORD) + 1)) + ffd.nFileSizeLow;
}
info.path = path;
info.name = ffd.cFileName;
info.create_timestamp = COMBINE_TIME(ffd.ftCreationTime);
info.access_timestamp = COMBINE_TIME(ffd.ftLastAccessTime);

21
src/xenia/base/string.cc
View File

@ -9,12 +9,27 @@
#include "xenia/base/string.h"
// TODO(benvanik): when GCC finally gets codecvt, use that.
#if XE_PLATFORM_LINUX
// codecvt existence check
#ifdef __clang__
// using clang
#if (__clang_major__ < 4) // 3.3 has it but I think we need at least 4 anyway
// insufficient clang version
#define NO_CODECVT 1
#else
#include <codecvt>
#endif // XE_PLATFORM_LINUX
#endif
#elif defined(__GNUC__) || defined(__GNUG__)
// using gcc
#if (__GNUC__ < 5)
// insufficient clang version
#define NO_CODECVT 1
#else
#include <codecvt>
#endif
// since the windows 10 sdk is required, this shouldn't be an issue
#elif defined(_MSC_VER)
#include <codecvt>
#endif
#include <cctype>
#include <cstring>

526
src/xenia/vfs/devices/stfs_container_device.cc
View File

@ -10,6 +10,7 @@
#include "xenia/vfs/devices/stfs_container_device.h"
#include <algorithm>
#include <queue>
#include <vector>
#include "xenia/base/logging.h"
@ -59,87 +60,88 @@ StfsContainerDevice::StfsContainerDevice(const std::string& mount_path,
StfsContainerDevice::~StfsContainerDevice() = default;
bool StfsContainerDevice::Initialize() {
if (filesystem::IsFolder(local_path_)) {
// Was given a folder. Try to find the file in
// local_path\TITLE_ID\000D0000\HASH_OF_42_CHARS
// We take care to not die if there are additional files around.
bool found_alternative = false;
auto files = filesystem::ListFiles(local_path_);
for (auto& file : files) {
if (file.type != filesystem::FileInfo::Type::kDirectory ||
file.name.size() != 8) {
continue;
}
auto child_path = xe::join_paths(local_path_, file.name);
auto child_files = filesystem::ListFiles(child_path);
for (auto& child_file : child_files) {
if (child_file.type != filesystem::FileInfo::Type::kDirectory ||
child_file.name != L"000D0000") {
continue;
}
auto stfs_path = xe::join_paths(child_path, child_file.name);
auto stfs_files = filesystem::ListFiles(stfs_path);
for (auto& stfs_file : stfs_files) {
if (stfs_file.type != filesystem::FileInfo::Type::kFile ||
stfs_file.name.size() != 42) {
continue;
}
// Probably it!
local_path_ = xe::join_paths(stfs_path, stfs_file.name);
found_alternative = true;
break;
}
if (found_alternative) {
break;
}
}
if (found_alternative) {
break;
}
}
// Resolve a valid STFS file if a directory is given.
if (filesystem::IsFolder(local_path_) && !ResolveFromFolder(local_path_)) {
XELOGE("Could not resolve an STFS container given path %s",
xe::to_string(local_path_).c_str());
return false;
}
if (!filesystem::PathExists(local_path_)) {
XELOGE("STFS container does not exist");
XELOGE("Path to STFS container does not exist: %s",
xe::to_string(local_path_).c_str());
return false;
}
mmap_ = MappedMemory::Open(local_path_, MappedMemory::Mode::kRead);
if (!mmap_) {
XELOGE("STFS container could not be mapped");
return false;
}
uint8_t* map_ptr = mmap_->data();
auto result = ReadHeaderAndVerify(map_ptr);
if (result != Error::kSuccess) {
XELOGE("STFS header read/verification failed: %d", result);
// Map the data file(s)
auto map_result = MapFiles();
if (map_result != Error::kSuccess) {
XELOGE("Failed to map STFS container: %d", map_result);
return false;
}
switch (header_.descriptor_type) {
case StfsDescriptorType::kStfs:
result = ReadAllEntriesSTFS(map_ptr);
return ReadSTFS() == Error::kSuccess;
break;
case StfsDescriptorType::kSvod:
if (!(header_.svod_volume_descriptor.device_features &
kFeatureHasEnhancedGDFLayout)) {
XELOGE("STFS SVOD does not have GDF layout!");
return false;
}
result = ReadAllEntriesEGDF(map_ptr);
break;
return ReadSVOD() == Error::kSuccess;
default:
// Shouldn't reach here.
XELOGE("Unknown STFS Descriptor Type: %d", header_.descriptor_type);
return false;
}
if (result != Error::kSuccess) {
XELOGE("STFS entry reading failed: %d", result);
return false;
}
StfsContainerDevice::Error StfsContainerDevice::MapFiles() {
// Map the file containing the STFS Header and read it.
XELOGI("Mapping STFS Header File: %s", xe::to_string(local_path_).c_str());
auto header_map = MappedMemory::Open(local_path_, MappedMemory::Mode::kRead);
auto header_result = ReadHeaderAndVerify(header_map->data());
if (header_result != Error::kSuccess) {
XELOGE("Error reading STFS Header: %d", header_result);
return header_result;
}
return true;
// If the STFS package is a single file, the header is self contained and
// we don't need to map any extra files.
// NOTE: data_file_count is 0 for STFS and 1 for SVOD
if (header_.data_file_count <= 1) {
XELOGI("STFS container is a single file.");
mmap_.emplace(std::make_pair(0, std::move(header_map)));
return Error::kSuccess;
}
// If the STFS package is multi-file, it is an SVOD system. We need to map
// the files in the .data folder and can discard the header.
auto data_fragment_path = local_path_ + L".data";
if (!filesystem::PathExists(data_fragment_path)) {
XELOGE("STFS container is multi-file, but path %s does not exist.",
xe::to_string(data_fragment_path).c_str());
return Error::kErrorFileMismatch;
}
// Ensure data fragment files are sorted
auto fragment_files = filesystem::ListFiles(data_fragment_path);
std::sort(fragment_files.begin(), fragment_files.end(),
[](filesystem::FileInfo& left, filesystem::FileInfo& right) {
return left.name < right.name;
});
if (fragment_files.size() != header_.data_file_count) {
XELOGE("SVOD expecting %d data fragments, but %d are present.",
header_.data_file_count, fragment_files.size());
return Error::kErrorFileMismatch;
}
for (size_t i = 0; i < fragment_files.size(); i++) {
auto file = fragment_files.at(i);
auto path = xe::join_paths(file.path, file.name);
auto data = MappedMemory::Open(path, MappedMemory::Mode::kRead);
mmap_.emplace(std::make_pair(i, std::move(data)));
}
XELOGI("SVOD successfully mapped %d files.", fragment_files.size());
return Error::kSuccess;
}
void StfsContainerDevice::Dump(StringBuffer* string_buffer) {
@ -196,87 +198,189 @@ StfsContainerDevice::Error StfsContainerDevice::ReadHeaderAndVerify(
return Error::kSuccess;
}
StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesEGDF(
const uint8_t* map_ptr) {
// Verify (and scan) the GDF magic first.
const uint8_t* p = map_ptr + BlockToOffsetSTFS(0);
if (std::memcmp(p, "MICROSOFT*XBOX*MEDIA", 20) != 0) {
return Error::kErrorDamagedFile;
StfsContainerDevice::Error StfsContainerDevice::ReadSVOD() {
// SVOD Systems can have different layouts. The root block is
// denoted by the magic "MICROSOFT*XBOX*MEDIA" and is always in
// the first "actual" data fragment of the system.
auto data = mmap_.at(0)->data();
const char* MEDIA_MAGIC = "MICROSOFT*XBOX*MEDIA";
// Check for EDGF layout
auto layout = &header_.svod_volume_descriptor.layout_type;
auto features = header_.svod_volume_descriptor.device_features;
bool has_egdf_layout = features & kFeatureHasEnhancedGDFLayout;
if (has_egdf_layout) {
// The STFS header has specified that this SVOD system uses the EGDF layout.
// We can expect the magic block to be located immediately after the hash
// blocks. We also offset block address calculation by 0x1000 by shifting
// block indices by +0x2.
if (memcmp(data + 0x2000, MEDIA_MAGIC, 20) == 0) {
base_offset_ = 0x0000;
magic_offset_ = 0x2000;
*layout = kEnhancedGDFLayout;
XELOGI("SVOD uses an EGDF layout. Magic block present at 0x2000.");
} else {
XELOGE("SVOD uses an EGDF layout, but the magic block was not found.");
return Error::kErrorFileMismatch;
}
} else if (memcmp(data + 0x12000, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0x12000, it is likely using an XSF
// layout. This is usually due to converting the game using a third-party
// tool, as most of them use a nulled XSF as a template.
base_offset_ = 0x10000;
magic_offset_ = 0x12000;
// Check for XSF Header
const char* XSF_MAGIC = "XSF";
if (memcmp(data + 0x2000, XSF_MAGIC, 3) == 0) {
*layout = kXSFLayout;
XELOGI("SVOD uses an XSF layout. Magic block present at 0x12000.");
XELOGI("Game was likely converted using a third-party tool.");
} else {
*layout = kUnknownLayout;
XELOGI("SVOD appears to use an XSF layout, but no header is present.");
XELOGI("SVOD magic block found at 0x12000");
}
} else if (memcmp(data + 0xD000, MEDIA_MAGIC, 20) == 0) {
// If the SVOD's magic block is at 0xD000, it most likely means that it is
// a single-file system. The STFS Header is 0xB000 bytes , and the remaining
// 0x2000 is from hash tables. In most cases, these will be STFS, not SVOD.
base_offset_ = 0xB000;
magic_offset_ = 0xD000;
// Check for single file system
if (header_.data_file_count == 1) {
*layout = kSingleFileLayout;
XELOGI("SVOD is a single file. Magic block present at 0xD000.");
} else {
*layout = kUnknownLayout;
XELOGE(
"SVOD is not a single file, but the magic block was found at "
"0xD000.");
}
} else {
XELOGE("Could not locate SVOD magic block.");
return Error::kErrorReadError;
}
uint32_t root_sector = xe::load<uint32_t>(p + 0x14);
uint32_t root_size = xe::load<uint32_t>(p + 0x18);
// Parse the root directory
uint8_t* magic_block = data + magic_offset_;
uint32_t root_block = xe::load<uint32_t>(magic_block + 0x14);
uint32_t root_size = xe::load<uint32_t>(magic_block + 0x18);
uint32_t root_creation_date = xe::load<uint32_t>(magic_block + 0x1C);
uint32_t root_creation_time = xe::load<uint32_t>(magic_block + 0x20);
uint64_t root_creation_timestamp =
decode_fat_timestamp(root_creation_date, root_creation_time);
auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get());
auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_);
root_entry->attributes_ = kFileAttributeDirectory;
root_entry->access_timestamp_ = root_creation_timestamp;
root_entry->create_timestamp_ = root_creation_timestamp;
root_entry->write_timestamp_ = root_creation_timestamp;
root_entry_ = std::unique_ptr<Entry>(root_entry);
const uint8_t* buffer = map_ptr + BlockToOffsetEGDF(root_sector);
return ReadEntryEGDF(buffer, 0, root_entry) ? Error::kSuccess
: Error::kErrorDamagedFile;
// Traverse all child entries
return ReadEntrySVOD(root_block, 0, root_entry);
}
bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer,
uint16_t entry_ordinal,
StfsContainerEntry* parent) {
const uint8_t* p = buffer + (entry_ordinal * 4);
StfsContainerDevice::Error StfsContainerDevice::ReadEntrySVOD(
uint32_t block, uint32_t ordinal, StfsContainerEntry* parent) {
// For games with a large amount of files, the ordinal offset can overrun
// the current block and potentially hit a hash block.
size_t ordinal_offset = ordinal * 0x4;
size_t block_offset = ordinal_offset / 0x800;
size_t true_ordinal_offset = ordinal_offset % 0x800;
uint16_t node_l = xe::load<uint16_t>(p + 0);
uint16_t node_r = xe::load<uint16_t>(p + 2);
uint32_t sector = xe::load<uint32_t>(p + 4);
uint32_t length = xe::load<uint32_t>(p + 8);
uint8_t attributes = xe::load<uint8_t>(p + 12);
uint8_t name_length = xe::load<uint8_t>(p + 13);
auto name = reinterpret_cast<const char*>(p + 14);
// Calculate the file & address of the block
size_t entry_address, entry_file;
BlockToOffsetSVOD(block + block_offset, &entry_address, &entry_file);
entry_address += true_ordinal_offset;
if (node_l && !ReadEntryEGDF(buffer, node_l, parent)) {
return false;
// Read block's descriptor
auto data = mmap_.at(entry_file)->data() + entry_address;
uint16_t node_l = xe::load<uint16_t>(data + 0x00);
uint16_t node_r = xe::load<uint16_t>(data + 0x02);
uint32_t data_block = xe::load<uint32_t>(data + 0x04);
uint32_t length = xe::load<uint32_t>(data + 0x08);
uint8_t attributes = xe::load<uint8_t>(data + 0x0C);
uint8_t name_length = xe::load<uint8_t>(data + 0x0D);
auto name = reinterpret_cast<const char*>(data + 0x0E);
auto name_str = std::string(name, name_length);
// Read the left node
if (node_l) {
auto node_result = ReadEntrySVOD(block, node_l, parent);
if (node_result != Error::kSuccess) {
return node_result;
}
}
auto entry = StfsContainerEntry::Create(
this, parent, std::string(name, name_length), mmap_.get());
// Read file & address of block's data
size_t data_address, data_file;
BlockToOffsetSVOD(data_block, &data_address, &data_file);
// Create the entry
// NOTE: SVOD entries don't have timestamps for individual files, which can
// cause issues when decrypting games. Using the root entry's timestamp
// solves this issues.
auto entry = StfsContainerEntry::Create(this, parent, name_str, &mmap_);
if (attributes & kFileAttributeDirectory) {
// Folder.
// Entry is a directory
entry->attributes_ = kFileAttributeDirectory | kFileAttributeReadOnly;
entry->data_offset_ = 0;
entry->data_size_ = 0;
entry->block_ = block;
entry->access_timestamp_ = root_entry_->create_timestamp();
entry->create_timestamp_ = root_entry_->create_timestamp();
entry->write_timestamp_ = root_entry_->create_timestamp();
if (length) {
// Not a leaf - read in children.
uint8_t* folder_ptr = mmap_->data() + BlockToOffsetEGDF(sector);
if (!ReadEntryEGDF(folder_ptr, 0, entry.get())) {
return false;
// If length is greater than 0, traverse the directory's children
auto directory_result = ReadEntrySVOD(data_block, 0, entry.get());
if (directory_result != Error::kSuccess) {
return directory_result;
}
}
} else {
// Regular file.
// Entry is a file
entry->attributes_ = kFileAttributeNormal | kFileAttributeReadOnly;
entry->size_ = length;
entry->allocation_size_ = xe::round_up(length, bytes_per_sector());
entry->data_offset_ = BlockToOffsetEGDF(sector);
entry->data_offset_ = data_address;
entry->data_size_ = length;
entry->block_ = data_block;
entry->access_timestamp_ = root_entry_->create_timestamp();
entry->create_timestamp_ = root_entry_->create_timestamp();
entry->write_timestamp_ = root_entry_->create_timestamp();
// Fill in all block records, sector by sector.
if (entry->attributes() & X_FILE_ATTRIBUTE_NORMAL) {
uint32_t sector_index = sector;
uint32_t block_index = data_block;
size_t remaining_size = xe::round_up(length, 0x800);
size_t last_record = -1;
size_t last_offset = -1;
while (remaining_size) {
size_t block_size = 0x800;
size_t offset = BlockToOffsetEGDF(sector_index);
sector_index++;
remaining_size -= block_size;
const size_t BLOCK_SIZE = 0x800;
size_t offset, file_index;
BlockToOffsetSVOD(block_index, &offset, &file_index);
block_index++;
remaining_size -= BLOCK_SIZE;
if (offset - last_offset == 0x800) {
// Consecutive, so append to last entry.
entry->block_list_[last_record].length += block_size;
entry->block_list_[last_record].length += BLOCK_SIZE;
last_offset = offset;
continue;
}
entry->block_list_.push_back({offset, block_size});
entry->block_list_.push_back({file_index, offset, BLOCK_SIZE});
last_record = entry->block_list_.size() - 1;
last_offset = offset;
}
@ -285,17 +389,82 @@ bool StfsContainerDevice::ReadEntryEGDF(const uint8_t* buffer,
parent->children_.emplace_back(std::move(entry));
// Read next file in the list.
if (node_r && !ReadEntryEGDF(buffer, node_r, parent)) {
return false;
// Read the right node.
if (node_r) {
auto node_result = ReadEntrySVOD(block, node_r, parent);
if (node_result != Error::kSuccess) {
return node_result;
}
}
return true;
return Error::kSuccess;
}
StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
const uint8_t* map_ptr) {
auto root_entry = new StfsContainerEntry(this, nullptr, "", mmap_.get());
void StfsContainerDevice::BlockToOffsetSVOD(size_t block, size_t* out_address,
size_t* out_file_index) {
// SVOD Systems use hash blocks for integrity checks. These hash blocks
// cause blocks to be discontinuous in memory, and must be accounted for.
// - Each data block is 0x800 bytes in length
// - Every group of 0x198 data blocks is preceded a Level0 hash table.
// Level0 tables contain 0xCC hashes, each representing two data blocks.
// The total size of each Level0 hash table is 0x1000 bytes in length.
// - Every 0xA1C4 Level0 hash tables is preceded by a Level1 hash table.
// Level1 tables contain 0xCB hashes, each representing two Level0 hashes.
// The total size of each Level1 hash table is 0x1000 bytes in length.
// - Files are split into fragments of 0xA290000 bytes in length,
// consisting of 0x14388 data blocks, 0xCB Level0 hash tables, and 0x1
// Level1 hash table.
const size_t BLOCK_SIZE = 0x800;
const size_t HASH_BLOCK_SIZE = 0x1000;
const size_t BLOCKS_PER_L0_HASH = 0x198;
const size_t HASHES_PER_L1_HASH = 0xA1C4;
const size_t BLOCKS_PER_FILE = 0x14388;
const size_t MAX_FILE_SIZE = 0xA290000;
const size_t BLOCK_OFFSET = header_.svod_volume_descriptor.data_block_offset;
const SvodLayoutType LAYOUT = header_.svod_volume_descriptor.layout_type;
// Resolve the true block address and file index
size_t true_block = block - (BLOCK_OFFSET * 2);
if (LAYOUT == kEnhancedGDFLayout) {
// EGDF has an 0x1000 byte offset, which is two blocks
true_block += 0x2;
}
size_t file_block = true_block % BLOCKS_PER_FILE;
size_t file_index = true_block / BLOCKS_PER_FILE;
size_t offset = 0;
// Calculate offset caused by Level0 Hash Tables
size_t level0_table_count = (file_block / BLOCKS_PER_L0_HASH) + 1;
offset += level0_table_count * HASH_BLOCK_SIZE;
// Calculate offset caused by Level1 Hash Tables
size_t level1_table_count = (level0_table_count / HASHES_PER_L1_HASH) + 1;
offset += level1_table_count * HASH_BLOCK_SIZE;
// For single-file SVOD layouts, include the size of the header in the offset.
if (LAYOUT == kSingleFileLayout) {
offset += base_offset_;
}
size_t block_address = (file_block * BLOCK_SIZE) + offset;
// If the offset causes the block address to overrun the file, round it.
if (block_address >= MAX_FILE_SIZE) {
file_index += 1;
block_address %= MAX_FILE_SIZE;
block_address += 0x2000;
}
*out_address = block_address;
*out_file_index = file_index;
}
StfsContainerDevice::Error StfsContainerDevice::ReadSTFS() {
auto data = mmap_.at(0)->data();
auto root_entry = new StfsContainerEntry(this, nullptr, "", &mmap_);
root_entry->attributes_ = kFileAttributeDirectory;
root_entry_ = std::unique_ptr<Entry>(root_entry);
@ -305,7 +474,7 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
auto& volume_descriptor = header_.stfs_volume_descriptor;
uint32_t table_block_index = volume_descriptor.file_table_block_number;
for (size_t n = 0; n < volume_descriptor.file_table_block_count; n++) {
const uint8_t* p = map_ptr + BlockToOffsetSTFS(table_block_index);
const uint8_t* p = data + BlockToOffsetSTFS(table_block_index);
for (size_t m = 0; m < 0x1000 / 0x40; m++) {
const uint8_t* filename = p; // 0x28b
if (filename[0] == 0) {
@ -333,11 +502,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
parent_entry = all_entries[path_indicator];
}
auto entry = StfsContainerEntry::Create(
this, parent_entry,
std::string(reinterpret_cast<const char*>(filename),
filename_length_flags & 0x3F),
mmap_.get());
std::string name_str(reinterpret_cast<const char*>(filename),
filename_length_flags & 0x3F);
auto entry =
StfsContainerEntry::Create(this, parent_entry, name_str, &mmap_);
// bit 0x40 = consecutive blocks (not fragmented?)
if (filename_length_flags & 0x80) {
entry->attributes_ = kFileAttributeDirectory;
@ -367,11 +536,11 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
size_t block_size =
std::min(static_cast<size_t>(0x1000), remaining_size);
size_t offset = BlockToOffsetSTFS(block_index);
entry->block_list_.push_back({offset, block_size});
entry->block_list_.push_back({0, offset, block_size});
remaining_size -= block_size;
auto block_hash = GetBlockHash(map_ptr, block_index, 0);
auto block_hash = GetBlockHash(data, block_index, 0);
if (table_size_shift_ && block_hash.info < 0x80) {
block_hash = GetBlockHash(map_ptr, block_index, 1);
block_hash = GetBlockHash(data, block_index, 1);
}
block_index = block_hash.next_block_index;
info = block_hash.info;
@ -381,9 +550,9 @@ StfsContainerDevice::Error StfsContainerDevice::ReadAllEntriesSTFS(
parent_entry->children_.emplace_back(std::move(entry));
}
auto block_hash = GetBlockHash(map_ptr, table_block_index, 0);
auto block_hash = GetBlockHash(data, table_block_index, 0);
if (table_size_shift_ && block_hash.info < 0x80) {
block_hash = GetBlockHash(map_ptr, table_block_index, 1);
block_hash = GetBlockHash(data, table_block_index, 1);
}
table_block_index = block_hash.next_block_index;
if (table_block_index == 0xFFFFFF) {
@ -402,41 +571,25 @@ size_t StfsContainerDevice::BlockToOffsetSTFS(uint64_t block_index) {
block_shift = package_type_ == StfsPackageType::kCon ? 1 : 0;
}
if (header_.descriptor_type == StfsDescriptorType::kStfs) {
// For every level there is a hash table
// Level 0: hash table of next 170 blocks
// Level 1: hash table of next 170 hash tables
// Level 2: hash table of next 170 level 1 hash tables
// And so on...
uint64_t base = kSTFSHashSpacing;
block = block_index;
for (uint32_t i = 0; i < 3; i++) {
block += (block_index + (base << block_shift)) / (base << block_shift);
if (block_index < base) {
break;
}
base *= kSTFSHashSpacing;
// For every level there is a hash table
// Level 0: hash table of next 170 blocks
// Level 1: hash table of next 170 hash tables
// Level 2: hash table of next 170 level 1 hash tables
// And so on...
uint64_t base = kSTFSHashSpacing;
block = block_index;
for (uint32_t i = 0; i < 3; i++) {
block += (block_index + (base << block_shift)) / (base << block_shift);
if (block_index < base) {
break;
}
} else if (header_.descriptor_type == StfsDescriptorType::kSvod) {
// Level 0: Hashes for the next 204 blocks
// Level 1: Hashes for the next 203 hash blocks + 1 for the next level 0
// 10......[204 blocks].....0.....[204 blocks].....0
// There are 0xA1C4 (41412) blocks for every level 1 hash table.
block = block_index;
block += (block_index + 204) / 204; // Level 0
block += (block_index + 204 * 203) / (204 * 203); // Level 1
base *= kSTFSHashSpacing;
}
return xe::round_up(header_.header_size, 0x1000) + (block << 12);
}
size_t StfsContainerDevice::BlockToOffsetEGDF(uint64_t sector) {
size_t offset = BlockToOffsetSTFS(
(sector / 2) - header_.svod_volume_descriptor.data_block_offset + 1);
return offset + ((sector & 0x1) << 11); // Sectors are 0x800 bytes.
}
StfsContainerDevice::BlockHash StfsContainerDevice::GetBlockHash(
const uint8_t* map_ptr, uint32_t block_index, uint32_t table_offset) {
uint32_t record = block_index % 0xAA;
@ -496,11 +649,6 @@ bool StfsHeader::Read(const uint8_t* p) {
header_size = xe::load_and_swap<uint32_t>(p + 0x340);
content_type = (StfsContentType)xe::load_and_swap<uint32_t>(p + 0x344);
metadata_version = xe::load_and_swap<uint32_t>(p + 0x348);
if (metadata_version > 1) {
// This is a variant of thumbnail data/etc.
// Can just ignore it for now (until we parse thumbnails).
XELOGW("STFSContainer doesn't support version %d yet", metadata_version);
}
content_size = xe::load_and_swap<uint32_t>(p + 0x34C);
media_id = xe::load_and_swap<uint32_t>(p + 0x354);
version = xe::load_and_swap<uint32_t>(p + 0x358);
@ -541,8 +689,70 @@ bool StfsHeader::Read(const uint8_t* p) {
title_thumbnail_image_size = xe::load_and_swap<uint32_t>(p + 0x1716);
std::memcpy(thumbnail_image, p + 0x171A, 0x4000);
std::memcpy(title_thumbnail_image, p + 0x571A, 0x4000);
// Metadata v2 Fields
if (metadata_version == 2) {
std::memcpy(series_id, p + 0x3B1, 0x10);
std::memcpy(season_id, p + 0x3C1, 0x10);
season_number = xe::load_and_swap<uint16_t>(p + 0x3D1);
episode_number = xe::load_and_swap<uint16_t>(p + 0x3D5);
for (size_t n = 0; n < 0x300 / 2; n++) {
additonal_display_names[n] =
xe::load_and_swap<uint16_t>(p + 0x541A + n * 2);
additional_display_descriptions[n] =
xe::load_and_swap<uint16_t>(p + 0x941A + n * 2);
}
}
return true;
}
const char* StfsContainerDevice::ReadMagic(const std::wstring& path) {
auto map = MappedMemory::Open(path, MappedMemory::Mode::kRead, 0, 4);
auto magic_data = xe::load<uint32_t>(map->data());
auto magic_bytes = static_cast<char*>(static_cast<void*>(&magic_data));
return std::move(magic_bytes);
}
bool StfsContainerDevice::ResolveFromFolder(const std::wstring& path) {
// Scan through folders until a file with magic is found
std::queue<filesystem::FileInfo> queue;
filesystem::FileInfo folder;
filesystem::GetInfo(local_path_, &folder);
queue.push(folder);
while (!queue.empty()) {
auto current_file = queue.front();
queue.pop();
if (current_file.type == filesystem::FileInfo::Type::kDirectory) {
auto path = xe::join_paths(current_file.path, current_file.name);
auto child_files = filesystem::ListFiles(path);
for (auto file : child_files) {
queue.push(file);
}
} else {
// Try to read the file's magic
auto path = xe::join_paths(current_file.path, current_file.name);
auto magic = ReadMagic(path);
if (memcmp(magic, "LIVE", 4) == 0 || memcmp(magic, "PIRS", 4) == 0 ||
memcmp(magic, "CON ", 4) == 0) {
local_path_ = xe::join_paths(current_file.path, current_file.name);
XELOGI("STFS Package found: %s", xe::to_string(local_path_).c_str());
return true;
}
}
}
if (local_path_ == path) {
// Could not find a suitable container file
return false;
}
return true;
}
} // namespace vfs
} // namespace xe
} // namespace xe

41
src/xenia/vfs/devices/stfs_container_device.h
View File

@ -10,6 +10,7 @@
#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_
#define XENIA_VFS_DEVICES_STFS_CONTAINER_DEVICE_H_
#include <map>
#include <memory>
#include <string>
@ -91,6 +92,13 @@ enum SvodDeviceFeatures {
kFeatureHasEnhancedGDFLayout = 0x40,
};
enum SvodLayoutType {
kUnknownLayout = 0x0,
kEnhancedGDFLayout = 0x1,
kXSFLayout = 0x2,
kSingleFileLayout = 0x4,
};
struct SvodVolumeDescriptor {
bool Read(const uint8_t* p);
@ -103,6 +111,8 @@ struct SvodVolumeDescriptor {
uint32_t data_block_count;
uint32_t data_block_offset;
// 0x5 padding bytes...
SvodLayoutType layout_type;
};
class StfsHeader {
@ -143,6 +153,14 @@ class StfsHeader {
uint32_t title_thumbnail_image_size;
uint8_t thumbnail_image[0x4000];
uint8_t title_thumbnail_image[0x4000];
// Metadata v2 Fields
uint8_t series_id[0x10];
uint8_t season_id[0x10];
int16_t season_number;
int16_t episode_number;
wchar_t additonal_display_names[0x300 / 2];
wchar_t additional_display_descriptions[0x300 / 2];
};
class StfsContainerDevice : public Device {
@ -156,7 +174,7 @@ class StfsContainerDevice : public Device {
Entry* ResolvePath(std::string path) override;
uint32_t total_allocation_units() const override {
return uint32_t(mmap_->size() / sectors_per_allocation_unit() /
return uint32_t(mmap_total_size_ / sectors_per_allocation_unit() /
bytes_per_sector());
}
uint32_t available_allocation_units() const override { return 0; }
@ -178,23 +196,30 @@ class StfsContainerDevice : public Device {
};
const uint32_t kSTFSHashSpacing = 170;
const uint32_t kSVODHashSpacing = 204;
const char* ReadMagic(const std::wstring& path);
bool ResolveFromFolder(const std::wstring& path);
Error MapFiles();
Error ReadHeaderAndVerify(const uint8_t* map_ptr);
Error ReadAllEntriesEGDF(const uint8_t* map_ptr);
bool ReadEntryEGDF(const uint8_t* buffer, uint16_t entry_ordinal,
StfsContainerEntry* parent);
Error ReadAllEntriesSTFS(const uint8_t* map_ptr);
Error ReadSVOD();
Error ReadEntrySVOD(uint32_t sector, uint32_t ordinal,
StfsContainerEntry* parent);
void BlockToOffsetSVOD(size_t sector, size_t* address, size_t* file_index);
Error ReadSTFS();
size_t BlockToOffsetSTFS(uint64_t block);
size_t BlockToOffsetEGDF(uint64_t block);
BlockHash GetBlockHash(const uint8_t* map_ptr, uint32_t block_index,
uint32_t table_offset);
std::wstring local_path_;
std::unique_ptr<MappedMemory> mmap_;
std::map<size_t, std::unique_ptr<MappedMemory>> mmap_;
size_t mmap_total_size_;
size_t base_offset_;
size_t magic_offset_;
std::unique_ptr<Entry> root_entry_;
StfsPackageType package_type_;
StfsHeader header_;

10
src/xenia/vfs/devices/stfs_container_entry.cc
View File

@ -8,15 +8,17 @@
*/
#include "xenia/vfs/devices/stfs_container_entry.h"
#include "xenia/base/math.h"
#include "xenia/vfs/devices/stfs_container_file.h"
#include <map>
namespace xe {
namespace vfs {
StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
std::string path, MappedMemory* mmap)
std::string path,
MultifileMemoryMap* mmap)
: Entry(device, parent, path),
mmap_(mmap),
data_offset_(0),
@ -25,7 +27,7 @@ StfsContainerEntry::StfsContainerEntry(Device* device, Entry* parent,
StfsContainerEntry::~StfsContainerEntry() = default;
std::unique_ptr<StfsContainerEntry> StfsContainerEntry::Create(
Device* device, Entry* parent, std::string name, MappedMemory* mmap) {
Device* device, Entry* parent, std::string name, MultifileMemoryMap* mmap) {
auto path = xe::join_paths(parent->path(), name);
auto entry = std::make_unique<StfsContainerEntry>(device, parent, path, mmap);
@ -38,4 +40,4 @@ X_STATUS StfsContainerEntry::Open(uint32_t desired_access, File** out_file) {
}
} // namespace vfs
} // namespace xe
} // namespace xe

15
src/xenia/vfs/devices/stfs_container_entry.h
View File

@ -10,6 +10,7 @@
#ifndef XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
#define XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
#include <map>
#include <string>
#include <vector>
@ -20,27 +21,30 @@
namespace xe {
namespace vfs {
typedef std::map<size_t, std::unique_ptr<MappedMemory>> MultifileMemoryMap;
class StfsContainerDevice;
class StfsContainerEntry : public Entry {
public:
StfsContainerEntry(Device* device, Entry* parent, std::string path,
MappedMemory* mmap);
MultifileMemoryMap* mmap);
~StfsContainerEntry() override;
static std::unique_ptr<StfsContainerEntry> Create(Device* device,
Entry* parent,
std::string name,
MappedMemory* mmap);
MultifileMemoryMap* mmap);
MappedMemory* mmap() const { return mmap_; }
MultifileMemoryMap* mmap() const { return mmap_; }
size_t data_offset() const { return data_offset_; }
size_t data_size() const { return data_size_; }
size_t block() const { return block_; }
X_STATUS Open(uint32_t desired_access, File** out_file) override;
struct BlockRecord {
size_t file;
size_t offset;
size_t length;
};
@ -49,13 +53,14 @@ class StfsContainerEntry : public Entry {
private:
friend class StfsContainerDevice;
MappedMemory* mmap_;
MultifileMemoryMap* mmap_;
size_t data_offset_;
size_t data_size_;
size_t block_;
std::vector<BlockRecord> block_list_;
};
} // namespace vfs
} // namespace xe
#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_
#endif // XENIA_VFS_DEVICES_STFS_CONTAINER_ENTRY_H_

5
src/xenia/vfs/devices/stfs_container_file.cc
View File

@ -34,7 +34,6 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
}
size_t src_offset = 0;
uint8_t* src = entry_->mmap()->data();
uint8_t* p = reinterpret_cast<uint8_t*>(buffer);
size_t remaining_length =
std::min(buffer_length, entry_->size() - byte_offset);
@ -48,6 +47,8 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
continue;
}
uint8_t* src = entry_->mmap()->at(record.file)->data();
size_t read_offset =
(byte_offset > src_offset) ? byte_offset - src_offset : 0;
size_t read_length =
@ -66,4 +67,4 @@ X_STATUS StfsContainerFile::ReadSync(void* buffer, size_t buffer_length,
}
} // namespace vfs
} // namespace xe
} // namespace xe

2
third_party/catch vendored

@ -1 +1 @@
Subproject commit a0ada2e935324db5c951571c52c9b07307ea422f
Subproject commit 6860c8def0ba7559bf077515b7a7ff63ad3444f8