From 11ba6e45aba70325315260f300b061d1b103f170 Mon Sep 17 00:00:00 2001 From: Nekotekina Date: Fri, 29 Jan 2021 13:32:19 +0300 Subject: [PATCH] Add MSELF support to SPRX precompilation. Add ppu_precompile() function in PPUThread.cpp Co-authored-by: Eladash --- rpcs3/Emu/Cell/PPUThread.cpp | 214 +++++++++++++++++++++++++++++++++++ rpcs3/Emu/System.cpp | 98 +--------------- rpcs3/Loader/mself.hpp | 44 +++++++ 3 files changed, 261 insertions(+), 95 deletions(-) create mode 100644 rpcs3/Loader/mself.hpp diff --git a/rpcs3/Emu/Cell/PPUThread.cpp b/rpcs3/Emu/Cell/PPUThread.cpp index 419dfdead7..74ee0cc6fe 100644 --- a/rpcs3/Emu/Cell/PPUThread.cpp +++ b/rpcs3/Emu/Cell/PPUThread.cpp @@ -1,6 +1,10 @@ #include "stdafx.h" #include "Utilities/JIT.h" +#include "Utilities/StrUtil.h" #include "Crypto/sha1.h" +#include "Crypto/unself.h" +#include "Loader/ELF.h" +#include "Loader/mself.hpp" #include "Emu/perf_meter.hpp" #include "Emu/Memory/vm_reservation.h" #include "Emu/Memory/vm_locking.h" @@ -74,6 +78,8 @@ extern u64 get_guest_system_time(); extern atomic_t g_watchdog_hold_ctr; extern atomic_t g_progr; +extern atomic_t g_progr_ftotal; +extern atomic_t g_progr_fdone; extern atomic_t g_progr_ptotal; extern atomic_t g_progr_pdone; @@ -113,6 +119,8 @@ extern void ppu_initialize(); extern void ppu_finalize(const ppu_module& info); extern void ppu_initialize(const ppu_module& info); static void ppu_initialize2(class jit_compiler& jit, const ppu_module& module_part, const std::string& cache_path, const std::string& obj_name); +extern void ppu_unload_prx(const lv2_prx&); +extern std::shared_ptr ppu_load_prx(const ppu_prx_object&, const std::string&); extern void ppu_execute_syscall(ppu_thread& ppu, u64 code); static bool ppu_break(ppu_thread& ppu, ppu_opcode_t op); @@ -1949,6 +1957,76 @@ namespace } #endif +namespace +{ + // Read-only file view starting with specified offset (for MSELF) + struct file_view : fs::file_base + { + const fs::file m_file; + const u64 m_off; + u64 m_pos; + + explicit file_view(fs::file&& _file, u64 offset) + : m_file(std::move(_file)) + , m_off(offset) + , m_pos(0) + { + } + + ~file_view() override + { + } + + fs::stat_t stat() override + { + return m_file.stat(); + } + + bool trunc(u64 length) override + { + return false; + } + + u64 read(void* buffer, u64 size) override + { + const u64 old_pos = m_file.pos(); + m_file.seek(m_off + m_pos); + const u64 result = m_file.read(buffer, size); + ensure(old_pos == m_file.seek(old_pos)); + + m_pos += result; + return result; + } + + u64 write(const void* buffer, u64 size) override + { + return 0; + } + + u64 seek(s64 offset, fs::seek_mode whence) override + { + const s64 new_pos = + whence == fs::seek_set ? offset : + whence == fs::seek_cur ? offset + m_pos : + whence == fs::seek_end ? offset + size() : -1; + + if (new_pos < 0) + { + fs::g_tls_error = fs::error::inval; + return -1; + } + + m_pos = new_pos; + return m_pos; + } + + u64 size() override + { + return m_file.size(); + } + }; +} + extern void ppu_finalize(const ppu_module& info) { // Get cache path for this executable @@ -1986,6 +2064,142 @@ extern void ppu_finalize(const ppu_module& info) #endif } +extern void ppu_precompile(std::vector& dir_queue) +{ + std::vector> file_queue; + file_queue.reserve(2000); + + // Initialize progress dialog + g_progr = "Scanning directories for SPRX libraries..."; + + // Find all .sprx files recursively (TODO: process .mself files) + for (usz i = 0; i < dir_queue.size(); i++) + { + if (Emu.IsStopped()) + { + break; + } + + ppu_log.notice("Scanning directory: %s", dir_queue[i]); + + for (auto&& entry : fs::dir(dir_queue[i])) + { + if (Emu.IsStopped()) + { + break; + } + + if (entry.is_directory) + { + if (entry.name != "." && entry.name != "..") + { + dir_queue.emplace_back(dir_queue[i] + entry.name + '/'); + } + + continue; + } + + // Check .sprx filename + if (fmt::to_upper(entry.name).ends_with(".SPRX")) + { + // Get full path + file_queue.emplace_back(dir_queue[i] + entry.name, 0); + g_progr_ftotal++; + continue; + } + + // Check .mself filename + if (fmt::to_upper(entry.name).ends_with(".MSELF")) + { + if (fs::file mself{dir_queue[i] + entry.name}) + { + mself_header hdr{}; + + if (mself.read(hdr) && hdr.get_count(mself.size())) + { + for (u32 i = 0; i < hdr.count; i++) + { + mself_record rec{}; + + if (mself.read(rec) && rec.get_pos(mself.size())) + { + std::string name = rec.name; + + if (fmt::to_upper(name).ends_with(".SPRX")) + { + // .sprx inside .mself found + file_queue.emplace_back(dir_queue[i] + entry.name, rec.off); + g_progr_ftotal++; + } + } + else + { + ppu_log.error("MSELF file is possibly truncated"); + break; + } + } + } + } + } + } + } + + g_progr = "Compiling PPU modules"; + + atomic_t fnext = 0; + + shared_mutex sprx_mtx; + + named_thread_group workers("SPRX Worker ", utils::get_thread_count(), [&] + { + for (usz func_i = fnext++; func_i < file_queue.size(); func_i = fnext++) + { + const auto& path = std::as_const(file_queue)[func_i].first; + + ppu_log.notice("Trying to load SPRX: %s", path); + + // Load MSELF or SPRX + fs::file src{path}; + + if (u64 off = file_queue[func_i].second) + { + // Adjust offset for MSELF + src.reset(std::make_unique(std::move(src), off)); + } + + // Some files may fail to decrypt due to the lack of klic + src = decrypt_self(std::move(src)); + + const ppu_prx_object obj = src; + + if (obj == elf_error::ok) + { + std::unique_lock lock(sprx_mtx); + + if (auto prx = ppu_load_prx(obj, path)) + { + lock.unlock(); + ppu_initialize(*prx); + idm::remove(idm::last_id()); + lock.lock(); + ppu_unload_prx(*prx); + lock.unlock(); + ppu_finalize(*prx); + g_progr_fdone++; + continue; + } + } + + ppu_log.error("Failed to load SPRX '%s' (%s)", path, obj.get_error()); + g_progr_fdone++; + continue; + } + }); + + // Join every thread + workers.join(); +} + extern void ppu_initialize() { const auto _main = g_fxo->get(); diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index 3b09d07ad2..0626034ded 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -66,6 +66,7 @@ atomic_t g_watchdog_hold_ctr{0}; extern void ppu_load_exec(const ppu_exec_object&); extern void spu_load_exec(const spu_exec_object&); +extern void ppu_precompile(std::vector& dir_queue); extern void ppu_initialize(const ppu_module&); extern void ppu_finalize(const ppu_module&); extern void ppu_unload_prx(const lv2_prx&); @@ -1115,55 +1116,10 @@ game_boot_result Emulator::Load(const std::string& title_id, bool add_only, bool } } - std::vector> file_queue; - file_queue.reserve(2000); - - // Initialize progress dialog - g_progr = "Scanning directories for SPRX libraries..."; - - // Find all .sprx files recursively (TODO: process .mself files) - for (usz i = 0; i < dir_queue.size(); i++) - { - if (Emu.IsStopped()) - { - break; - } - - sys_log.notice("Scanning directory: %s", dir_queue[i]); - - for (auto&& entry : fs::dir(dir_queue[i])) - { - if (Emu.IsStopped()) - { - break; - } - - if (entry.is_directory) - { - if (entry.name != "." && entry.name != "..") - { - dir_queue.emplace_back(dir_queue[i] + entry.name + '/'); - } - - continue; - } - - // Check .sprx filename - if (fmt::to_upper(entry.name).ends_with(".SPRX")) - { - // Get full path - file_queue.emplace_back(dir_queue[i] + entry.name, 0); - g_progr_ftotal++; - } - } - } - - g_progr = "Compiling PPU modules"; - if (std::string path = m_path + "/USRDIR/EBOOT.BIN"; fs::is_file(path)) { // Compile EBOOT.BIN first - sys_log.notice("Trying to load EBOOT.BIN: %s", path); + ppu_log.notice("Trying to load EBOOT.BIN: %s", path); fs::file src{path}; @@ -1194,55 +1150,7 @@ game_boot_result Emulator::Load(const std::string& title_id, bool add_only, bool ensure(vm::falloc(0x10000, 0xf0000, vm::main)); } - atomic_t fnext = 0; - - shared_mutex sprx_mtx; - - named_thread_group workers("SPRX Worker ", GetMaxThreads(), [&] - { - for (usz func_i = fnext++; func_i < file_queue.size(); func_i = fnext++) - { - const auto& path = std::as_const(file_queue)[func_i].first; - - sys_log.notice("Trying to load SPRX: %s", path); - - // Load MSELF or SPRX - fs::file src{path}; - - if (file_queue[func_i].second == 0) - { - // Some files may fail to decrypt due to the lack of klic - src = decrypt_self(std::move(src)); - } - - const ppu_prx_object obj = src; - - if (obj == elf_error::ok) - { - std::unique_lock lock(sprx_mtx); - - if (auto prx = ppu_load_prx(obj, path)) - { - lock.unlock(); - ppu_initialize(*prx); - idm::remove(idm::last_id()); - lock.lock(); - ppu_unload_prx(*prx); - lock.unlock(); - ppu_finalize(*prx); - g_progr_fdone++; - continue; - } - } - - sys_log.error("Failed to load SPRX '%s' (%s)", path, obj.get_error()); - g_progr_fdone++; - continue; - } - }); - - // Join every thread - workers.join(); + ppu_precompile(dir_queue); // Exit "process" Emu.CallAfter([] diff --git a/rpcs3/Loader/mself.hpp b/rpcs3/Loader/mself.hpp new file mode 100644 index 0000000000..637b377d7f --- /dev/null +++ b/rpcs3/Loader/mself.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include "util/types.hpp" +#include "util/endian.hpp" + +struct mself_header +{ + nse_t magic; // "MSF\x00" + be_t ver; // 1 + be_t size; // File size + be_t count; // Number of records + be_t header_size; // ??? + u8 reserved[0x28]; + + u32 get_count(u64 file_size) + { + // Fast sanity check + if (magic != "MSF"_u32 || ver != u32{1} || this->size != file_size) [[unlikely]] + return 0; + + return count; + } +}; + +CHECK_SIZE(mself_header, 0x40); + +struct mself_record +{ + char name[0x20]; + be_t off; + be_t size; + u8 reserved[0x10]; + + u64 get_pos(u64 file_size) + { + // Fast sanity check + if (off < file_size && off + size <= file_size) [[likely]] + return off; + + return 0; + } +}; + +CHECK_SIZE(mself_record, 0x40);