diff --git a/Utilities/File.cpp b/Utilities/File.cpp index 58951da672..c47c1c503e 100644 --- a/Utilities/File.cpp +++ b/Utilities/File.cpp @@ -2255,8 +2255,22 @@ fs::file fs::make_gather(std::vector files) return result; } -fs::pending_file::pending_file(std::string_view path) +bool fs::pending_file::open(std::string_view path) { + file.close(); + + if (!m_path.empty()) + { + fs::remove_file(m_path); + } + + if (path.empty()) + { + fs::g_tls_error = fs::error::noent; + m_dest.clear(); + return false; + } + do { m_path = fmt::format(u8"%s/$%s.%s.tmp", get_parent_dir(path), path.substr(path.find_last_of(fs::delim) + 1), fmt::base57(utils::get_unique_tsc())); @@ -2270,6 +2284,8 @@ fs::pending_file::pending_file(std::string_view path) m_path.clear(); } while (fs::g_tls_error == fs::error::exist); // Only retry if failed due to existing file + + return file.operator bool(); } fs::pending_file::~pending_file() @@ -2284,7 +2300,7 @@ fs::pending_file::~pending_file() bool fs::pending_file::commit(bool overwrite) { - if (!file || m_path.empty()) + if (m_path.empty()) { fs::g_tls_error = fs::error::noent; return false; @@ -2292,7 +2308,11 @@ bool fs::pending_file::commit(bool overwrite) // The temporary file's contents must be on disk before rename #ifndef _WIN32 - file.sync(); + if (file) + { + file.sync(); + } + #endif file.close(); diff --git a/Utilities/File.h b/Utilities/File.h index 5832e0f594..b3ac96698f 100644 --- a/Utilities/File.h +++ b/Utilities/File.h @@ -674,12 +674,24 @@ namespace fs // This is meant to modify files atomically, overwriting is likely bool commit(bool overwrite = true); + bool open(std::string_view path); + + pending_file() noexcept = default; + + pending_file(std::string_view path) noexcept + { + open(path); + } - pending_file(std::string_view path); pending_file(const pending_file&) = delete; pending_file& operator=(const pending_file&) = delete; ~pending_file(); + const std::string& get_temp_path() const + { + return m_path; + } + private: std::string m_path{}; // Pending file path std::string m_dest{}; // Destination file path diff --git a/rpcs3/Emu/Cell/PPUThread.cpp b/rpcs3/Emu/Cell/PPUThread.cpp index b69dca2405..7cb8d1bbad 100644 --- a/rpcs3/Emu/Cell/PPUThread.cpp +++ b/rpcs3/Emu/Cell/PPUThread.cpp @@ -3493,13 +3493,26 @@ namespace // Read-only file view starting with specified offset (for MSELF) struct file_view : fs::file_base { - const fs::file m_file; + const fs::file m_storage; + const fs::file& m_file; const u64 m_off; + const u64 m_max_size; u64 m_pos; - explicit file_view(fs::file&& _file, u64 offset) - : m_file(std::move(_file)) + explicit file_view(const fs::file& _file, u64 offset, u64 max_size) noexcept + : m_storage(fs::file()) + , m_file(_file) , m_off(offset) + , m_max_size(max_size) + , m_pos(0) + { + } + + explicit file_view(fs::file&& _file, u64 offset, u64 max_size) noexcept + : m_storage(std::move(_file)) + , m_file(m_storage) + , m_off(offset) + , m_max_size(max_size) , m_pos(0) { } @@ -3520,18 +3533,14 @@ namespace u64 read(void* buffer, u64 size) override { - const u64 old_pos = m_file.pos(); - m_file.seek(m_off + m_pos); - const u64 result = m_file.read(buffer, size); - ensure(old_pos == m_file.seek(old_pos)); - + const u64 result = file_view::read_at(m_pos, buffer, size); m_pos += result; return result; } u64 read_at(u64 offset, void* buffer, u64 size) override { - return m_file.read_at(offset + m_off, buffer, size); + return m_file.read_at(m_off + m_pos, buffer, std::min(size, utils::sub_saturate(m_max_size, m_pos))); } u64 write(const void*, u64) override @@ -3563,10 +3572,17 @@ namespace }; } -extern fs::file make_file_view(fs::file&& _file, u64 offset) +extern fs::file make_file_view(const fs::file& _file, u64 offset, u64 max_size = umax) { fs::file file; - file.reset(std::make_unique(std::move(_file), offset)); + file.reset(std::make_unique(_file, offset, max_size)); + return file; +} + +extern fs::file make_file_view(fs::file&& _file, u64 offset, u64 max_size = umax) +{ + fs::file file; + file.reset(std::make_unique(std::move(_file), offset, max_size)); return file; } @@ -3835,7 +3851,7 @@ extern void ppu_precompile(std::vector& dir_queue, std::vector(std::move(src), off)); + src = make_file_view(std::move(src), offset); // Adjust path for MSELF too fmt::append(path, "_x%x", off); diff --git a/rpcs3/Emu/Cell/lv2/sys_overlay.cpp b/rpcs3/Emu/Cell/lv2/sys_overlay.cpp index e63716c1cf..f987905fa8 100644 --- a/rpcs3/Emu/Cell/lv2/sys_overlay.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_overlay.cpp @@ -68,7 +68,7 @@ static error_code overlay_load_module(vm::ptr ovlmid, const std::string& vp return CELL_OK; } -fs::file make_file_view(fs::file&&, u64); +fs::file make_file_view(fs::file&& file, u64 offset, u64 size); std::shared_ptr lv2_overlay::load(utils::serial& ar) { @@ -82,7 +82,7 @@ std::shared_ptr lv2_overlay::load(utils::serial& ar) if (file) { u128 klic = g_fxo->get().last_key(); - file = make_file_view(std::move(file), offset); + file = make_file_view(std::move(file), offset, umax); ovlm = ppu_load_overlay(ppu_exec_object{ decrypt_self(std::move(file), reinterpret_cast(&klic)) }, false, path, 0, &ar).first; ensure(ovlm); } diff --git a/rpcs3/Emu/Cell/lv2/sys_prx.cpp b/rpcs3/Emu/Cell/lv2/sys_prx.cpp index a63ea9fbc3..d16c617e34 100644 --- a/rpcs3/Emu/Cell/lv2/sys_prx.cpp +++ b/rpcs3/Emu/Cell/lv2/sys_prx.cpp @@ -286,7 +286,7 @@ static error_code prx_load_module(const std::string& vpath, u64 flags, vm::ptr lv2_prx::load(utils::serial& ar) { @@ -319,7 +319,7 @@ std::shared_ptr lv2_prx::load(utils::serial& ar) if (file) { u128 klic = g_fxo->get().last_key(); - file = make_file_view(std::move(file), offset); + file = make_file_view(std::move(file), offset, umax); prx = ppu_load_prx(ppu_prx_object{ decrypt_self(std::move(file), reinterpret_cast(&klic)) }, false, path, 0, &ar); prx->m_loaded_flags = std::move(loaded_flags); prx->m_external_loaded_flags = std::move(external_flags); diff --git a/rpcs3/Emu/System.cpp b/rpcs3/Emu/System.cpp index e10f833712..760e9d7ccb 100644 --- a/rpcs3/Emu/System.cpp +++ b/rpcs3/Emu/System.cpp @@ -40,6 +40,7 @@ #include "../Crypto/unself.h" #include "util/logs.hpp" #include "util/serialization.hpp" +#include "savestate_utils.hpp" #include #include @@ -87,6 +88,8 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_ extern void send_close_home_menu_cmds(); +fs::file make_file_view(const fs::file& file, u64 offset, u64 size); + fs::file g_tty; atomic_t g_tty_size{0}; std::array, 16> g_tty_input; @@ -711,9 +714,8 @@ bool Emulator::BootRsxCapture(const std::string& path) std::unique_ptr frame = std::make_unique(); utils::serial load; + load.m_file_handler = make_uncompressed_serialization_file_handler(std::move(in_file)); load.set_reading_state(); - in_file.read(load.data, in_file.size()); - load.data.shrink_to_fit(); load(*frame); in_file.close(); @@ -883,9 +885,8 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, { m_ar = std::make_shared(); m_ar->set_reading_state(); - save.seek(0); - save.read(m_ar->data, save.size()); - m_ar->data.shrink_to_fit(); + + m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save)); } m_boot_source_type = CELL_GAME_GAMETYPE_SYS; @@ -946,22 +947,29 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, return game_boot_result::savestate_corrupted; } - if (header.LE_format != (std::endian::native == std::endian::little) || header.offset >= m_ar->data.size()) + if (header.LE_format != (std::endian::native == std::endian::little) || header.offset >= m_ar->get_size(header.offset)) { return game_boot_result::savestate_corrupted; } g_cfg.savestate.state_inspection_mode.set(header.state_inspection_support); - // Emulate seek operation (please avoid using in other places) - m_ar->pos = header.offset; - - if (!is_savestate_version_compatible(m_ar->operator std::vector>(), true)) { - return game_boot_result::savestate_version_unsupported; - } + // Read data on another container to keep the existing data + utils::serial ar_temp; + ar_temp.set_reading_state(); + ar_temp.swap_handler(*m_ar); + ar_temp.seek_pos(header.offset); + ar_temp.m_avoid_large_prefetch = true; - m_ar->pos = sizeof(file_header); // Restore position + if (!is_savestate_version_compatible(ar_temp.operator std::vector>(), true)) + { + return game_boot_result::savestate_version_unsupported; + } + + // Restore file handler + ar_temp.swap_handler(*m_ar); + } argv.clear(); klic.clear(); @@ -1006,8 +1014,9 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch, if (size) { fs::remove_all(path, false); - ensure(tar_object(fs::file(&m_ar->data[m_ar->pos], size)).extract(path)); - m_ar->pos += size; + m_ar->breathe(true); + ensure(tar_object(make_file_view(*static_cast(m_ar->m_file_handler.get())->m_file, m_ar->data_offset, size)).extract(path)); + m_ar->seek_pos(m_ar->pos + size, size >= 4096); } }; @@ -2831,10 +2840,34 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s std::unique_ptr to_ar; + fs::pending_file file; + std::string path; + + while (savestate) + { + path = get_savestate_file(m_title_id, m_path, 0, 0); + + if (!fs::create_path(fs::get_parent_dir(path))) + { + sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error); + savestate = false; + break; + } + + if (!file.open(path)) + { + sys_log.error("Failed to create savestate temporary file! (path='%s', %s)", file.get_temp_path(), fs::g_tls_error); + savestate = false; + break; + } + + to_ar = std::make_unique(); + to_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(file.file)); + break; + } + if (savestate) { - to_ar = std::make_unique(); - // Savestate thread named_thread emu_state_cap_thread("Emu State Capture Thread", [&]() { @@ -2851,12 +2884,33 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s // Avoid duplicating TAR object memory because it can be very large auto save_tar = [&](const std::string& path) { + const usz old_data_start = ar.data_offset; + const usz old_pos = ar.seek_end(); + + ar.breathe(); + ar(usz{}); // Reserve memory to be patched later with correct size - const usz old_size = ar.data.size(); - ar.data = tar_object::save_directory(path, std::move(ar.data)); - ar.seek_end(); - const usz tar_size = ar.data.size() - old_size; - std::memcpy(ar.data.data() + old_size - sizeof(usz), &tar_size, sizeof(usz)); + tar_object::save_directory(path, ar); + + const usz new_pos = ar.seek_end(); + const usz tar_size = new_pos - old_pos - sizeof(usz); + + // Check if breathe() actually did something, in this case memory needs to be discarded + const bool was_emptied = old_data_start != ar.data_offset; + + if (was_emptied) + { + ensure(ar.data_offset > old_data_start); + + // Write to file directly (slower) + ar.m_file_handler->handle_file_op(ar, old_pos, sizeof(tar_size), &tar_size); + } + else + { + // If noty written to file, simply write to memory + std::memcpy(ar.data.data() + old_pos - old_data_start, &tar_size, sizeof(usz)); + } + sys_log.success("Saved the contents of directory '%s' (size=0x%x)", path, tar_size); }; @@ -2900,8 +2954,8 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s ar("RPCS3SAV"_u64); ar(std::endian::native == std::endian::little); ar(g_cfg.savestate.state_inspection_mode.get()); - ar(std::array{}); // Reserved for future use ar(usz{0}); // Offset of versioning data, to be overwritten at the end of saving + ar(std::array{}); // Reserved for future use if (auto dir = vfs::get("/dev_bdvd/PS3_GAME"); fs::is_dir(dir) && !fs::is_file(fs::get_parent_dir(dir) + "/PS3_DISC.SFB")) { @@ -2953,24 +3007,29 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s if (savestate) { - const std::string path = get_savestate_file(m_title_id, m_path, 0, 0); - - if (!fs::create_path(fs::get_parent_dir(path))) - { - sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error); - } - - fs::pending_file file(path); - // Identifer -> version std::vector> used_serial = read_used_savestate_versions(); auto& ar = *to_ar; + const usz pos = ar.seek_end(); - std::memcpy(&ar.data[10], &pos, 8);// Set offset + + // Patch offset with a direct write + ar.m_file_handler->handle_file_op(ar, 10, sizeof(usz), &pos); + + // Write the version data at the end ar(used_serial); - if (!file.file || (file.file.write(ar.data), !file.commit())) + // Final file write, the file is ready to be committed + ar.breathe(true); + +#ifndef _WIN32 + // The temporary file's contents must be on disk before rename + // Flush to file + ar.m_file_handler->handle_file_op(ar, umax, umax, nullptr); +#endif + + if (!file.commit()) { sys_log.error("Failed to write savestate to file! (path='%s', %s)", path, fs::g_tls_error); savestate = false; diff --git a/rpcs3/Emu/savestate_utils.cpp b/rpcs3/Emu/savestate_utils.cpp index 7934892ccf..13e8ddd525 100644 --- a/rpcs3/Emu/savestate_utils.cpp +++ b/rpcs3/Emu/savestate_utils.cpp @@ -1,9 +1,11 @@ #include "stdafx.h" #include "util/types.hpp" -#include "util/serialization.hpp" #include "util/logs.hpp" +#include "util/asm.hpp" #include "Utilities/File.h" +#include "Utilities/StrFmt.h" #include "system_config.h" +#include "savestate_utils.hpp" #include "System.h" @@ -11,6 +13,14 @@ LOG_CHANNEL(sys_log, "SYS"); +template <> +void fmt_class_string::format(std::string& out, u64 arg) +{ + const utils::serial& ar = get_object(arg); + + fmt::append(out, "{ %s, 0x%x/0%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size()); +} + struct serial_ver_t { bool used = false; @@ -79,7 +89,7 @@ SERIALIZATION_VER(sys_io, 23, 1) SERIALIZATION_VER(LLE, 24, 1) SERIALIZATION_VER(HLE, 25, 1) -std::vector> get_savestate_versioning_data(const fs::file& file) +std::vector> get_savestate_versioning_data(fs::file&& file) { if (!file) { @@ -105,11 +115,10 @@ std::vector> get_savestate_versioning_data(const fs::file& f return {}; } - file.seek(offs); - utils::serial ar; ar.set_reading_state(); - file.read(ar.data, fsize - offs); + ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file)); + ar.seek_pos(offs); return ar; } @@ -172,9 +181,9 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_ return fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT"; } -bool is_savestate_compatible(const fs::file& file) +bool is_savestate_compatible(fs::file&& file) { - return is_savestate_version_compatible(get_savestate_versioning_data(file), false); + return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file)), false); } std::vector> read_used_savestate_versions() @@ -199,7 +208,7 @@ bool boot_last_savestate(bool testing) { if (!g_cfg.savestate.suspend_emu && !Emu.GetTitleID().empty() && (Emu.IsRunning() || Emu.GetStatus() == system_state::paused)) { - extern bool is_savestate_compatible(const fs::file& file); + extern bool is_savestate_compatible(fs::file&& file); const std::string save_dir = fs::get_cache_dir() + "/savestates/"; @@ -252,3 +261,149 @@ bool boot_last_savestate(bool testing) return false; } + +bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) +{ + if (ar.is_writing()) + { + if (data) + { + m_file->seek(pos); + m_file->write(data, size); + return true; + } + + m_file->seek(ar.data_offset); + m_file->write(ar.data); + + if (pos == umax && size == umax) + { + // Request to flush the file to disk + m_file->sync(); + } + + ar.data_offset += ar.data.size(); + ar.data.clear(); + ar.seek_end(); + return true; + } + + if (!size) + { + return true; + } + + if (pos == 0 && size == umax) + { + // Discard loaded data until pos if profitable + const usz limit = ar.data_offset + ar.data.size(); + + if (ar.pos > ar.data_offset && ar.pos < limit) + { + const usz may_discard_bytes = ar.pos - ar.data_offset; + const usz moved_byte_count_on_discard = limit - ar.pos; + + // Cheeck profitability (check recycled memory and std::memmove costs) + if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3)) + { + ar.data_offset += may_discard_bytes; + ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes); + + if (ar.data.capacity() >= 0x200'0000) + { + // Discard memory + ar.data.shrink_to_fit(); + } + } + + return true; + } + + // Discard all loaded data + ar.data_offset += ar.data.size(); + ar.data.clear(); + + if (ar.data.capacity() >= 0x200'0000) + { + // Discard memory + ar.data.shrink_to_fit(); + } + + return true; + } + + if (~size < pos) + { + // Overflow + return false; + } + + if (ar.data.empty() && pos != ar.pos) + { + // Relocate instead oof over-fetch + ar.seek_pos(pos); + } + + const usz read_pre_buffer = utils::sub_saturate(ar.data_offset, pos); + + if (read_pre_buffer) + { + // Read past data + // Harsh operation on performance, luckily rare and not typically needed + // Also this may would be disallowed when moving to compressed files + // This may be a result of wrong usage of breathe() function + ar.data.resize(ar.data.size() + read_pre_buffer); + std::memmove(ar.data.data() + read_pre_buffer, ar.data.data(), ar.data.size() - read_pre_buffer); + ensure(m_file->read_at(pos, ar.data.data(), read_pre_buffer) == read_pre_buffer); + ar.data_offset -= read_pre_buffer; + } + + const usz read_past_buffer = utils::sub_saturate(pos + size, ar.data_offset + ar.data.size()); + + if (read_past_buffer) + { + // Read proceeding data + // More lightweight operation, this is the common operation + // Allowed to fail, if memory is truly needed an assert would take place later + const usz old_size = ar.data.size(); + + // Try to prefetch data by reading more than requested + ar.data.resize(std::max({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} })); + ar.data.resize(m_file->read_at(old_size + ar.data_offset, data ? const_cast(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size); + } + + return true; +} + + +usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const +{ + if (ar.is_writing()) + { + return m_file->size(); + } + + const usz memory_available = ar.data_offset + ar.data.size(); + + if (memory_available >= recommended) + { + // Avoid calling size() if possible + return memory_available; + } + + return std::max(m_file->size(), memory_available); +} + +namespace stx +{ + extern void serial_breathe(utils::serial& ar) + { + ar.breathe(); + } +} + +// MSVC bug workaround, see above similar case +extern void serial_breathe(utils::serial& ar) +{ + ::stx::serial_breathe(ar); +} diff --git a/rpcs3/Emu/savestate_utils.hpp b/rpcs3/Emu/savestate_utils.hpp new file mode 100644 index 0000000000..c3aa8d10d0 --- /dev/null +++ b/rpcs3/Emu/savestate_utils.hpp @@ -0,0 +1,45 @@ +#include "util/serialization.hpp" + +namespace fs +{ + class file; +} + +// Uncompressed file serialization handler +struct uncompressed_serialization_file_handler : utils::serialization_file_handler +{ + const std::unique_ptr m_file_storage; + const std::add_pointer_t m_file; + + explicit uncompressed_serialization_file_handler(fs::file&& file) noexcept + : utils::serialization_file_handler() + , m_file_storage(std::make_unique(std::move(file))) + , m_file(m_file_storage.get()) + { + } + + explicit uncompressed_serialization_file_handler(const fs::file& file) noexcept + : utils::serialization_file_handler() + , m_file_storage(nullptr) + , m_file(std::addressof(file)) + { + } + + // Handle file read and write requests + bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override; + + // Get available memory or file size + // Preferably memory size if is already greater/equal to recommended to avoid additional file ops + usz get_size(const utils::serial& ar, usz recommended) const override; +}; + +inline std::unique_ptr make_uncompressed_serialization_file_handler(fs::file&& file) +{ + return std::make_unique(std::move(file)); +} + +inline std::unique_ptr make_uncompressed_serialization_file_handler(const fs::file& file) +{ + return std::make_unique(file); +} + diff --git a/rpcs3/Loader/TAR.cpp b/rpcs3/Loader/TAR.cpp index 6eb48f973d..74148260eb 100644 --- a/rpcs3/Loader/TAR.cpp +++ b/rpcs3/Loader/TAR.cpp @@ -8,6 +8,7 @@ #include "TAR.h" #include "util/asm.hpp" +#include "util/serialization.hpp" #include @@ -258,14 +259,14 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs) return true; } -std::vector tar_object::save_directory(const std::string& src_dir, std::vector&& init, const process_func& func, std::string full_path) +void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func, std::string full_path) { const std::string& target_path = full_path.empty() ? src_dir : full_path; fs::stat_t stat{}; if (!fs::get_stat(target_path, stat)) { - return std::move(init); + return; } if (stat.is_directory) @@ -276,13 +277,13 @@ std::vector tar_object::save_directory(const std::string& src_dir, std::vect { if (entry.name.find_first_not_of('.') == umax) continue; - init = save_directory(src_dir, std::move(init), func, target_path + '/' + entry.name); + save_directory(src_dir, ar, func, target_path + '/' + entry.name); has_items = true; } if (has_items) { - return std::move(init); + return; } } @@ -304,34 +305,34 @@ std::vector tar_object::save_directory(const std::string& src_dir, std::vect std::string saved_path{target_path.data() + src_dir.size(), target_path.size() - src_dir.size()}; - const u64 old_size = init.size(); - init.resize(old_size + sizeof(TARHeader)); + const u64 old_size = ar.data.size(); + ar.data.resize(old_size + sizeof(TARHeader)); if (!stat.is_directory) { fs::file fd(target_path); - const u64 old_size2 = init.size(); + const u64 old_size2 = ar.data.size(); if (func) { // Use custom function for file saving if provided // Allows for example to compress PNG files as JPEG in the TAR itself - if (!func(fd, saved_path, std::move(init))) + if (!func(fd, saved_path, ar)) { // Revert (this entry should not be included if func returns false) - init.resize(old_size); - return std::move(init); + ar.data.resize(old_size); + return; } } else { - init.resize(init.size() + stat.size); - ensure(fd.read(init.data() + old_size2, stat.size) == stat.size); + ar.data.resize(ar.data.size() + stat.size); + ensure(fd.read(ar.data.data() + old_size2, stat.size) == stat.size); } // Align - init.resize(old_size2 + utils::align(init.size() - old_size2, 512)); + ar.data.resize(old_size2 + utils::align(ar.data.size() - old_size2, 512)); fd.close(); fs::utime(target_path, stat.atime, stat.mtime); @@ -352,8 +353,10 @@ std::vector tar_object::save_directory(const std::string& src_dir, std::vect write_octal(header.padding, stat.atime); header.filetype = stat.is_directory ? '5' : '0'; - std::memcpy(init.data() + old_size, &header, sizeof(header)); - return std::move(init); + std::memcpy(ar.data.data() + old_size, &header, sizeof(header)); + + // TAR is an old format which does not depend on previous data so memory ventilation is trivial here + ar.breathe(); } bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file) diff --git a/rpcs3/Loader/TAR.h b/rpcs3/Loader/TAR.h index 6a60734297..352220d497 100644 --- a/rpcs3/Loader/TAR.h +++ b/rpcs3/Loader/TAR.h @@ -22,6 +22,11 @@ namespace fs class file; } +namespace utils +{ + struct serial; +} + class tar_object { const fs::file& m_file; @@ -38,13 +43,13 @@ public: fs::file get_file(const std::string& path); - using process_func = std::function&&)>; + using process_func = std::function; // Extract all files in archive to destination (as VFS if is_vfs is true) // Allow to optionally specify explicit mount point (which may be directory meant for extraction) bool extract(std::string prefix_path = {}, bool is_vfs = false); - static std::vector save_directory(const std::string& src_dir, std::vector&& init = std::vector{}, const process_func& func = {}, std::string append_path = {}); + static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::string append_path = {}); }; bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file = {}); diff --git a/rpcs3/emucore.vcxproj b/rpcs3/emucore.vcxproj index bcadb9051e..cc7c60f51d 100644 --- a/rpcs3/emucore.vcxproj +++ b/rpcs3/emucore.vcxproj @@ -607,6 +607,7 @@ + diff --git a/rpcs3/emucore.vcxproj.filters b/rpcs3/emucore.vcxproj.filters index 087b89c120..5a11ef535f 100644 --- a/rpcs3/emucore.vcxproj.filters +++ b/rpcs3/emucore.vcxproj.filters @@ -2389,6 +2389,9 @@ Crypto + + Emu + diff --git a/rpcs3/rpcs3qt/game_list_frame.cpp b/rpcs3/rpcs3qt/game_list_frame.cpp index 1e902fe03f..e8f3e68ff1 100644 --- a/rpcs3/rpcs3qt/game_list_frame.cpp +++ b/rpcs3/rpcs3qt/game_list_frame.cpp @@ -1078,7 +1078,7 @@ void game_list_frame::ShowContextMenu(const QPoint &pos) }); } - extern bool is_savestate_compatible(const fs::file& file); + extern bool is_savestate_compatible(fs::file&& file); if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate))) { diff --git a/rpcs3/util/fixed_typemap.hpp b/rpcs3/util/fixed_typemap.hpp index d296777b35..5442c4175f 100644 --- a/rpcs3/util/fixed_typemap.hpp +++ b/rpcs3/util/fixed_typemap.hpp @@ -251,6 +251,12 @@ namespace stx *m_order++ = data; *m_info++ = &type; m_init[id] = true; + + if (ar) + { + extern void serial_breathe(utils::serial& ar); + serial_breathe(*ar); + } } } @@ -310,7 +316,8 @@ namespace stx } } - void save(utils::serial& ar) + template requires (std::is_same_v) + void save(T& ar) { if (!is_init()) { @@ -332,7 +339,11 @@ namespace stx // Save data in forward order for (u32 i = _max; i; i--) { - if (auto save = (*std::prev(m_info, i))->save) save(*std::prev(m_order, i), ar); + if (auto save = (*std::prev(m_info, i))->save) + { + save(*std::prev(m_order, i), ar); + ar.breathe(); + } } } diff --git a/rpcs3/util/serialization.hpp b/rpcs3/util/serialization.hpp index cb127af9c3..542f84e462 100644 --- a/rpcs3/util/serialization.hpp +++ b/rpcs3/util/serialization.hpp @@ -8,13 +8,13 @@ namespace utils template concept FastRandomAccess = requires (T& obj) { - std::data(obj)[0]; + std::data(obj)[std::size(obj)]; }; template concept Reservable = requires (T& obj) { - obj.reserve(0); + obj.reserve(std::size(obj)); }; template @@ -32,15 +32,29 @@ namespace utils template concept ListAlike = requires (T& obj) { obj.insert(obj.end(), std::declval()); }; + struct serial; + + struct serialization_file_handler + { + serialization_file_handler() = default; + virtual ~serialization_file_handler() = default; + + virtual bool handle_file_op(serial& ar, usz pos, usz size, const void* data = nullptr) = 0; + virtual usz get_size(const utils::serial& ar, usz recommended = umax) const = 0; + }; + struct serial { std::vector data; + usz data_offset = 0; usz pos = 0; bool m_is_writing = true; + bool m_avoid_large_prefetch = false; + std::unique_ptr m_file_handler; - serial() = default; + serial() noexcept = default; serial(const serial&) = delete; - ~serial() = default; + ~serial() noexcept = default; // Checks if this instance is currently used for serialization bool is_writing() const @@ -58,21 +72,44 @@ namespace utils } } - bool raw_serialize(const void* ptr, usz size) + template requires (std::is_convertible_v, const void*>) + bool raw_serialize(Func&& memory_provider, usz size) { + if (!size) + { + return true; + } + + // Overflow check + ensure(~pos >= size); + if (is_writing()) { - data.insert(data.begin() + pos, static_cast(ptr), static_cast(ptr) + size); + ensure(pos >= data_offset); + const auto ptr = reinterpret_cast(memory_provider()); + data.insert(data.begin() + pos - data_offset, ptr, ptr + size); pos += size; return true; } - ensure(data.size() - pos >= size); - std::memcpy(const_cast(ptr), data.data() + pos, size); + if (data.empty() || pos < data_offset || pos + size > data.size() + data_offset) + { + // Load from file + ensure(m_file_handler); + ensure(m_file_handler->handle_file_op(*this, pos, size, nullptr)); + ensure(!data.empty() && pos >= data_offset && pos + size <= data.size() + data_offset); + } + + std::memcpy(const_cast(static_cast(memory_provider())), data.data() + pos - data_offset, size); pos += size; return true; } + bool raw_serialize(const void* ptr, usz size) + { + return raw_serialize(FN(ptr), size); + } + template requires Integral bool serialize_vle(T&& value) { @@ -157,19 +194,15 @@ namespace utils return true; } - obj.clear(); - usz size = 0; if (!deserialize_vle(size)) { return false; } - obj.resize(size); - if constexpr (Bitcopy) { - if (!raw_serialize(obj.data(), sizeof(obj[0]) * size)) + if (!raw_serialize([&](){ obj.resize(size); return obj.data(); }, sizeof(obj[0]) * size)) { obj.clear(); return false; @@ -177,6 +210,10 @@ namespace utils } else { + // TODO: Postpone resizing to after file bounds checks + obj.clear(); + obj.resize(size); + for (auto&& value : obj) { if (!serialize(value)) @@ -306,7 +343,9 @@ namespace utils } m_is_writing = false; + m_avoid_large_prefetch = false; pos = 0; + data_offset = 0; } // Reset to empty serialization manager @@ -315,23 +354,53 @@ namespace utils data.clear(); m_is_writing = true; pos = 0; + data_offset = 0; + m_file_handler.reset(); } usz seek_end(usz backwards = 0) { - ensure(pos >= backwards); - pos = data.size() - backwards; + ensure(data.size() + data_offset >= backwards); + pos = data.size() + data_offset - backwards; return pos; } + usz seek_pos(usz val, bool empty_data = false) + { + const usz old_pos = std::exchange(pos, val); + + if (empty_data || data.empty()) + { + // Relocate future data + data.clear(); + data_offset = pos; + } + + return old_pos; + } + usz pad_from_end(usz forwards) { ensure(is_writing()); - pos = data.size(); - data.resize(pos + forwards); + pos = data.size() + data_offset; + data.resize(data.size() + forwards); return pos; } + // Allow for memory saving operations: if writing, flush to file if too large. If reading, discard memory (not implemented). + // Execute only if past memory is known to not going be reused + void breathe(bool forced = false) + { + if (!forced && (!m_file_handler || (data.size() < 0x20'0000 && pos >= data_offset))) + { + // Let's not do anything if less than 32MB + return; + } + + ensure(m_file_handler); + ensure(m_file_handler->handle_file_op(*this, 0, umax, nullptr)); + } + template requires (std::is_copy_constructible_v>) && (std::is_constructible_v> || Bitcopy> || std::is_constructible_v, stx::exact_t> || TupleAlike>) operator T() @@ -364,6 +433,16 @@ namespace utils } } + void swap_handler(serial& ar) + { + std::swap(ar.m_file_handler, this->m_file_handler); + } + + usz get_size(usz recommended = umax) const + { + return m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size(); + } + template requires (std::is_copy_constructible_v && std::is_constructible_v && Bitcopy) std::pair try_read() { @@ -372,10 +451,11 @@ namespace utils return {}; } - const usz left = data.size() - pos; + const usz end_pos = pos + sizeof(T); + const usz size = get_size(end_pos); using type = std::remove_const_t; - if (left >= sizeof(type)) + if (size >= end_pos) { u8 buf[sizeof(type)]{}; ensure(raw_serialize(buf, sizeof(buf))); @@ -389,7 +469,8 @@ namespace utils // Used when an invalid state is encountered somewhere in a place we can't check success code such as constructor) bool is_valid() const { - return pos <= data.size(); + // TODO + return true; } }; }