ShuriZma
/
rpcs3
mirror of https://github.com/RPCS3/rpcs3.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Savestates: Implement initial RAM ventilation system

This commit is contained in:
Eladash 2023-10-29 01:46:52 +02:00 committed by Elad Ashkenazi
parent 2db607c716
commit 66d01b688c
15 changed files with 513 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Utilities/File.cpp
View File

@ -2255,8 +2255,22 @@ fs::file fs::make_gather(std::vector<fs::file> files)
return result;
}
fs::pending_file::pending_file(std::string_view path)
bool fs::pending_file::open(std::string_view path)
{
file.close();
if (!m_path.empty())
{
fs::remove_file(m_path);
}
if (path.empty())
{
fs::g_tls_error = fs::error::noent;
m_dest.clear();
return false;
}
do
{
m_path = fmt::format(u8"%s/＄%s.%s.tmp", get_parent_dir(path), path.substr(path.find_last_of(fs::delim) + 1), fmt::base57(utils::get_unique_tsc()));
@ -2270,6 +2284,8 @@ fs::pending_file::pending_file(std::string_view path)
m_path.clear();
}
while (fs::g_tls_error == fs::error::exist); // Only retry if failed due to existing file
return file.operator bool();
}
fs::pending_file::~pending_file()
@ -2284,7 +2300,7 @@ fs::pending_file::~pending_file()
bool fs::pending_file::commit(bool overwrite)
{
if (!file || m_path.empty())
if (m_path.empty())
{
fs::g_tls_error = fs::error::noent;
return false;
@ -2292,7 +2308,11 @@ bool fs::pending_file::commit(bool overwrite)
// The temporary file's contents must be on disk before rename
#ifndef _WIN32
if (file)
{
file.sync();
}
#endif
file.close();

14
Utilities/File.h
View File

@ -674,12 +674,24 @@ namespace fs
// This is meant to modify files atomically, overwriting is likely
bool commit(bool overwrite = true);
bool open(std::string_view path);
pending_file() noexcept = default;
pending_file(std::string_view path) noexcept
{
open(path);
}
pending_file(std::string_view path);
pending_file(const pending_file&) = delete;
pending_file& operator=(const pending_file&) = delete;
~pending_file();
const std::string& get_temp_path() const
{
return m_path;
}
private:
std::string m_path{}; // Pending file path
std::string m_dest{}; // Destination file path

40
rpcs3/Emu/Cell/PPUThread.cpp
View File

@ -3493,13 +3493,26 @@ namespace
// Read-only file view starting with specified offset (for MSELF)
struct file_view : fs::file_base
{
const fs::file m_file;
const fs::file m_storage;
const fs::file& m_file;
const u64 m_off;
const u64 m_max_size;
u64 m_pos;
explicit file_view(fs::file&& _file, u64 offset)
: m_file(std::move(_file))
explicit file_view(const fs::file& _file, u64 offset, u64 max_size) noexcept
: m_storage(fs::file())
, m_file(_file)
, m_off(offset)
, m_max_size(max_size)
, m_pos(0)
{
}
explicit file_view(fs::file&& _file, u64 offset, u64 max_size) noexcept
: m_storage(std::move(_file))
, m_file(m_storage)
, m_off(offset)
, m_max_size(max_size)
, m_pos(0)
{
}
@ -3520,18 +3533,14 @@ namespace
u64 read(void* buffer, u64 size) override
{
const u64 old_pos = m_file.pos();
m_file.seek(m_off + m_pos);
const u64 result = m_file.read(buffer, size);
ensure(old_pos == m_file.seek(old_pos));
const u64 result = file_view::read_at(m_pos, buffer, size);
m_pos += result;
return result;
}
u64 read_at(u64 offset, void* buffer, u64 size) override
{
return m_file.read_at(offset + m_off, buffer, size);
return m_file.read_at(m_off + m_pos, buffer, std::min<u64>(size, utils::sub_saturate<u64>(m_max_size, m_pos)));
}
u64 write(const void*, u64) override
@ -3563,10 +3572,17 @@ namespace
};
}
extern fs::file make_file_view(fs::file&& _file, u64 offset)
extern fs::file make_file_view(const fs::file& _file, u64 offset, u64 max_size = umax)
{
fs::file file;
file.reset(std::make_unique<file_view>(std::move(_file), offset));
file.reset(std::make_unique<file_view>(_file, offset, max_size));
return file;
}
extern fs::file make_file_view(fs::file&& _file, u64 offset, u64 max_size = umax)
{
fs::file file;
file.reset(std::make_unique<file_view>(std::move(_file), offset, max_size));
return file;
}
@ -3835,7 +3851,7 @@ extern void ppu_precompile(std::vector<std::string>& dir_queue, std::vector<ppu_
if (u64 off = offset)
{
// Adjust offset for MSELF
src.reset(std::make_unique<file_view>(std::move(src), off));
src = make_file_view(std::move(src), offset);
// Adjust path for MSELF too
fmt::append(path, "_x%x", off);

4
rpcs3/Emu/Cell/lv2/sys_overlay.cpp
View File

@ -68,7 +68,7 @@ static error_code overlay_load_module(vm::ptr<u32> ovlmid, const std::string& vp
return CELL_OK;
}
fs::file make_file_view(fs::file&&, u64);
fs::file make_file_view(fs::file&& file, u64 offset, u64 size);
std::shared_ptr<void> lv2_overlay::load(utils::serial& ar)
{
@ -82,7 +82,7 @@ std::shared_ptr<void> lv2_overlay::load(utils::serial& ar)
if (file)
{
u128 klic = g_fxo->get<loaded_npdrm_keys>().last_key();
file = make_file_view(std::move(file), offset);
file = make_file_view(std::move(file), offset, umax);
ovlm = ppu_load_overlay(ppu_exec_object{ decrypt_self(std::move(file), reinterpret_cast<u8*>(&klic)) }, false, path, 0, &ar).first;
ensure(ovlm);
}

4
rpcs3/Emu/Cell/lv2/sys_prx.cpp
View File

@ -286,7 +286,7 @@ static error_code prx_load_module(const std::string& vpath, u64 flags, vm::ptr<s
return not_an_error(idm::last_id());
}
fs::file make_file_view(fs::file&& _file, u64 offset);
fs::file make_file_view(fs::file&& file, u64 offset, u64 size);
std::shared_ptr<void> lv2_prx::load(utils::serial& ar)
{
@ -319,7 +319,7 @@ std::shared_ptr<void> lv2_prx::load(utils::serial& ar)
if (file)
{
u128 klic = g_fxo->get<loaded_npdrm_keys>().last_key();
file = make_file_view(std::move(file), offset);
file = make_file_view(std::move(file), offset, umax);
prx = ppu_load_prx(ppu_prx_object{ decrypt_self(std::move(file), reinterpret_cast<u8*>(&klic)) }, false, path, 0, &ar);
prx->m_loaded_flags = std::move(loaded_flags);
prx->m_external_loaded_flags = std::move(external_flags);

121
rpcs3/Emu/System.cpp
View File

@ -40,6 +40,7 @@
#include "../Crypto/unself.h"
#include "util/logs.hpp"
#include "util/serialization.hpp"
#include "savestate_utils.hpp"
#include <fstream>
#include <memory>
@ -87,6 +88,8 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_
extern void send_close_home_menu_cmds();
fs::file make_file_view(const fs::file& file, u64 offset, u64 size);
fs::file g_tty;
atomic_t<s64> g_tty_size{0};
std::array<std::deque<std::string>, 16> g_tty_input;
@ -711,9 +714,8 @@ bool Emulator::BootRsxCapture(const std::string& path)
std::unique_ptr<rsx::frame_capture_data> frame = std::make_unique<rsx::frame_capture_data>();
utils::serial load;
load.m_file_handler = make_uncompressed_serialization_file_handler(std::move(in_file));
load.set_reading_state();
in_file.read(load.data, in_file.size());
load.data.shrink_to_fit();
load(*frame);
in_file.close();
@ -883,9 +885,8 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
{
m_ar = std::make_shared<utils::serial>();
m_ar->set_reading_state();
save.seek(0);
save.read(m_ar->data, save.size());
m_ar->data.shrink_to_fit();
m_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(save));
}
m_boot_source_type = CELL_GAME_GAMETYPE_SYS;
@ -946,22 +947,29 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
return game_boot_result::savestate_corrupted;
}
if (header.LE_format != (std::endian::native == std::endian::little) || header.offset >= m_ar->data.size())
if (header.LE_format != (std::endian::native == std::endian::little) || header.offset >= m_ar->get_size(header.offset))
{
return game_boot_result::savestate_corrupted;
}
g_cfg.savestate.state_inspection_mode.set(header.state_inspection_support);
// Emulate seek operation (please avoid using in other places)
m_ar->pos = header.offset;
{
// Read data on another container to keep the existing data
utils::serial ar_temp;
ar_temp.set_reading_state();
ar_temp.swap_handler(*m_ar);
ar_temp.seek_pos(header.offset);
ar_temp.m_avoid_large_prefetch = true;
if (!is_savestate_version_compatible(m_ar->operator std::vector<std::pair<u16, u16>>(), true))
if (!is_savestate_version_compatible(ar_temp.operator std::vector<std::pair<u16, u16>>(), true))
{
return game_boot_result::savestate_version_unsupported;
}
m_ar->pos = sizeof(file_header); // Restore position
// Restore file handler
ar_temp.swap_handler(*m_ar);
}
argv.clear();
klic.clear();
@ -1006,8 +1014,9 @@ game_boot_result Emulator::Load(const std::string& title_id, bool is_disc_patch,
if (size)
{
fs::remove_all(path, false);
ensure(tar_object(fs::file(&m_ar->data[m_ar->pos], size)).extract(path));
m_ar->pos += size;
m_ar->breathe(true);
ensure(tar_object(make_file_view(*static_cast<uncompressed_serialization_file_handler*>(m_ar->m_file_handler.get())->m_file, m_ar->data_offset, size)).extract(path));
m_ar->seek_pos(m_ar->pos + size, size >= 4096);
}
};
@ -2831,10 +2840,34 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
std::unique_ptr<utils::serial> to_ar;
fs::pending_file file;
std::string path;
while (savestate)
{
path = get_savestate_file(m_title_id, m_path, 0, 0);
if (!fs::create_path(fs::get_parent_dir(path)))
{
sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error);
savestate = false;
break;
}
if (!file.open(path))
{
sys_log.error("Failed to create savestate temporary file! (path='%s', %s)", file.get_temp_path(), fs::g_tls_error);
savestate = false;
break;
}
to_ar = std::make_unique<utils::serial>();
to_ar->m_file_handler = make_uncompressed_serialization_file_handler(std::move(file.file));
break;
}
if (savestate)
{
to_ar = std::make_unique<utils::serial>();
// Savestate thread
named_thread emu_state_cap_thread("Emu State Capture Thread", [&]()
{
@ -2851,12 +2884,33 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
// Avoid duplicating TAR object memory because it can be very large
auto save_tar = [&](const std::string& path)
{
const usz old_data_start = ar.data_offset;
const usz old_pos = ar.seek_end();
ar.breathe();
ar(usz{}); // Reserve memory to be patched later with correct size
const usz old_size = ar.data.size();
ar.data = tar_object::save_directory(path, std::move(ar.data));
ar.seek_end();
const usz tar_size = ar.data.size() - old_size;
std::memcpy(ar.data.data() + old_size - sizeof(usz), &tar_size, sizeof(usz));
tar_object::save_directory(path, ar);
const usz new_pos = ar.seek_end();
const usz tar_size = new_pos - old_pos - sizeof(usz);
// Check if breathe() actually did something, in this case memory needs to be discarded
const bool was_emptied = old_data_start != ar.data_offset;
if (was_emptied)
{
ensure(ar.data_offset > old_data_start);
// Write to file directly (slower)
ar.m_file_handler->handle_file_op(ar, old_pos, sizeof(tar_size), &tar_size);
}
else
{
// If noty written to file, simply write to memory
std::memcpy(ar.data.data() + old_pos - old_data_start, &tar_size, sizeof(usz));
}
sys_log.success("Saved the contents of directory '%s' (size=0x%x)", path, tar_size);
};
@ -2900,8 +2954,8 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
ar("RPCS3SAV"_u64);
ar(std::endian::native == std::endian::little);
ar(g_cfg.savestate.state_inspection_mode.get());
ar(std::array<u8, 32>{}); // Reserved for future use
ar(usz{0}); // Offset of versioning data, to be overwritten at the end of saving
ar(std::array<u8, 32>{}); // Reserved for future use
if (auto dir = vfs::get("/dev_bdvd/PS3_GAME"); fs::is_dir(dir) && !fs::is_file(fs::get_parent_dir(dir) + "/PS3_DISC.SFB"))
{
@ -2953,24 +3007,29 @@ void Emulator::Kill(bool allow_autoexit, bool savestate, savestate_stage* save_s
if (savestate)
{
const std::string path = get_savestate_file(m_title_id, m_path, 0, 0);
if (!fs::create_path(fs::get_parent_dir(path)))
{
sys_log.error("Failed to create savestate directory! (path='%s', %s)", fs::get_parent_dir(path), fs::g_tls_error);
}
fs::pending_file file(path);
// Identifer -> version
std::vector<std::pair<u16, u16>> used_serial = read_used_savestate_versions();
auto& ar = *to_ar;
const usz pos = ar.seek_end();
std::memcpy(&ar.data[10], &pos, 8);// Set offset
// Patch offset with a direct write
ar.m_file_handler->handle_file_op(ar, 10, sizeof(usz), &pos);
// Write the version data at the end
ar(used_serial);
if (!file.file || (file.file.write(ar.data), !file.commit()))
// Final file write, the file is ready to be committed
ar.breathe(true);
#ifndef _WIN32
// The temporary file's contents must be on disk before rename
// Flush to file
ar.m_file_handler->handle_file_op(ar, umax, umax, nullptr);
#endif
if (!file.commit())
{
sys_log.error("Failed to write savestate to file! (path='%s', %s)", path, fs::g_tls_error);
savestate = false;

171
rpcs3/Emu/savestate_utils.cpp
View File

@ -1,9 +1,11 @@
#include "stdafx.h"
#include "util/types.hpp"
#include "util/serialization.hpp"
#include "util/logs.hpp"
#include "util/asm.hpp"
#include "Utilities/File.h"
#include "Utilities/StrFmt.h"
#include "system_config.h"
#include "savestate_utils.hpp"
#include "System.h"
@ -11,6 +13,14 @@
LOG_CHANNEL(sys_log, "SYS");
template <>
void fmt_class_string<utils::serial>::format(std::string& out, u64 arg)
{
const utils::serial& ar = get_object(arg);
fmt::append(out, "{ %s, 0x%x/0%x, memory=0x%x }", ar.is_writing() ? "writing" : "reading", ar.pos, ar.data_offset + ar.data.size(), ar.data.size());
}
struct serial_ver_t
{
bool used = false;
@ -79,7 +89,7 @@ SERIALIZATION_VER(sys_io, 23, 1)
SERIALIZATION_VER(LLE, 24, 1)
SERIALIZATION_VER(HLE, 25, 1)
std::vector<std::pair<u16, u16>> get_savestate_versioning_data(const fs::file& file)
std::vector<std::pair<u16, u16>> get_savestate_versioning_data(fs::file&& file)
{
if (!file)
{
@ -105,11 +115,10 @@ std::vector<std::pair<u16, u16>> get_savestate_versioning_data(const fs::file& f
return {};
}
file.seek(offs);
utils::serial ar;
ar.set_reading_state();
file.read(ar.data, fsize - offs);
ar.m_file_handler = make_uncompressed_serialization_file_handler(std::move(file));
ar.seek_pos(offs);
return ar;
}
@ -172,9 +181,9 @@ std::string get_savestate_file(std::string_view title_id, std::string_view boot_
return fs::get_cache_dir() + "/savestates/" + title + "/" + title + '_' + prefix + '_' + save_id + ".SAVESTAT";
}
bool is_savestate_compatible(const fs::file& file)
bool is_savestate_compatible(fs::file&& file)
{
return is_savestate_version_compatible(get_savestate_versioning_data(file), false);
return is_savestate_version_compatible(get_savestate_versioning_data(std::move(file)), false);
}
std::vector<std::pair<u16, u16>> read_used_savestate_versions()
@ -199,7 +208,7 @@ bool boot_last_savestate(bool testing)
{
if (!g_cfg.savestate.suspend_emu && !Emu.GetTitleID().empty() && (Emu.IsRunning() || Emu.GetStatus() == system_state::paused))
{
extern bool is_savestate_compatible(const fs::file& file);
extern bool is_savestate_compatible(fs::file&& file);
const std::string save_dir = fs::get_cache_dir() + "/savestates/";
@ -252,3 +261,149 @@ bool boot_last_savestate(bool testing)
return false;
}
bool uncompressed_serialization_file_handler::handle_file_op(utils::serial& ar, usz pos, usz size, const void* data)
{
if (ar.is_writing())
{
if (data)
{
m_file->seek(pos);
m_file->write(data, size);
return true;
}
m_file->seek(ar.data_offset);
m_file->write(ar.data);
if (pos == umax && size == umax)
{
// Request to flush the file to disk
m_file->sync();
}
ar.data_offset += ar.data.size();
ar.data.clear();
ar.seek_end();
return true;
}
if (!size)
{
return true;
}
if (pos == 0 && size == umax)
{
// Discard loaded data until pos if profitable
const usz limit = ar.data_offset + ar.data.size();
if (ar.pos > ar.data_offset && ar.pos < limit)
{
const usz may_discard_bytes = ar.pos - ar.data_offset;
const usz moved_byte_count_on_discard = limit - ar.pos;
// Cheeck profitability (check recycled memory and std::memmove costs)
if (may_discard_bytes >= 0x50'0000 || (may_discard_bytes >= 0x20'0000 && moved_byte_count_on_discard / may_discard_bytes < 3))
{
ar.data_offset += may_discard_bytes;
ar.data.erase(ar.data.begin(), ar.data.begin() + may_discard_bytes);
if (ar.data.capacity() >= 0x200'0000)
{
// Discard memory
ar.data.shrink_to_fit();
}
}
return true;
}
// Discard all loaded data
ar.data_offset += ar.data.size();
ar.data.clear();
if (ar.data.capacity() >= 0x200'0000)
{
// Discard memory
ar.data.shrink_to_fit();
}
return true;
}
if (~size < pos)
{
// Overflow
return false;
}
if (ar.data.empty() && pos != ar.pos)
{
// Relocate instead oof over-fetch
ar.seek_pos(pos);
}
const usz read_pre_buffer = utils::sub_saturate<usz>(ar.data_offset, pos);
if (read_pre_buffer)
{
// Read past data
// Harsh operation on performance, luckily rare and not typically needed
// Also this may would be disallowed when moving to compressed files
// This may be a result of wrong usage of breathe() function
ar.data.resize(ar.data.size() + read_pre_buffer);
std::memmove(ar.data.data() + read_pre_buffer, ar.data.data(), ar.data.size() - read_pre_buffer);
ensure(m_file->read_at(pos, ar.data.data(), read_pre_buffer) == read_pre_buffer);
ar.data_offset -= read_pre_buffer;
}
const usz read_past_buffer = utils::sub_saturate<usz>(pos + size, ar.data_offset + ar.data.size());
if (read_past_buffer)
{
// Read proceeding data
// More lightweight operation, this is the common operation
// Allowed to fail, if memory is truly needed an assert would take place later
const usz old_size = ar.data.size();
// Try to prefetch data by reading more than requested
ar.data.resize(std::max<usz>({ ar.data.capacity(), ar.data.size() + read_past_buffer * 3 / 2, ar.m_avoid_large_prefetch ? usz{4096} : usz{0x10'0000} }));
ar.data.resize(m_file->read_at(old_size + ar.data_offset, data ? const_cast<void*>(data) : ar.data.data() + old_size, ar.data.size() - old_size) + old_size);
}
return true;
}
usz uncompressed_serialization_file_handler::get_size(const utils::serial& ar, usz recommended) const
{
if (ar.is_writing())
{
return m_file->size();
}
const usz memory_available = ar.data_offset + ar.data.size();
if (memory_available >= recommended)
{
// Avoid calling size() if possible
return memory_available;
}
return std::max<usz>(m_file->size(), memory_available);
}
namespace stx
{
extern void serial_breathe(utils::serial& ar)
{
ar.breathe();
}
}
// MSVC bug workaround, see above similar case
extern void serial_breathe(utils::serial& ar)
{
::stx::serial_breathe(ar);
}

45
rpcs3/Emu/savestate_utils.hpp Normal file
View File

@ -0,0 +1,45 @@
#include "util/serialization.hpp"
namespace fs
{
class file;
}
// Uncompressed file serialization handler
struct uncompressed_serialization_file_handler : utils::serialization_file_handler
{
const std::unique_ptr<fs::file> m_file_storage;
const std::add_pointer_t<const fs::file> m_file;
explicit uncompressed_serialization_file_handler(fs::file&& file) noexcept
: utils::serialization_file_handler()
, m_file_storage(std::make_unique<fs::file>(std::move(file)))
, m_file(m_file_storage.get())
{
}
explicit uncompressed_serialization_file_handler(const fs::file& file) noexcept
: utils::serialization_file_handler()
, m_file_storage(nullptr)
, m_file(std::addressof(file))
{
}
// Handle file read and write requests
bool handle_file_op(utils::serial& ar, usz pos, usz size, const void* data) override;
// Get available memory or file size
// Preferably memory size if is already greater/equal to recommended to avoid additional file ops
usz get_size(const utils::serial& ar, usz recommended) const override;
};
inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(fs::file&& file)
{
return std::make_unique<uncompressed_serialization_file_handler>(std::move(file));
}
inline std::unique_ptr<uncompressed_serialization_file_handler> make_uncompressed_serialization_file_handler(const fs::file& file)
{
return std::make_unique<uncompressed_serialization_file_handler>(file);
}

33
rpcs3/Loader/TAR.cpp
View File

@ -8,6 +8,7 @@
#include "TAR.h"
#include "util/asm.hpp"
#include "util/serialization.hpp"
#include <charconv>
@ -258,14 +259,14 @@ bool tar_object::extract(std::string prefix_path, bool is_vfs)
return true;
}
std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vector<u8>&& init, const process_func& func, std::string full_path)
void tar_object::save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func, std::string full_path)
{
const std::string& target_path = full_path.empty() ? src_dir : full_path;
fs::stat_t stat{};
if (!fs::get_stat(target_path, stat))
{
return std::move(init);
return;
}
if (stat.is_directory)
@ -276,13 +277,13 @@ std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vect
{
if (entry.name.find_first_not_of('.') == umax) continue;
init = save_directory(src_dir, std::move(init), func, target_path + '/' + entry.name);
save_directory(src_dir, ar, func, target_path + '/' + entry.name);
has_items = true;
}
if (has_items)
{
return std::move(init);
return;
}
}
@ -304,34 +305,34 @@ std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vect
std::string saved_path{target_path.data() + src_dir.size(), target_path.size() - src_dir.size()};
const u64 old_size = init.size();
init.resize(old_size + sizeof(TARHeader));
const u64 old_size = ar.data.size();
ar.data.resize(old_size + sizeof(TARHeader));
if (!stat.is_directory)
{
fs::file fd(target_path);
const u64 old_size2 = init.size();
const u64 old_size2 = ar.data.size();
if (func)
{
// Use custom function for file saving if provided
// Allows for example to compress PNG files as JPEG in the TAR itself
if (!func(fd, saved_path, std::move(init)))
if (!func(fd, saved_path, ar))
{
// Revert (this entry should not be included if func returns false)
init.resize(old_size);
return std::move(init);
ar.data.resize(old_size);
return;
}
}
else
{
init.resize(init.size() + stat.size);
ensure(fd.read(init.data() + old_size2, stat.size) == stat.size);
ar.data.resize(ar.data.size() + stat.size);
ensure(fd.read(ar.data.data() + old_size2, stat.size) == stat.size);
}
// Align
init.resize(old_size2 + utils::align(init.size() - old_size2, 512));
ar.data.resize(old_size2 + utils::align(ar.data.size() - old_size2, 512));
fd.close();
fs::utime(target_path, stat.atime, stat.mtime);
@ -352,8 +353,10 @@ std::vector<u8> tar_object::save_directory(const std::string& src_dir, std::vect
write_octal(header.padding, stat.atime);
header.filetype = stat.is_directory ? '5' : '0';
std::memcpy(init.data() + old_size, &header, sizeof(header));
return std::move(init);
std::memcpy(ar.data.data() + old_size, &header, sizeof(header));
// TAR is an old format which does not depend on previous data so memory ventilation is trivial here
ar.breathe();
}
bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file)

9
rpcs3/Loader/TAR.h
View File

@ -22,6 +22,11 @@ namespace fs
class file;
}
namespace utils
{
struct serial;
}
class tar_object
{
const fs::file& m_file;
@ -38,13 +43,13 @@ public:
fs::file get_file(const std::string& path);
using process_func = std::function<bool(const fs::file&, std::string&, std::vector<u8>&&)>;
using process_func = std::function<bool(const fs::file&, std::string&, utils::serial&)>;
// Extract all files in archive to destination (as VFS if is_vfs is true)
// Allow to optionally specify explicit mount point (which may be directory meant for extraction)
bool extract(std::string prefix_path = {}, bool is_vfs = false);
static std::vector<u8> save_directory(const std::string& src_dir, std::vector<u8>&& init = std::vector<u8>{}, const process_func& func = {}, std::string append_path = {});
static void save_directory(const std::string& src_dir, utils::serial& ar, const process_func& func = {}, std::string append_path = {});
};
bool extract_tar(const std::string& file_path, const std::string& dir_path, fs::file file = {});

1
rpcs3/emucore.vcxproj
View File

@ -607,6 +607,7 @@
<ClInclude Include="Emu\RSX\Overlays\Shaders\shader_loading_dialog_native.h" />
<ClInclude Include="Emu\RSX\RSXDisAsm.h" />
<ClInclude Include="Emu\RSX\RSXZCULL.h" />
<ClInclude Include="Emu\savestate_utils.hpp" />
<ClInclude Include="Emu\system_progress.hpp" />
<ClInclude Include="Emu\system_utils.hpp" />
<ClInclude Include="Emu\title.h" />

3
rpcs3/emucore.vcxproj.filters
View File

@ -2389,6 +2389,9 @@
<ClInclude Include="Crypto\unzip.h">
<Filter>Crypto</Filter>
</ClInclude>
<ClInclude Include="Emu\savestate_utils.hpp">
<Filter>Emu</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<None Include="Emu\RSX\Program\GLSLSnippets\GPUDeswizzle.glsl">

2
rpcs3/rpcs3qt/game_list_frame.cpp
View File

@ -1078,7 +1078,7 @@ void game_list_frame::ShowContextMenu(const QPoint &pos)
});
}
extern bool is_savestate_compatible(const fs::file& file);
extern bool is_savestate_compatible(fs::file&& file);
if (const std::string sstate = get_savestate_file(current_game.serial, current_game.path, 0, 0); is_savestate_compatible(fs::file(sstate)))
{

15
rpcs3/util/fixed_typemap.hpp
View File

@ -251,6 +251,12 @@ namespace stx
*m_order++ = data;
*m_info++ = &type;
m_init[id] = true;
if (ar)
{
extern void serial_breathe(utils::serial& ar);
serial_breathe(*ar);
}
}
}
@ -310,7 +316,8 @@ namespace stx
}
}
void save(utils::serial& ar)
template <typename T> requires (std::is_same_v<T&, utils::serial&>)
void save(T& ar)
{
if (!is_init())
{
@ -332,7 +339,11 @@ namespace stx
// Save data in forward order
for (u32 i = _max; i; i--)
{
if (auto save = (*std::prev(m_info, i))->save) save(*std::prev(m_order, i), ar);
if (auto save = (*std::prev(m_info, i))->save)
{
save(*std::prev(m_order, i), ar);
ar.breathe();
}
}
}

121
rpcs3/util/serialization.hpp
View File

@ -8,13 +8,13 @@ namespace utils
template <typename T>
concept FastRandomAccess = requires (T& obj)
{
std::data(obj)[0];
std::data(obj)[std::size(obj)];
};
template <typename T>
concept Reservable = requires (T& obj)
{
obj.reserve(0);
obj.reserve(std::size(obj));
};
template <typename T>
@ -32,15 +32,29 @@ namespace utils
template <typename T>
concept ListAlike = requires (T& obj) { obj.insert(obj.end(), std::declval<typename T::value_type>()); };
struct serial;
struct serialization_file_handler
{
serialization_file_handler() = default;
virtual ~serialization_file_handler() = default;
virtual bool handle_file_op(serial& ar, usz pos, usz size, const void* data = nullptr) = 0;
virtual usz get_size(const utils::serial& ar, usz recommended = umax) const = 0;
};
struct serial
{
std::vector<u8> data;
usz data_offset = 0;
usz pos = 0;
bool m_is_writing = true;
bool m_avoid_large_prefetch = false;
std::unique_ptr<serialization_file_handler> m_file_handler;
serial() = default;
serial() noexcept = default;
serial(const serial&) = delete;
~serial() = default;
~serial() noexcept = default;
// Checks if this instance is currently used for serialization
bool is_writing() const
@ -58,21 +72,44 @@ namespace utils
}
}
bool raw_serialize(const void* ptr, usz size)
template <typename Func> requires (std::is_convertible_v<std::invoke_result_t<Func>, const void*>)
bool raw_serialize(Func&& memory_provider, usz size)
{
if (!size)
{
return true;
}
// Overflow check
ensure(~pos >= size);
if (is_writing())
{
data.insert(data.begin() + pos, static_cast<const u8*>(ptr), static_cast<const u8*>(ptr) + size);
ensure(pos >= data_offset);
const auto ptr = reinterpret_cast<const u8*>(memory_provider());
data.insert(data.begin() + pos - data_offset, ptr, ptr + size);
pos += size;
return true;
}
ensure(data.size() - pos >= size);
std::memcpy(const_cast<void*>(ptr), data.data() + pos, size);
if (data.empty() || pos < data_offset || pos + size > data.size() + data_offset)
{
// Load from file
ensure(m_file_handler);
ensure(m_file_handler->handle_file_op(*this, pos, size, nullptr));
ensure(!data.empty() && pos >= data_offset && pos + size <= data.size() + data_offset);
}
std::memcpy(const_cast<void*>(static_cast<const void*>(memory_provider())), data.data() + pos - data_offset, size);
pos += size;
return true;
}
bool raw_serialize(const void* ptr, usz size)
{
return raw_serialize(FN(ptr), size);
}
template <typename T> requires Integral<T>
bool serialize_vle(T&& value)
{
@ -157,19 +194,15 @@ namespace utils
return true;
}
obj.clear();
usz size = 0;
if (!deserialize_vle(size))
{
return false;
}
obj.resize(size);
if constexpr (Bitcopy<typename T::value_type>)
{
if (!raw_serialize(obj.data(), sizeof(obj[0]) * size))
if (!raw_serialize([&](){ obj.resize(size); return obj.data(); }, sizeof(obj[0]) * size))
{
obj.clear();
return false;
@ -177,6 +210,10 @@ namespace utils
}
else
{
// TODO: Postpone resizing to after file bounds checks
obj.clear();
obj.resize(size);
for (auto&& value : obj)
{
if (!serialize(value))
@ -306,7 +343,9 @@ namespace utils
}
m_is_writing = false;
m_avoid_large_prefetch = false;
pos = 0;
data_offset = 0;
}
// Reset to empty serialization manager
@ -315,23 +354,53 @@ namespace utils
data.clear();
m_is_writing = true;
pos = 0;
data_offset = 0;
m_file_handler.reset();
}
usz seek_end(usz backwards = 0)
{
ensure(pos >= backwards);
pos = data.size() - backwards;
ensure(data.size() + data_offset >= backwards);
pos = data.size() + data_offset - backwards;
return pos;
}
usz seek_pos(usz val, bool empty_data = false)
{
const usz old_pos = std::exchange(pos, val);
if (empty_data || data.empty())
{
// Relocate future data
data.clear();
data_offset = pos;
}
return old_pos;
}
usz pad_from_end(usz forwards)
{
ensure(is_writing());
pos = data.size();
data.resize(pos + forwards);
pos = data.size() + data_offset;
data.resize(data.size() + forwards);
return pos;
}
// Allow for memory saving operations: if writing, flush to file if too large. If reading, discard memory (not implemented).
// Execute only if past memory is known to not going be reused
void breathe(bool forced = false)
{
if (!forced && (!m_file_handler || (data.size() < 0x20'0000 && pos >= data_offset)))
{
// Let's not do anything if less than 32MB
return;
}
ensure(m_file_handler);
ensure(m_file_handler->handle_file_op(*this, 0, umax, nullptr));
}
template <typename T> requires (std::is_copy_constructible_v<std::remove_const_t<T>>) && (std::is_constructible_v<std::remove_const_t<T>> || Bitcopy<std::remove_const_t<T>> ||
std::is_constructible_v<std::remove_const_t<T>, stx::exact_t<serial&>> || TupleAlike<std::remove_const_t<T>>)
operator T()
@ -364,6 +433,16 @@ namespace utils
}
}
void swap_handler(serial& ar)
{
std::swap(ar.m_file_handler, this->m_file_handler);
}
usz get_size(usz recommended = umax) const
{
return m_file_handler ? m_file_handler->get_size(*this, recommended) : data_offset + data.size();
}
template <typename T> requires (std::is_copy_constructible_v<T> && std::is_constructible_v<T> && Bitcopy<T>)
std::pair<bool, T> try_read()
{
@ -372,10 +451,11 @@ namespace utils
return {};
}
const usz left = data.size() - pos;
const usz end_pos = pos + sizeof(T);
const usz size = get_size(end_pos);
using type = std::remove_const_t<T>;
if (left >= sizeof(type))
if (size >= end_pos)
{
u8 buf[sizeof(type)]{};
ensure(raw_serialize(buf, sizeof(buf)));
@ -389,7 +469,8 @@ namespace utils
// Used when an invalid state is encountered somewhere in a place we can't check success code such as constructor)
bool is_valid() const
{
return pos <= data.size();
// TODO
return true;
}
};
}