From d94922002b8a5b37d0dcb30b5d2662abc20a6187 Mon Sep 17 00:00:00 2001 From: Techjar Date: Thu, 18 Oct 2018 04:33:05 -0400 Subject: [PATCH 1/4] NetPlay: Implement chunked data transfer This sends arbitrary packets in chunks to be reassembled at the other end, allowing large data transfers to be speed-limited and interleaved with other packets being sent. It also enables tracking the progress of large data transfers. --- Source/Core/Common/TraversalClient.cpp | 11 +- Source/Core/Core/Config/NetplaySettings.cpp | 5 + Source/Core/Core/Config/NetplaySettings.h | 3 + Source/Core/Core/NetPlayClient.cpp | 74 ++++++- Source/Core/Core/NetPlayClient.h | 20 +- Source/Core/Core/NetPlayProto.h | 10 + Source/Core/Core/NetPlayServer.cpp | 193 +++++++++++++++++- Source/Core/Core/NetPlayServer.h | 48 ++++- Source/Core/DolphinQt/CMakeLists.txt | 1 + Source/Core/DolphinQt/DolphinQt.vcxproj | 3 + .../NetPlay/ChunkedProgressDialog.cpp | 123 +++++++++++ .../DolphinQt/NetPlay/ChunkedProgressDialog.h | 41 ++++ .../Core/DolphinQt/NetPlay/NetPlayDialog.cpp | 26 +++ Source/Core/DolphinQt/NetPlay/NetPlayDialog.h | 7 + .../DolphinQt/NetPlay/NetPlaySetupDialog.cpp | 32 ++- .../DolphinQt/NetPlay/NetPlaySetupDialog.h | 2 + 16 files changed, 569 insertions(+), 30 deletions(-) create mode 100644 Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.cpp create mode 100644 Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.h diff --git a/Source/Core/Common/TraversalClient.cpp b/Source/Core/Common/TraversalClient.cpp index 34c16ddff4..4edf31a045 100644 --- a/Source/Core/Common/TraversalClient.cpp +++ b/Source/Core/Common/TraversalClient.cpp @@ -10,6 +10,7 @@ #include "Common/Logging/Log.h" #include "Common/MsgHandler.h" #include "Common/Random.h" +#include "Core/NetPlayProto.h" TraversalClient::TraversalClient(ENetHost* netHost, const std::string& server, const u16 port) : m_NetHost(netHost), m_Server(server), m_port(port) @@ -316,11 +317,11 @@ bool EnsureTraversalClient(const std::string& server, u16 server_port, u16 liste g_OldListenPort = listen_port; ENetAddress addr = {ENET_HOST_ANY, listen_port}; - ENetHost* host = enet_host_create(&addr, // address - 50, // peerCount - 1, // channelLimit - 0, // incomingBandwidth - 0); // outgoingBandwidth + ENetHost* host = enet_host_create(&addr, // address + 50, // peerCount + NetPlay::CHANNEL_COUNT, // channelLimit + 0, // incomingBandwidth + 0); // outgoingBandwidth if (!host) { g_MainNetHost.reset(); diff --git a/Source/Core/Core/Config/NetplaySettings.cpp b/Source/Core/Core/Config/NetplaySettings.cpp index 167244fd9d..c3ac30a0b2 100644 --- a/Source/Core/Core/Config/NetplaySettings.cpp +++ b/Source/Core/Core/Config/NetplaySettings.cpp @@ -33,6 +33,11 @@ const ConfigInfo NETPLAY_USE_UPNP{{System::Main, "NetPlay", "UseUPNP"}, fa const ConfigInfo NETPLAY_ENABLE_QOS{{System::Main, "NetPlay", "EnableQoS"}, true}; +const ConfigInfo NETPLAY_ENABLE_CHUNKED_UPLOAD_LIMIT{ + {System::Main, "NetPlay", "EnableChunkedUploadLimit"}, false}; +const ConfigInfo NETPLAY_CHUNKED_UPLOAD_LIMIT{{System::Main, "NetPlay", "ChunkedUploadLimit"}, + 3000}; + const ConfigInfo NETPLAY_BUFFER_SIZE{{System::Main, "NetPlay", "BufferSize"}, 5}; const ConfigInfo NETPLAY_CLIENT_BUFFER_SIZE{{System::Main, "NetPlay", "BufferSizeClient"}, 1}; diff --git a/Source/Core/Core/Config/NetplaySettings.h b/Source/Core/Core/Config/NetplaySettings.h index c44877101d..1a7103c590 100644 --- a/Source/Core/Core/Config/NetplaySettings.h +++ b/Source/Core/Core/Config/NetplaySettings.h @@ -30,6 +30,9 @@ extern const ConfigInfo NETPLAY_USE_UPNP; extern const ConfigInfo NETPLAY_ENABLE_QOS; +extern const ConfigInfo NETPLAY_ENABLE_CHUNKED_UPLOAD_LIMIT; +extern const ConfigInfo NETPLAY_CHUNKED_UPLOAD_LIMIT; + extern const ConfigInfo NETPLAY_BUFFER_SIZE; extern const ConfigInfo NETPLAY_CLIENT_BUFFER_SIZE; diff --git a/Source/Core/Core/NetPlayClient.cpp b/Source/Core/Core/NetPlayClient.cpp index 6cc5d84528..3a13aa99c5 100644 --- a/Source/Core/Core/NetPlayClient.cpp +++ b/Source/Core/Core/NetPlayClient.cpp @@ -112,7 +112,7 @@ NetPlayClient::NetPlayClient(const std::string& address, const u16 port, NetPlay if (!traversal_config.use_traversal) { // Direct Connection - m_client = enet_host_create(nullptr, 1, 3, 0, 0); + m_client = enet_host_create(nullptr, 1, CHANNEL_COUNT, 0, 0); if (m_client == nullptr) { @@ -124,7 +124,7 @@ NetPlayClient::NetPlayClient(const std::string& address, const u16 port, NetPlay enet_address_set_host(&addr, address.c_str()); addr.port = port; - m_server = enet_host_connect(m_client, &addr, 3, 0); + m_server = enet_host_connect(m_client, &addr, CHANNEL_COUNT, 0); if (m_server == nullptr) { @@ -338,6 +338,61 @@ unsigned int NetPlayClient::OnData(sf::Packet& packet) } break; + case NP_MSG_CHUNKED_DATA_START: + { + u32 cid; + packet >> cid; + std::string title; + packet >> title; + u64 data_size = Common::PacketReadU64(packet); + + m_chunked_data_receive_queue.emplace(cid, sf::Packet{}); + + std::vector players; + players.push_back(m_local_player->pid); + m_dialog->ShowChunkedProgressDialog(title, data_size, players); + } + break; + + case NP_MSG_CHUNKED_DATA_END: + { + u32 cid; + packet >> cid; + + OnData(m_chunked_data_receive_queue[cid]); + m_chunked_data_receive_queue.erase(m_chunked_data_receive_queue.find(cid)); + m_dialog->HideChunkedProgressDialog(); + + sf::Packet complete_packet; + complete_packet << static_cast(NP_MSG_CHUNKED_DATA_COMPLETE); + complete_packet << cid; + Send(complete_packet, CHUNKED_DATA_CHANNEL); + } + break; + + case NP_MSG_CHUNKED_DATA_PAYLOAD: + { + u32 cid; + packet >> cid; + + while (!packet.endOfPacket()) + { + u8 byte; + packet >> byte; + m_chunked_data_receive_queue[cid] << byte; + } + + m_dialog->SetChunkedProgress(m_local_player->pid, + m_chunked_data_receive_queue[cid].getDataSize()); + + sf::Packet progress_packet; + progress_packet << static_cast(NP_MSG_CHUNKED_DATA_PROGRESS); + progress_packet << cid; + progress_packet << sf::Uint64{m_chunked_data_receive_queue[cid].getDataSize()}; + Send(progress_packet, CHUNKED_DATA_CHANNEL); + } + break; + case NP_MSG_PAD_MAPPING: { for (PadMapping& mapping : m_pad_map) @@ -1050,11 +1105,11 @@ unsigned int NetPlayClient::OnData(sf::Packet& packet) return 0; } -void NetPlayClient::Send(const sf::Packet& packet) +void NetPlayClient::Send(const sf::Packet& packet, const u8 channel_id) { ENetPacket* epac = enet_packet_create(packet.getData(), packet.getDataSize(), ENET_PACKET_FLAG_RELIABLE); - enet_peer_send(m_server, 0, epac); + enet_peer_send(m_server, channel_id, epac); } void NetPlayClient::DisplayPlayersPing() @@ -1104,11 +1159,11 @@ void NetPlayClient::Disconnect() m_server = nullptr; } -void NetPlayClient::SendAsync(sf::Packet&& packet) +void NetPlayClient::SendAsync(sf::Packet&& packet, const u8 channel_id) { { std::lock_guard lkq(m_crit.async_queue_write); - m_async_queue.Push(std::move(packet)); + m_async_queue.Push(AsyncQueueEntry{std::move(packet), channel_id}); } ENetUtil::WakeupThread(m_client); } @@ -1136,7 +1191,10 @@ void NetPlayClient::ThreadFunc() net = enet_host_service(m_client, &netEvent, 250); while (!m_async_queue.Empty()) { - Send(m_async_queue.Front()); + { + auto& e = m_async_queue.Front(); + Send(e.packet, e.channel_id); + } m_async_queue.Pop(); } if (net > 0) @@ -1557,7 +1615,7 @@ void NetPlayClient::OnConnectReady(ENetAddress addr) if (m_connection_state == ConnectionState::WaitingForTraversalClientConnectReady) { m_connection_state = ConnectionState::Connecting; - enet_host_connect(m_client, &addr, 0, 0); + enet_host_connect(m_client, &addr, CHANNEL_COUNT, 0); } } diff --git a/Source/Core/Core/NetPlayClient.h b/Source/Core/Core/NetPlayClient.h index 6b3746c2b6..b1d74181f3 100644 --- a/Source/Core/Core/NetPlayClient.h +++ b/Source/Core/Core/NetPlayClient.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include "Common/CommonTypes.h" @@ -60,6 +62,11 @@ public: virtual void SetMD5Progress(int pid, int progress) = 0; virtual void SetMD5Result(int pid, const std::string& result) = 0; virtual void AbortMD5() = 0; + + virtual void ShowChunkedProgressDialog(const std::string& title, u64 data_size, + const std::vector& players) = 0; + virtual void HideChunkedProgressDialog() = 0; + virtual void SetChunkedProgress(int pid, u64 progress) = 0; }; enum class PlayerGameStatus @@ -85,7 +92,7 @@ class NetPlayClient : public TraversalClientClient { public: void ThreadFunc(); - void SendAsync(sf::Packet&& packet); + void SendAsync(sf::Packet&& packet, u8 channel_id = DEFAULT_CHANNEL); NetPlayClient(const std::string& address, const u16 port, NetPlayUI* dialog, const std::string& name, const NetTraversalConfig& traversal_config); @@ -130,6 +137,12 @@ public: void AdjustPadBufferSize(unsigned int size); protected: + struct AsyncQueueEntry + { + sf::Packet packet; + u8 channel_id; + }; + void ClearBuffers(); struct @@ -140,7 +153,7 @@ protected: std::recursive_mutex async_queue_write; } m_crit; - Common::SPSCQueue m_async_queue; + Common::SPSCQueue m_async_queue; std::array, 4> m_pad_buffer; std::array, 4> m_wiimote_buffer; @@ -203,7 +216,7 @@ private: void AddPadStateToPacket(int in_game_pad, const GCPadStatus& np, sf::Packet& packet); void SendWiimoteState(int in_game_pad, const NetWiimote& nw); unsigned int OnData(sf::Packet& packet); - void Send(const sf::Packet& packet); + void Send(const sf::Packet& packet, u8 channel_id = DEFAULT_CHANNEL); void Disconnect(); bool Connect(); void ComputeMD5(const std::string& file_identifier); @@ -233,6 +246,7 @@ private: u16 m_sync_ar_codes_count = 0; u16 m_sync_ar_codes_success_count = 0; bool m_sync_ar_codes_complete = false; + std::unordered_map m_chunked_data_receive_queue; u64 m_initial_rtc = 0; u32 m_timebase_frame = 0; diff --git a/Source/Core/Core/NetPlayProto.h b/Source/Core/Core/NetPlayProto.h index a45bc8f96d..18711e9a43 100644 --- a/Source/Core/Core/NetPlayProto.h +++ b/Source/Core/Core/NetPlayProto.h @@ -110,6 +110,12 @@ enum NP_MSG_CHAT_MESSAGE = 0x30, + NP_MSG_CHUNKED_DATA_START = 0x40, + NP_MSG_CHUNKED_DATA_END = 0x41, + NP_MSG_CHUNKED_DATA_PAYLOAD = 0x42, + NP_MSG_CHUNKED_DATA_PROGRESS = 0x43, + NP_MSG_CHUNKED_DATA_COMPLETE = 0x44, + NP_MSG_PAD_DATA = 0x60, NP_MSG_PAD_MAPPING = 0x61, NP_MSG_PAD_BUFFER = 0x62, @@ -179,6 +185,10 @@ enum constexpr u32 NETPLAY_LZO_IN_LEN = 1024 * 64; constexpr u32 NETPLAY_LZO_OUT_LEN = NETPLAY_LZO_IN_LEN + (NETPLAY_LZO_IN_LEN / 16) + 64 + 3; +constexpr size_t CHUNKED_DATA_UNIT_SIZE = 16384; +constexpr u8 CHANNEL_COUNT = 2; +constexpr u8 DEFAULT_CHANNEL = 0; +constexpr u8 CHUNKED_DATA_CHANNEL = 1; using NetWiimote = std::vector; using MessageId = u8; diff --git a/Source/Core/Core/NetPlayServer.cpp b/Source/Core/Core/NetPlayServer.cpp index dc1c05896d..c2e018ff70 100644 --- a/Source/Core/Core/NetPlayServer.cpp +++ b/Source/Core/Core/NetPlayServer.cpp @@ -5,6 +5,7 @@ #include "Core/NetPlayServer.h" #include +#include #include #include #include @@ -66,6 +67,10 @@ NetPlayServer::~NetPlayServer() if (is_connected) { m_do_loop = false; + m_chunked_data_event.Set(); + m_chunked_data_complete_event.Set(); + if (m_chunked_data_thread.joinable()) + m_chunked_data_thread.join(); m_thread.join(); enet_host_destroy(m_server); @@ -118,7 +123,7 @@ NetPlayServer::NetPlayServer(const u16 port, const bool forward_port, ENetAddress serverAddr; serverAddr.host = ENET_HOST_ANY; serverAddr.port = port; - m_server = enet_host_create(&serverAddr, 10, 3, 0, 0); + m_server = enet_host_create(&serverAddr, 10, CHANNEL_COUNT, 0, 0); if (m_server != nullptr) m_server->intercept = ENetUtil::InterceptCallback; } @@ -128,6 +133,7 @@ NetPlayServer::NetPlayServer(const u16 port, const bool forward_port, m_do_loop = true; m_thread = std::thread(&NetPlayServer::ThreadFunc, this); m_target_buffer_size = 5; + m_chunked_data_thread = std::thread(&NetPlayServer::ChunkedDataThreadFunc, this); #ifdef USE_UPNP if (forward_port) @@ -164,7 +170,16 @@ void NetPlayServer::ThreadFunc() { { std::lock_guard lkp(m_crit.players); - SendToClients(m_async_queue.Front()); + auto& e = m_async_queue.Front(); + if (e.target_mode == TargetMode::Only) + { + if (m_players.find(e.target_pid) != m_players.end()) + Send(m_players.at(e.target_pid).socket, e.packet, e.channel_id); + } + else + { + SendToClients(e.packet, e.target_pid, e.channel_id); + } } m_async_queue.Pop(); } @@ -529,15 +544,47 @@ void NetPlayServer::SetHostInputAuthority(const bool enable) AdjustPadBufferSize(m_target_buffer_size); } -void NetPlayServer::SendAsyncToClients(sf::Packet&& packet) +void NetPlayServer::SendAsync(sf::Packet&& packet, const PlayerId pid, const u8 channel_id) { { std::lock_guard lkq(m_crit.async_queue_write); - m_async_queue.Push(std::move(packet)); + m_async_queue.Push(AsyncQueueEntry{std::move(packet), pid, TargetMode::Only, channel_id}); } ENetUtil::WakeupThread(m_server); } +void NetPlayServer::SendAsyncToClients(sf::Packet&& packet, const PlayerId skip_pid, + const u8 channel_id) +{ + { + std::lock_guard lkq(m_crit.async_queue_write); + m_async_queue.Push( + AsyncQueueEntry{std::move(packet), skip_pid, TargetMode::AllExcept, channel_id}); + } + ENetUtil::WakeupThread(m_server); +} + +void NetPlayServer::SendChunked(sf::Packet&& packet, const PlayerId pid, const std::string& title) +{ + { + std::lock_guard lkq(m_crit.chunked_data_queue_write); + m_chunked_data_queue.Push( + ChunkedDataQueueEntry{std::move(packet), pid, TargetMode::Only, title}); + } + m_chunked_data_event.Set(); +} + +void NetPlayServer::SendChunkedToClients(sf::Packet&& packet, const PlayerId skip_pid, + const std::string& title) +{ + { + std::lock_guard lkq(m_crit.chunked_data_queue_write); + m_chunked_data_queue.Push( + ChunkedDataQueueEntry{std::move(packet), skip_pid, TargetMode::AllExcept, title}); + } + m_chunked_data_event.Set(); +} + // called from ---NETPLAY--- thread unsigned int NetPlayServer::OnData(sf::Packet& packet, Client& player) { @@ -566,6 +613,29 @@ unsigned int NetPlayServer::OnData(sf::Packet& packet, Client& player) } break; + case NP_MSG_CHUNKED_DATA_PROGRESS: + { + u32 cid; + packet >> cid; + u64 progress = Common::PacketReadU64(packet); + + m_dialog->SetChunkedProgress(player.pid, progress); + } + break; + + case NP_MSG_CHUNKED_DATA_COMPLETE: + { + u32 cid; + packet >> cid; + + if (m_chunked_data_complete_count.find(cid) != m_chunked_data_complete_count.end()) + { + m_chunked_data_complete_count[cid]++; + m_chunked_data_complete_event.Set(); + } + } + break; + case NP_MSG_PAD_DATA: { // if this is pad data from the last game still being received, ignore it @@ -1611,22 +1681,23 @@ u64 NetPlayServer::GetInitialNetPlayRTC() const } // called from multiple threads -void NetPlayServer::SendToClients(const sf::Packet& packet, const PlayerId skip_pid) +void NetPlayServer::SendToClients(const sf::Packet& packet, const PlayerId skip_pid, + const u8 channel_id) { for (auto& p : m_players) { if (p.second.pid && p.second.pid != skip_pid) { - Send(p.second.socket, packet); + Send(p.second.socket, packet, channel_id); } } } -void NetPlayServer::Send(ENetPeer* socket, const sf::Packet& packet) +void NetPlayServer::Send(ENetPeer* socket, const sf::Packet& packet, const u8 channel_id) { ENetPacket* epac = enet_packet_create(packet.getData(), packet.getDataSize(), ENET_PACKET_FLAG_RELIABLE); - enet_peer_send(socket, 0, epac); + enet_peer_send(socket, channel_id, epac); } void NetPlayServer::KickPlayer(PlayerId player) @@ -1714,4 +1785,110 @@ std::vector> NetPlayServer::GetInterfaceList result.emplace_back(std::make_pair("!local!", "127.0.0.1")); return result; } + +// called from ---Chunked Data--- thread +void NetPlayServer::ChunkedDataThreadFunc() +{ + while (m_do_loop) + { + m_chunked_data_event.Wait(); + + while (!m_chunked_data_queue.Empty()) + { + if (!m_do_loop) + return; + auto& e = m_chunked_data_queue.Front(); + const u32 id = m_next_chunked_data_id++; + + m_chunked_data_complete_count[id] = 0; + size_t player_count; + { + std::vector players; + if (e.target_mode == TargetMode::Only) + { + players.push_back(e.target_pid); + } + else + { + for (auto& pl : m_players) + { + if (pl.second.pid != e.target_pid) + players.push_back(pl.second.pid); + } + } + player_count = players.size(); + + sf::Packet pac; + pac << static_cast(NP_MSG_CHUNKED_DATA_START); + pac << id << e.title << sf::Uint64{e.packet.getDataSize()}; + + ChunkedDataSend(std::move(pac), e.target_pid, e.target_mode); + + if (e.target_mode == TargetMode::AllExcept && e.target_pid == 1) + m_dialog->ShowChunkedProgressDialog(e.title, e.packet.getDataSize(), players); + } + + const bool enable_limit = Config::Get(Config::NETPLAY_ENABLE_CHUNKED_UPLOAD_LIMIT); + const float bytes_per_second = + (std::max(Config::Get(Config::NETPLAY_CHUNKED_UPLOAD_LIMIT), 1u) / 8.0f) * 1024.0f; + const std::chrono::duration send_interval(CHUNKED_DATA_UNIT_SIZE / bytes_per_second); + size_t index = 0; + do + { + if (!m_do_loop) + return; + if (e.target_mode == TargetMode::Only) + { + if (m_players.find(e.target_pid) == m_players.end()) + break; + } + + auto start = std::chrono::steady_clock::now(); + + sf::Packet pac; + pac << static_cast(NP_MSG_CHUNKED_DATA_PAYLOAD); + pac << id; + size_t len = std::min(CHUNKED_DATA_UNIT_SIZE, e.packet.getDataSize() - index); + pac.append(static_cast(e.packet.getData()) + index, len); + + ChunkedDataSend(std::move(pac), e.target_pid, e.target_mode); + index += CHUNKED_DATA_UNIT_SIZE; + + if (enable_limit) + { + std::chrono::duration delta = std::chrono::steady_clock::now() - start; + std::this_thread::sleep_for(send_interval - delta); + } + } while (index < e.packet.getDataSize()); + + { + sf::Packet pac; + pac << static_cast(NP_MSG_CHUNKED_DATA_END); + pac << id; + ChunkedDataSend(std::move(pac), e.target_pid, e.target_mode); + } + + while (m_chunked_data_complete_count[id] < player_count && m_do_loop) + m_chunked_data_complete_event.Wait(); + m_chunked_data_complete_count.erase(m_chunked_data_complete_count.find(id)); + m_dialog->HideChunkedProgressDialog(); + + m_chunked_data_queue.Pop(); + } + } +} + +// called from ---Chunked Data--- thread +void NetPlayServer::ChunkedDataSend(sf::Packet&& packet, const PlayerId pid, + const TargetMode target_mode) +{ + if (target_mode == TargetMode::Only) + { + SendAsync(std::move(packet), pid, CHUNKED_DATA_CHANNEL); + } + else + { + SendAsyncToClients(std::move(packet), pid, CHUNKED_DATA_CHANNEL); + } +} } // namespace NetPlay diff --git a/Source/Core/Core/NetPlayServer.h b/Source/Core/Core/NetPlayServer.h index ce75d2605d..e16d7c1107 100644 --- a/Source/Core/Core/NetPlayServer.h +++ b/Source/Core/Core/NetPlayServer.h @@ -13,6 +13,8 @@ #include #include #include +#include +#include "Common/Event.h" #include "Common/QoSSession.h" #include "Common/SPSCQueue.h" #include "Common/Timer.h" @@ -29,7 +31,12 @@ class NetPlayServer : public TraversalClientClient { public: void ThreadFunc(); - void SendAsyncToClients(sf::Packet&& packet); + void SendAsync(sf::Packet&& packet, PlayerId pid, u8 channel_id = DEFAULT_CHANNEL); + void SendAsyncToClients(sf::Packet&& packet, PlayerId skip_pid = 0, + u8 channel_id = DEFAULT_CHANNEL); + void SendChunked(sf::Packet&& packet, PlayerId pid, const std::string& title = ""); + void SendChunkedToClients(sf::Packet&& packet, PlayerId skip_pid = 0, + const std::string& title = ""); NetPlayServer(u16 port, bool forward_port, const NetTraversalConfig& traversal_config); ~NetPlayServer(); @@ -84,6 +91,28 @@ private: bool IsHost() const { return pid == 1; } }; + enum class TargetMode + { + Only, + AllExcept + }; + + struct AsyncQueueEntry + { + sf::Packet packet; + PlayerId target_pid; + TargetMode target_mode; + u8 channel_id; + }; + + struct ChunkedDataQueueEntry + { + sf::Packet packet; + PlayerId target_pid; + TargetMode target_mode; + std::string title; + }; + bool SyncSaveData(); bool SyncCodes(); void CheckSyncAndStartGame(); @@ -93,8 +122,9 @@ private: u64 GetInitialNetPlayRTC() const; - void SendToClients(const sf::Packet& packet, const PlayerId skip_pid = 0); - void Send(ENetPeer* socket, const sf::Packet& packet); + void SendToClients(const sf::Packet& packet, PlayerId skip_pid = 0, + u8 channel_id = DEFAULT_CHANNEL); + void Send(ENetPeer* socket, const sf::Packet& packet, u8 channel_id = DEFAULT_CHANNEL); unsigned int OnConnect(ENetPeer* socket); unsigned int OnDisconnect(const Client& player); unsigned int OnData(sf::Packet& packet, Client& player); @@ -105,6 +135,8 @@ private: void UpdatePadMapping(); void UpdateWiimoteMapping(); std::vector> GetInterfaceListInternal() const; + void ChunkedDataThreadFunc(); + void ChunkedDataSend(sf::Packet&& packet, PlayerId pid, const TargetMode target_mode); NetSettings m_settings; @@ -138,11 +170,19 @@ private: // lock order std::recursive_mutex players; std::recursive_mutex async_queue_write; + std::recursive_mutex chunked_data_queue_write; } m_crit; + Common::SPSCQueue m_async_queue; + Common::SPSCQueue m_chunked_data_queue; + std::string m_selected_game; std::thread m_thread; - Common::SPSCQueue m_async_queue; + Common::Event m_chunked_data_event; + Common::Event m_chunked_data_complete_event; + std::thread m_chunked_data_thread; + u32 m_next_chunked_data_id; + std::unordered_map m_chunked_data_complete_count; ENetHost* m_server = nullptr; TraversalClient* m_traversal_client = nullptr; diff --git a/Source/Core/DolphinQt/CMakeLists.txt b/Source/Core/DolphinQt/CMakeLists.txt index 86a73c34ed..f7a86cbfe7 100644 --- a/Source/Core/DolphinQt/CMakeLists.txt +++ b/Source/Core/DolphinQt/CMakeLists.txt @@ -93,6 +93,7 @@ add_executable(dolphin-emu GameList/ListProxyModel.cpp GCMemcardManager.cpp QtUtils/BlockUserInputFilter.cpp + NetPlay/ChunkedProgressDialog.cpp NetPlay/GameListDialog.cpp NetPlay/MD5Dialog.cpp NetPlay/NetPlayDialog.cpp diff --git a/Source/Core/DolphinQt/DolphinQt.vcxproj b/Source/Core/DolphinQt/DolphinQt.vcxproj index ae85412043..b4125f8232 100644 --- a/Source/Core/DolphinQt/DolphinQt.vcxproj +++ b/Source/Core/DolphinQt/DolphinQt.vcxproj @@ -141,6 +141,7 @@ + @@ -174,6 +175,7 @@ + @@ -350,6 +352,7 @@ + diff --git a/Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.cpp b/Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.cpp new file mode 100644 index 0000000000..2a295a0298 --- /dev/null +++ b/Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.cpp @@ -0,0 +1,123 @@ +// Copyright 2017 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DolphinQt/NetPlay/ChunkedProgressDialog.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "Common/StringUtil.h" + +#include "Core/NetPlayClient.h" +#include "Core/NetPlayServer.h" + +#include "DolphinQt/Settings.h" + +static QString GetPlayerNameFromPID(int pid) +{ + QString player_name = QObject::tr("Invalid Player ID"); + auto client = Settings::Instance().GetNetPlayClient(); + if (!client) + return player_name; + + for (const auto* player : client->GetPlayers()) + { + if (player->pid == pid) + { + player_name = QString::fromStdString(player->name); + break; + } + } + return player_name; +} + +ChunkedProgressDialog::ChunkedProgressDialog(QWidget* parent) : QDialog(parent) +{ + CreateWidgets(); + ConnectWidgets(); + setWindowTitle(tr("Data Transfer")); + setWindowFlags(windowFlags() & ~Qt::WindowContextHelpButtonHint); +} + +void ChunkedProgressDialog::CreateWidgets() +{ + m_main_layout = new QVBoxLayout; + m_progress_box = new QGroupBox; + m_progress_layout = new QVBoxLayout; + + m_progress_box->setLayout(m_progress_layout); + + m_main_layout->addWidget(m_progress_box); + setLayout(m_main_layout); +} + +void ChunkedProgressDialog::ConnectWidgets() +{ +} + +void ChunkedProgressDialog::show(const QString& title, const u64 data_size, + const std::vector& players) +{ + m_progress_box->setTitle(title); + m_data_size = data_size; + + for (auto& pair : m_progress_bars) + { + m_progress_layout->removeWidget(pair.second); + pair.second->deleteLater(); + } + + for (auto& pair : m_status_labels) + { + m_progress_layout->removeWidget(pair.second); + pair.second->deleteLater(); + } + + m_progress_bars.clear(); + m_status_labels.clear(); + + auto client = Settings::Instance().GetNetPlayClient(); + if (!client) + return; + + for (const auto* player : client->GetPlayers()) + { + if (std::find(players.begin(), players.end(), player->pid) == players.end()) + continue; + + m_progress_bars[player->pid] = new QProgressBar; + m_status_labels[player->pid] = new QLabel; + + m_progress_layout->addWidget(m_progress_bars[player->pid]); + m_progress_layout->addWidget(m_status_labels[player->pid]); + } + + QDialog::show(); +} + +void ChunkedProgressDialog::SetProgress(const int pid, const u64 progress) +{ + QString player_name = GetPlayerNameFromPID(pid); + + if (!m_status_labels.count(pid)) + return; + + const float acquired = progress / 1024.0f / 1024.0f; + const float total = m_data_size / 1024.0f / 1024.0f; + const int prog = std::lround((static_cast(progress) / m_data_size) * 100.0f); + + m_status_labels[pid]->setText(tr("%1[%2]: %3/%4 MiB") + .arg(player_name, QString::number(pid), + QString::fromStdString(StringFromFormat("%.2f", acquired)), + QString::fromStdString(StringFromFormat("%.2f", total)))); + m_progress_bars[pid]->setValue(prog); +} diff --git a/Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.h b/Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.h new file mode 100644 index 0000000000..794c26ac3e --- /dev/null +++ b/Source/Core/DolphinQt/NetPlay/ChunkedProgressDialog.h @@ -0,0 +1,41 @@ +// Copyright 2017 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include + +#include + +#include "Common/CommonTypes.h" + +class QGroupBox; +class QLabel; +class QProgressBar; +class QVBoxLayout; +class QWidget; + +class ChunkedProgressDialog : public QDialog +{ + Q_OBJECT +public: + explicit ChunkedProgressDialog(QWidget* parent); + + void show(const QString& title, u64 data_size, const std::vector& players); + void SetProgress(int pid, u64 progress); + +private: + void CreateWidgets(); + void ConnectWidgets(); + + std::map m_progress_bars; + std::map m_status_labels; + u64 m_data_size = 0; + + QGroupBox* m_progress_box; + QVBoxLayout* m_progress_layout; + QVBoxLayout* m_main_layout; +}; diff --git a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp index 111c1f18e2..9780773b8a 100644 --- a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp +++ b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp @@ -43,6 +43,7 @@ #include "Core/NetPlayServer.h" #include "DolphinQt/GameList/GameListModel.h" +#include "DolphinQt/NetPlay/ChunkedProgressDialog.h" #include "DolphinQt/NetPlay/GameListDialog.h" #include "DolphinQt/NetPlay/MD5Dialog.h" #include "DolphinQt/NetPlay/PadMappingDialog.h" @@ -68,6 +69,7 @@ NetPlayDialog::NetPlayDialog(QWidget* parent) m_pad_mapping = new PadMappingDialog(this); m_md5_dialog = new MD5Dialog(this); + m_chunked_progress_dialog = new ChunkedProgressDialog(this); ResetExternalIP(); CreateChatLayout(); @@ -1046,3 +1048,27 @@ void NetPlayDialog::AbortMD5() m_md5_button->setEnabled(true); }); } + +void NetPlayDialog::ShowChunkedProgressDialog(const std::string& title, const u64 data_size, + const std::vector& players) +{ + QueueOnObject(this, [this, title, data_size, players] { + if (m_chunked_progress_dialog->isVisible()) + m_chunked_progress_dialog->close(); + + m_chunked_progress_dialog->show(QString::fromStdString(title), data_size, players); + }); +} + +void NetPlayDialog::HideChunkedProgressDialog() +{ + QueueOnObject(this, [this] { m_chunked_progress_dialog->close(); }); +} + +void NetPlayDialog::SetChunkedProgress(const int pid, const u64 progress) +{ + QueueOnObject(this, [this, pid, progress] { + if (m_chunked_progress_dialog->isVisible()) + m_chunked_progress_dialog->SetProgress(pid, progress); + }); +} diff --git a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h index c2fbc0df72..b3a17908c7 100644 --- a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h +++ b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h @@ -10,6 +10,7 @@ #include "Core/NetPlayClient.h" #include "VideoCommon/OnScreenDisplay.h" +class ChunkedProgressDialog; class MD5Dialog; class GameListModel; class PadMappingDialog; @@ -67,6 +68,11 @@ public: void SetMD5Progress(int pid, int progress) override; void SetMD5Result(int pid, const std::string& result) override; void AbortMD5() override; + + void ShowChunkedProgressDialog(const std::string& title, u64 data_size, + const std::vector& players) override; + void HideChunkedProgressDialog() override; + void SetChunkedProgress(int pid, u64 progress) override; signals: void Boot(const QString& filename); void Stop(); @@ -122,6 +128,7 @@ private: QGridLayout* m_main_layout; MD5Dialog* m_md5_dialog; + ChunkedProgressDialog* m_chunked_progress_dialog; PadMappingDialog* m_pad_mapping; std::string m_current_game; Common::Lazy m_external_ip_address; diff --git a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp index 25f44ba4db..6da3a09578 100644 --- a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp +++ b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.cpp @@ -35,6 +35,8 @@ NetPlaySetupDialog::NetPlaySetupDialog(QWidget* parent) int connect_port = Config::Get(Config::NETPLAY_CONNECT_PORT); int host_port = Config::Get(Config::NETPLAY_HOST_PORT); int host_listen_port = Config::Get(Config::NETPLAY_LISTEN_PORT); + bool enable_chunked_upload_limit = Config::Get(Config::NETPLAY_ENABLE_CHUNKED_UPLOAD_LIMIT); + u32 chunked_upload_limit = Config::Get(Config::NETPLAY_CHUNKED_UPLOAD_LIMIT); #ifdef USE_UPNP bool use_upnp = Config::Get(Config::NETPLAY_USE_UPNP); @@ -50,6 +52,10 @@ NetPlaySetupDialog::NetPlaySetupDialog(QWidget* parent) m_host_force_port_box->setValue(host_listen_port); m_host_force_port_box->setEnabled(false); + m_host_chunked_upload_limit_check->setChecked(enable_chunked_upload_limit); + m_host_chunked_upload_limit_box->setValue(chunked_upload_limit); + m_host_chunked_upload_limit_box->setEnabled(enable_chunked_upload_limit); + OnConnectionTypeChanged(m_connection_type->currentIndex()); ConnectWidgets(); @@ -101,6 +107,8 @@ void NetPlaySetupDialog::CreateMainLayout() m_host_port_box = new QSpinBox; m_host_force_port_check = new QCheckBox(tr("Force Listen Port:")); m_host_force_port_box = new QSpinBox; + m_host_chunked_upload_limit_check = new QCheckBox(tr("Limit Chunked Upload Speed:")); + m_host_chunked_upload_limit_box = new QSpinBox; #ifdef USE_UPNP m_host_upnp = new QCheckBox(tr("Forward port (UPnP)")); @@ -110,6 +118,12 @@ void NetPlaySetupDialog::CreateMainLayout() m_host_port_box->setMaximum(65535); m_host_force_port_box->setMaximum(65535); + m_host_chunked_upload_limit_box->setRange(1, 1000000); + m_host_chunked_upload_limit_box->setSingleStep(100); + m_host_chunked_upload_limit_box->setSuffix(QStringLiteral(" kbps")); + + m_host_chunked_upload_limit_check->setToolTip(tr( + "This will limit the speed of chunked uploading per client, which is used for save sync.")); host_layout->addWidget(m_host_port_label, 0, 0); host_layout->addWidget(m_host_port_box, 0, 1); @@ -119,7 +133,9 @@ void NetPlaySetupDialog::CreateMainLayout() host_layout->addWidget(m_host_games, 1, 0, 1, -1); host_layout->addWidget(m_host_force_port_check, 2, 0); host_layout->addWidget(m_host_force_port_box, 2, 1, Qt::AlignLeft); - host_layout->addWidget(m_host_button, 2, 2, Qt::AlignRight); + host_layout->addWidget(m_host_chunked_upload_limit_check, 3, 0); + host_layout->addWidget(m_host_chunked_upload_limit_box, 3, 1, Qt::AlignLeft); + host_layout->addWidget(m_host_button, 2, 2, 2, 1, Qt::AlignRight); host_widget->setLayout(host_layout); @@ -163,7 +179,14 @@ void NetPlaySetupDialog::ConnectWidgets() connect(m_host_games, &QListWidget::itemDoubleClicked, this, &NetPlaySetupDialog::accept); connect(m_host_force_port_check, &QCheckBox::toggled, - [this](int value) { m_host_force_port_box->setEnabled(value); }); + [this](bool value) { m_host_force_port_box->setEnabled(value); }); + connect(m_host_chunked_upload_limit_check, &QCheckBox::toggled, this, [this](bool value) { + m_host_chunked_upload_limit_box->setEnabled(value); + SaveSettings(); + }); + connect(m_host_chunked_upload_limit_box, + static_cast(&QSpinBox::valueChanged), this, + &NetPlaySetupDialog::SaveSettings); #ifdef USE_UPNP connect(m_host_upnp, &QCheckBox::stateChanged, this, &NetPlaySetupDialog::SaveSettings); #endif @@ -191,6 +214,11 @@ void NetPlaySetupDialog::SaveSettings() if (m_host_force_port_check->isChecked()) Config::SetBaseOrCurrent(Config::NETPLAY_LISTEN_PORT, static_cast(m_host_force_port_box->value())); + + Config::SetBaseOrCurrent(Config::NETPLAY_ENABLE_CHUNKED_UPLOAD_LIMIT, + m_host_chunked_upload_limit_check->isChecked()); + Config::SetBaseOrCurrent(Config::NETPLAY_CHUNKED_UPLOAD_LIMIT, + m_host_chunked_upload_limit_box->value()); } void NetPlaySetupDialog::OnConnectionTypeChanged(int index) diff --git a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h index af718c5afb..7f5456848b 100644 --- a/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h +++ b/Source/Core/DolphinQt/NetPlay/NetPlaySetupDialog.h @@ -63,6 +63,8 @@ private: QPushButton* m_host_button; QCheckBox* m_host_force_port_check; QSpinBox* m_host_force_port_box; + QCheckBox* m_host_chunked_upload_limit_check; + QSpinBox* m_host_chunked_upload_limit_box; #ifdef USE_UPNP QCheckBox* m_host_upnp; From 673074830991e589e8b2ab39f890f3cce40993f6 Mon Sep 17 00:00:00 2001 From: Techjar Date: Fri, 19 Oct 2018 01:41:45 -0400 Subject: [PATCH 2/4] NetPlay: Use chunked data transfer for save sync --- Source/Core/Core/NetPlayServer.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Source/Core/Core/NetPlayServer.cpp b/Source/Core/Core/NetPlayServer.cpp index c2e018ff70..915eb78781 100644 --- a/Source/Core/Core/NetPlayServer.cpp +++ b/Source/Core/Core/NetPlayServer.cpp @@ -1252,7 +1252,8 @@ bool NetPlayServer::SyncSaveData() pac << static_cast(SYNC_SAVE_DATA_NOTIFY); pac << save_count; - SendAsyncToClients(std::move(pac)); + // send this on the chunked data channel to ensure it's sequenced properly + SendAsyncToClients(std::move(pac), 0, CHUNKED_DATA_CHANNEL); } if (save_count == 0) @@ -1295,7 +1296,9 @@ bool NetPlayServer::SyncSaveData() pac << sf::Uint64{0}; } - SendAsyncToClients(std::move(pac)); + SendChunkedToClients( + std::move(pac), 1, + StringFromFormat("Memory Card %c Synchronization", is_slot_a ? 'A' : 'B')); } else if (SConfig::GetInstance().m_EXIDevice[i] == ExpansionInterface::EXIDEVICE_MEMORYCARDFOLDER) @@ -1327,7 +1330,9 @@ bool NetPlayServer::SyncSaveData() pac << static_cast(0); } - SendAsyncToClients(std::move(pac)); + SendChunkedToClients( + std::move(pac), 1, + StringFromFormat("GCI Folder %c Synchronization", is_slot_a ? 'A' : 'B')); } } @@ -1387,7 +1392,7 @@ bool NetPlayServer::SyncSaveData() pac << false; // save does not exist } - SendAsyncToClients(std::move(pac)); + SendChunkedToClients(std::move(pac), 1, "Wii Save Synchronization"); } return true; From f4eb4fab08be581dddf7253a128bebce10ec3019 Mon Sep 17 00:00:00 2001 From: Techjar Date: Tue, 13 Nov 2018 17:42:12 -0500 Subject: [PATCH 3/4] HW/WiiSave: Fix reversed condition in WriteFiles This didn't make any sense, as it would only attempt to create the directory if it already existed, and would simply fail if it didn't exist. --- Source/Core/Core/HW/WiiSave.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Core/Core/HW/WiiSave.cpp b/Source/Core/Core/HW/WiiSave.cpp index ddf9eb7f72..575f04a643 100644 --- a/Source/Core/Core/HW/WiiSave.cpp +++ b/Source/Core/Core/HW/WiiSave.cpp @@ -143,7 +143,7 @@ public: else if (file.type == SaveFile::Type::Directory) { const FS::Result meta = m_fs->GetMetadata(*m_uid, *m_gid, path); - if (!meta || meta->is_file) + if (meta && meta->is_file) return false; const FS::ResultCode result = m_fs->CreateDirectory(*m_uid, *m_gid, path, 0, modes); From b06b7e5686429f30b46e2c422cd9d988477f6268 Mon Sep 17 00:00:00 2001 From: Techjar Date: Tue, 2 Oct 2018 09:16:12 -0400 Subject: [PATCH 4/4] NetPlay: Add full Wii save sync This adds the ability to sync all Wii saves, instead of only the selected game. Useful for cases like launching a game though GeckoOS. --- Source/Core/Core/Config/NetplaySettings.cpp | 2 + Source/Core/Core/Config/NetplaySettings.h | 1 + Source/Core/Core/NetPlayClient.cpp | 61 ++++++---- Source/Core/Core/NetPlayProto.h | 7 +- Source/Core/Core/NetPlayServer.cpp | 113 +++++++++++------- Source/Core/Core/WiiRoot.cpp | 37 +++++- .../Core/DolphinQt/NetPlay/NetPlayDialog.cpp | 13 ++ Source/Core/DolphinQt/NetPlay/NetPlayDialog.h | 1 + 8 files changed, 169 insertions(+), 66 deletions(-) diff --git a/Source/Core/Core/Config/NetplaySettings.cpp b/Source/Core/Core/Config/NetplaySettings.cpp index c3ac30a0b2..6abedb7f82 100644 --- a/Source/Core/Core/Config/NetplaySettings.cpp +++ b/Source/Core/Core/Config/NetplaySettings.cpp @@ -53,5 +53,7 @@ const ConfigInfo NETPLAY_STRICT_SETTINGS_SYNC{{System::Main, "NetPlay", "S false}; const ConfigInfo NETPLAY_HOST_INPUT_AUTHORITY{{System::Main, "NetPlay", "HostInputAuthority"}, false}; +const ConfigInfo NETPLAY_SYNC_ALL_WII_SAVES{{System::Main, "NetPlay", "SyncAllWiiSaves"}, + false}; } // namespace Config diff --git a/Source/Core/Core/Config/NetplaySettings.h b/Source/Core/Core/Config/NetplaySettings.h index 1a7103c590..a7d9cf667d 100644 --- a/Source/Core/Core/Config/NetplaySettings.h +++ b/Source/Core/Core/Config/NetplaySettings.h @@ -44,5 +44,6 @@ extern const ConfigInfo NETPLAY_RECORD_INPUTS; extern const ConfigInfo NETPLAY_REDUCE_POLLING_RATE; extern const ConfigInfo NETPLAY_STRICT_SETTINGS_SYNC; extern const ConfigInfo NETPLAY_HOST_INPUT_AUTHORITY; +extern const ConfigInfo NETPLAY_SYNC_ALL_WII_SAVES; } // namespace Config diff --git a/Source/Core/Core/NetPlayClient.cpp b/Source/Core/Core/NetPlayClient.cpp index 3a13aa99c5..adc9a86e9d 100644 --- a/Source/Core/Core/NetPlayClient.cpp +++ b/Source/Core/Core/NetPlayClient.cpp @@ -62,6 +62,7 @@ namespace NetPlay static std::mutex crit_netplay_client; static NetPlayClient* netplay_client = nullptr; static std::unique_ptr s_wii_sync_fs; +static std::vector s_wii_sync_titles; static bool s_si_poll_batching; // called from ---GUI--- thread @@ -608,6 +609,7 @@ unsigned int NetPlayClient::OnData(sf::Packet& packet) packet >> m_net_settings.m_SyncSaveData; packet >> m_net_settings.m_SaveDataRegion; packet >> m_net_settings.m_SyncCodes; + packet >> m_net_settings.m_SyncAllWiiSaves; m_net_settings.m_IsHosting = m_local_player->IsHost(); m_net_settings.m_HostInputAuthority = m_host_input_authority; @@ -787,13 +789,6 @@ unsigned int NetPlayClient::OnData(sf::Packet& packet) if (m_local_player->IsHost()) return 0; - const auto game = m_dialog->FindGameFile(m_selected_game); - if (game == nullptr) - { - SyncSaveDataResponse(true); // whatever, we won't be booting anyways - return 0; - } - const std::string path = File::GetUserPath(D_USER_IDX) + "Wii" GC_MEMCARD_NETPLAY DIR_SEP; if (File::Exists(path) && !File::DeleteDirRecursively(path)) @@ -804,16 +799,25 @@ unsigned int NetPlayClient::OnData(sf::Packet& packet) } auto temp_fs = std::make_unique(path); - temp_fs->CreateDirectory(IOS::PID_KERNEL, IOS::PID_KERNEL, - Common::GetTitleDataPath(game->GetTitleID()), 0, - {IOS::HLE::FS::Mode::ReadWrite, IOS::HLE::FS::Mode::ReadWrite, - IOS::HLE::FS::Mode::ReadWrite}); - auto save = WiiSave::MakeNandStorage(temp_fs.get(), game->GetTitleID()); + std::vector titles; - bool exists; - packet >> exists; - if (exists) + u32 save_count; + packet >> save_count; + for (u32 n = 0; n < save_count; n++) { + u64 title_id = Common::PacketReadU64(packet); + titles.push_back(title_id); + temp_fs->CreateDirectory(IOS::PID_KERNEL, IOS::PID_KERNEL, + Common::GetTitleDataPath(title_id), 0, + {IOS::HLE::FS::Mode::ReadWrite, IOS::HLE::FS::Mode::ReadWrite, + IOS::HLE::FS::Mode::ReadWrite}); + auto save = WiiSave::MakeNandStorage(temp_fs.get(), title_id); + + bool exists; + packet >> exists; + if (!exists) + continue; + // Header WiiSave::Header header; packet >> header.tid; @@ -879,7 +883,7 @@ unsigned int NetPlayClient::OnData(sf::Packet& packet) } } - SetWiiSyncFS(std::move(temp_fs)); + SetWiiSyncData(std::move(temp_fs), titles); SyncSaveDataResponse(true); } break; @@ -1925,7 +1929,7 @@ bool NetPlayClient::StopGame() // stop game m_dialog->StopGame(); - ClearWiiSyncFS(); + ClearWiiSyncData(); return true; } @@ -2126,12 +2130,18 @@ IOS::HLE::FS::FileSystem* GetWiiSyncFS() return s_wii_sync_fs.get(); } -void SetWiiSyncFS(std::unique_ptr fs) +const std::vector& GetWiiSyncTitles() { - s_wii_sync_fs = std::move(fs); + return s_wii_sync_titles; } -void ClearWiiSyncFS() +void SetWiiSyncData(std::unique_ptr fs, const std::vector& titles) +{ + s_wii_sync_fs = std::move(fs); + s_wii_sync_titles.insert(s_wii_sync_titles.end(), titles.begin(), titles.end()); +} + +void ClearWiiSyncData() { // We're just assuming it will always be here because it is const std::string path = File::GetUserPath(D_USER_IDX) + "Wii" GC_MEMCARD_NETPLAY DIR_SEP; @@ -2139,6 +2149,7 @@ void ClearWiiSyncFS() File::DeleteDirRecursively(path); s_wii_sync_fs.reset(); + s_wii_sync_titles.clear(); } void SetSIPollBatching(bool state) @@ -2152,6 +2163,16 @@ void SendPowerButtonEvent() netplay_client->SendPowerButtonEvent(); } +bool IsSyncingAllWiiSaves() +{ + std::lock_guard lk(crit_netplay_client); + + if (netplay_client) + return netplay_client->GetNetSettings().m_SyncAllWiiSaves; + + return false; +} + void NetPlay_Enable(NetPlayClient* const np) { std::lock_guard lk(crit_netplay_client); diff --git a/Source/Core/Core/NetPlayProto.h b/Source/Core/Core/NetPlayProto.h index 18711e9a43..2e33069930 100644 --- a/Source/Core/Core/NetPlayProto.h +++ b/Source/Core/Core/NetPlayProto.h @@ -79,6 +79,7 @@ struct NetSettings bool m_SyncSaveData; bool m_SyncCodes; std::string m_SaveDataRegion; + bool m_SyncAllWiiSaves; bool m_IsHosting; bool m_HostInputAuthority; }; @@ -202,8 +203,10 @@ bool IsNetPlayRunning(); // IsNetPlayRunning() must be true before calling this. const NetSettings& GetNetSettings(); IOS::HLE::FS::FileSystem* GetWiiSyncFS(); -void SetWiiSyncFS(std::unique_ptr fs); -void ClearWiiSyncFS(); +const std::vector& GetWiiSyncTitles(); +void SetWiiSyncData(std::unique_ptr fs, const std::vector& titles); +void ClearWiiSyncData(); void SetSIPollBatching(bool state); void SendPowerButtonEvent(); +bool IsSyncingAllWiiSaves(); } // namespace NetPlay diff --git a/Source/Core/Core/NetPlayServer.cpp b/Source/Core/Core/NetPlayServer.cpp index 915eb78781..7e6ee2f863 100644 --- a/Source/Core/Core/NetPlayServer.cpp +++ b/Source/Core/Core/NetPlayServer.cpp @@ -42,7 +42,9 @@ #include "Core/HW/Sram.h" #include "Core/HW/WiiSave.h" #include "Core/HW/WiiSaveStructs.h" +#include "Core/IOS/ES/ES.h" #include "Core/IOS/FS/FileSystem.h" +#include "Core/IOS/IOS.h" #include "Core/NetPlayClient.h" //for NetPlayUI #include "DiscIO/Enums.h" #include "InputCommon/GCPadStatus.h" @@ -1202,6 +1204,7 @@ bool NetPlayServer::StartGame() spac << m_settings.m_SyncSaveData; spac << region; spac << m_settings.m_SyncCodes; + spac << m_settings.m_SyncAllWiiSaves; SendAsyncToClients(std::move(spac)); @@ -1240,7 +1243,8 @@ bool NetPlayServer::SyncSaveData() bool wii_save = false; if (m_settings.m_CopyWiiSave && (game->GetPlatform() == DiscIO::Platform::WiiDisc || - game->GetPlatform() == DiscIO::Platform::WiiWAD)) + game->GetPlatform() == DiscIO::Platform::WiiWAD || + game->GetPlatform() == DiscIO::Platform::ELFOrDOL)) { wii_save = true; save_count++; @@ -1339,58 +1343,87 @@ bool NetPlayServer::SyncSaveData() if (wii_save) { const auto configured_fs = IOS::HLE::FS::MakeFileSystem(IOS::HLE::FS::Location::Configured); - const auto save = WiiSave::MakeNandStorage(configured_fs.get(), game->GetTitleID()); + + std::vector> saves; + if (m_settings.m_SyncAllWiiSaves) + { + IOS::HLE::Kernel ios; + for (const u64 title : ios.GetES()->GetInstalledTitles()) + { + auto save = WiiSave::MakeNandStorage(configured_fs.get(), title); + saves.push_back(std::make_pair(title, std::move(save))); + } + } + else if (game->GetPlatform() == DiscIO::Platform::WiiDisc || + game->GetPlatform() == DiscIO::Platform::WiiWAD) + { + auto save = WiiSave::MakeNandStorage(configured_fs.get(), game->GetTitleID()); + saves.push_back(std::make_pair(game->GetTitleID(), std::move(save))); + } + + std::vector titles; sf::Packet pac; pac << static_cast(NP_MSG_SYNC_SAVE_DATA); pac << static_cast(SYNC_SAVE_DATA_WII); + pac << static_cast(saves.size()); - if (save->SaveExists()) + for (const auto& pair : saves) { - const std::optional header = save->ReadHeader(); - const std::optional bk_header = save->ReadBkHeader(); - const std::optional> files = save->ReadFiles(); - if (!header || !bk_header || !files) - return false; + pac << sf::Uint64{pair.first}; + titles.push_back(pair.first); + const auto& save = pair.second; - pac << true; // save exists - - // Header - pac << sf::Uint64{header->tid}; - pac << header->banner_size << header->permissions << header->unk1; - for (size_t i = 0; i < header->md5.size(); i++) - pac << header->md5[i]; - pac << header->unk2; - for (size_t i = 0; i < header->banner_size; i++) - pac << header->banner[i]; - - // BkHeader - pac << bk_header->size << bk_header->magic << bk_header->ngid << bk_header->number_of_files - << bk_header->size_of_files << bk_header->unk1 << bk_header->unk2 - << bk_header->total_size; - for (size_t i = 0; i < bk_header->unk3.size(); i++) - pac << bk_header->unk3[i]; - pac << sf::Uint64{bk_header->tid}; - for (size_t i = 0; i < bk_header->mac_address.size(); i++) - pac << bk_header->mac_address[i]; - - // Files - for (const WiiSave::Storage::SaveFile& file : *files) + if (save->SaveExists()) { - pac << file.mode << file.attributes << static_cast(file.type) << file.path; + const std::optional header = save->ReadHeader(); + const std::optional bk_header = save->ReadBkHeader(); + const std::optional> files = save->ReadFiles(); + if (!header || !bk_header || !files) + return false; - if (file.type == WiiSave::Storage::SaveFile::Type::File) + pac << true; // save exists + + // Header + pac << sf::Uint64{header->tid}; + pac << header->banner_size << header->permissions << header->unk1; + for (size_t i = 0; i < header->md5.size(); i++) + pac << header->md5[i]; + pac << header->unk2; + for (size_t i = 0; i < header->banner_size; i++) + pac << header->banner[i]; + + // BkHeader + pac << bk_header->size << bk_header->magic << bk_header->ngid << bk_header->number_of_files + << bk_header->size_of_files << bk_header->unk1 << bk_header->unk2 + << bk_header->total_size; + for (size_t i = 0; i < bk_header->unk3.size(); i++) + pac << bk_header->unk3[i]; + pac << sf::Uint64{bk_header->tid}; + for (size_t i = 0; i < bk_header->mac_address.size(); i++) + pac << bk_header->mac_address[i]; + + // Files + for (const WiiSave::Storage::SaveFile& file : *files) { - const std::optional>& data = *file.data; - if (!data || !CompressBufferIntoPacket(*data, pac)) - return false; + pac << file.mode << file.attributes << static_cast(file.type) << file.path; + + if (file.type == WiiSave::Storage::SaveFile::Type::File) + { + const std::optional>& data = *file.data; + if (!data || !CompressBufferIntoPacket(*data, pac)) + return false; + } } } + else + { + pac << false; // save does not exist + } } - else - { - pac << false; // save does not exist - } + + // Set titles for host-side loading in WiiRoot + SetWiiSyncData(nullptr, titles); SendChunkedToClients(std::move(pac), 1, "Wii Save Synchronization"); } diff --git a/Source/Core/Core/WiiRoot.cpp b/Source/Core/Core/WiiRoot.cpp index ef6cea8fe5..1e846eca55 100644 --- a/Source/Core/Core/WiiRoot.cpp +++ b/Source/Core/Core/WiiRoot.cpp @@ -17,6 +17,7 @@ #include "Common/StringUtil.h" #include "Core/ConfigManager.h" #include "Core/HW/WiiSave.h" +#include "Core/IOS/ES/ES.h" #include "Core/IOS/FS/FileSystem.h" #include "Core/IOS/IOS.h" #include "Core/IOS/Uids.h" @@ -30,6 +31,16 @@ namespace FS = IOS::HLE::FS; static std::string s_temp_wii_root; +static void CopySave(FS::FileSystem* source, FS::FileSystem* dest, const u64 title_id) +{ + dest->CreateDirectory(IOS::PID_KERNEL, IOS::PID_KERNEL, Common::GetTitleDataPath(title_id), 0, + {IOS::HLE::FS::Mode::ReadWrite, IOS::HLE::FS::Mode::ReadWrite, + IOS::HLE::FS::Mode::ReadWrite}); + const auto source_save = WiiSave::MakeNandStorage(source, title_id); + const auto dest_save = WiiSave::MakeNandStorage(dest, title_id); + WiiSave::Copy(source_save.get(), dest_save.get()); +} + static void InitializeDeterministicWiiSaves(FS::FileSystem* session_fs) { const u64 title_id = SConfig::GetInstance().GetTitleID(); @@ -53,10 +64,28 @@ static void InitializeDeterministicWiiSaves(FS::FileSystem* session_fs) { // Copy the current user's save to the Blank NAND auto* sync_fs = NetPlay::GetWiiSyncFS(); - const auto user_save = - WiiSave::MakeNandStorage(sync_fs ? sync_fs : configured_fs.get(), title_id); - const auto session_save = WiiSave::MakeNandStorage(session_fs, title_id); - WiiSave::Copy(user_save.get(), session_save.get()); + auto& sync_titles = NetPlay::GetWiiSyncTitles(); + if (sync_fs) + { + for (const u64 title : sync_titles) + { + CopySave(sync_fs, session_fs, title); + } + } + else + { + if (NetPlay::IsSyncingAllWiiSaves()) + { + for (const u64 title : sync_titles) + { + CopySave(configured_fs.get(), session_fs, title); + } + } + else + { + CopySave(configured_fs.get(), session_fs, title_id); + } + } } } diff --git a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp index 9780773b8a..fb79123d88 100644 --- a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp +++ b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.cpp @@ -85,6 +85,7 @@ NetPlayDialog::NetPlayDialog(QWidget* parent) const bool reduce_polling_rate = Config::Get(Config::NETPLAY_REDUCE_POLLING_RATE); const bool strict_settings_sync = Config::Get(Config::NETPLAY_STRICT_SETTINGS_SYNC); const bool host_input_authority = Config::Get(Config::NETPLAY_HOST_INPUT_AUTHORITY); + const bool sync_all_wii_saves = Config::Get(Config::NETPLAY_SYNC_ALL_WII_SAVES); m_buffer_size_box->setValue(buffer_size); m_save_sd_box->setChecked(write_save_sdcard_data); @@ -95,6 +96,7 @@ NetPlayDialog::NetPlayDialog(QWidget* parent) m_reduce_polling_rate_box->setChecked(reduce_polling_rate); m_strict_settings_sync_box->setChecked(strict_settings_sync); m_host_input_authority_box->setChecked(host_input_authority); + m_sync_all_wii_saves_box->setChecked(sync_all_wii_saves); ConnectWidgets(); @@ -127,6 +129,7 @@ void NetPlayDialog::CreateMainLayout() m_strict_settings_sync_box = new QCheckBox(tr("Strict Settings Sync")); m_host_input_authority_box = new QCheckBox(tr("Host Input Authority")); m_sync_codes_box = new QCheckBox(tr("Sync Codes")); + m_sync_all_wii_saves_box = new QCheckBox(tr("Sync All Wii Saves")); m_buffer_label = new QLabel(tr("Buffer:")); m_quit_button = new QPushButton(tr("Quit")); m_splitter = new QSplitter(Qt::Horizontal); @@ -200,6 +203,7 @@ void NetPlayDialog::CreateMainLayout() options_boxes->addWidget(m_save_sd_box); options_boxes->addWidget(m_load_wii_box); options_boxes->addWidget(m_sync_save_data_box); + options_boxes->addWidget(m_sync_all_wii_saves_box); options_boxes->addWidget(m_sync_codes_box); options_boxes->addWidget(m_record_input_box); options_boxes->addWidget(m_reduce_polling_rate_box); @@ -339,6 +343,9 @@ void NetPlayDialog::ConnectWidgets() } }); + connect(m_sync_save_data_box, &QCheckBox::stateChanged, this, + [this](bool checked) { m_sync_all_wii_saves_box->setEnabled(checked); }); + // SaveSettings() - Save Hosting-Dialog Settings connect(m_buffer_size_box, static_cast(&QSpinBox::valueChanged), this, @@ -351,6 +358,7 @@ void NetPlayDialog::ConnectWidgets() connect(m_reduce_polling_rate_box, &QCheckBox::stateChanged, this, &NetPlayDialog::SaveSettings); connect(m_strict_settings_sync_box, &QCheckBox::stateChanged, this, &NetPlayDialog::SaveSettings); connect(m_host_input_authority_box, &QCheckBox::stateChanged, this, &NetPlayDialog::SaveSettings); + connect(m_sync_all_wii_saves_box, &QCheckBox::stateChanged, this, &NetPlayDialog::SaveSettings); } void NetPlayDialog::OnChat() @@ -464,6 +472,8 @@ void NetPlayDialog::OnStart() settings.m_StrictSettingsSync = m_strict_settings_sync_box->isChecked(); settings.m_SyncSaveData = m_sync_save_data_box->isChecked(); settings.m_SyncCodes = m_sync_codes_box->isChecked(); + settings.m_SyncAllWiiSaves = + m_sync_all_wii_saves_box->isChecked() && m_sync_save_data_box->isChecked(); // Unload GameINI to restore things to normal Config::RemoveLayer(Config::LayerType::GlobalGame); @@ -517,6 +527,7 @@ void NetPlayDialog::show(std::string nickname, bool use_traversal) m_reduce_polling_rate_box->setHidden(!is_hosting); m_strict_settings_sync_box->setHidden(!is_hosting); m_host_input_authority_box->setHidden(!is_hosting); + m_sync_all_wii_saves_box->setHidden(!is_hosting); m_kick_button->setHidden(!is_hosting); m_assign_ports_button->setHidden(!is_hosting); m_md5_button->setHidden(!is_hosting); @@ -817,6 +828,7 @@ void NetPlayDialog::SetOptionsEnabled(bool enabled) m_reduce_polling_rate_box->setEnabled(enabled); m_strict_settings_sync_box->setEnabled(enabled); m_host_input_authority_box->setEnabled(enabled); + m_sync_all_wii_saves_box->setEnabled(enabled && m_sync_save_data_box->isChecked()); } m_record_input_box->setEnabled(enabled); @@ -1011,6 +1023,7 @@ void NetPlayDialog::SaveSettings() Config::SetBase(Config::NETPLAY_REDUCE_POLLING_RATE, m_reduce_polling_rate_box->isChecked()); Config::SetBase(Config::NETPLAY_STRICT_SETTINGS_SYNC, m_strict_settings_sync_box->isChecked()); Config::SetBase(Config::NETPLAY_HOST_INPUT_AUTHORITY, m_host_input_authority_box->isChecked()); + Config::SetBase(Config::NETPLAY_SYNC_ALL_WII_SAVES, m_sync_all_wii_saves_box->isChecked()); } void NetPlayDialog::ShowMD5Dialog(const std::string& file_identifier) diff --git a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h index b3a17908c7..3cad57c795 100644 --- a/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h +++ b/Source/Core/DolphinQt/NetPlay/NetPlayDialog.h @@ -123,6 +123,7 @@ private: QCheckBox* m_reduce_polling_rate_box; QCheckBox* m_strict_settings_sync_box; QCheckBox* m_host_input_authority_box; + QCheckBox* m_sync_all_wii_saves_box; QPushButton* m_quit_button; QSplitter* m_splitter;