HTTPDownloader: Add support for progress updates/cancelling

common/HTTPDownloader.cpp

@@ -18,6 +18,7 @@
 #include "common/HTTPDownloader.h"
 #include "common/Assertions.h"
 #include "common/Console.h"
+#include "common/ProgressCallback.h"
 #include "common/StringUtil.h"
 #include "common/Timer.h"
 #include "common/Threading.h"
@@ -47,13 +48,14 @@ void HTTPDownloader::SetMaxActiveRequests(u32 max_active_requests)
 	m_max_active_requests = max_active_requests;
 }
 
-void HTTPDownloader::CreateRequest(std::string url, Request::Callback callback)
+void HTTPDownloader::CreateRequest(std::string url, Request::Callback callback, ProgressCallback* progress)
 {
 	Request* req = InternalCreateRequest();
 	req->parent = this;
 	req->type = Request::Type::Get;
 	req->url = std::move(url);
 	req->callback = std::move(callback);
+	req->progress = progress;
 	req->start_time = Common::Timer::GetCurrentValue();
 
 	std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
@@ -66,7 +68,7 @@ void HTTPDownloader::CreateRequest(std::string url, Request::Callback callback)
 	LockedAddRequest(req);
 }
 
-void HTTPDownloader::CreatePostRequest(std::string url, std::string post_data, Request::Callback callback)
+void HTTPDownloader::CreatePostRequest(std::string url, std::string post_data, Request::Callback callback, ProgressCallback* progress)
 {
 	Request* req = InternalCreateRequest();
 	req->parent = this;
@@ -74,6 +76,7 @@ void HTTPDownloader::CreatePostRequest(std::string url, std::string post_data, R
 	req->url = std::move(url);
 	req->post_data = std::move(post_data);
 	req->callback = std::move(callback);
+	req->progress = progress;
 	req->start_time = Common::Timer::GetCurrentValue();
 
 	std::unique_lock<std::mutex> lock(m_pending_http_request_lock);
@@ -107,8 +110,8 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
 			continue;
 		}
 
-		if (req->state == Request::State::Started && current_time >= req->start_time &&
+		if ((req->state == Request::State::Started || req->state == Request::State::Receiving) &&
-			Common::Timer::ConvertValueToSeconds(current_time - req->start_time) >= m_timeout)
+			current_time >= req->start_time && Common::Timer::ConvertValueToSeconds(current_time - req->start_time) >= m_timeout)
 		{
 			// request timed out
 			Console.Error("Request for '%s' timed out", req->url.c_str());
@@ -117,7 +120,24 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
 			m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
 			lock.unlock();
 
-			req->callback(HTTP_STATUS_TIMEOUT, req->content_type, Request::Data());
+			req->callback(HTTP_STATUS_TIMEOUT, std::string(), Request::Data());
+
+			CloseRequest(req);
+
+			lock.lock();
+			continue;
+		}
+		else if ((req->state == Request::State::Started || req->state == Request::State::Receiving) && req->progress &&
+			req->progress->IsCancelled())
+		{
+			// request timed out
+			Console.Error("Request for '%s' cancelled", req->url.c_str());
+
+			req->state.store(Request::State::Cancelled);
+			m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
+			lock.unlock();
+
+			req->callback(HTTP_STATUS_CANCELLED, std::string(), Request::Data());
 
 			CloseRequest(req);
 
@@ -127,6 +147,17 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
 
 		if (req->state != Request::State::Complete)
 		{
+			if (req->progress)
+			{
+				const u32 size = static_cast<u32>(req->data.size());
+				if (size != req->last_progress_update)
+				{
+					req->last_progress_update = size;
+					req->progress->SetProgressRange(req->content_length);
+					req->progress->SetProgressValue(req->last_progress_update);
+				}
+			}
+
 			active_requests++;
 			index++;
 			continue;

common/HTTPDownloader.h

@@ -1,5 +1,5 @@
 /*  PCSX2 - PS2 Emulator for PCs
- *  Copyright (C) 2002-2022  PCSX2 Dev Team
+ *  Copyright (C) 2002-2023 PCSX2 Dev Team
  *
  *  PCSX2 is free software: you can redistribute it and/or modify it under the terms
  *  of the GNU Lesser General Public License as published by the Free Software Found-
@@ -14,15 +14,20 @@
  */
 
 #pragma once
+
 #include "common/Pcsx2Defs.h"
+
 #include <atomic>
 #include <functional>
 #include <memory>
 #include <mutex>
 #include <string>
 #include <string_view>
+#include <variant>
 #include <vector>
 
+class ProgressCallback;
+
 class HTTPDownloader
 {
 public:
@@ -56,6 +61,7 @@ public:
 
 		HTTPDownloader* parent;
 		Callback callback;
+		ProgressCallback* progress;
 		std::string url;
 		std::string post_data;
 		std::string content_type;
@@ -63,6 +69,7 @@ public:
 		u64 start_time;
 		s32 status_code = 0;
 		u32 content_length = 0;
+		u32 last_progress_update = 0;
 		Type type = Type::Get;
 		std::atomic<State> state{State::Pending};
 	};
@@ -70,7 +77,7 @@ public:
 	HTTPDownloader();
 	virtual ~HTTPDownloader();
 
-	static std::unique_ptr<HTTPDownloader> Create(const char* user_agent = DEFAULT_USER_AGENT);
+	static std::unique_ptr<HTTPDownloader> Create(std::string user_agent = DEFAULT_USER_AGENT);
 	static std::string URLEncode(const std::string_view& str);
 	static std::string URLDecode(const std::string_view& str);
 	static std::string GetExtensionForContentType(const std::string& content_type);
@@ -78,8 +85,8 @@ public:
 	void SetTimeout(float timeout);
 	void SetMaxActiveRequests(u32 max_active_requests);
 
-	void CreateRequest(std::string url, Request::Callback callback);
+	void CreateRequest(std::string url, Request::Callback callback, ProgressCallback* progress = nullptr);
-	void CreatePostRequest(std::string url, std::string post_data, Request::Callback callback);
+	void CreatePostRequest(std::string url, std::string post_data, Request::Callback callback, ProgressCallback* progress = nullptr);
 	void PollRequests();
 	void WaitForAllRequests();
 	bool HasAnyRequests();

common/HTTPDownloaderCurl.cpp

@@ -39,10 +39,10 @@ HTTPDownloaderCurl::~HTTPDownloaderCurl()
 		curl_multi_cleanup(m_multi_handle);
 }
 
-std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(const char* user_agent)
+std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(std::string user_agent)
 {
 	std::unique_ptr<HTTPDownloaderCurl> instance(std::make_unique<HTTPDownloaderCurl>());
-	if (!instance->Initialize(user_agent))
+	if (!instance->Initialize(std::move(user_agent)))
 		return {};
 
 	return instance;
@@ -51,7 +51,7 @@ std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(const char* user_agent)
 static bool s_curl_initialized = false;
 static std::once_flag s_curl_initialized_once_flag;
 
-bool HTTPDownloaderCurl::Initialize(const char* user_agent)
+bool HTTPDownloaderCurl::Initialize(std::string user_agent)
 {
 	if (!s_curl_initialized)
 	{
@@ -79,7 +79,7 @@ bool HTTPDownloaderCurl::Initialize(const char* user_agent)
 		return false;
 	}
 
-	m_user_agent = user_agent;
+	m_user_agent = std::move(user_agent);
 	return true;
 }
 
@@ -90,7 +90,16 @@ size_t HTTPDownloaderCurl::WriteCallback(char* ptr, size_t size, size_t nmemb, v
 	const size_t transfer_size = size * nmemb;
 	const size_t new_size = current_size + transfer_size;
 	req->data.resize(new_size);
+	req->start_time = Common::Timer::GetCurrentValue();
 	std::memcpy(&req->data[current_size], ptr, transfer_size);
+
+	if (req->content_length == 0)
+	{
+		curl_off_t length;
+		if (curl_easy_getinfo(req->handle, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &length) == CURLE_OK)
+			req->content_length = static_cast<u32>(length);
+	}
+
 	return nmemb;
 }
 
@@ -174,8 +183,9 @@ bool HTTPDownloaderCurl::StartRequest(HTTPDownloader::Request* request)
 	curl_easy_setopt(req->handle, CURLOPT_USERAGENT, m_user_agent.c_str());
 	curl_easy_setopt(req->handle, CURLOPT_WRITEFUNCTION, &HTTPDownloaderCurl::WriteCallback);
 	curl_easy_setopt(req->handle, CURLOPT_WRITEDATA, req);
-	curl_easy_setopt(req->handle, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(req->handle, CURLOPT_NOSIGNAL, 1L);
 	curl_easy_setopt(req->handle, CURLOPT_PRIVATE, req);
+	curl_easy_setopt(req->handle, CURLOPT_FOLLOWLOCATION, 1L);
 
 	if (request->type == Request::Type::Post)
 	{

common/HTTPDownloaderCurl.h

@@ -28,7 +28,7 @@ public:
 	HTTPDownloaderCurl();
 	~HTTPDownloaderCurl() override;
 
-	bool Initialize(const char* user_agent);
+	bool Initialize(std::string user_agent);
 
 protected:
 	Request* InternalCreateRequest() override;

common/HTTPDownloaderWinHTTP.cpp

@@ -20,6 +20,7 @@
 #include "common/Console.h"
 #include "common/StringUtil.h"
 #include "common/Timer.h"
+
 #include <VersionHelpers.h>
 #include <algorithm>
 
@@ -39,16 +40,16 @@ HTTPDownloaderWinHttp::~HTTPDownloaderWinHttp()
 	}
 }
 
-std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(const char* user_agent)
+std::unique_ptr<HTTPDownloader> HTTPDownloader::Create(std::string user_agent)
 {
 	std::unique_ptr<HTTPDownloaderWinHttp> instance(std::make_unique<HTTPDownloaderWinHttp>());
-	if (!instance->Initialize(user_agent))
+	if (!instance->Initialize(std::move(user_agent)))
 		return {};
 
 	return instance;
 }
 
-bool HTTPDownloaderWinHttp::Initialize(const char* user_agent)
+bool HTTPDownloaderWinHttp::Initialize(std::string user_agent)
 {
 	const DWORD dwAccessType = WINHTTP_ACCESS_TYPE_AUTOMATIC_PROXY;
 

common/HTTPDownloaderWinHTTP.h

@@ -26,7 +26,7 @@ public:
 	HTTPDownloaderWinHttp();
 	~HTTPDownloaderWinHttp() override;
 
-	bool Initialize(const char* user_agent);
+	bool Initialize(std::string user_agent);
 
 protected:
 	Request* InternalCreateRequest() override;
@@ -46,5 +46,7 @@ private:
 	static void CALLBACK HTTPStatusCallback(HINTERNET hInternet, DWORD_PTR dwContext, DWORD dwInternetStatus,
 		LPVOID lpvStatusInformation, DWORD dwStatusInformationLength);
 
+	static bool CheckCancelled(Request* request);
+
 	HINTERNET m_hSession = NULL;
 };

common/ProgressCallback.h

@@ -82,8 +82,8 @@ public:
 	virtual void PushState() override;
 	virtual void PopState() override;
 
-	virtual bool IsCancelled() const override;
+	bool IsCancelled() const;
-	virtual bool IsCancellable() const override;
+	bool IsCancellable() const;
 
 	virtual void SetCancellable(bool cancellable) override;
 	virtual void SetStatusText(const char* text) override;

pcsx2/Achievements.cpp

@@ -28,7 +28,6 @@
 #include "IopMem.h"
 #include "MTGS.h"
 #include "Memory.h"
-#include "SysForwardDefs.h"
 #include "VMManager.h"
 #include "svnrev.h"
 #include "vtlb.h"
@@ -135,7 +134,6 @@ namespace Achievements
 	static void EnsureCacheDirectoriesExist();
 	static void ClearGameInfo();
 	static void ClearGameHash();
-	static std::string GetUserAgent();
 	static void BeginLoadingScreen(const char* text, bool* was_running_idle);
 	static void EndLoadingScreen(bool was_running_idle);
 	static std::string_view GetELFNameForHash(const std::string& elf_path);
@@ -248,19 +246,6 @@ std::unique_lock<std::recursive_mutex> Achievements::GetLock()
 	return std::unique_lock(s_achievements_mutex);
 }
 
-std::string Achievements::GetUserAgent()
-{
-	std::string ret;
-	if (!PCSX2_isReleaseVersion && GIT_TAGGED_COMMIT)
-		ret = fmt::format("PCSX2 Nightly - {} ({})", GIT_TAG, GetOSVersionString());
-	else if (!PCSX2_isReleaseVersion)
-		ret = fmt::format("PCSX2 {} ({})", GIT_REV, GetOSVersionString());
-	else
-		ret = fmt::format("PCSX2 {}.{}.{}-{} ({})", PCSX2_VersionHi, PCSX2_VersionMid, PCSX2_VersionLo, SVN_REV, GetOSVersionString());
-
-	return ret;
-}
-
 void Achievements::BeginLoadingScreen(const char* text, bool* was_running_idle)
 {
 	MTGS::RunOnGSThread(&ImGuiManager::InitializeFullscreenUI);
@@ -476,7 +461,7 @@ bool Achievements::Initialize()
 
 bool Achievements::CreateClient(rc_client_t** client, std::unique_ptr<HTTPDownloader>* http)
 {
-	*http = HTTPDownloader::Create(GetUserAgent().c_str());
+	*http = HTTPDownloader::Create(Host::GetHTTPUserAgent());
 	if (!*http)
 	{
 		Host::ReportErrorAsync("Achievements Error", "Failed to create HTTPDownloader, cannot use achievements");
@@ -2918,7 +2903,7 @@ void Achievements::SwitchToRAIntegration()
 void Achievements::RAIntegration::InitializeRAIntegration(void* main_window_handle)
 {
 	RA_InitClient((HWND)main_window_handle, "PCSX2", GIT_TAG);
-	RA_SetUserAgentDetail(Achievements::GetUserAgent().c_str());
+	RA_SetUserAgentDetail(Host::GetHTTPUserAgent().c_str());
 
 	RA_InstallSharedFunctions(RACallbackIsActive, RACallbackCauseUnpause, RACallbackCausePause, RACallbackRebuildMenu,
 		RACallbackEstimateTitle, RACallbackResetEmulator, RACallbackLoadROM);

pcsx2/GameList.cpp

@@ -1286,7 +1286,7 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
 		return false;
 	}
 
-	std::unique_ptr<HTTPDownloader> downloader(HTTPDownloader::Create());
+	std::unique_ptr<HTTPDownloader> downloader(HTTPDownloader::Create(Host::GetHTTPUserAgent()));
 	if (!downloader)
 	{
 		progress->DisplayError("Failed to create HTTP downloader.");

pcsx2/Host.cpp

@@ -19,7 +19,9 @@
 #include "GS/Renderers/HW/GSTextureReplacements.h"
 #include "Host.h"
 #include "LayeredSettingsInterface.h"
+#include "SysForwardDefs.h"
 #include "VMManager.h"
+#include "svnrev.h"
 
 #include "common/Assertions.h"
 #include "common/CrashHandler.h"
@@ -28,6 +30,8 @@
 #include "common/Path.h"
 #include "common/StringUtil.h"
 
+#include "fmt/format.h"
+
 #include <cstdarg>
 #include <shared_mutex>
 
@@ -168,6 +172,19 @@ bool Host::ConfirmFormattedMessage(const std::string_view& title, const char* fo
 	return ConfirmMessage(title, message);
 }
 
+std::string Host::GetHTTPUserAgent()
+{
+	std::string ret;
+	if (!PCSX2_isReleaseVersion && GIT_TAGGED_COMMIT)
+		ret = fmt::format("PCSX2 Nightly - {} ({})", GIT_TAG, GetOSVersionString());
+	else if (!PCSX2_isReleaseVersion)
+		ret = fmt::format("PCSX2 {} ({})", GIT_REV, GetOSVersionString());
+	else
+		ret = fmt::format("PCSX2 {}.{}.{}-{} ({})", PCSX2_VersionHi, PCSX2_VersionMid, PCSX2_VersionLo, SVN_REV, GetOSVersionString());
+
+	return ret;
+}
+
 std::unique_lock<std::mutex> Host::GetSettingsLock()
 {
 	return std::unique_lock<std::mutex>(s_settings_mutex);

pcsx2/Host.h

@@ -99,6 +99,9 @@ namespace Host
 	/// Requests shut down of the current virtual machine.
 	void RequestVMShutdown(bool allow_confirm, bool allow_save_state, bool default_save_state);
 
+	/// Returns the user agent to use for HTTP requests.
+	std::string GetHTTPUserAgent();
+
 	/// Base setting retrieval, bypasses layers.
 	std::string GetBaseStringSettingValue(const char* section, const char* key, const char* default_value = "");
 	bool GetBaseBoolSettingValue(const char* section, const char* key, bool default_value = false);