ShuriZma
/
duckstation
mirror of https://github.com/stenzek/duckstation
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

GameList: Handle non-extension-suffixed urls based on content type

This commit is contained in:
Connor McLaughlin 2022-09-18 13:00:55 +10:00
parent cc0127d5ed
commit d9722516c3
9 changed files with 211 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/common/file_system.cpp
View File

@ -145,6 +145,37 @@ std::string Path::SanitizeFileName(const std::string_view& str, bool strip_slash
return ret;
}
void Path::SanitizeFileName(std::string* str, bool strip_slashes /* = true */)
{
const size_t len = str->length();
char small_buf[128];
std::unique_ptr<char[]> large_buf;
char* str_copy = small_buf;
if (len >= std::size(small_buf))
{
large_buf = std::make_unique<char[]>(len + 1);
str_copy = large_buf.get();
}
std::memcpy(str_copy, str->c_str(), sizeof(char) * (len + 1));
str->clear();
size_t pos = 0;
while (pos < len)
{
char32_t ch;
pos += StringUtil::DecodeUTF8(str_copy + pos, pos - len, &ch);
ch = FileSystemCharacterIsSane(ch, strip_slashes) ? ch : U'_';
StringUtil::EncodeAndAppendUTF8(*str, ch);
}
#ifdef _WIN32
// Windows: Can't end filename with a period.
if (str->length() > 0 && str->back() == '.')
str->back() = '_';
#endif
}
bool Path::IsAbsolute(const std::string_view& path)
{
#ifdef _WIN32

98
src/common/http_downloader.cpp
View File

@ -1,6 +1,7 @@
#include "http_downloader.h"
#include "assert.h"
#include "log.h"
#include "string_util.h"
#include "timer.h"
Log_SetChannel(HTTPDownloader);
@ -100,7 +101,7 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
m_pending_http_requests.erase(m_pending_http_requests.begin() + index);
lock.unlock();
req->callback(-1, Request::Data());
req->callback(-1, std::string(), Request::Data());
CloseRequest(req);
@ -122,7 +123,7 @@ void HTTPDownloader::LockedPollRequests(std::unique_lock<std::mutex>& lock)
// run callback with lock unheld
lock.unlock();
req->callback(req->status_code, req->data);
req->callback(req->status_code, std::move(req->content_type), std::move(req->data));
CloseRequest(req);
lock.lock();
}
@ -253,4 +254,97 @@ std::string HTTPDownloader::URLDecode(const std::string_view& str)
return std::string(str);
}
std::string HTTPDownloader::GetExtensionForContentType(const std::string& content_type)
{
// Based on https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
static constexpr const char* table[][2] = {
{"audio/aac", "aac"},
{"application/x-abiword", "abw"},
{"application/x-freearc", "arc"},
{"image/avif", "avif"},
{"video/x-msvideo", "avi"},
{"application/vnd.amazon.ebook", "azw"},
{"application/octet-stream", "bin"},
{"image/bmp", "bmp"},
{"application/x-bzip", "bz"},
{"application/x-bzip2", "bz2"},
{"application/x-cdf", "cda"},
{"application/x-csh", "csh"},
{"text/css", "css"},
{"text/csv", "csv"},
{"application/msword", "doc"},
{"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"},
{"application/vnd.ms-fontobject", "eot"},
{"application/epub+zip", "epub"},
{"application/gzip", "gz"},
{"image/gif", "gif"},
{"text/html", "htm"},
{"image/vnd.microsoft.icon", "ico"},
{"text/calendar", "ics"},
{"application/java-archive", "jar"},
{"image/jpeg", "jpg"},
{"text/javascript", "js"},
{"application/json", "json"},
{"application/ld+json", "jsonld"},
{"audio/midi audio/x-midi", "mid"},
{"text/javascript", "mjs"},
{"audio/mpeg", "mp3"},
{"video/mp4", "mp4"},
{"video/mpeg", "mpeg"},
{"application/vnd.apple.installer+xml", "mpkg"},
{"application/vnd.oasis.opendocument.presentation", "odp"},
{"application/vnd.oasis.opendocument.spreadsheet", "ods"},
{"application/vnd.oasis.opendocument.text", "odt"},
{"audio/ogg", "oga"},
{"video/ogg", "ogv"},
{"application/ogg", "ogx"},
{"audio/opus", "opus"},
{"font/otf", "otf"},
{"image/png", "png"},
{"application/pdf", "pdf"},
{"application/x-httpd-php", "php"},
{"application/vnd.ms-powerpoint", "ppt"},
{"application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx"},
{"application/vnd.rar", "rar"},
{"application/rtf", "rtf"},
{"application/x-sh", "sh"},
{"image/svg+xml", "svg"},
{"application/x-tar", "tar"},
{"image/tiff", "tif"},
{"video/mp2t", "ts"},
{"font/ttf", "ttf"},
{"text/plain", "txt"},
{"application/vnd.visio", "vsd"},
{"audio/wav", "wav"},
{"audio/webm", "weba"},
{"video/webm", "webm"},
{"image/webp", "webp"},
{"font/woff", "woff"},
{"font/woff2", "woff2"},
{"application/xhtml+xml", "xhtml"},
{"application/vnd.ms-excel", "xls"},
{"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"},
{"application/xml", "xml"},
{"text/xml", "xml"},
{"application/vnd.mozilla.xul+xml", "xul"},
{"application/zip", "zip"},
{"video/3gpp", "3gp"},
{"audio/3gpp", "3gp"},
{"video/3gpp2", "3g2"},
{"audio/3gpp2", "3g2"},
{"application/x-7z-compressed", "7z"},
};
std::string ret;
for (size_t i = 0; i < std::size(table); i++)
{
if (StringUtil::Strncasecmp(table[i][0], content_type.data(), content_type.length()) == 0)
{
ret = table[i][1];
break;
}
}
return ret;
}
} // namespace Common

4
src/common/http_downloader.h
View File

@ -21,7 +21,7 @@ public:
struct Request
{
using Data = std::vector<u8>;
using Callback = std::function<void(s32 status_code, const Data& data)>;
using Callback = std::function<void(s32 status_code, std::string content_type, Data data)>;
enum class Type
{
@ -42,6 +42,7 @@ public:
Callback callback;
std::string url;
std::string post_data;
std::string content_type;
Data data;
u64 start_time;
s32 status_code = 0;
@ -56,6 +57,7 @@ public:
static std::unique_ptr<HTTPDownloader> Create(const char* user_agent = DEFAULT_USER_AGENT);
static std::string URLEncode(const std::string_view& str);
static std::string URLDecode(const std::string_view& str);
static std::string GetExtensionForContentType(const std::string& content_type);
void SetTimeout(float timeout);
void SetMaxActiveRequests(u32 max_active_requests);

7
src/common/http_downloader_curl.cpp
View File

@ -88,6 +88,11 @@ void HTTPDownloaderCurl::ProcessRequest(Request* req)
long response_code = 0;
curl_easy_getinfo(req->handle, CURLINFO_RESPONSE_CODE, &response_code);
req->status_code = static_cast<s32>(response_code);
char* content_type = nullptr;
if (!curl_easy_getinfo(req->handle, CURLINFO_CONTENT_TYPE, &content_type) && content_type)
req->content_type = content_type;
Log_DevPrintf("Request for '%s' returned status code %d and %zu bytes", req->url.c_str(), req->status_code,
req->data.size());
}
@ -159,4 +164,4 @@ void HTTPDownloaderCurl::CloseRequest(HTTPDownloader::Request* request)
req->closed.store(true);
}
} // namespace FrontendCommon
} // namespace Common

20
src/common/http_downloader_winhttp.cpp
View File

@ -130,6 +130,20 @@ void CALLBACK HTTPDownloaderWinHttp::HTTPStatusCallback(HINTERNET hRequest, DWOR
req->content_length = 0;
}
DWORD content_type_length = 0;
if (!WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_TYPE, WINHTTP_HEADER_NAME_BY_INDEX,
WINHTTP_NO_OUTPUT_BUFFER, &content_type_length, WINHTTP_NO_HEADER_INDEX) &&
GetLastError() == ERROR_INSUFFICIENT_BUFFER && content_type_length >= sizeof(content_type_length))
{
std::wstring content_type_wstring;
content_type_wstring.resize((content_type_length / sizeof(wchar_t)) - 1);
if (WinHttpQueryHeaders(hRequest, WINHTTP_QUERY_CONTENT_TYPE, WINHTTP_HEADER_NAME_BY_INDEX,
content_type_wstring.data(), &content_type_length, WINHTTP_NO_HEADER_INDEX))
{
req->content_type = StringUtil::WideStringToUTF8String(content_type_wstring);
}
}
Log_DevPrintf("Status code %d, content-length is %u", req->status_code, req->content_length);
req->data.reserve(req->content_length);
req->state = Request::State::Receiving;
@ -224,7 +238,7 @@ bool HTTPDownloaderWinHttp::StartRequest(HTTPDownloader::Request* request)
if (!WinHttpCrackUrl(url_wide.c_str(), static_cast<DWORD>(url_wide.size()), 0, &uc))
{
Log_ErrorPrintf("WinHttpCrackUrl() failed: %u", GetLastError());
req->callback(-1, req->data);
req->callback(-1, std::string(), req->data);
delete req;
return false;
}
@ -236,7 +250,7 @@ bool HTTPDownloaderWinHttp::StartRequest(HTTPDownloader::Request* request)
if (!req->hConnection)
{
Log_ErrorPrintf("Failed to start HTTP request for '%s': %u", req->url.c_str(), GetLastError());
req->callback(-1, req->data);
req->callback(-1, std::string(), req->data);
delete req;
return false;
}
@ -297,4 +311,4 @@ void HTTPDownloaderWinHttp::CloseRequest(HTTPDownloader::Request* request)
delete req;
}
} // namespace FrontendCommon
} // namespace Common

1
src/common/path.h
View File

@ -23,6 +23,7 @@ void Canonicalize(std::string* path);
/// Sanitizes a filename for use in a filesystem.
std::string SanitizeFileName(const std::string_view& str, bool strip_slashes = true);
void SanitizeFileName(std::string* str, bool strip_slashes = true);
/// Returns true if the specified path is an absolute path (C:\Path on Windows or /path on Unix).
bool IsAbsolute(const std::string_view& path);

3
src/duckstation-qt/qtprogresscallback.cpp
View File

@ -115,7 +115,8 @@ void QtModalProgressCallback::checkForDelayedShow()
}
}
QtAsyncProgressThread::QtAsyncProgressThread(QWidget* parent) : QThread(parent) {}
// NOTE: We deliberately don't set the thread parent, because otherwise we can't move it.
QtAsyncProgressThread::QtAsyncProgressThread(QWidget* parent) : QThread() {}
QtAsyncProgressThread::~QtAsyncProgressThread() = default;

59
src/frontend-common/achievements.cpp
View File

@ -67,26 +67,29 @@ static Achievement* GetMutableAchievementByID(u32 id);
static void ClearGameInfo(bool clear_achievements = true, bool clear_leaderboards = true);
static void ClearGameHash();
static std::string GetUserAgent();
static void LoginCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void LoginASyncCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void LoginCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void LoginASyncCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void SendLogin(const char* username, const char* password, Common::HTTPDownloader* http_downloader,
Common::HTTPDownloader::Request::Callback callback);
static void DownloadImage(std::string url, std::string cache_filename);
static void DisplayAchievementSummary();
static void GetUserUnlocksCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void GetUserUnlocksCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data);
static void GetUserUnlocks();
static void GetPatchesCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void GetLbInfoCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void GetPatchesCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void GetLbInfoCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void GetPatches(u32 game_id);
static std::string GetGameHash(CDImage* image);
static void SetChallengeMode(bool enabled);
static void SendGetGameId();
static void GetGameIdCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void SendPlayingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void GetGameIdCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void SendPlayingCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void UpdateRichPresence();
static void SendPingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void UnlockAchievementCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void SubmitLeaderboardCallback(s32 status_code, Common::HTTPDownloader::Request::Data data);
static void SendPingCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data);
static void UnlockAchievementCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data);
static void SubmitLeaderboardCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data);
static bool s_active = false;
static bool s_logged_in = false;
@ -177,7 +180,7 @@ public:
if (error != RC_OK)
{
FormattedError("%s failed: error %d (%s)", RAPIStructName<T>(), error, rc_error_str(error));
callback(-1, Common::HTTPDownloader::Request::Data());
callback(-1, std::string(), Common::HTTPDownloader::Request::Data());
return;
}
@ -825,7 +828,7 @@ void Achievements::EnsureCacheDirectoriesExist()
}
}
void Achievements::LoginCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::LoginCallback(s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data)
{
std::unique_lock lock(s_achievements_mutex);
@ -858,11 +861,12 @@ void Achievements::LoginCallback(s32 status_code, Common::HTTPDownloader::Reques
}
}
void Achievements::LoginASyncCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::LoginASyncCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
ImGuiFullscreen::CloseBackgroundProgressDialog("cheevos_async_login");
LoginCallback(status_code, std::move(data));
LoginCallback(status_code, std::move(content_type), std::move(data));
}
void Achievements::SendLogin(const char* username, const char* password, Common::HTTPDownloader* http_downloader,
@ -943,7 +947,8 @@ void Achievements::Logout()
void Achievements::DownloadImage(std::string url, std::string cache_filename)
{
auto callback = [cache_filename](s32 status_code, Common::HTTPDownloader::Request::Data data) {
auto callback = [cache_filename](s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data) {
if (status_code != HTTP_OK)
return;
@ -998,7 +1003,8 @@ void Achievements::DisplayAchievementSummary()
});
}
void Achievements::GetUserUnlocksCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::GetUserUnlocksCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1046,7 +1052,8 @@ void Achievements::GetUserUnlocks()
request.Send(GetUserUnlocksCallback);
}
void Achievements::GetPatchesCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::GetPatchesCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1185,7 +1192,8 @@ void Achievements::GetPatchesCallback(s32 status_code, Common::HTTPDownloader::R
}
}
void Achievements::GetLbInfoCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::GetLbInfoCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1275,7 +1283,8 @@ std::string Achievements::GetGameHash(CDImage* image)
return hash_str;
}
void Achievements::GetGameIdCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::GetGameIdCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1393,7 +1402,8 @@ void Achievements::SendGetGameId()
request.Send(GetGameIdCallback);
}
void Achievements::SendPlayingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::SendPlayingCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1445,7 +1455,8 @@ void Achievements::UpdateRichPresence()
Host::OnAchievementsRefreshed();
}
void Achievements::SendPingCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::SendPingCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1634,7 +1645,8 @@ void Achievements::DeactivateAchievement(Achievement* achievement)
Log_DevPrintf("Deactivated achievement %s (%u)", achievement->title.c_str(), achievement->id);
}
void Achievements::UnlockAchievementCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::UnlockAchievementCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;
@ -1649,7 +1661,8 @@ void Achievements::UnlockAchievementCallback(s32 status_code, Common::HTTPDownlo
response.new_player_score);
}
void Achievements::SubmitLeaderboardCallback(s32 status_code, Common::HTTPDownloader::Request::Data data)
void Achievements::SubmitLeaderboardCallback(s32 status_code, std::string content_type,
Common::HTTPDownloader::Request::Data data)
{
if (!System::IsValid())
return;

22
src/frontend-common/game_list.cpp
View File

@ -779,7 +779,7 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
std::string filename(Common::HTTPDownloader::URLDecode(url));
downloader->CreateRequest(
std::move(url), [use_serial, &save_callback, entry_path = std::move(entry_path),
filename = std::move(filename)](s32 status_code, Common::HTTPDownloader::Request::Data data) {
filename = std::move(filename)](s32 status_code, std::string content_type, Common::HTTPDownloader::Request::Data data) {
if (status_code != Common::HTTPDownloader::HTTP_OK || data.empty())
return;
@ -788,12 +788,26 @@ bool GameList::DownloadCovers(const std::vector<std::string>& url_templates, boo
if (!entry || !GetCoverImagePathForEntry(entry).empty())
return;
std::string write_path(GetNewCoverImagePathForEntry(entry, filename.c_str(), use_serial));
// prefer the content type from the response for the extension
// otherwise, if it's missing, and the request didn't have an extension.. fall back to jpegs.
std::string template_filename;
std::string content_type_extension(Common::HTTPDownloader::GetExtensionForContentType(content_type));
// don't treat the domain name as an extension..
const std::string::size_type last_slash = filename.find('/');
const std::string::size_type last_dot = filename.find('.');
if (!content_type_extension.empty())
template_filename = fmt::format("cover.{}", content_type_extension);
else if (last_slash != std::string::npos && last_dot != std::string::npos && last_dot > last_slash)
template_filename = Path::GetFileName(filename);
else
template_filename = "cover.jpg";
std::string write_path(GetNewCoverImagePathForEntry(entry, template_filename.c_str(), use_serial));
if (write_path.empty())
return;
FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size());
if (save_callback)
if (FileSystem::WriteBinaryFile(write_path.c_str(), data.data(), data.size()) && save_callback)
save_callback(entry, std::move(write_path));
});
downloader->WaitForAllRequests();