/* Copyright 2022 flyinghead This file is part of Flycast. Flycast is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. Flycast is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with Flycast. If not, see . */ #include "gamesdb.h" #include "oslib/http_client.h" #include "stdclass.h" #include "emulator.h" #define APIKEY "3fcc5e726a129924972be97abfd577ac5311f8f12398a9d9bcb5a377d4656fa8" std::string TheGamesDb::makeUrl(const std::string& endpoint) { return "https://api.thegamesdb.net/v1/" + endpoint + "?apikey=" APIKEY; } bool TheGamesDb::initialize(const std::string& saveDirectory) { if (!Scraper::initialize(saveDirectory)) return false; http::init(); boxartCache.clear(); return true; } TheGamesDb::~TheGamesDb() { http::term(); } void TheGamesDb::copyFile(const std::string& from, const std::string& to) { FILE *ffrom = nowide::fopen(from.c_str(), "rb"); if (ffrom == nullptr) { WARN_LOG(COMMON, "Can't open %s: error %d", from.c_str(), errno); return; } FILE *fto = nowide::fopen(to.c_str(), "wb"); if (fto == nullptr) { WARN_LOG(COMMON, "Can't open %s: error %d", to.c_str(), errno); fclose(ffrom); return; } u8 buffer[4096]; while (true) { int l = fread(buffer, 1, sizeof(buffer), ffrom); if (l == 0) break; fwrite(buffer, 1, l, fto); } fclose(ffrom); fclose(fto); } json TheGamesDb::httpGet(const std::string& url) { if (getTimeMs() < blackoutPeriod) throw std::runtime_error(""); blackoutPeriod = 0; DEBUG_LOG(COMMON, "TheGameDb: GET %s", url.c_str()); std::vector receivedData; int status = http::get(url, receivedData); bool success = http::success(status); if (status == 403) // hit rate-limit cap blackoutPeriod = getTimeMs() + 60 * 1000; else if (!success) blackoutPeriod = getTimeMs() + 1000; if (!success || receivedData.empty()) throw std::runtime_error("http error"); std::string content((const char *)&receivedData[0], receivedData.size()); DEBUG_LOG(COMMON, "TheGameDb: received [%s]", content.c_str()); json v = json::parse(content); int code = v["code"]; if (!http::success(code)) { // TODO can this happen? http status should be the same std::string status; try { status = v.at("status"); } catch (const json::exception& e) { } throw std::runtime_error(std::string("TheGamesDB error ") + std::to_string(code) + ": " + status); } return v; } void TheGamesDb::fetchPlatforms() { if (dreamcastPlatformId != 0 && arcadePlatformId != 0) return; auto getPlatformId = [this](const std::string& platform) { std::string url = makeUrl("Platforms/ByPlatformName") + "&name=" + platform; json v = httpGet(url); try { const json& array = v.at("data").at("platforms"); for (const auto& o : array) { try { std::string name = o.at("name"); if (name == "Sega Dreamcast") dreamcastPlatformId = o.at("id"); else if (name == "Arcade") arcadePlatformId = o.at("id"); } catch (const json::exception& e) { } } } catch (const json::exception& e) { } }; getPlatformId("Dreamcast"); getPlatformId("Arcade"); if (dreamcastPlatformId == 0 || arcadePlatformId == 0) throw std::runtime_error("can't find dreamcast or arcade platform id"); } void TheGamesDb::parseBoxart(GameBoxart& item, const json& j, int gameId) { std::string baseUrl; try { baseUrl = j.at("base_url").at("thumb"); } catch (const json::exception& e) { try { baseUrl = j.at("base_url").at("small"); } catch (const json::exception& e) { return; } } if (!j.contains("data") && !j.contains("images")) // No boxart return; const json& images = j.contains("data") ? j.at("data") : j.at("images"); try { const json& dataArray = images.at(std::to_string(gameId)); std::string imagePath; for (const auto& o : dataArray) { try { // Look for boxart if (o.at("type") != "boxart") continue; } catch (const json::exception& e) { continue; } try { // We want the front side if specified if (o.at("side") == "back") continue; } catch (const json::exception& e) { // ignore if not found } try { imagePath = o.at("filename").get(); break; } catch (const json::exception& e) { // continue if not found } } if (imagePath.empty()) { // Use titlescreen for (const auto& o : dataArray) { try { if (o.at("type") != "titlescreen") continue; imagePath = o.at("filename").get(); break; } catch (const json::exception& e) { // continue if not found } } } if (imagePath.empty()) { // Use screenshot for (const auto& o : dataArray) { try { if (o.at("type") != "screenshot") continue; imagePath = o.at("filename").get(); break; } catch (const json::exception& e) { // continue if not found } } } if (!imagePath.empty()) { // Build the full URL and get from cache or download std::string url = baseUrl + imagePath; std::string filename = makeUniqueFilename("dummy.jpg"); // thegamesdb returns some images as png, but they are really jpeg auto cached = boxartCache.find(url); if (cached != boxartCache.end()) { copyFile(cached->second, filename); item.setBoxartPath(filename); item.boxartUrl = url; } else { if (downloadImage(url, filename)) { item.setBoxartPath(filename); item.boxartUrl = url; boxartCache[url] = filename; } } } } catch (const json::exception& e) { // No boxart for this game } } bool TheGamesDb::parseGameInfo(const json& gameArray, const json& boxartArray, GameBoxart& item, const std::string& diskId) { for (const auto& game : gameArray) { if (!diskId.empty()) { bool found = false; try { for (const auto& uid : game.at("uids")) if (uid.at("uid") == diskId) { found = true; break; } } catch (const json::exception& e) { } if (!found) continue; } // Name try { item.name = game.at("game_title"); } catch (const json::exception& e) { } // Release date try { item.releaseDate = game.at("release_date"); } catch (const json::exception& e) { } // Overview try { item.overview = game.at("overview"); } catch (const json::exception& e) { } // GameDB id int id; try { id = game.at("id"); } catch (const json::exception& e) { return true; } // Boxart parseBoxart(item, boxartArray, id); if (item.boxartPath.empty()) { std::string imgUrl = makeUrl("Games/Images") + "&games_id=" + std::to_string(id); json images = httpGet(imgUrl); try { parseBoxart(item, images.at("data"), id); } catch (const json::exception& e) { } } return true; } return false; } bool TheGamesDb::fetchGameInfo(GameBoxart& item, const std::string& url, const std::string& diskId) { json v = httpGet(url); try { const json& array = v.at("data").at("games"); const json& boxart = v.at("include").at("boxart"); return parseGameInfo(array, boxart, item, diskId); } catch (const json::exception& e) { return false; } } void TheGamesDb::scrape(GameBoxart& item) { if (item.searchName.empty()) // invalid rom or disk return; fetchPlatforms(); // Ignore default disk ids used by kos and katana if (!item.uniqueId.empty() && item.uniqueId != "T0000" && item.uniqueId != "T0000M") { std::string url = makeUrl("Games/ByGameUniqueID") + "&fields=overview,uids&include=boxart&filter%5Bplatform%5D=" + std::to_string(dreamcastPlatformId) + "&uid=" + http::urlEncode(item.uniqueId); if (fetchGameInfo(item, url, item.uniqueId)) item.scraped = true; } if (!item.scraped) fetchByName(item); item.scraped = true; } void TheGamesDb::fetchByName(GameBoxart& item) { if (item.searchName.empty()) return; int platform = getGamePlatform(item.fileName); std::string url = makeUrl("Games/ByGameName") + "&fields=overview&include=boxart&filter%5Bplatform%5D="; if (platform == DC_PLATFORM_DREAMCAST) url += std::to_string(dreamcastPlatformId); else url += std::to_string(arcadePlatformId); url += "&name=" + http::urlEncode(item.searchName); if (fetchGameInfo(item, url)) item.scraped = true; } void TheGamesDb::fetchByUids(std::vector& items) { std::string uidCriteria; for (const GameBoxart& item : items) { if (item.scraped || item.uniqueId.empty()) continue; if (!uidCriteria.empty()) uidCriteria += ','; uidCriteria += item.uniqueId; } if (uidCriteria.empty()) return; std::string url = makeUrl("Games/ByGameUniqueID") + "&fields=overview,uids&include=boxart&filter%5Bplatform%5D=" + std::to_string(dreamcastPlatformId) + "&uid=" + http::urlEncode(uidCriteria); json v = httpGet(url); try { const json& array = v.at("data").at("games"); const json& boxartArray = v.at("include").at("boxart"); for (GameBoxart& item : items) { if (!item.scraped && !item.uniqueId.empty() && parseGameInfo(array, boxartArray, item, item.uniqueId)) item.scraped = true; } } catch (const json::exception& e) { } } void TheGamesDb::scrape(std::vector& items) { if (getTimeMs() < blackoutPeriod) throw std::runtime_error(""); blackoutPeriod = 0.0; fetchPlatforms(); fetchByUids(items); for (GameBoxart& item : items) { if (!item.scraped) { if (!item.searchName.empty()) fetchByName(item); else if (item.gamePath.empty()) { std::string localPath = makeUniqueFilename("dreamcast_logo_grey.png"); std::string biosArtUrl{ "https://flyinghead.github.io/flycast-builds/dreamcast_logo_grey.png" }; if (downloadImage(biosArtUrl, localPath)) { item.setBoxartPath(localPath); item.boxartUrl = biosArtUrl; } } item.scraped = true; } } }