boxart: batch uid searches. Bios image. More arcade rom name changes

core/hw/naomi/naomi_roms.cpp

@@ -259,11 +259,11 @@ Game Games[] =
         NULL,
         kick4csh_eeprom_dump
     },
-    // Marvel Vs. Capcom 2 New Age of Heroes (Export, Korea, Rev A)
+    // Marvel vs. Capcom 2 New Age of Heroes (Export, Korea, Rev A)
     {
         "mvsc2",
         NULL,
-        "Marvel Vs. Capcom 2 New Age of Heroes (Export, Korea)",
+        "Marvel vs. Capcom 2 New Age of Heroes (Export, Korea)",
         0x08800000,
         0xc18b6e7c,
         NULL,
@@ -927,11 +927,11 @@ Game Games[] =
         &alienfnt_inputs,
         alienfnt_eeprom_dump
     },
-    // Capcom Vs. SNK Millennium Fight 2000 (JPN, USA, EXP, KOR, AUS) (Rev C)
+    // Capcom vs. SNK Millennium Fight 2000 (JPN, USA, EXP, KOR, AUS) (Rev C)
     {
         "capsnk",
         NULL,
-        "Capcom Vs. SNK Millennium Fight 2000 (Rev C)",
+        "Capcom vs. SNK Millennium Fight 2000 (Rev C)",
         0x07800000,
         0x00000000,
         NULL,
@@ -952,11 +952,11 @@ Game Games[] =
         &capcom_4btn_inputs,
 		capsnk_eeprom_dump,
     },
-    // Capcom Vs. SNK Millennium Fight 2000 (Rev A)
+    // Capcom vs. SNK Millennium Fight 2000 (Rev A)
     {
         "capsnka",
         "capsnk",
-        "Capcom Vs. SNK Millennium Fight 2000 (Rev A)",
+        "Capcom vs. SNK Millennium Fight 2000 (Rev A)",
         0x07800000,
         0x00000000,
         NULL,
@@ -977,11 +977,11 @@ Game Games[] =
         &capcom_4btn_inputs,
 		capsnk_eeprom_dump,
     },
-    // Capcom Vs. SNK Millennium Fight 2000
+    // Capcom vs. SNK Millennium Fight 2000
     {
         "capsnkb",
         "capsnk",
-        "Capcom Vs. SNK Millennium Fight 2000",
+        "Capcom vs. SNK Millennium Fight 2000",
         0x07800000,
         0x00000000,
         NULL,
@@ -1849,11 +1849,11 @@ Game Games[] =
 		&ggx_inputs,
 		// error message at boot with free play eeprom
     },
-    // Mobile Suit Gundam: Federation Vs. Zeon
+    // Mobile Suit Gundam: Federation vs. Zeon
     {
         "gundmct",
         NULL,
-        "Mobile Suit Gundam: Federation Vs. Zeon",
+        "Mobile Suit Gundam: Federation vs. Zeon",
         0x0a800000,
         0x000e8010,
         NULL,
@@ -1964,11 +1964,11 @@ Game Games[] =
         nullptr,
         &shot12_inputs,
     },
-    // Heavy Metal Geomatrix (JPN, USA, EUR, ASI, AUS) (Rev B)
+    // Heavy Metal: Geomatrix (JPN, USA, EUR, ASI, AUS) (Rev B)
     {
         "hmgeo",
         NULL,
-        "Heavy Metal Geomatrix",
+        "Heavy Metal: Geomatrix",
         0x06000000,
         0x00038510,
         NULL,
@@ -2352,11 +2352,11 @@ Game Games[] =
             { NULL, 0, 0 },
         }
     },
-    // Marvel Vs. Capcom 2 New Age of Heroes (USA, Rev A)
+    // Marvel vs. Capcom 2 New Age of Heroes (USA, Rev A)
     {
         "mvsc2u",
         "mvsc2",
-        "Marvel Vs. Capcom 2 New Age of Heroes (USA)",
+        "Marvel vs. Capcom 2 New Age of Heroes (USA)",
         0x07800000,
         0x0002c840,
         NULL,
@@ -3266,11 +3266,11 @@ Game Games[] =
 		NULL,
 		&marine_fishing_inputs,
     },
-    // Spawn In the Demon's Hand (JPN, USA, EUR, ASI, AUS) (Rev B)
+    // Spawn: In the Demon's Hand (JPN, USA, EUR, ASI, AUS) (Rev B)
     {
         "spawn",
         NULL,
-        "Spawn In the Demon's Hand",
+        "Spawn: In the Demon's Hand",
         0x05800000,
         0x00078d01,
         NULL,
@@ -4644,11 +4644,11 @@ Game Games[] =
 		nullptr,
 		confmiss_eeprom_dump,
     },
-    // Capcom Vs. SNK Millennium Fight 2000 Pro (Japan)
+    // Capcom vs. SNK Millennium Fight 2000 Pro (Japan)
     {
         "cvsgd",
         NULL,
-        "Capcom Vs. SNK Millennium Fight 2000 Pro (Japan)",
+        "Capcom vs. SNK Millennium Fight 2000 Pro (Japan)",
         0x4000,
         0,
         "naomi",
@@ -4662,13 +4662,13 @@ Game Games[] =
 		&capcom_4btn_inputs,
 		cvsgd_eeprom_dump,
     },
-    // Capcom Vs. SNK 2 Mark Of The Millennium 2001
+    // Capcom vs. SNK 2 Mark Of The Millennium 2001
     // ver 010804
     // with Japan BIOS will be shown 010705, likely forgot / was not cared to update it
     {
         "cvs2",
         NULL,
-        "Capcom Vs. SNK 2 Mark Of The Millennium 2001 (USA)",
+        "Capcom vs. SNK 2 Mark Of The Millennium 2001 (USA)",
         0x4000,
         0,
         "naomi",
@@ -4682,12 +4682,12 @@ Game Games[] =
         &capcom_6btn_inputs,
 		cvs2_eeprom_dump,
     },
-    // Capcom Vs. SNK 2 Millionaire Fighting 2001 (Rev A)
+    // Capcom vs. SNK 2 Millionaire Fighting 2001 (Rev A)
     // ver 010705
     {
         "cvs2mf",
         "cvs2",
-        "Capcom Vs. SNK 2 Millionaire Fighting 2001 (Japan, Rev A)",
+        "Capcom vs. SNK 2 Millionaire Fighting 2001 (Japan, Rev A)",
         0x4000,
         0,
         "naomi",
@@ -4872,11 +4872,11 @@ Game Games[] =
 		&guilty_gear_inputs,
 		ggxxsla_eeprom_dump,
     },
-    // Mobile Suit Gundam: Federation Vs. Zeon
+    // Mobile Suit Gundam: Federation vs. Zeon
     {
         "gundmgd",
         NULL,
-        "Mobile Suit Gundam: Federation Vs. Zeon",
+        "Mobile Suit Gundam: Federation vs. Zeon",
         0x4000,
         0,
         "naomi",
@@ -4891,11 +4891,11 @@ Game Games[] =
 		&shot1234_inputs,
 		gundmct_eeprom_dump
     },
-    // Mobile Suit Gundam: Federation Vs. Zeon DX (USA, Japan)
+    // Mobile Suit Gundam: Federation vs. Zeon DX (USA, Japan)
     {
         "gundmxgd",
         NULL,
-        "Mobile Suit Gundam: Federation Vs. Zeon DX",
+        "Mobile Suit Gundam: Federation vs. Zeon DX",
         0x4000,
         0,
         "naomi",

core/rend/boxart/boxart.cpp

@@ -18,6 +18,8 @@
  */
 #include "boxart.h"
 #include "gamesdb.h"
+#include "../game_scanner.h"
+#include <chrono>
 
 static std::string getGameFileName(const std::string& path)
 {
@@ -33,56 +35,80 @@ const GameBoxart *Boxart::getBoxart(const GameMedia& media)
 {
 	loadDatabase();
 	std::string fileName = getGameFileName(media.path);
+	const GameBoxart *boxart = nullptr;
 	{
 		std::lock_guard<std::mutex> guard(mutex);
 		auto it = games.find(fileName);
 		if (it != games.end())
-			return &it->second;
+			boxart = &it->second;
-		else
+		else if (config::FetchBoxart)
-			return nullptr;
+		{
+			GameBoxart box;
+			box.fileName = fileName;
+			box.gamePath = media.path;
+			box.name = media.name;
+			box.searchName = media.gameName;	// for arcade games
+			games[box.fileName] = box;
+			toFetch.push_back(box);
+		}
 	}
+	if (config::FetchBoxart)
+		fetchBoxart();
+	return boxart;
 }
 
-std::future<const GameBoxart *> Boxart::fetchBoxart(const GameMedia& media)
+void Boxart::fetchBoxart()
 {
-	if (scraper == nullptr)
+	if (fetching.valid() && fetching.wait_for(std::chrono::seconds(0)) == std::future_status::ready)
-	{
+		fetching.get();
-		scraper = std::unique_ptr<Scraper>(new TheGamesDb());
+	if (fetching.valid())
-		if (!scraper->initialize(getSaveDirectory()))
+		return;
+	if (toFetch.empty())
+		return;
+	fetching = std::async(std::launch::async, [this]() {
+		if (scraper == nullptr)
 		{
-			WARN_LOG(COMMON, "thegamesdb scraper initialization failed");
+			scraper = std::unique_ptr<Scraper>(new TheGamesDb());
-			scraper.reset();
+			if (!scraper->initialize(getSaveDirectory()))
-		}
+			{
-	}
+				WARN_LOG(COMMON, "thegamesdb scraper initialization failed");
-	return std::async(std::launch::async, [this, media]() {
+				scraper.reset();
-		std::string fileName = getGameFileName(media.path);
+				return;
-		const GameBoxart *rv = nullptr;
+			}
-		if (scraper != nullptr)
+		}
-		{
+		std::vector<GameBoxart> boxart;
-			GameBoxart boxart;
+		{
-			boxart.fileName = fileName;
+			std::lock_guard<std::mutex> guard(mutex);
-			boxart.gamePath = media.path;
+			size_t size = std::min(toFetch.size(), (size_t)10);
-			boxart.name = trim_trailing_ws(media.gameName);
+			boxart = std::vector<GameBoxart>(toFetch.begin(), toFetch.begin() + size);
-			DEBUG_LOG(COMMON, "Scraping %s -> %s", media.name.c_str(), boxart.name.c_str());
+			toFetch.erase(toFetch.begin(), toFetch.begin() + size);
-			try {
+		}
-				scraper->scrape(boxart);
+		DEBUG_LOG(COMMON, "Scraping %d games", (int)boxart.size());
-				{
+		try {
-					std::lock_guard<std::mutex> guard(mutex);
+			scraper->scrape(boxart);
-					games[fileName] = boxart;
+			{
-					rv = &games[fileName];
+				std::lock_guard<std::mutex> guard(mutex);
-				}
+				for (const GameBoxart& b : boxart)
-				databaseDirty = true;
+					if (b.scraped)
-			} catch (const std::exception& e) {
+						games[b.fileName] = b;
-				if (*e.what() != '\0')
+			}
-					INFO_LOG(COMMON, "thegamesdb error: %s", e.what());
+			databaseDirty = true;
+		} catch (const std::exception& e) {
+			if (*e.what() != '\0')
+				INFO_LOG(COMMON, "thegamesdb error: %s", e.what());
+			{
+				// put back items into toFetch array
+				std::lock_guard<std::mutex> guard(mutex);
+				toFetch.insert(toFetch.begin(), boxart.begin(), boxart.end());
 			}
 		}
-		return rv;
 	});
 }
 
 void Boxart::saveDatabase()
 {
+	if (fetching.valid())
+		fetching.get();
 	if (!databaseDirty)
 		return;
 	std::string db_name = getSaveDirectory() + DB_NAME;

core/rend/boxart/boxart.h

@@ -18,18 +18,18 @@
  */
 #pragma once
 #include "scraper.h"
-#include "../game_scanner.h"
 #include "stdclass.h"
 #include <unordered_map>
 #include <memory>
 #include <future>
 #include <mutex>
 
+struct GameMedia;
+
 class Boxart
 {
 public:
 	const GameBoxart *getBoxart(const GameMedia& media);
-	std::future<const GameBoxart *> fetchBoxart(const GameMedia& media);
 	void saveDatabase();
 
 private:
@@ -37,6 +37,7 @@ private:
 	std::string getSaveDirectory() const {
 		return get_writable_data_path("/boxart/");
 	}
+	void fetchBoxart();
 
 	std::unordered_map<std::string, GameBoxart> games;
 	std::mutex mutex;
@@ -44,5 +45,8 @@ private:
 	bool databaseLoaded = false;
 	bool databaseDirty = false;
 
+	std::vector<GameBoxart> toFetch;
+	std::future<void> fetching;
+
 	static constexpr char const *DB_NAME = "flycast-gamedb.json";
 };

core/rend/boxart/gamesdb.cpp

@@ -74,8 +74,14 @@ void TheGamesDb::copyFile(const std::string& from, const std::string& to)
 	fclose(fto);
 }
 
-bool TheGamesDb::httpGet(const std::string& url, std::vector<u8>& receivedData)
+json TheGamesDb::httpGet(const std::string& url)
 {
+	if (os_GetSeconds() < blackoutPeriod)
+		throw std::runtime_error("");
+	blackoutPeriod = 0.0;
+
+	DEBUG_LOG(COMMON, "TheGameDb: GET %s", url.c_str());
+	std::vector<u8> receivedData;
 	int status = http::get(url, receivedData);
 	bool success = http::success(status);
 	if (status == 403)
@@ -83,8 +89,26 @@ bool TheGamesDb::httpGet(const std::string& url, std::vector<u8>& receivedData)
 		blackoutPeriod = os_GetSeconds() + 60.0;
 	else if (!success)
 		blackoutPeriod = os_GetSeconds() + 1.0;
+	if (!success)
+		throw std::runtime_error("http error");
 
-	return success;
+	std::string content((const char *)&receivedData[0], receivedData.size());
+	DEBUG_LOG(COMMON, "TheGameDb: received [%s]", content.c_str());
+
+	json v = json::parse(content);
+	int code = v["code"];
+	if (!http::success(code))
+	{
+		// TODO can this happen? http status should be the same
+		std::string status;
+		try {
+			status = v["status"];
+		} catch (const json::exception& e) {
+		}
+		throw std::runtime_error(std::string("TheGamesDB error ") + std::to_string(code) + ": " + status);
+	}
+
+	return v;
 }
 
 void TheGamesDb::fetchPlatforms()
@@ -95,14 +119,7 @@ void TheGamesDb::fetchPlatforms()
 	auto getPlatformId = [this](const std::string& platform)
 	{
 		std::string url = makeUrl("Platforms/ByPlatformName") + "&name=" + platform;
-		DEBUG_LOG(COMMON, "TheGameDb: GET %s", url.c_str());
+		json v = httpGet(url);
-
-		std::vector<u8> receivedData;
-		if (!httpGet(url, receivedData))
-			throw std::runtime_error("http error");
-
-		std::string content((const char *)&receivedData[0], receivedData.size());
-		json v = json::parse(content);
 
 		const json& array = v["data"]["platforms"];
 
@@ -207,34 +224,9 @@ void TheGamesDb::parseBoxart(GameBoxart& item, const json& j, int gameId)
 	}
 }
 
-bool TheGamesDb::fetchGameInfo(GameBoxart& item, const std::string& url, const std::string& diskId)
+bool TheGamesDb::parseGameInfo(const json& gameArray, const json& boxartArray, GameBoxart& item, const std::string& diskId)
 {
-	DEBUG_LOG(COMMON, "TheGameDb: GET %s", url.c_str());
+	for (const auto& game : gameArray)
-	std::vector<u8> receivedData;
-	if (!httpGet(url, receivedData))
-		throw std::runtime_error("http error");
-
-	std::string content((const char *)&receivedData[0], receivedData.size());
-	DEBUG_LOG(COMMON, "TheGameDb: received [%s]", content.c_str());
-
-	json v = json::parse(content);
-
-	int code = v["code"];
-	if (!http::success(code))
-	{
-		// TODO can this happen? http status should be the same
-		std::string status;
-		try {
-			status = v["status"];
-		} catch (const json::exception& e) {
-		}
-		throw std::runtime_error(std::string("TheGamesDB error ") + std::to_string(code) + ": " + status);
-	}
-	json array = v["data"]["games"];
-	if (array.empty())
-		return false;
-
-	for (const auto& game : array)
 	{
 		if (!diskId.empty())
 		{
@@ -274,49 +266,50 @@ bool TheGamesDb::fetchGameInfo(GameBoxart& item, const std::string& url, const s
 		}
 
 		// Boxart
-		parseBoxart(item, v["include"]["boxart"], id);
+		parseBoxart(item, boxartArray, id);
 
 		if (item.boxartPath.empty())
 		{
 			std::string imgUrl = makeUrl("Games/Images") + "&games_id=" + std::to_string(id);
-			DEBUG_LOG(COMMON, "TheGameDb: GET %s", imgUrl.c_str());
+			json images = httpGet(imgUrl);
-			if (!httpGet(imgUrl, receivedData))
-				throw std::runtime_error("http error");
-			content = std::string((const char *)&receivedData[0], receivedData.size());
-			DEBUG_LOG(COMMON, "TheGameDb: received [%s]", content.c_str());
-			json images = json::parse(content);
 			parseBoxart(item, images["data"], id);
 		}
-		break;
+		return true;
 	}
-
+	return false;
-	return true;
 }
 
-void TheGamesDb::scrape(GameBoxart& item)
+bool TheGamesDb::fetchGameInfo(GameBoxart& item, const std::string& url, const std::string& diskId)
 {
-	scrape(item, "");
+	json v = httpGet(url);
+	json& array = v["data"]["games"];
+	if (array.empty())
+		return false;
+
+	return parseGameInfo(array, v["include"]["boxart"], item, diskId);
 }
 
-void TheGamesDb::scrape(GameBoxart& item, const std::string& diskId)
+void TheGamesDb::getUidAndSearchName(GameBoxart& media)
 {
-	if (os_GetSeconds() < blackoutPeriod)
+	int platform = getGamePlatform(media.gamePath.c_str());
-		throw std::runtime_error("");
-	blackoutPeriod = 0.0;
-
-	item.found = false;
-	int platform = getGamePlatform(item.gamePath.c_str());
-	std::string gameName;
-	std::string uniqueId;
 	if (platform == DC_PLATFORM_DREAMCAST)
 	{
+		if (media.gamePath.empty())
+		{
+			// Dreamcast BIOS
+			media.uniqueId.clear();
+			media.searchName.clear();
+			return;
+		}
 		Disc *disc;
 		try {
-			disc = OpenDisc(item.gamePath.c_str());
+			disc = OpenDisc(media.gamePath.c_str());
 		} catch (const std::exception& e) {
-			WARN_LOG(COMMON, "Can't open disk %s: %s", item.gamePath.c_str(), e.what());
+			WARN_LOG(COMMON, "Can't open disk %s: %s", media.gamePath.c_str(), e.what());
 			// No need to retry if the disk is invalid/corrupted
-			item.scraped = true;
+			media.scraped = true;
+			media.uniqueId.clear();
+			media.searchName.clear();
 			return;
 		}
 
@@ -335,64 +328,138 @@ void TheGamesDb::scrape(GameBoxart& item, const std::string& diskId)
 		memcpy(&diskId, sector, sizeof(diskId));
 		delete disc;
 
-		uniqueId = trim_trailing_ws(std::string(diskId.product_number, sizeof(diskId.product_number)));
+		media.uniqueId = trim_trailing_ws(std::string(diskId.product_number, sizeof(diskId.product_number)));
 
-		gameName = trim_trailing_ws(std::string(diskId.software_name, sizeof(diskId.software_name)));
+		media.searchName = trim_trailing_ws(std::string(diskId.software_name, sizeof(diskId.software_name)));
-		if (gameName.empty())
+		if (media.searchName.empty())
-			gameName = item.name;
+			media.searchName = media.name;
 
 		if (diskId.area_symbols[0] != '\0')
 		{
+			media.region = 0;
 			if (diskId.area_symbols[0] == 'J')
-				item.region |= GameBoxart::JAPAN;
+				media.region |= GameBoxart::JAPAN;
 			if (diskId.area_symbols[1] == 'U')
-				item.region |= GameBoxart::USA;
+				media.region |= GameBoxart::USA;
 			if (diskId.area_symbols[2] == 'E')
-				item.region |= GameBoxart::EUROPE;
+				media.region |= GameBoxart::EUROPE;
 		}
+		else
+			media.region = GameBoxart::JAPAN | GameBoxart::USA | GameBoxart::EUROPE;
 	}
 	else
 	{
-		gameName = item.name;
+		media.uniqueId.clear();
-		size_t spos = gameName.find('/');
+		// Use first one in case of alternate names (Virtua Tennis / Power Smash)
+		size_t spos = media.searchName.find('/');
 		if (spos != std::string::npos)
-			gameName = trim_trailing_ws(gameName.substr(0, spos));
+			media.searchName = trim_trailing_ws(media.searchName.substr(0, spos));
-		while (!gameName.empty())
+		// Delete trailing (...) and [...]
+		while (!media.searchName.empty())
 		{
 			size_t pos{ std::string::npos };
-			if (gameName.back() == ')')
+			if (media.searchName.back() == ')')
-				pos = gameName.find_last_of('(');
+				pos = media.searchName.find_last_of('(');
-			else if (gameName.back() == ']')
+			else if (media.searchName.back() == ']')
-				pos = gameName.find_last_of('[');
+				pos = media.searchName.find_last_of('[');
 			if (pos == std::string::npos)
 				break;
-			gameName = trim_trailing_ws(gameName.substr(0, pos));
+			media.searchName = trim_trailing_ws(media.searchName.substr(0, pos));
 		}
 	}
+}
 
+void TheGamesDb::scrape(GameBoxart& item)
+{
+	item.found = false;
+	getUidAndSearchName(item);
+	if (item.searchName.empty())
+		// invalid rom or disk
+		return;
 	fetchPlatforms();
 
-	if (!uniqueId.empty())
+	if (!item.uniqueId.empty())
 	{
-		std::string url = makeUrl("Games/ByGameUniqueID") + "&fields=overview,uids&include=boxart&filter%5Bplatform%5D=";
+		std::string url = makeUrl("Games/ByGameUniqueID") + "&fields=overview,uids&include=boxart&filter%5Bplatform%5D="
-		if (platform == DC_PLATFORM_DREAMCAST)
+			+ std::to_string(dreamcastPlatformId) + "&uid=" + http::urlEncode(item.uniqueId);
-			url += std::to_string(dreamcastPlatformId);
+		if (fetchGameInfo(item, url, item.uniqueId))
-		else
+			item.scraped = item.found = true;
-			url += std::to_string(arcadePlatformId);
+	}
-		// Can be batched, separated by commas
+	if (!item.scraped)
-		url += "&uid=" + http::urlEncode(uniqueId);
+		fetchByName(item);
 
-		item.found = fetchGameInfo(item, url, diskId);
-	}
-	if (!item.found)
-	{
-		std::string url = makeUrl("Games/ByGameName") + "&fields=overview&include=boxart&filter%5Bplatform%5D=";
-		if (platform == DC_PLATFORM_DREAMCAST)
-			url += std::to_string(dreamcastPlatformId);
-		else
-			url += std::to_string(arcadePlatformId);
-		url += "&name=" + http::urlEncode(gameName);
-		item.found = fetchGameInfo(item, url);
-	}
 	item.scraped = true;
 }
+
+void TheGamesDb::fetchByName(GameBoxart& item)
+{
+	if (item.searchName.empty())
+		return;
+	int platform = getGamePlatform(item.gamePath.c_str());
+	std::string url = makeUrl("Games/ByGameName") + "&fields=overview&include=boxart&filter%5Bplatform%5D=";
+	if (platform == DC_PLATFORM_DREAMCAST)
+		url += std::to_string(dreamcastPlatformId);
+	else
+		url += std::to_string(arcadePlatformId);
+	url += "&name=" + http::urlEncode(item.searchName);
+	if (fetchGameInfo(item, url))
+		item.scraped = item.found = true;
+}
+
+void TheGamesDb::fetchByUids(std::vector<GameBoxart>& items)
+{
+	std::string uidCriteria;
+	for (const GameBoxart& item : items)
+	{
+		if (item.scraped || item.uniqueId.empty())
+			continue;
+		if (!uidCriteria.empty())
+			uidCriteria += ',';
+		uidCriteria += item.uniqueId;
+	}
+	if (uidCriteria.empty())
+		return;
+	std::string url = makeUrl("Games/ByGameUniqueID") + "&fields=overview,uids&include=boxart&filter%5Bplatform%5D="
+			+ std::to_string(dreamcastPlatformId) + "&uid=" + http::urlEncode(uidCriteria);
+	json v = httpGet(url);
+	json& array = v["data"]["games"];
+	if (array.empty())
+		return;
+	json& boxartArray = v["include"]["boxart"];
+	for (GameBoxart& item : items)
+	{
+		if (!item.scraped && !item.uniqueId.empty() && parseGameInfo(array, boxartArray, item, item.uniqueId))
+			item.scraped = item.found = true;
+	}
+}
+
+void TheGamesDb::scrape(std::vector<GameBoxart>& items)
+{
+	if (os_GetSeconds() < blackoutPeriod)
+		throw std::runtime_error("");
+	blackoutPeriod = 0.0;
+
+	fetchPlatforms();
+	for (GameBoxart& item : items)
+	{
+		if (!item.scraped)
+			item.found = false;
+		getUidAndSearchName(item);
+	}
+	fetchByUids(items);
+	for (GameBoxart& item : items)
+	{
+		if (!item.scraped)
+		{
+			if (!item.searchName.empty())
+				fetchByName(item);
+			else if (item.gamePath.empty())
+			{
+				std::string localPath = makeUniqueFilename("dreamcast_logo_grey.png");
+				if (downloadImage("https://flyinghead.github.io/flycast-builds/dreamcast_logo_grey.png", localPath))
+					item.boxartPath = localPath;
+			}
+		}
+		item.scraped = true;
+	}
+}

core/rend/boxart/gamesdb.h

@@ -27,19 +27,23 @@ class TheGamesDb : public Scraper
 public:
 	bool initialize(const std::string& saveDirectory) override;
 	void scrape(GameBoxart& item) override;
+	void scrape(std::vector<GameBoxart>& items) override;
 	~TheGamesDb();
 
 private:
-	void scrape(GameBoxart& item, const std::string& diskId);
 	void fetchPlatforms();
 	bool fetchGameInfo(GameBoxart& item, const std::string& url, const std::string& diskId = "");
 	std::string makeUrl(const std::string& endpoint);
 	void copyFile(const std::string& from, const std::string& to);
-	bool httpGet(const std::string& url, std::vector<u8>& receivedData);
+	json httpGet(const std::string& url);
 	void parseBoxart(GameBoxart& item, const json& j, int gameId);
+	void getUidAndSearchName(GameBoxart& media);
+	void fetchByUids(std::vector<GameBoxart>& items);
+	void fetchByName(GameBoxart& item);
+	bool parseGameInfo(const json& gameArray, const json& boxartArray, GameBoxart& item, const std::string& diskId);
 
-	int dreamcastPlatformId;
+	int dreamcastPlatformId = 0;
-	int arcadePlatformId;
+	int arcadePlatformId = 0;
 	double blackoutPeriod = 0.0;
 
 	std::map<std::string, std::string> boxartCache;	// key: url, value: local file path

core/rend/boxart/scraper.h

@@ -31,6 +31,7 @@ struct GameBoxart
 	std::string fileName;
 	std::string name;
 	std::string uniqueId;
+	std::string searchName;
 	u32 region = 0;
 	std::string releaseDate;
 	std::string overview;
@@ -50,10 +51,10 @@ struct GameBoxart
 		json j = {
 			{ "file_name", fileName },
 			{ "name", name },
+			{ "unique_id", uniqueId },
 			{ "region", region },
 			{ "release_date", releaseDate },
 			{ "overview", overview },
-			{ "game_path", gamePath },
 			{ "screenshot_path", screenshotPath },
 			{ "fanart_path", fanartPath },
 			{ "boxart_path", boxartPath },
@@ -83,10 +84,10 @@ struct GameBoxart
 	{
 		loadProperty(fileName, j, "file_name");
 		loadProperty(name, j, "name");
+		loadProperty(uniqueId, j, "unique_id");
 		loadProperty(region, j, "region");
 		loadProperty(releaseDate, j, "release_date");
 		loadProperty(overview, j, "overview");
-		loadProperty(gamePath, j, "game_path");
 		loadProperty(screenshotPath, j, "screenshot_path");
 		loadProperty(fanartPath, j, "fanart_path");
 		loadProperty(boxartPath, j, "boxart_path");
@@ -102,7 +103,14 @@ public:
 		this->saveDirectory = saveDirectory;
 		return true;
 	}
+
 	virtual void scrape(GameBoxart& item) = 0;
+
+	virtual void scrape(std::vector<GameBoxart>& items) {
+		for (GameBoxart& item : items)
+			scrape(item);
+	}
+
 	virtual ~Scraper() = default;
 
 protected:

core/rend/gui.cpp

@@ -2295,7 +2295,65 @@ inline static void gui_display_demo()
 	ImGui::ShowDemoWindow();
 }
 
-static std::future<const GameBoxart *> futureBoxart;
+static void gameTooltip(const std::string& tip)
+{
+    if (ImGui::IsItemHovered())
+    {
+        ImGui::BeginTooltip();
+        ImGui::PushTextWrapPos(ImGui::GetFontSize() * 25.0f);
+        ImGui::TextUnformatted(tip.c_str());
+        ImGui::PopTextWrapPos();
+        ImGui::EndTooltip();
+    }
+}
+
+static bool getGameImage(const GameBoxart *art, ImTextureID& textureId, bool allowLoad)
+{
+	textureId = ImTextureID{};
+	if (art->boxartPath.empty())
+		return false;
+
+	// Get the boxart texture. Load it if needed.
+	textureId = imguiDriver->getTexture(art->boxartPath);
+	if (textureId == ImTextureID() && allowLoad)
+	{
+		int width, height;
+		u8 *imgData = loadImage(art->boxartPath, width, height);
+		if (imgData != nullptr)
+		{
+			try {
+				textureId = imguiDriver->updateTextureAndAspectRatio(art->boxartPath, imgData, width, height);
+			} catch (const std::exception&) {
+				// vulkan can throw during resizing
+			}
+			free(imgData);
+		}
+		return true;
+	}
+	return false;
+}
+
+static bool gameImageButton(ImTextureID textureId, const std::string& tooltip)
+{
+	float ar = imguiDriver->getAspectRatio(textureId);
+	ImVec2 uv0 { 0.f, 0.f };
+	ImVec2 uv1 { 1.f, 1.f };
+	if (ar > 1)
+	{
+		uv0.y = -(ar - 1) / 2;
+		uv1.y = 1 + (ar - 1) / 2;
+	}
+	else if (ar != 0)
+	{
+		ar = 1 / ar;
+		uv0.x = -(ar - 1) / 2;
+		uv1.x = 1 + (ar - 1) / 2;
+	}
+	bool pressed = ImGui::ImageButton(textureId, ScaledVec2(200, 200) - ImGui::GetStyle().FramePadding * 2, uv0, uv1);
+	gameTooltip(tooltip);
+
+    return pressed;
+}
 
 static void gui_display_content()
 {
@@ -2344,20 +2402,35 @@ static void gui_display_content()
 			ImGui::PushStyleVar(ImGuiStyleVar_ItemSpacing, ScaledVec2(8, 20));
 
 		int counter = 0;
+		int loadedImages = 0;
 		if (gui_state != GuiState::SelectDisk && filter.PassFilter("Dreamcast BIOS"))
 		{
 			ImGui::PushID("bios");
 			bool pressed;
 			if (config::BoxartDisplayMode)
-				pressed = ImGui::Button("Dreamcast BIOS", ScaledVec2(200, 200));
+			{
+				ImTextureID textureId{};
+				GameMedia game;
+				const GameBoxart *art = boxart.getBoxart(game);
+				if (art != nullptr)
+				{
+					if (getGameImage(art, textureId, loadedImages < 10))
+						loadedImages++;
+				}
+				if (textureId != ImTextureID())
+					pressed = gameImageButton(textureId, "Dreamcast BIOS");
+				else
+					pressed = ImGui::Button("Dreamcast BIOS", ScaledVec2(200, 200));
+			}
 			else
+			{
 				pressed = ImGui::Selectable("Dreamcast BIOS");
+			}
 			if (pressed)
 				gui_start_game("");
 			ImGui::PopID();
 			counter++;
 		}
-		int loadedImages = 0;
 		{
 			scanner.get_mutex().lock();
 			for (const auto& game : scanner.get_game_list())
@@ -2375,15 +2448,6 @@ static void gui_display_content()
 				if (config::BoxartDisplayMode)
 				{
 					art = boxart.getBoxart(game);
-					if (art == nullptr && config::FetchBoxart)
-					{
-						if (futureBoxart.valid() && futureBoxart.wait_for(std::chrono::seconds(0)) == std::future_status::ready) {
-							futureBoxart.get();
-							art = boxart.getBoxart(game);
-						}
-						if (art == nullptr && !futureBoxart.valid())
-							futureBoxart = boxart.fetchBoxart(game);
-					}
 					if (art != nullptr)
 						gameName = art->name;
 				}
@@ -2397,55 +2461,19 @@ static void gui_display_content()
 							ImGui::SameLine();
 						counter++;
 						ImTextureID textureId{};
-						if (art != nullptr && !art->boxartPath.empty())
+						if (art != nullptr)
 						{
-							// Get the boxart texture. Load it if needed.
+							// Get the boxart texture. Load it if needed (max 10 per frame).
-							textureId = imguiDriver->getTexture(art->boxartPath);
+							if (getGameImage(art, textureId, loadedImages < 10))
-							if (textureId == ImTextureID() && loadedImages < 10)
-							{
-								// Load 10 images max per frame
 								loadedImages++;
-								int width, height;
-								u8 *imgData = loadImage(art->boxartPath, width, height);
-								if (imgData != nullptr)
-								{
-									try {
-										textureId = imguiDriver->updateTextureAndAspectRatio(art->boxartPath, imgData, width, height);
-									} catch (const std::exception&) {
-										// vulkan can throw during resizing
-									}
-									free(imgData);
-								}
-							}
 						}
 						if (textureId != ImTextureID())
-						{
+							pressed = gameImageButton(textureId, game.name);
-							float ar = imguiDriver->getAspectRatio(textureId);
-							ImVec2 uv0 { 0.f, 0.f };
-							ImVec2 uv1 { 1.f, 1.f };
-							if (ar > 1)
-							{
-								uv0.y = -(ar - 1) / 2;
-								uv1.y = 1 + (ar - 1) / 2;
-							}
-							else if (ar != 0)
-							{
-								ar = 1 / ar;
-								uv0.x = -(ar - 1) / 2;
-								uv1.x = 1 + (ar - 1) / 2;
-							}
-							pressed = ImGui::ImageButton(textureId, ScaledVec2(200, 200) - ImGui::GetStyle().FramePadding * 2, uv0, uv1);
-						}
 						else
+						{
 							pressed = ImGui::Button(gameName.c_str(), ScaledVec2(200, 200));
-					    if (ImGui::IsItemHovered())
+							gameTooltip(game.name);
-					    {
+						}
-					        ImGui::BeginTooltip();
-					        ImGui::PushTextWrapPos(ImGui::GetFontSize() * 25.0f);
-					        ImGui::TextUnformatted(game.name.c_str());
-					        ImGui::PopTextWrapPos();
-					        ImGui::EndTooltip();
-					    }
 					}
 					else
 					{
@@ -2855,8 +2883,6 @@ void gui_error(const std::string& what)
 
 void gui_save()
 {
-	if (futureBoxart.valid())
-		futureBoxart.get();
 	boxart.saveDatabase();
 }