GameDatabase: Switch to YAML

This commit is contained in:
Stenzek 2024-02-04 02:36:25 +10:00
parent 5c08fa9d00
commit d7a1c447c6
No known key found for this signature in database
9 changed files with 426 additions and 400 deletions

View File

@ -3,13 +3,15 @@ name: GameDB Lint
on: on:
pull_request: pull_request:
paths: paths:
- 'data/resources/gamedb.json' - 'data/resources/gamedb.yaml'
- 'data/resources/discdb.yaml'
push: push:
branches: branches:
- master - master
- dev - dev
paths: paths:
- 'data/resources/gamedb.json' - 'data/resources/gamedb.yaml'
- 'data/resources/discdb.yaml'
workflow_dispatch: workflow_dispatch:
jobs: jobs:
@ -25,9 +27,12 @@ jobs:
shell: bash shell: bash
run: | run: |
sudo apt-get update sudo apt-get update
sudo apt-get -y install python3-demjson sudo apt-get -y install yamllint
- name: Check GameDB - name: Check GameDB
shell: bash shell: bash
run: | run: yamllint -c extras/yamllint-config.yaml -s -f github data/resources/gamedb.yaml
jsonlint -s data/resources/gamedb.json
- name: Check DiscDB
shell: bash
run: yamllint -c extras/yamllint-config.yaml -s -f github data/resources/discdb.yaml

View File

@ -0,0 +1,16 @@
extends: default
rules:
line-length:
max: 200
indentation:
spaces: 2
indent-sequences: true
document-start:
present: false
document-end:
present: false
comments:
require-starting-space: true
min-spaces-from-content: 1

View File

@ -131,7 +131,7 @@ target_precompile_headers(core PRIVATE "pch.h")
target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..") target_include_directories(core PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/..")
target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..") target_include_directories(core PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/..")
target_link_libraries(core PUBLIC Threads::Threads common util ZLIB::ZLIB) target_link_libraries(core PUBLIC Threads::Threads common util ZLIB::ZLIB)
target_link_libraries(core PRIVATE stb xxhash imgui rapidjson rcheevos) target_link_libraries(core PRIVATE stb xxhash imgui rapidyaml rcheevos)
if(CPU_ARCH_X64) if(CPU_ARCH_X64)
target_compile_definitions(core PUBLIC "ENABLE_RECOMPILER=1" "ENABLE_NEWREC=1" "ENABLE_MMAP_FASTMEM=1") target_compile_definitions(core PUBLIC "ENABLE_RECOMPILER=1" "ENABLE_NEWREC=1" "ENABLE_MMAP_FASTMEM=1")

View File

@ -10,7 +10,11 @@
<PreprocessorDefinitions Condition="('$(Platform)'=='x64' Or '$(Platform)'=='ARM64')">ENABLE_MMAP_FASTMEM=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="('$(Platform)'=='x64' Or '$(Platform)'=='ARM64')">ENABLE_MMAP_FASTMEM=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PreprocessorDefinitions Condition="('$(Platform)'=='x64' Or '$(Platform)'=='ARM64')">ENABLE_NEWREC=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="('$(Platform)'=='x64' Or '$(Platform)'=='ARM64')">ENABLE_NEWREC=1;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\zlib\include;$(SolutionDir)dep\rcheevos\include;$(SolutionDir)dep\rapidjson\include;$(SolutionDir)dep\discord-rpc\include</AdditionalIncludeDirectories> <AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\zlib\include;$(SolutionDir)dep\rcheevos\include;$(SolutionDir)dep\discord-rpc\include</AdditionalIncludeDirectories>
<PreprocessorDefinitions>%(PreprocessorDefinitions);C4_NO_DEBUG_BREAK=1</PreprocessorDefinitions>
<AdditionalIncludeDirectories>%(AdditionalIncludeDirectories);$(SolutionDir)dep\rapidyaml\include;$(SolutionDir)dep\rapidjson\include</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Platform)'!='ARM64'">%(AdditionalIncludeDirectories);$(SolutionDir)dep\rainterface</AdditionalIncludeDirectories> <AdditionalIncludeDirectories Condition="'$(Platform)'!='ARM64'">%(AdditionalIncludeDirectories);$(SolutionDir)dep\rainterface</AdditionalIncludeDirectories>
<AdditionalIncludeDirectories Condition="'$(Platform)'=='x64'">%(AdditionalIncludeDirectories);$(SolutionDir)dep\xbyak\xbyak</AdditionalIncludeDirectories> <AdditionalIncludeDirectories Condition="'$(Platform)'=='x64'">%(AdditionalIncludeDirectories);$(SolutionDir)dep\xbyak\xbyak</AdditionalIncludeDirectories>

View File

@ -172,6 +172,9 @@
<ProjectReference Include="..\..\dep\rainterface\rainterface.vcxproj" Condition="'$(Platform)'!='ARM64'"> <ProjectReference Include="..\..\dep\rainterface\rainterface.vcxproj" Condition="'$(Platform)'!='ARM64'">
<Project>{e4357877-d459-45c7-b8f6-dcbb587bb528}</Project> <Project>{e4357877-d459-45c7-b8f6-dcbb587bb528}</Project>
</ProjectReference> </ProjectReference>
<ProjectReference Include="..\..\dep\rapidyaml\rapidyaml.vcxproj">
<Project>{1ad23a8a-4c20-434c-ae6b-0e07759eeb1e}</Project>
</ProjectReference>
<ProjectReference Include="..\..\dep\rcheevos\rcheevos.vcxproj"> <ProjectReference Include="..\..\dep\rcheevos\rcheevos.vcxproj">
<Project>{4ba0a6d4-3ae1-42b2-9347-096fd023ff64}</Project> <Project>{4ba0a6d4-3ae1-42b2-9347-096fd023ff64}</Project>
</ProjectReference> </ProjectReference>

View File

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2019-2023 Connor McLaughlin <stenzek@gmail.com> // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "game_database.h" #include "game_database.h"
@ -16,22 +16,18 @@
#include "common/string_util.h" #include "common/string_util.h"
#include "common/timer.h" #include "common/timer.h"
#include "rapidjson/document.h" #include "ryml.hpp"
#include "rapidjson/error/en.h"
#include <iomanip> #include <iomanip>
#include <memory> #include <memory>
#include <optional> #include <optional>
#include <sstream> #include <sstream>
#include <type_traits>
#include "IconsFontAwesome5.h" #include "IconsFontAwesome5.h"
Log_SetChannel(GameDatabase); Log_SetChannel(GameDatabase);
#ifdef _WIN32
#include "common/windows_headers.h"
#endif
namespace GameDatabase { namespace GameDatabase {
enum : u32 enum : u32
@ -46,35 +42,51 @@ static const Entry* GetEntryForId(const std::string_view& code);
static bool LoadFromCache(); static bool LoadFromCache();
static bool SaveToCache(); static bool SaveToCache();
static bool LoadGameDBJson(); static void SetRymlCallbacks();
static bool ParseJsonEntry(Entry* entry, const rapidjson::Value& value); static bool LoadGameDBYaml();
static bool ParseJsonCodes(u32 index, const rapidjson::Value& value); static bool ParseYamlEntry(Entry* entry, const ryml::ConstNodeRef& value);
static bool ParseYamlCodes(u32 index, const ryml::ConstNodeRef& value, std::string_view serial);
static bool LoadTrackHashes(); static bool LoadTrackHashes();
static std::array<const char*, static_cast<u32>(GameDatabase::Trait::Count)> s_trait_names = {{ static constexpr const std::array<const char*, static_cast<int>(CompatibilityRating::Count)>
"ForceInterpreter", s_compatibility_rating_names = {
"ForceSoftwareRenderer", {"Unknown", "DoesntBoot", "CrashesInIntro", "CrashesInGame", "GraphicalAudioIssues", "NoIssues"}};
"ForceSoftwareRendererForReadbacks",
"ForceInterlacing", static constexpr const std::array<const char*, static_cast<size_t>(CompatibilityRating::Count)>
"DisableTrueColor", s_compatibility_rating_display_names = {{TRANSLATE_NOOP("GameListCompatibilityRating", "Unknown"),
"DisableUpscaling", TRANSLATE_NOOP("GameListCompatibilityRating", "Doesn't Boot"),
"DisableTextureFiltering", TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In Intro"),
"DisableScaledDithering", TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In-Game"),
"DisableForceNTSCTimings", TRANSLATE_NOOP("GameListCompatibilityRating", "Graphical/Audio Issues"),
"DisableWidescreen", TRANSLATE_NOOP("GameListCompatibilityRating", "No Issues")}};
"DisablePGXP",
"DisablePGXPCulling", static constexpr const std::array<const char*, static_cast<u32>(GameDatabase::Trait::Count)> s_trait_names = {{
"DisablePGXPTextureCorrection", "forceInterpreter",
"DisablePGXPColorCorrection", "forceSoftwareRenderer",
"DisablePGXPDepthBuffer", "forceSoftwareRendererForReadbacks",
"ForcePGXPVertexCache", "forceInterlacing",
"ForcePGXPCPUMode", "disableTrueColor",
"ForceRecompilerMemoryExceptions", "disableUpscaling",
"ForceRecompilerICache", "disableTextureFiltering",
"ForceRecompilerLUTFastmem", "disableScaledDithering",
"IsLibCryptProtected", "disableForceNTSCTimings",
"disableWidescreen",
"disablePGXP",
"disablePGXPCulling",
"disablePGXPTextureCorrection",
"disablePGXPColorCorrection",
"disablePGXPDepthBuffer",
"forcePGXPVertexCache",
"forcePGXPCPUMode",
"forceRecompilerMemoryExceptions",
"forceRecompilerICache",
"forceRecompilerLUTFastmem",
"isLibCryptProtected",
}}; }};
static constexpr const char* GAMEDB_YAML_FILENAME = "gamedb.yaml";
static constexpr const char* DISCDB_YAML_FILENAME = "discdb.yaml";
static bool s_loaded = false; static bool s_loaded = false;
static bool s_track_hashes_loaded = false; static bool s_track_hashes_loaded = false;
@ -84,6 +96,94 @@ static PreferUnorderedStringMap<u32> s_code_lookup;
static TrackHashesMap s_track_hashes_map; static TrackHashesMap s_track_hashes_map;
} // namespace GameDatabase } // namespace GameDatabase
// RapidYAML utility routines.
ALWAYS_INLINE std::string_view to_stringview(const c4::csubstr& s)
{
return std::string_view(s.data(), s.size());
}
ALWAYS_INLINE std::string_view to_stringview(const c4::substr& s)
{
return std::string_view(s.data(), s.size());
}
ALWAYS_INLINE c4::csubstr to_csubstr(const std::string_view& sv)
{
return c4::csubstr(sv.data(), sv.length());
}
static bool GetStringFromObject(const ryml::ConstNodeRef& object, std::string_view key, std::string* dest)
{
dest->clear();
const ryml::ConstNodeRef member = object.find_child(to_csubstr(key));
if (!member.valid())
return false;
const c4::csubstr val = member.val();
if (!val.empty())
dest->assign(val.data(), val.size());
return true;
}
template<typename T>
static bool GetUIntFromObject(const ryml::ConstNodeRef& object, std::string_view key, T* dest)
{
*dest = 0;
const ryml::ConstNodeRef member = object.find_child(to_csubstr(key));
if (!member.valid())
return false;
const c4::csubstr val = member.val();
if (val.empty())
{
Log_ErrorFmt("Unexpected empty value in {}", key);
return false;
}
const std::optional<T> opt_value = StringUtil::FromChars<T>(to_stringview(val));
if (!opt_value.has_value())
{
Log_ErrorFmt("Unexpected non-uint value in {}", key);
return false;
}
*dest = opt_value.value();
return true;
}
template<typename T>
static std::optional<T> GetOptionalTFromObject(const ryml::ConstNodeRef& object, std::string_view key)
{
std::optional<T> ret;
const ryml::ConstNodeRef member = object.find_child(to_csubstr(key));
if (member.valid())
{
const c4::csubstr val = member.val();
if (!val.empty())
{
ret = StringUtil::FromChars<T>(to_stringview(val));
if (!ret.has_value())
{
if constexpr (std::is_floating_point_v<T>)
Log_ErrorFmt("Unexpected non-float value in {}", key);
else if constexpr (std::is_integral_v<T>)
Log_ErrorFmt("Unexpected non-int value in {}", key);
}
}
else
{
Log_ErrorFmt("Unexpected empty value in {}", key);
}
}
return ret;
}
void GameDatabase::EnsureLoaded() void GameDatabase::EnsureLoaded()
{ {
if (s_loaded) if (s_loaded)
@ -98,11 +198,11 @@ void GameDatabase::EnsureLoaded()
s_entries = {}; s_entries = {};
s_code_lookup = {}; s_code_lookup = {};
LoadGameDBJson(); LoadGameDBYaml();
SaveToCache(); SaveToCache();
} }
Log_InfoPrintf("Database load took %.2f ms", timer.GetTimeMilliseconds()); Log_InfoFmt("Database load took {:.0f}ms", timer.GetTimeMilliseconds());
} }
void GameDatabase::Unload() void GameDatabase::Unload()
@ -198,30 +298,16 @@ GameDatabase::Entry* GameDatabase::GetMutableEntry(const std::string_view& seria
return nullptr; return nullptr;
} }
const char* GameDatabase::GetTraitName(Trait trait)
{
DebugAssert(trait < Trait::Count);
return s_trait_names[static_cast<u32>(trait)];
}
const char* GameDatabase::GetCompatibilityRatingName(CompatibilityRating rating) const char* GameDatabase::GetCompatibilityRatingName(CompatibilityRating rating)
{ {
static std::array<const char*, static_cast<int>(CompatibilityRating::Count)> names = { return s_compatibility_rating_names[static_cast<int>(rating)];
{"Unknown", "DoesntBoot", "CrashesInIntro", "CrashesInGame", "GraphicalAudioIssues", "NoIssues"}};
return names[static_cast<int>(rating)];
} }
const char* GameDatabase::GetCompatibilityRatingDisplayName(CompatibilityRating rating) const char* GameDatabase::GetCompatibilityRatingDisplayName(CompatibilityRating rating)
{ {
static constexpr std::array<const char*, static_cast<size_t>(CompatibilityRating::Count)> names = {
{TRANSLATE_NOOP("GameListCompatibilityRating", "Unknown"),
TRANSLATE_NOOP("GameListCompatibilityRating", "Doesn't Boot"),
TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In Intro"),
TRANSLATE_NOOP("GameListCompatibilityRating", "Crashes In-Game"),
TRANSLATE_NOOP("GameListCompatibilityRating", "Graphical/Audio Issues"),
TRANSLATE_NOOP("GameListCompatibilityRating", "No Issues")}};
return (rating >= CompatibilityRating::Unknown && rating < CompatibilityRating::Count) ? return (rating >= CompatibilityRating::Unknown && rating < CompatibilityRating::Count) ?
Host::TranslateToCString("GameListCompatibilityRating", names[static_cast<int>(rating)]) : Host::TranslateToCString("GameListCompatibilityRating",
s_compatibility_rating_display_names[static_cast<int>(rating)]) :
""; "";
} }
@ -580,7 +666,7 @@ bool GameDatabase::LoadFromCache()
return false; return false;
} }
const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.json", false).value_or(0); const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.yaml", false).value_or(0);
u32 signature, version, num_entries, num_codes; u32 signature, version, num_entries, num_codes;
u64 file_gamedb_ts; u64 file_gamedb_ts;
@ -674,7 +760,7 @@ bool GameDatabase::LoadFromCache()
bool GameDatabase::SaveToCache() bool GameDatabase::SaveToCache()
{ {
const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.json", false).value_or(0); const u64 gamedb_ts = Host::GetResourceFileTimestamp("gamedb.yaml", false).value_or(0);
std::unique_ptr<ByteStream> stream( std::unique_ptr<ByteStream> stream(
ByteStream::OpenFile(GetCacheFile().c_str(), BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_WRITE | ByteStream::OpenFile(GetCacheFile().c_str(), BYTESTREAM_OPEN_CREATE | BYTESTREAM_OPEN_WRITE |
@ -742,166 +828,81 @@ bool GameDatabase::SaveToCache()
return true; return true;
} }
////////////////////////////////////////////////////////////////////////// void GameDatabase::SetRymlCallbacks()
// JSON Parsing
//////////////////////////////////////////////////////////////////////////
static bool GetStringFromObject(const rapidjson::Value& object, const char* key, std::string* dest)
{ {
dest->clear(); ryml::Callbacks callbacks = ryml::get_callbacks();
auto member = object.FindMember(key); callbacks.m_error = [](const char* msg, size_t msg_len, ryml::Location loc, void* userdata) {
if (member == object.MemberEnd() || !member->value.IsString()) Log_ErrorFmt("Parse error at {}:{} (bufpos={}): {}", loc.line, loc.col, loc.offset, std::string_view(msg, msg_len));
return false; };
ryml::set_callbacks(callbacks);
dest->assign(member->value.GetString(), member->value.GetStringLength()); c4::set_error_callback(
return true; [](const char* msg, size_t msg_size) { Log_ErrorFmt("C4 error: {}", std::string_view(msg, msg_size)); });
} }
static bool GetBoolFromObject(const rapidjson::Value& object, const char* key, bool* dest) bool GameDatabase::LoadGameDBYaml()
{ {
*dest = false; Common::Timer timer;
auto member = object.FindMember(key); const std::optional<std::string> gamedb_data = Host::ReadResourceFileToString(GAMEDB_YAML_FILENAME, false);
if (member == object.MemberEnd() || !member->value.IsBool())
return false;
*dest = member->value.GetBool();
return true;
}
template<typename T>
static bool GetUIntFromObject(const rapidjson::Value& object, const char* key, T* dest)
{
*dest = 0;
auto member = object.FindMember(key);
if (member == object.MemberEnd() || !member->value.IsUint())
return false;
*dest = static_cast<T>(member->value.GetUint());
return true;
}
static bool GetArrayOfStringsFromObject(const rapidjson::Value& object, const char* key, std::vector<std::string>* dest)
{
dest->clear();
auto member = object.FindMember(key);
if (member == object.MemberEnd() || !member->value.IsArray())
return false;
for (const rapidjson::Value& str : member->value.GetArray())
{
if (str.IsString())
{
dest->emplace_back(str.GetString(), str.GetStringLength());
}
}
return true;
}
template<typename T>
static std::optional<T> GetOptionalIntFromObject(const rapidjson::Value& object, const char* key)
{
auto member = object.FindMember(key);
if (member == object.MemberEnd() || !member->value.IsInt())
return std::nullopt;
return static_cast<T>(member->value.GetInt());
}
template<typename T>
static std::optional<T> GetOptionalUIntFromObject(const rapidjson::Value& object, const char* key)
{
auto member = object.FindMember(key);
if (member == object.MemberEnd() || !member->value.IsUint())
return std::nullopt;
return static_cast<T>(member->value.GetUint());
}
static std::optional<float> GetOptionalFloatFromObject(const rapidjson::Value& object, const char* key)
{
auto member = object.FindMember(key);
if (member == object.MemberEnd() || !member->value.IsFloat())
return std::nullopt;
return member->value.GetFloat();
}
bool GameDatabase::LoadGameDBJson()
{
std::optional<std::string> gamedb_data(Host::ReadResourceFileToString("gamedb.json", false));
if (!gamedb_data.has_value()) if (!gamedb_data.has_value())
{ {
Log_ErrorPrintf("Failed to read game database"); Log_ErrorPrint("Failed to read game database");
return false; return false;
} }
// TODO: Parse in-place, avoid string allocations. SetRymlCallbacks();
std::unique_ptr<rapidjson::Document> json = std::make_unique<rapidjson::Document>();
json->Parse(gamedb_data->c_str(), gamedb_data->size());
if (json->HasParseError())
{
Log_ErrorPrintf("Failed to parse game database: %s at offset %zu",
rapidjson::GetParseError_En(json->GetParseError()), json->GetErrorOffset());
return false;
}
if (!json->IsArray()) const ryml::Tree tree = ryml::parse_in_arena(to_csubstr(GAMEDB_YAML_FILENAME), to_csubstr(gamedb_data.value()));
{ const ryml::ConstNodeRef root = tree.rootref();
Log_ErrorPrintf("Document is not an array"); s_entries.reserve(root.num_children());
return false;
}
const auto& jarray = json->GetArray(); for (const ryml::ConstNodeRef& current : root.children())
s_entries.reserve(jarray.Size());
for (const rapidjson::Value& current : json->GetArray())
{ {
// TODO: binary sort // TODO: binary sort
const u32 index = static_cast<u32>(s_entries.size()); const u32 index = static_cast<u32>(s_entries.size());
Entry& entry = s_entries.emplace_back(); Entry& entry = s_entries.emplace_back();
if (!ParseJsonEntry(&entry, current)) if (!ParseYamlEntry(&entry, current))
{ {
s_entries.pop_back(); s_entries.pop_back();
continue; continue;
} }
ParseJsonCodes(index, current); ParseYamlCodes(index, current, entry.serial);
} }
Log_InfoPrintf("Loaded %zu entries and %zu codes from database", s_entries.size(), s_code_lookup.size()); ryml::reset_callbacks();
return true;
Log_InfoFmt("Loaded {} entries and {} codes from database in {:.0f}ms.", s_entries.size(), s_code_lookup.size(),
timer.GetTimeMilliseconds());
return !s_entries.empty();
} }
bool GameDatabase::ParseJsonEntry(Entry* entry, const rapidjson::Value& value) bool GameDatabase::ParseYamlEntry(Entry* entry, const ryml::ConstNodeRef& value)
{ {
if (!value.IsObject()) entry->serial = to_stringview(value.key());
if (entry->serial.empty())
{ {
Log_WarningPrintf("entry is not an object"); Log_ErrorPrint("Missing serial for entry.");
return false; return false;
} }
if (!GetStringFromObject(value, "serial", &entry->serial) || !GetStringFromObject(value, "name", &entry->title) || GetStringFromObject(value, "name", &entry->title);
entry->serial.empty())
if (const ryml::ConstNodeRef metadata = value.find_child(to_csubstr("metadata")); metadata.valid())
{ {
Log_ErrorPrintf("Missing serial or title for entry"); GetStringFromObject(metadata, "genre", &entry->genre);
return false; GetStringFromObject(metadata, "developer", &entry->developer);
} GetStringFromObject(metadata, "publisher", &entry->publisher);
GetStringFromObject(value, "genre", &entry->genre); GetUIntFromObject(metadata, "minPlayers", &entry->min_players);
GetStringFromObject(value, "developer", &entry->developer); GetUIntFromObject(metadata, "maxPlayers", &entry->max_players);
GetStringFromObject(value, "publisher", &entry->publisher); GetUIntFromObject(metadata, "minBlocks", &entry->min_blocks);
GetUIntFromObject(metadata, "maxBlocks", &entry->max_blocks);
GetUIntFromObject(value, "minPlayers", &entry->min_players);
GetUIntFromObject(value, "maxPlayers", &entry->max_players);
GetUIntFromObject(value, "minBlocks", &entry->min_blocks);
GetUIntFromObject(value, "maxBlocks", &entry->max_blocks);
entry->release_date = 0; entry->release_date = 0;
{ {
std::string release_date; std::string release_date;
if (GetStringFromObject(value, "releaseDate", &release_date)) if (GetStringFromObject(metadata, "releaseDate", &release_date))
{ {
std::istringstream iss(release_date); std::istringstream iss(release_date);
struct tm parsed_time = {}; struct tm parsed_time = {};
@ -917,26 +918,27 @@ bool GameDatabase::ParseJsonEntry(Entry* entry, const rapidjson::Value& value)
} }
} }
} }
}
entry->supported_controllers = static_cast<u16>(~0u); entry->supported_controllers = static_cast<u16>(~0u);
const auto controllers = value.FindMember("controllers");
if (controllers != value.MemberEnd()) if (const ryml::ConstNodeRef controllers = value.find_child(to_csubstr("controllers"));
{ controllers.valid() && controllers.has_children())
if (controllers->value.IsArray())
{ {
bool first = true; bool first = true;
for (const rapidjson::Value& controller : controllers->value.GetArray()) for (const ryml::ConstNodeRef& controller : controllers.children())
{ {
if (!controller.IsString()) const std::string_view controller_str = to_stringview(controller.val());
if (controller_str.empty())
{ {
Log_WarningPrintf("controller is not a string"); Log_WarningFmt("controller is not a string in {}", entry->serial);
return false; return false;
} }
std::optional<ControllerType> ctype = Settings::ParseControllerTypeName(controller.GetString()); std::optional<ControllerType> ctype = Settings::ParseControllerTypeName(controller_str);
if (!ctype.has_value()) if (!ctype.has_value())
{ {
Log_WarningPrintf("Invalid controller type '%s'", controller.GetString()); Log_WarningFmt("Invalid controller type {} in {}", controller_str, entry->serial);
continue; continue;
} }
@ -949,122 +951,135 @@ bool GameDatabase::ParseJsonEntry(Entry* entry, const rapidjson::Value& value)
entry->supported_controllers |= (1u << static_cast<u16>(ctype.value())); entry->supported_controllers |= (1u << static_cast<u16>(ctype.value()));
} }
} }
if (const ryml::ConstNodeRef compatibility = value.find_child(to_csubstr("compatibility"));
compatibility.valid() && compatibility.has_children())
{
const ryml::ConstNodeRef rating = compatibility.find_child(to_csubstr("rating"));
if (rating.valid())
{
const std::string_view rating_str = to_stringview(rating.val());
const auto iter = std::find(s_compatibility_rating_names.begin(), s_compatibility_rating_names.end(), rating_str);
if (iter != s_compatibility_rating_names.end())
{
const size_t rating_idx = static_cast<size_t>(std::distance(s_compatibility_rating_names.begin(), iter));
DebugAssert(rating_idx < static_cast<size_t>(CompatibilityRating::Count));
entry->compatibility = static_cast<CompatibilityRating>(rating_idx);
}
else else
{ {
Log_WarningPrintf("controllers is not an array"); Log_WarningFmt("Unknown compatibility rating {} in {}", rating_str, entry->serial);
}
} }
} }
const auto compatibility = value.FindMember("compatibility"); if (const ryml::ConstNodeRef traits = value.find_child(to_csubstr("traits")); traits.valid() && traits.has_children())
if (compatibility != value.MemberEnd())
{ {
if (compatibility->value.IsObject()) for (const ryml::ConstNodeRef& trait : traits.children())
{ {
u32 rating; const std::string_view trait_str = to_stringview(trait.val());
if (GetUIntFromObject(compatibility->value, "rating", &rating) && if (trait_str.empty())
rating < static_cast<u32>(CompatibilityRating::Count))
{ {
entry->compatibility = static_cast<CompatibilityRating>(rating); Log_WarningFmt("Empty trait in {}", entry->serial);
continue;
} }
}
else const auto iter = std::find(s_trait_names.begin(), s_trait_names.end(), trait_str);
if (iter == s_trait_names.end())
{ {
Log_WarningPrintf("compatibility is not an object"); Log_WarningFmt("Unknown trait {} in {}", trait_str, entry->serial);
continue;
}
const size_t trait_idx = static_cast<size_t>(std::distance(s_trait_names.begin(), iter));
DebugAssert(trait_idx < static_cast<size_t>(Trait::Count));
entry->traits[trait_idx] = true;
} }
} }
const auto traits = value.FindMember("traits"); if (const ryml::ConstNodeRef settings = value.find_child(to_csubstr("settings"));
if (traits != value.MemberEnd()) settings.valid() && settings.has_children())
{ {
if (traits->value.IsObject()) entry->display_active_start_offset = GetOptionalTFromObject<s16>(settings, "displayActiveStartOffset");
{ entry->display_active_end_offset = GetOptionalTFromObject<s16>(settings, "displayActiveEndOffset");
const auto& traitsobj = traits->value; entry->display_line_start_offset = GetOptionalTFromObject<s8>(settings, "displayLineStartOffset");
for (u32 trait = 0; trait < static_cast<u32>(Trait::Count); trait++) entry->display_line_end_offset = GetOptionalTFromObject<s8>(settings, "displayLineEndOffset");
{ entry->dma_max_slice_ticks = GetOptionalTFromObject<u32>(settings, "dmaMaxSliceTicks");
bool bvalue; entry->dma_halt_ticks = GetOptionalTFromObject<u32>(settings, "dmaHaltTicks");
if (GetBoolFromObject(traitsobj, s_trait_names[trait], &bvalue) && bvalue) entry->gpu_fifo_size = GetOptionalTFromObject<u32>(settings, "gpuFIFOSize");
entry->traits[trait] = bvalue; entry->gpu_max_run_ahead = GetOptionalTFromObject<u32>(settings, "gpuMaxRunAhead");
entry->gpu_pgxp_tolerance = GetOptionalTFromObject<float>(settings, "gpuPGXPTolerance");
entry->gpu_pgxp_depth_threshold = GetOptionalTFromObject<float>(settings, "gpuPGXPDepthThreshold");
} }
entry->display_active_start_offset = GetOptionalIntFromObject<s16>(traitsobj, "DisplayActiveStartOffset"); if (const ryml::ConstNodeRef disc_set = value.find_child("discSet"); disc_set.valid() && disc_set.has_children())
entry->display_active_end_offset = GetOptionalIntFromObject<s16>(traitsobj, "DisplayActiveEndOffset");
entry->display_line_start_offset = GetOptionalIntFromObject<s8>(traitsobj, "DisplayLineStartOffset");
entry->display_line_end_offset = GetOptionalIntFromObject<s8>(traitsobj, "DisplayLineEndOffset");
entry->dma_max_slice_ticks = GetOptionalUIntFromObject<u32>(traitsobj, "DMAMaxSliceTicks");
entry->dma_halt_ticks = GetOptionalUIntFromObject<u32>(traitsobj, "DMAHaltTicks");
entry->gpu_fifo_size = GetOptionalUIntFromObject<u32>(traitsobj, "GPUFIFOSize");
entry->gpu_max_run_ahead = GetOptionalUIntFromObject<u32>(traitsobj, "GPUMaxRunAhead");
entry->gpu_pgxp_tolerance = GetOptionalFloatFromObject(traitsobj, "GPUPGXPTolerance");
entry->gpu_pgxp_depth_threshold = GetOptionalFloatFromObject(traitsobj, "GPUPGXPDepthThreshold");
}
else
{ {
Log_WarningPrintf("traits is not an object"); GetStringFromObject(disc_set, "name", &entry->disc_set_name);
}
if (const ryml::ConstNodeRef set_serials = disc_set.find_child("serials");
set_serials.valid() && set_serials.has_children())
{
entry->disc_set_serials.reserve(set_serials.num_children());
for (const ryml::ConstNodeRef& serial : set_serials)
{
const std::string_view serial_str = to_stringview(serial.val());
if (serial_str.empty())
{
Log_WarningFmt("Empty disc set serial in {}", entry->serial);
continue;
} }
GetStringFromObject(value, "discSetName", &entry->disc_set_name); if (std::find(entry->disc_set_serials.begin(), entry->disc_set_serials.end(), serial_str) !=
const auto disc_set_serials = value.FindMember("discSetSerials"); entry->disc_set_serials.end())
if (disc_set_serials != value.MemberEnd())
{ {
if (disc_set_serials->value.IsArray()) Log_WarningFmt("Duplicate serial {} in disc set serials for {}", serial_str, entry->serial);
{ continue;
const auto disc_set_serials_array = disc_set_serials->value.GetArray();
entry->disc_set_serials.reserve(disc_set_serials_array.Size());
for (const rapidjson::Value& serial : disc_set_serials_array)
{
if (serial.IsString())
{
entry->disc_set_serials.emplace_back(serial.GetString(), serial.GetStringLength());
} }
else
{ entry->disc_set_serials.emplace_back(serial_str);
Log_WarningPrintf("discSetSerial is not a string");
} }
} }
} }
else
{
Log_WarningPrintf("discSetSerials is not an array");
}
}
return true; return true;
} }
bool GameDatabase::ParseJsonCodes(u32 index, const rapidjson::Value& value) bool GameDatabase::ParseYamlCodes(u32 index, const ryml::ConstNodeRef& value, std::string_view serial)
{ {
auto member = value.FindMember("codes"); const ryml::ConstNodeRef& codes = value.find_child(to_csubstr("codes"));
if (member == value.MemberEnd()) if (!codes.valid() || !codes.has_children())
{ {
Log_WarningPrintf("codes member is missing"); // use serial instead
auto iter = s_code_lookup.find(serial);
if (iter != s_code_lookup.end())
{
Log_WarningFmt("Duplicate code '{}'", serial);
return false; return false;
} }
if (!member->value.IsArray()) s_code_lookup.emplace(serial, index);
{ return true;
Log_WarningPrintf("codes is not an array");
return false;
} }
u32 added = 0; u32 added = 0;
for (const rapidjson::Value& current_code : member->value.GetArray()) for (const ryml::ConstNodeRef& current_code : codes)
{ {
if (!current_code.IsString()) const std::string_view current_code_str = to_stringview(current_code.val());
if (current_code_str.empty())
{ {
Log_WarningPrintf("code is not a string"); Log_WarningFmt("code is not a string in {}", serial);
continue; continue;
} }
const std::string_view code(current_code.GetString(), current_code.GetStringLength()); auto iter = s_code_lookup.find(current_code_str);
auto iter = s_code_lookup.find(code);
if (iter != s_code_lookup.end()) if (iter != s_code_lookup.end())
{ {
Log_WarningPrintf("Duplicate code '%.*s'", static_cast<int>(code.size()), code.data()); Log_WarningFmt("Duplicate code '{}' in {}", current_code_str, serial);
continue; continue;
} }
s_code_lookup.emplace(code, index); s_code_lookup.emplace(current_code_str, index);
added++; added++;
} }
@ -1082,105 +1097,84 @@ void GameDatabase::EnsureTrackHashesMapLoaded()
bool GameDatabase::LoadTrackHashes() bool GameDatabase::LoadTrackHashes()
{ {
std::optional<std::string> gamedb_data(Host::ReadResourceFileToString("gamedb.json", false)); Common::Timer load_timer;
std::optional<std::string> gamedb_data(Host::ReadResourceFileToString(DISCDB_YAML_FILENAME, false));
if (!gamedb_data.has_value()) if (!gamedb_data.has_value())
{ {
Log_ErrorPrintf("Failed to read game database"); Log_ErrorPrint("Failed to read game database");
return false; return false;
} }
SetRymlCallbacks();
// TODO: Parse in-place, avoid string allocations. // TODO: Parse in-place, avoid string allocations.
std::unique_ptr<rapidjson::Document> json = std::make_unique<rapidjson::Document>(); const ryml::Tree tree = ryml::parse_in_arena(to_csubstr(DISCDB_YAML_FILENAME), to_csubstr(gamedb_data.value()));
json->Parse(gamedb_data->c_str(), gamedb_data->size()); const ryml::ConstNodeRef root = tree.rootref();
if (json->HasParseError())
{
Log_ErrorPrintf("Failed to parse game database: %s at offset %zu",
rapidjson::GetParseError_En(json->GetParseError()), json->GetErrorOffset());
return false;
}
if (!json->IsArray())
{
Log_ErrorPrintf("Document is not an array");
return false;
}
s_track_hashes_map = {}; s_track_hashes_map = {};
for (const rapidjson::Value& current : json->GetArray()) size_t serials = 0;
for (const ryml::ConstNodeRef& current : root.children())
{ {
if (!current.IsObject()) const std::string_view serial = to_stringview(current.key());
if (serial.empty() || !current.has_children())
{ {
Log_WarningPrintf("entry is not an object"); Log_WarningPrint("entry is not an object");
continue; continue;
} }
std::vector<std::string> codes; const ryml::ConstNodeRef track_data = current.find_child(to_csubstr("trackData"));
if (!GetArrayOfStringsFromObject(current, "codes", &codes)) if (!track_data.valid() || !track_data.has_children())
{ {
Log_WarningPrintf("codes member is missing"); Log_WarningFmt("trackData is missing in {}", serial);
continue; continue;
} }
auto track_data = current.FindMember("track_data"); u32 revision = 0;
if (track_data == current.MemberEnd()) for (const ryml::ConstNodeRef& track_revisions : track_data.children())
{ {
Log_WarningPrintf("track_data member is missing"); const ryml::ConstNodeRef tracks = track_revisions.find_child(to_csubstr("tracks"));
if (!tracks.valid() || !tracks.has_children())
{
Log_WarningFmt("tracks member is missing in {}", serial);
continue; continue;
} }
if (!track_data->value.IsArray()) std::string revision_string;
GetStringFromObject(track_revisions, "version", &revision_string);
for (const ryml::ConstNodeRef& track : tracks)
{ {
Log_WarningPrintf("track_data is not an array"); const ryml::ConstNodeRef md5 = track.find_child("md5");
std::string_view md5_str;
if (!md5.valid() || (md5_str = to_stringview(md5.val())).empty())
{
Log_WarningFmt("md5 is missing in track in {}", serial);
continue; continue;
} }
uint32_t revision = 0; const std::optional<CDImageHasher::Hash> md5o = CDImageHasher::HashFromString(md5_str);
for (const rapidjson::Value& track_revisions : track_data->value.GetArray()) if (md5o.has_value())
{ {
if (!track_revisions.IsObject()) s_track_hashes_map.emplace(std::piecewise_construct, std::forward_as_tuple(md5o.value()),
{ std::forward_as_tuple(std::string(serial), revision_string, revision));
Log_WarningPrintf("track_data is not an array of object");
continue;
} }
else
auto tracks = track_revisions.FindMember("tracks");
if (tracks == track_revisions.MemberEnd())
{ {
Log_WarningPrintf("tracks member is missing"); Log_WarningFmt("invalid md5 in {}", serial);
continue;
}
if (!tracks->value.IsArray())
{
Log_WarningPrintf("tracks is not an array");
continue;
}
std::string revisionString;
GetStringFromObject(track_revisions, "version", &revisionString);
for (const rapidjson::Value& track : tracks->value.GetArray())
{
auto md5_field = track.FindMember("md5");
if (md5_field == track.MemberEnd() || !md5_field->value.IsString())
{
continue;
}
auto md5 = CDImageHasher::HashFromString(
std::string_view(md5_field->value.GetString(), md5_field->value.GetStringLength()));
if (md5)
{
s_track_hashes_map.emplace(std::piecewise_construct, std::forward_as_tuple(md5.value()),
std::forward_as_tuple(codes, revisionString, revision));
} }
} }
revision++; revision++;
} }
serials++;
} }
return true; ryml::reset_callbacks();
Log_InfoFmt("Loaded {} track hashes from {} serials in {:.0f}ms.", s_track_hashes_map.size(), serials,
load_timer.GetTimeMilliseconds());
return !s_track_hashes_map.empty();
} }
const GameDatabase::TrackHashesMap& GameDatabase::GetTrackHashesMap() const GameDatabase::TrackHashesMap& GameDatabase::GetTrackHashesMap()

View File

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com> // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#pragma once #pragma once
@ -98,16 +98,14 @@ const Entry* GetEntryForSerial(const std::string_view& serial);
std::string GetSerialForDisc(CDImage* image); std::string GetSerialForDisc(CDImage* image);
std::string GetSerialForPath(const char* path); std::string GetSerialForPath(const char* path);
const char* GetTraitName(Trait trait);
const char* GetCompatibilityRatingName(CompatibilityRating rating); const char* GetCompatibilityRatingName(CompatibilityRating rating);
const char* GetCompatibilityRatingDisplayName(CompatibilityRating rating); const char* GetCompatibilityRatingDisplayName(CompatibilityRating rating);
/// Map of track hashes for image verification /// Map of track hashes for image verification
struct TrackData struct TrackData
{ {
TrackData(std::vector<std::string> codes, std::string revisionString, uint32_t revision) TrackData(std::string serial_, std::string revision_str_, uint32_t revision_)
: codes(std::move(codes)), revisionString(revisionString), revision(revision) : serial(std::move(serial_)), revision_str(std::move(revision_str_)), revision(revision_)
{ {
} }
@ -115,12 +113,12 @@ struct TrackData
{ {
// 'revisionString' is deliberately ignored in comparisons as it's redundant with comparing 'revision'! Do not // 'revisionString' is deliberately ignored in comparisons as it's redundant with comparing 'revision'! Do not
// change! // change!
return left.codes == right.codes && left.revision == right.revision; return left.serial == right.serial && left.revision == right.revision;
} }
std::vector<std::string> codes; std::string serial;
std::string revisionString; std::string revision_str;
uint32_t revision; u32 revision;
}; };
using TrackHashesMap = std::multimap<CDImageHasher::Hash, TrackData>; using TrackHashesMap = std::multimap<CDImageHasher::Hash, TrackData>;

View File

@ -1,4 +1,4 @@
// SPDX-FileCopyrightText: 2019-2022 Connor McLaughlin <stenzek@gmail.com> // SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
// SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0) // SPDX-License-Identifier: (GPL-3.0 OR CC-BY-NC-ND-4.0)
#include "gamesummarywidget.h" #include "gamesummarywidget.h"
@ -13,7 +13,6 @@
#include "fmt/format.h" #include "fmt/format.h"
#include <QtConcurrent/QtConcurrent>
#include <QtCore/QFuture> #include <QtCore/QFuture>
#include <QtWidgets/QMessageBox> #include <QtWidgets/QMessageBox>
@ -219,13 +218,6 @@ void GameSummaryWidget::onComputeHashClicked()
return; return;
} }
#ifndef _DEBUGFAST
// Kick off hash preparation asynchronously, as building the map of results may take a while
// This breaks for DebugFast because of the iterator debug level mismatch.
QFuture<const GameDatabase::TrackHashesMap*> result =
QtConcurrent::run([]() { return &GameDatabase::GetTrackHashesMap(); });
#endif
QtModalProgressCallback progress_callback(this); QtModalProgressCallback progress_callback(this);
progress_callback.SetProgressRange(image->GetTrackCount()); progress_callback.SetProgressRange(image->GetTrackCount());
@ -259,6 +251,7 @@ void GameSummaryWidget::onComputeHashClicked()
if (calculate_hash_success) if (calculate_hash_success)
{ {
std::string found_revision; std::string found_revision;
std::string found_serial;
m_redump_search_keyword = CDImageHasher::HashToString(track_hashes.front()); m_redump_search_keyword = CDImageHasher::HashToString(track_hashes.front());
progress_callback.SetStatusText("Verifying hashes..."); progress_callback.SetStatusText("Verifying hashes...");
@ -270,11 +263,7 @@ void GameSummaryWidget::onComputeHashClicked()
// 2. For each data track match, try to match all audio tracks // 2. For each data track match, try to match all audio tracks
// If all match, assume this revision. Else, try other revisions, // If all match, assume this revision. Else, try other revisions,
// and accept the one with the most matches. // and accept the one with the most matches.
#ifndef _DEBUGFAST
const GameDatabase::TrackHashesMap& hashes_map = *result.result();
#else
const GameDatabase::TrackHashesMap& hashes_map = GameDatabase::GetTrackHashesMap(); const GameDatabase::TrackHashesMap& hashes_map = GameDatabase::GetTrackHashesMap();
#endif
auto data_track_matches = hashes_map.equal_range(track_hashes[0]); auto data_track_matches = hashes_map.equal_range(track_hashes[0]);
if (data_track_matches.first != data_track_matches.second) if (data_track_matches.first != data_track_matches.second)
@ -317,13 +306,30 @@ void GameSummaryWidget::onComputeHashClicked()
} }
} }
found_revision = best_data_match->second.revisionString; found_revision = best_data_match->second.revision_str;
found_serial = best_data_match->second.serial;
} }
QString text;
if (!found_revision.empty()) if (!found_revision.empty())
text = tr("Revision: %1").arg(found_revision.empty() ? tr("N/A") : QString::fromStdString(found_revision));
if (found_serial != m_ui.serial->text().toStdString())
{ {
m_ui.revision->setText( text =
tr("Revision: %1").arg(found_revision.empty() ? tr("N/A") : QString::fromStdString(found_revision))); tr("Serial Mismatch: %1 vs %2%3").arg(QString::fromStdString(found_serial)).arg(m_ui.serial->text()).arg(text);
}
if (!text.isEmpty())
{
if (m_ui.verifySpacer)
{
m_ui.verifyLayout->removeItem(m_ui.verifySpacer);
delete m_ui.verifySpacer;
m_ui.verifySpacer = nullptr;
}
m_ui.revision->setText(text);
m_ui.revision->setVisible(true); m_ui.revision->setVisible(true);
} }
} }

View File

@ -189,9 +189,9 @@
<widget class="QComboBox" name="inputProfile"/> <widget class="QComboBox" name="inputProfile"/>
</item> </item>
<item row="16" column="1"> <item row="16" column="1">
<layout class="QHBoxLayout" name="horizontalLayout_2" stretch="1,0,0"> <layout class="QHBoxLayout" name="verifyLayout" stretch="0,1,0">
<item> <item>
<spacer name="horizontalSpacer"> <spacer name="verifySpacer">
<property name="orientation"> <property name="orientation">
<enum>Qt::Horizontal</enum> <enum>Qt::Horizontal</enum>
</property> </property>
@ -207,7 +207,7 @@
<widget class="QLineEdit" name="revision"> <widget class="QLineEdit" name="revision">
<property name="minimumSize"> <property name="minimumSize">
<size> <size>
<width>100</width> <width>300</width>
<height>0</height> <height>0</height>
</size> </size>
</property> </property>