ShuriZma
/
flycast
mirror of https://github.com/flyinghead/flycast.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

boxart: extract 0GDTEX.PVR. Offline scraping 

extract iso9660 file system utility out of reios
.pvr limited parser
This commit is contained in:
Flyinghead 2022-07-21 10:43:45 +02:00
parent 807b8dea91
commit ad8b41adb0
15 changed files with 627 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
CMakeLists.txt
View File

@ -771,7 +771,9 @@ target_sources(${PROJECT_NAME} PRIVATE
core/imgread/cue.cpp
core/imgread/gdi.cpp
core/imgread/ImgReader.cpp
core/imgread/ioctl.cpp)
core/imgread/ioctl.cpp
core/imgread/isofs.cpp
core/imgread/isofs.h)
if(NOT LIBRETRO)
target_sources(${PROJECT_NAME} PRIVATE

112
core/imgread/isofs.cpp Normal file
View File

@ -0,0 +1,112 @@
/*
Copyright 2022 flyinghead
This file is part of Flycast.
Flycast is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Flycast is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Flycast. If not, see <https://www.gnu.org/licenses/>.
*/
#include "isofs.h"
#include "reios/iso9660.h"
static u32 decode_iso733(iso733_t v)
{
return ((v >> 56) & 0x000000FF)
| ((v >> 40) & 0x0000FF00)
| ((v >> 24) & 0x00FF0000)
| ((v >> 8) & 0xFF000000);
}
IsoFs::IsoFs(Disc *disc) : disc(disc)
{
baseFad = disc->GetBaseFAD();
}
IsoFs::Directory *IsoFs::getRoot()
{
u8 temp[2048];
disc->ReadSectors(baseFad + 16, 1, temp, 2048);
// Primary Volume Descriptor
const iso9660_pvd_t *pvd = (const iso9660_pvd_t *)temp;
Directory *root = new Directory(this);
if (pvd->type == 1 && !memcmp(pvd->id, ISO_STANDARD_ID, strlen(ISO_STANDARD_ID)) && pvd->version == 1)
{
u32 lba = decode_iso733(pvd->root_directory_record.extent);
u32 len = decode_iso733(pvd->root_directory_record.size);
len = ((len + 2047) / 2048) * 2048;
root->data.resize(len);
DEBUG_LOG(GDROM, "iso9660 root directory FAD: %d, len: %d", 150 + lba, len);
disc->ReadSectors(150 + lba, len / 2048, root->data.data(), 2048);
}
else {
WARN_LOG(GDROM, "iso9660 PVD NOT found");
root->data.resize(1);
root->data[0] = 0;
}
return root;
}
IsoFs::Entry *IsoFs::Directory::getEntry(const std::string& name)
{
std::string isoname = name + ';';
for (u32 i = 0; i < data.size(); )
{
const iso9660_dir_t *dir = (const iso9660_dir_t *)&data[i];
if (dir->length == 0)
break;
if ((u8)dir->filename.str[0] > isoname.size()
&& memcmp(dir->filename.str + 1, isoname.c_str(), isoname.size()) == 0)
{
DEBUG_LOG(GDROM, "Found %s at offset %X", name.c_str(), i);
u32 startFad = decode_iso733(dir->extent) + 150;
u32 len = decode_iso733(dir->size);
if ((dir->file_flags & ISO_DIRECTORY) == 0)
{
File *file = new File(fs);
file->startFad = startFad;
file->len = len;
return file;
}
else
{
Directory *directory = new Directory(fs);
directory->data.resize(len);
fs->disc->ReadSectors(startFad, len / 2048, directory->data.data(), 2048);
return directory;
}
}
i += dir->length;
}
return nullptr;
}
u32 IsoFs::File::read(u8 *buf, u32 size, u32 offset) const
{
size = std::min(size, len - offset);
u32 sectors = size / 2048;
fs->disc->ReadSectors(startFad + offset / 2048, sectors, buf, 2048);
size -= sectors * 2048;
if (size > 0)
{
u8 temp[2048];
fs->disc->ReadSectors(startFad + offset / 2048 + sectors, 1, temp, 2048);
memcpy(buf + sectors * 2048, temp, size);
}
return sectors * 2048 + size;
}

76
core/imgread/isofs.h Normal file
View File

@ -0,0 +1,76 @@
/*
Copyright 2022 flyinghead
This file is part of Flycast.
Flycast is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Flycast is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Flycast. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include "common.h"
class IsoFs
{
public:
class Entry
{
public:
virtual bool isDirectory() const = 0;
virtual ~Entry() = default;
protected:
Entry(IsoFs *fs) : fs(fs) {}
IsoFs *fs;
};
class Directory final : public Entry
{
public:
bool isDirectory() const override { return true; }
Entry *getEntry(const std::string& name);
private:
Directory(IsoFs *fs) : Entry(fs) {}
std::vector<u8> data;
friend class IsoFs;
};
class File final : public Entry
{
public:
bool isDirectory() const override { return false; }
u32 getSize() const { return len; }
u32 read(u8 *buf, u32 size, u32 offset = 0) const;
private:
File(IsoFs *fs) : Entry(fs) {}
u32 startFad = 0;
u32 len = 0;
friend class IsoFs;
};
IsoFs(Disc *disc);
Directory *getRoot();
private:
Disc *disc;
u32 baseFad;
};

32
core/reios/descrambl.cpp
View File

@ -7,25 +7,24 @@
*/
#include "descrambl.h"
#include "imgread/common.h"
#include <algorithm>
#define MAXCHUNK (2048*1024)
static unsigned int seed;
static u32 seed;
static void my_srand(unsigned int n)
static void my_srand(u32 n)
{
seed = n & 0xffff;
}
static unsigned int my_rand()
static u32 my_rand()
{
seed = (seed * 2109 + 9273) & 0x7fff;
return (seed + 0xc000) & 0xffff;
}
static void load_chunk(u8* &src, unsigned char *ptr, unsigned long sz)
static void load_chunk(const u8* &src, u8 *ptr, u32 sz)
{
verify(sz <= MAXCHUNK);
@ -53,32 +52,23 @@ static void load_chunk(u8* &src, unsigned char *ptr, unsigned long sz)
}
}
static void descrambl_buffer(u8* src, unsigned char *dst, unsigned long filesz)
void descrambl_buffer(const u8 *src, u8 *dst, u32 size)
{
unsigned long chunksz;
u32 chunksz;
my_srand(filesz);
my_srand(size);
/* Descramble 2 meg blocks for as long as possible, then
gradually reduce the window down to 32 bytes (1 slice) */
for (chunksz = MAXCHUNK; chunksz >= 32; chunksz >>= 1)
while (filesz >= chunksz)
while (size >= chunksz)
{
load_chunk(src, dst, chunksz);
filesz -= chunksz;
size -= chunksz;
dst += chunksz;
}
/* Load final incomplete slice */
if (filesz)
memcpy(dst, src, filesz);
}
void descrambl_file(u32 FAD, u32 file_size, u8* dst) {
u8* temp_file = new u8[file_size + 2048];
libGDR_ReadSector(temp_file, FAD, (file_size+2047) / 2048, 2048);
descrambl_buffer(temp_file, dst, file_size);
delete[] temp_file;
if (size)
memcpy(dst, src, size);
}

2
core/reios/descrambl.h
View File

@ -1,4 +1,4 @@
#pragma once
#include "types.h"
void descrambl_file(u32 FAD, u32 file_size, u8* dst);
void descrambl_buffer(const u8 *src, u8 *dst, u32 size);

152
core/reios/reios.cpp
View File

@ -21,12 +21,12 @@
#include "hw/holly/sb_mem.h"
#include "hw/holly/sb.h"
#include "hw/naomi/naomi_cart.h"
#include "iso9660.h"
#include "font.h"
#include "hw/aica/aica.h"
#include "hw/aica/aica_mem.h"
#include "hw/pvr/pvr_regs.h"
#include "imgread/common.h"
#include "imgread/isofs.h"
#include "oslib/oslib.h"
#include <map>
@ -50,120 +50,82 @@ static MemChip *flashrom;
static u32 base_fad = 45150;
static bool descrambl = false;
static u32 bootSectors;
extern Disc *disc;
static void reios_pre_init()
{
if (libGDR_GetDiscType() == GdRom) {
base_fad = 45150;
descrambl = false;
} else {
u8 ses[6];
libGDR_GetSessionInfo(ses, 0);
libGDR_GetSessionInfo(ses, ses[2]);
base_fad = (ses[3] << 16) | (ses[4] << 8) | (ses[5] << 0);
descrambl = true;
if (disc != nullptr)
{
base_fad = disc->GetBaseFAD();
descrambl = disc->type != GdRom;
}
}
static u32 decode_iso733(iso733_t v)
{
return ((v >> 56) & 0x000000FF)
| ((v >> 40) & 0x0000FF00)
| ((v >> 24) & 0x00FF0000)
| ((v >> 8) & 0xFF000000);
}
static bool reios_locate_bootfile(const char* bootfile)
{
reios_pre_init();
if (ip_meta.wince == '1' && descrambl)
{
ERROR_LOG(REIOS, "Unsupported CDI: wince == '1'");
return false;
}
// Load IP.BIN bootstrap
libGDR_ReadSector(GetMemPtr(0x8c008000, 0), base_fad, 16, 2048);
u32 data_len = 2048 * 1024;
u8* temp = new u8[data_len];
libGDR_ReadSector(temp, base_fad + 16, 1, 2048);
iso9660_pvd_t *pvd = (iso9660_pvd_t *)temp;
if (pvd->type == 1 && !memcmp(pvd->id, ISO_STANDARD_ID, strlen(ISO_STANDARD_ID)) && pvd->version == 1)
IsoFs isofs(disc);
std::unique_ptr<IsoFs::Directory> root(isofs.getRoot());
if (root == nullptr)
{
INFO_LOG(REIOS, "iso9660 PVD found");
u32 lba = decode_iso733(pvd->root_directory_record.extent);
u32 len = decode_iso733(pvd->root_directory_record.size);
data_len = ((len + 2047) / 2048) * 2048;
INFO_LOG(REIOS, "iso9660 root_directory, FAD: %d, len: %d", 150 + lba, data_len);
libGDR_ReadSector(temp, 150 + lba, data_len / 2048, 2048);
ERROR_LOG(REIOS, "ISO file system root not found");
return false;
}
else {
libGDR_ReadSector(temp, base_fad + 16, data_len / 2048, 2048);
}
int bootfile_len = strlen(bootfile);
while (bootfile_len > 0 && isspace(bootfile[bootfile_len - 1]))
bootfile_len--;
for (u32 i = 0; i < data_len; )
std::unique_ptr<IsoFs::Entry> bootEntry(root->getEntry(trim_trailing_ws(bootfile)));
if (bootEntry == nullptr || bootEntry->isDirectory())
{
iso9660_dir_t *dir = (iso9660_dir_t *)&temp[i];
if (dir->length == 0)
break;
ERROR_LOG(REIOS, "Boot file '%s' not found", bootfile);
return false;
}
IsoFs::File *bootFile = (IsoFs::File *)bootEntry.get();
if ((dir->file_flags & ISO_DIRECTORY) == 0 && memcmp(dir->filename.str + 1, bootfile, bootfile_len) == 0)
{
INFO_LOG(REIOS, "Found %.*s at offset %X", bootfile_len, bootfile, i);
u32 offset = 0;
u32 size = bootFile->getSize();
if (ip_meta.wince == '1')
{
bootFile->read(GetMemPtr(0x8ce01000, 2048), 2048);
offset = 2048;
size -= offset;
}
bootSectors = size / 2048;
u32 lba = decode_iso733(dir->extent) + 150;
u32 len = decode_iso733(dir->size);
if (ip_meta.wince == '1')
{
if (descrambl)
{
WARN_LOG(REIOS, "Unsupported CDI: wince == '1'");
delete[] temp;
return false;
}
libGDR_ReadSector(GetMemPtr(0x8ce01000, 0), lba, 1, 2048);
lba++;
len -= 2048;
}
INFO_LOG(REIOS, "file LBA: %d", lba);
INFO_LOG(REIOS, "file LEN: %d", len);
bootSectors = (len + 2047) / 2048;
if (descrambl)
descrambl_file(lba, len, GetMemPtr(0x8c010000, 0));
else
libGDR_ReadSector(GetMemPtr(0x8c010000, 0), lba, bootSectors, 2048);
delete[] temp;
u8 data[24] = {0};
// system id
for (u32 j = 0; j < 8; j++)
data[j] = _vmem_ReadMem8(0x0021a056 + j);
// system properties
for (u32 j = 0; j < 5; j++)
data[8 + j] = _vmem_ReadMem8(0x0021a000 + j);
// system settings
flash_syscfg_block syscfg{};
verify(static_cast<DCFlashChip*>(flashrom)->ReadBlock(FLASH_PT_USER, FLASH_USER_SYSCFG, &syscfg));
memcpy(&data[16], &syscfg.time_lo, 8);
memcpy(GetMemPtr(0x8c000068, sizeof(data)), data, sizeof(data));
return true;
}
i += dir->length;
if (descrambl)
{
std::vector<u8> buf(size);
bootFile->read(buf.data(), size, offset);
descrambl_buffer(buf.data(), GetMemPtr(0x8c010000, size), size);
}
else
{
bootFile->read(GetMemPtr(0x8c010000, size), size, offset);
}
delete[] temp;
return false;
u8 data[24] = {0};
// system id
for (u32 j = 0; j < 8; j++)
data[j] = _vmem_ReadMem8(0x0021a056 + j);
// system properties
for (u32 j = 0; j < 5; j++)
data[8 + j] = _vmem_ReadMem8(0x0021a000 + j);
// system settings
flash_syscfg_block syscfg{};
verify(static_cast<DCFlashChip*>(flashrom)->ReadBlock(FLASH_PT_USER, FLASH_USER_SYSCFG, &syscfg));
memcpy(&data[16], &syscfg.time_lo, 8);
memcpy(GetMemPtr(0x8c000068, sizeof(data)), data, sizeof(data));
return true;
}
ip_meta_t ip_meta;

7
core/rend/TexCache.cpp
View File

@ -13,7 +13,7 @@
#include <omp.h>
#endif
u8* vq_codebook;
const u8 *vq_codebook;
u32 palette_index;
bool KillTex=false;
u32 palette16_ram[1024];
@ -22,7 +22,6 @@ u32 pal_hash_256[4];
u32 pal_hash_16[64];
bool palette_updated;
extern bool pal_needs_update;
float fb_scale_x, fb_scale_y;
// Rough approximation of LoD bias from D adjust param, only used to increase LoD
const std::array<f32, 16> D_Adjust_LoD_Bias = {
@ -339,7 +338,7 @@ namespace directx {
#undef TEX_CONV_TABLE
static const PvrTexInfo *pvrTexInfo = opengl::pvrTexInfo;
static const u32 VQMipPoint[11] =
extern const u32 VQMipPoint[11] =
{
0x00000,//1
0x00001,//2
@ -353,7 +352,7 @@ static const u32 VQMipPoint[11] =
0x05556,//512
0x15556//1024
};
static const u32 OtherMipPoint[11] =
extern const u32 OtherMipPoint[11] =
{
0x00003,//1
0x00004,//2

49
core/rend/TexCache.h
View File

@ -9,7 +9,7 @@
#include <memory>
#include <unordered_map>
extern u8* vq_codebook;
extern const u8 *vq_codebook;
extern u32 palette_index;
extern u32 palette16_ram[1024];
extern u32 palette32_ram[1024];
@ -227,9 +227,9 @@ struct ConvertPlanar
using unpacked_type = typename Unpacker::unpacked_type;
static constexpr u32 xpp = 4;
static constexpr u32 ypp = 1;
static void Convert(PixelBuffer<unpacked_type> *pb, u8 *data)
static void Convert(PixelBuffer<unpacked_type> *pb, const u8 *data)
{
u16 *p_in = (u16 *)data;
const u16 *p_in = (const u16 *)data;
pb->prel(0, Unpacker::unpack(p_in[0]));
pb->prel(1, Unpacker::unpack(p_in[1]));
pb->prel(2, Unpacker::unpack(p_in[2]));
@ -243,10 +243,10 @@ struct ConvertPlanarYUV
using unpacked_type = u32;
static constexpr u32 xpp = 4;
static constexpr u32 ypp = 1;
static void Convert(PixelBuffer<u32> *pb, u8 *data)
static void Convert(PixelBuffer<u32> *pb, const u8 *data)
{
//convert 4x1 4444 to 4x1 8888
u32 *p_in = (u32 *)data;
const u32 *p_in = (const u32 *)data;
s32 Y0 = (p_in[0] >> 8) & 255; //
@ -280,9 +280,9 @@ struct ConvertTwiddle
using unpacked_type = typename Unpacker::unpacked_type;
static constexpr u32 xpp = 2;
static constexpr u32 ypp = 2;
static void Convert(PixelBuffer<unpacked_type> *pb, u8 *data)
static void Convert(PixelBuffer<unpacked_type> *pb, const u8 *data)
{
u16 *p_in = (u16 *)data;
const u16 *p_in = (const u16 *)data;
pb->prel(0, 0, Unpacker::unpack(p_in[0]));
pb->prel(0, 1, Unpacker::unpack(p_in[1]));
pb->prel(1, 0, Unpacker::unpack(p_in[2]));
@ -296,10 +296,10 @@ struct ConvertTwiddleYUV
using unpacked_type = u32;
static constexpr u32 xpp = 2;
static constexpr u32 ypp = 2;
static void Convert(PixelBuffer<u32> *pb, u8 *data)
static void Convert(PixelBuffer<u32> *pb, const u8 *data)
{
//convert 4x1 4444 to 4x1 8888
u16* p_in = (u16 *)data;
const u16* p_in = (const u16 *)data;
s32 Y0 = (p_in[0] >> 8) & 255; //
s32 Yu = (p_in[0] >> 0) & 255; //p_in[0]
@ -342,9 +342,9 @@ struct ConvertTwiddlePal4
using unpacked_type = typename Unpacker::unpacked_type;
static constexpr u32 xpp = 4;
static constexpr u32 ypp = 4;
static void Convert(PixelBuffer<unpacked_type> *pb, u8 *data)
static void Convert(PixelBuffer<unpacked_type> *pb, const u8 *data)
{
u8 *p_in = (u8 *)data;
const u8 *p_in = data;
pb->prel(0, 0, Unpacker::unpack(p_in[0] & 0xF));
pb->prel(0, 1, Unpacker::unpack((p_in[0] >> 4) & 0xF)); p_in++;
@ -374,9 +374,9 @@ struct ConvertTwiddlePal8
using unpacked_type = typename Unpacker::unpacked_type;
static constexpr u32 xpp = 2;
static constexpr u32 ypp = 4;
static void Convert(PixelBuffer<unpacked_type> *pb, u8 *data)
static void Convert(PixelBuffer<unpacked_type> *pb, const u8 *data)
{
u8* p_in = (u8 *)data;
const u8* p_in = (const u8 *)data;
pb->prel(0, 0, Unpacker::unpack(p_in[0])); p_in++;
pb->prel(0, 1, Unpacker::unpack(p_in[0])); p_in++;
@ -392,7 +392,7 @@ struct ConvertTwiddlePal8
//handler functions
template<class PixelConvertor>
void texture_PL(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in,u32 Width,u32 Height)
void texture_PL(PixelBuffer<typename PixelConvertor::unpacked_type>* pb, const u8* p_in, u32 Width, u32 Height)
{
pb->amove(0,0);
@ -403,7 +403,7 @@ void texture_PL(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in
{
for (u32 x=0;x<Width;x++)
{
u8* p = p_in;
const u8* p = p_in;
PixelConvertor::Convert(pb,p);
p_in+=8;
@ -414,7 +414,7 @@ void texture_PL(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in
}
template<class PixelConvertor>
void texture_TW(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in,u32 Width,u32 Height)
void texture_TW(PixelBuffer<typename PixelConvertor::unpacked_type>* pb, const u8* p_in, u32 Width, u32 Height)
{
pb->amove(0, 0);
@ -427,7 +427,7 @@ void texture_TW(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in
{
for (u32 x = 0; x < Width; x += PixelConvertor::xpp)
{
u8* p = &p_in[(twop(x, y, bcx, bcy) / divider) << 3];
const u8* p = &p_in[(twop(x, y, bcx, bcy) / divider) << 3];
PixelConvertor::Convert(pb, p);
pb->rmovex(PixelConvertor::xpp);
@ -437,7 +437,7 @@ void texture_TW(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in
}
template<class PixelConvertor>
void texture_VQ(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in,u32 Width,u32 Height)
void texture_VQ(PixelBuffer<typename PixelConvertor::unpacked_type>* pb, const u8* p_in, u32 Width, u32 Height)
{
p_in += 256 * 4 * 2; // Skip VQ codebook
pb->amove(0, 0);
@ -459,9 +459,9 @@ void texture_VQ(PixelBuffer<typename PixelConvertor::unpacked_type>* pb,u8* p_in
}
}
typedef void (*TexConvFP)(PixelBuffer<u16> *pb, u8 *p_in, u32 width, u32 height);
typedef void (*TexConvFP8)(PixelBuffer<u8> *pb, u8 *p_in, u32 width, u32 height);
typedef void (*TexConvFP32)(PixelBuffer<u32> *pb, u8 *p_in, u32 width, u32 height);
typedef void (*TexConvFP)(PixelBuffer<u16> *pb, const u8 *p_in, u32 width, u32 height);
typedef void (*TexConvFP8)(PixelBuffer<u8> *pb, const u8 *p_in, u32 width, u32 height);
typedef void (*TexConvFP32)(PixelBuffer<u32> *pb, const u8 *p_in, u32 width, u32 height);
//Planar
constexpr TexConvFP tex565_PL = texture_PL<ConvertPlanar<UnpackerNop<u16>>>;
@ -815,10 +815,3 @@ void dump_screenshot(u8 *buffer, u32 width, u32 height, bool alpha = false, u32
extern const std::array<f32, 16> D_Adjust_LoD_Bias;
#undef clamp
extern float fb_scale_x, fb_scale_y;
static inline void rend_set_fb_scale(float x, float y)
{
fb_scale_x = x;
fb_scale_y = y;
}

84
core/rend/boxart/boxart.cpp
View File

@ -35,25 +35,33 @@ const GameBoxart *Boxart::getBoxart(const GameMedia& media)
{
loadDatabase();
std::string fileName = getGameFileName(media.path);
const GameBoxart *boxart = nullptr;
GameBoxart *boxart = nullptr;
{
std::lock_guard<std::mutex> guard(mutex);
auto it = games.find(fileName);
if (it != games.end())
{
boxart = &it->second;
else if (config::FetchBoxart)
if (config::FetchBoxart && !boxart->busy && !boxart->scraped)
{
boxart->busy = true;
boxart->gamePath = media.path;
toFetch.push_back(*boxart);
}
}
else
{
GameBoxart box;
box.fileName = fileName;
box.gamePath = media.path;
box.name = media.name;
box.searchName = media.gameName; // for arcade games
box.busy = true;
games[box.fileName] = box;
toFetch.push_back(box);
}
}
if (config::FetchBoxart)
fetchBoxart();
fetchBoxart();
return boxart;
}
@ -66,12 +74,17 @@ void Boxart::fetchBoxart()
if (toFetch.empty())
return;
fetching = std::async(std::launch::async, [this]() {
if (scraper == nullptr)
if (offlineScraper == nullptr)
{
offlineScraper = std::unique_ptr<Scraper>(new OfflineScraper());
offlineScraper->initialize(getSaveDirectory());
}
if (config::FetchBoxart && scraper == nullptr)
{
scraper = std::unique_ptr<Scraper>(new TheGamesDb());
if (!scraper->initialize(getSaveDirectory()))
{
WARN_LOG(COMMON, "thegamesdb scraper initialization failed");
ERROR_LOG(COMMON, "thegamesdb scraper initialization failed");
scraper.reset();
return;
}
@ -84,22 +97,49 @@ void Boxart::fetchBoxart()
toFetch.erase(toFetch.begin(), toFetch.begin() + size);
}
DEBUG_LOG(COMMON, "Scraping %d games", (int)boxart.size());
try {
scraper->scrape(boxart);
{
std::lock_guard<std::mutex> guard(mutex);
for (const GameBoxart& b : boxart)
if (b.scraped)
offlineScraper->scrape(boxart);
{
std::lock_guard<std::mutex> guard(mutex);
for (GameBoxart& b : boxart)
if (b.scraped || b.parsed)
{
if (!config::FetchBoxart || b.scraped)
b.busy = false;
games[b.fileName] = b;
databaseDirty = true;
}
}
if (config::FetchBoxart)
{
try {
scraper->scrape(boxart);
{
std::lock_guard<std::mutex> guard(mutex);
for (GameBoxart& b : boxart)
{
b.busy = false;
games[b.fileName] = b;
}
databaseDirty = true;
} catch (const std::exception& e) {
if (*e.what() != '\0')
INFO_LOG(COMMON, "thegamesdb error: %s", e.what());
{
// put back items into toFetch array
std::lock_guard<std::mutex> guard(mutex);
toFetch.insert(toFetch.begin(), boxart.begin(), boxart.end());
}
}
databaseDirty = true;
} catch (const std::exception& e) {
if (*e.what() != '\0')
INFO_LOG(COMMON, "thegamesdb error: %s", e.what());
{
// put back failed items into toFetch array
std::lock_guard<std::mutex> guard(mutex);
for (GameBoxart& b : boxart)
if (b.scraped)
{
b.busy = false;
games[b.fileName] = b;
databaseDirty = true;
}
else
{
toFetch.push_back(b);
}
}
}
}
});
@ -124,7 +164,7 @@ void Boxart::saveDatabase()
{
std::lock_guard<std::mutex> guard(mutex);
for (const auto& game : games)
if (game.second.scraped)
if (game.second.scraped || game.second.parsed)
array.push_back(game.second.to_json());
}
std::string serialized = array.dump(4);

1
core/rend/boxart/boxart.h
View File

@ -42,6 +42,7 @@ private:
std::unordered_map<std::string, GameBoxart> games;
std::mutex mutex;
std::unique_ptr<Scraper> scraper;
std::unique_ptr<Scraper> offlineScraper;
bool databaseLoaded = false;
bool databaseDirty = false;

105
core/rend/boxart/gamesdb.cpp
View File

@ -18,10 +18,9 @@
*/
#include "gamesdb.h"
#include "http_client.h"
#include "reios/reios.h"
#include "imgread/common.h"
#include "stdclass.h"
#include "oslib/oslib.h"
#include "emulator.h"
#include <cctype>
#define APIKEY "3fcc5e726a129924972be97abfd577ac5311f8f12398a9d9bcb5a377d4656fa8"
@ -215,13 +214,13 @@ void TheGamesDb::parseBoxart(GameBoxart& item, const json& j, int gameId)
if (cached != boxartCache.end())
{
copyFile(cached->second, filename);
item.boxartPath = filename;
item.setBoxartPath(filename);
}
else
{
if (downloadImage(url, filename))
{
item.boxartPath = filename;
item.setBoxartPath(filename);
boxartCache[url] = filename;
}
}
@ -293,90 +292,8 @@ bool TheGamesDb::fetchGameInfo(GameBoxart& item, const std::string& url, const s
return parseGameInfo(array, v["include"]["boxart"], item, diskId);
}
void TheGamesDb::getUidAndSearchName(GameBoxart& media)
{
int platform = getGamePlatform(media.gamePath.c_str());
if (platform == DC_PLATFORM_DREAMCAST)
{
if (media.gamePath.empty())
{
// Dreamcast BIOS
media.uniqueId.clear();
media.searchName.clear();
return;
}
Disc *disc;
try {
disc = OpenDisc(media.gamePath.c_str());
} catch (const std::exception& e) {
WARN_LOG(COMMON, "Can't open disk %s: %s", media.gamePath.c_str(), e.what());
// No need to retry if the disk is invalid/corrupted
media.scraped = true;
media.uniqueId.clear();
media.searchName.clear();
return;
}
u32 base_fad;
if (disc->type == GdRom) {
base_fad = 45150;
} else {
u8 ses[6];
disc->GetSessionInfo(ses, 0);
disc->GetSessionInfo(ses, ses[2]);
base_fad = (ses[3] << 16) | (ses[4] << 8) | (ses[5] << 0);
}
u8 sector[2048];
disc->ReadSectors(base_fad, 1, sector, sizeof(sector));
ip_meta_t diskId;
memcpy(&diskId, sector, sizeof(diskId));
delete disc;
media.uniqueId = trim_trailing_ws(std::string(diskId.product_number, sizeof(diskId.product_number)));
media.searchName = trim_trailing_ws(std::string(diskId.software_name, sizeof(diskId.software_name)));
if (media.searchName.empty())
media.searchName = media.name;
if (diskId.area_symbols[0] != '\0')
{
media.region = 0;
if (diskId.area_symbols[0] == 'J')
media.region |= GameBoxart::JAPAN;
if (diskId.area_symbols[1] == 'U')
media.region |= GameBoxart::USA;
if (diskId.area_symbols[2] == 'E')
media.region |= GameBoxart::EUROPE;
}
else
media.region = GameBoxart::JAPAN | GameBoxart::USA | GameBoxart::EUROPE;
}
else
{
media.uniqueId.clear();
// Use first one in case of alternate names (Virtua Tennis / Power Smash)
size_t spos = media.searchName.find('/');
if (spos != std::string::npos)
media.searchName = trim_trailing_ws(media.searchName.substr(0, spos));
// Delete trailing (...) and [...]
while (!media.searchName.empty())
{
size_t pos{ std::string::npos };
if (media.searchName.back() == ')')
pos = media.searchName.find_last_of('(');
else if (media.searchName.back() == ']')
pos = media.searchName.find_last_of('[');
if (pos == std::string::npos)
break;
media.searchName = trim_trailing_ws(media.searchName.substr(0, pos));
}
}
}
void TheGamesDb::scrape(GameBoxart& item)
{
item.found = false;
getUidAndSearchName(item);
if (item.searchName.empty())
// invalid rom or disk
return;
@ -387,7 +304,7 @@ void TheGamesDb::scrape(GameBoxart& item)
std::string url = makeUrl("Games/ByGameUniqueID") + "&fields=overview,uids&include=boxart&filter%5Bplatform%5D="
+ std::to_string(dreamcastPlatformId) + "&uid=" + http::urlEncode(item.uniqueId);
if (fetchGameInfo(item, url, item.uniqueId))
item.scraped = item.found = true;
item.scraped = true;
}
if (!item.scraped)
fetchByName(item);
@ -407,7 +324,7 @@ void TheGamesDb::fetchByName(GameBoxart& item)
url += std::to_string(arcadePlatformId);
url += "&name=" + http::urlEncode(item.searchName);
if (fetchGameInfo(item, url))
item.scraped = item.found = true;
item.scraped = true;
}
void TheGamesDb::fetchByUids(std::vector<GameBoxart>& items)
@ -433,7 +350,7 @@ void TheGamesDb::fetchByUids(std::vector<GameBoxart>& items)
for (GameBoxart& item : items)
{
if (!item.scraped && !item.uniqueId.empty() && parseGameInfo(array, boxartArray, item, item.uniqueId))
item.scraped = item.found = true;
item.scraped = true;
}
}
@ -444,12 +361,6 @@ void TheGamesDb::scrape(std::vector<GameBoxart>& items)
blackoutPeriod = 0.0;
fetchPlatforms();
for (GameBoxart& item : items)
{
if (!item.scraped)
item.found = false;
getUidAndSearchName(item);
}
fetchByUids(items);
for (GameBoxart& item : items)
{
@ -461,9 +372,9 @@ void TheGamesDb::scrape(std::vector<GameBoxart>& items)
{
std::string localPath = makeUniqueFilename("dreamcast_logo_grey.png");
if (downloadImage("https://flyinghead.github.io/flycast-builds/dreamcast_logo_grey.png", localPath))
item.boxartPath = localPath;
item.setBoxartPath(localPath);
}
item.scraped = true;
}
item.scraped = true;
}
}

1
core/rend/boxart/gamesdb.h
View File

@ -37,7 +37,6 @@ private:
void copyFile(const std::string& from, const std::string& to);
json httpGet(const std::string& url);
void parseBoxart(GameBoxart& item, const json& j, int gameId);
void getUidAndSearchName(GameBoxart& media);
void fetchByUids(std::vector<GameBoxart>& items);
void fetchByName(GameBoxart& item);
bool parseGameInfo(const json& gameArray, const json& boxartArray, GameBoxart& item, const std::string& diskId);

143
core/rend/boxart/pvrparser.h Normal file
View File

@ -0,0 +1,143 @@
/*
Copyright 2022 flyinghead
This file is part of Flycast.
Flycast is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
Flycast is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Flycast. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include "types.h"
#include "rend/TexCache.h"
extern const u32 VQMipPoint[11];
extern const u32 OtherMipPoint[11];
enum PvrDataFormat {
PvrSquareTwiddled = 0x01,
PvrSquareTwiddledMipmaps = 0x02,
PvrVQ = 0x03,
PvrVQMipmaps = 0x04,
PvrPal4 = 0x05,
PvrPal4Mipmaps = 0x06,
PvrPal8 = 0x07,
PvrPal8Mipmaps = 0x08,
PvrRectangle = 0x09,
PvrStride = 0x0B,
PvrRectangleTwiddled = 0x0D,
PvrSmallVQ = 0x10,
PvrSmallVQMipmaps = 0x11,
PvrSquareTwiddledMipmapsAlt = 0x12,
};
static bool pvrParse(const u8 *data, u32 len, u32& width, u32& height, std::vector<u8>& out)
{
if (len < 16)
return false;
const u8 *p = data;
if (!memcmp("GBIX", p, 4))
{
p += 4;
u32 idxSize = *(u32 *)p;
p += 4 + idxSize;
}
if (memcmp("PVRT", p, 4))
{
WARN_LOG(COMMON, "Invalid PVR file: header not found");
return false;
}
p += 4;
u32 size = *(u32 *)p;
p += 4;
PixelFormat pixelFormat = (PixelFormat)*p++;
PvrDataFormat imgType = (PvrDataFormat)*p++;
p += 2;
width = *(u16 *)p;
p += 2;
height = *(u16 *)p;
p += 2;
::vq_codebook = p;
TexConvFP32 texConv;
switch (pixelFormat)
{
case Pixel1555:
if (imgType == PvrSquareTwiddled || imgType == PvrSquareTwiddledMipmaps
|| imgType == PvrRectangleTwiddled || imgType == PvrSquareTwiddledMipmapsAlt)
texConv = opengl::tex1555_TW32;
else if (imgType == PvrVQ || imgType == PvrVQMipmaps)
texConv = opengl::tex1555_VQ32;
else if (imgType == PvrRectangle || imgType == PvrStride)
texConv = opengl::tex1555_PL32;
else
{
WARN_LOG(COMMON, "Unsupported 1555 image type: %d", imgType);
return false;
}
break;
case Pixel565:
if (imgType == PvrSquareTwiddled || imgType == PvrSquareTwiddledMipmaps
|| imgType == PvrRectangleTwiddled || imgType == PvrSquareTwiddledMipmapsAlt)
texConv = opengl::tex565_TW32;
else if (imgType == PvrVQ || imgType == PvrVQMipmaps)
texConv = opengl::tex565_VQ32;
else if (imgType == PvrRectangle || imgType == PvrStride)
texConv = opengl::tex565_PL32;
else
{
WARN_LOG(COMMON, "Unsupported 565 image type: %d", imgType);
return false;
}
break;
case Pixel4444:
if (imgType == PvrSquareTwiddled || imgType == PvrSquareTwiddledMipmaps
|| imgType == PvrRectangleTwiddled || imgType == PvrSquareTwiddledMipmapsAlt)
texConv = opengl::tex4444_TW32;
else if (imgType == PvrVQ || imgType == PvrVQMipmaps)
texConv = opengl::tex4444_VQ32;
else if (imgType == PvrRectangle || imgType == PvrStride)
texConv = opengl::tex4444_PL32;
else
{
WARN_LOG(COMMON, "Unsupported 4444 image type: %d", imgType);
return false;
}
break;
default:
WARN_LOG(COMMON, "Unsupported PVR pixel type: %d", pixelFormat);
return false;
}
DEBUG_LOG(COMMON, "PVR file: size %d pixelFmt %d imgType %d w %d h %d\n", size, pixelFormat, imgType, width, height);
u32 texU = 3;
while (1u << texU < width)
texU++;
if (imgType == PvrSquareTwiddledMipmapsAlt)
// Hardcoding pixel size. Not correct for palette texs
p += OtherMipPoint[texU] * 2;
else if (imgType == PvrSquareTwiddledMipmaps)
// Hardcoding pixel size. Not correct for palette texs
p += (OtherMipPoint[texU] - 2) * 2;
else if (imgType == PvrVQMipmaps)
p += VQMipPoint[texU];
PixelBuffer<u32> pb;
pb.init(width, height);
texConv(&pb, p, width, height);
out.resize(width * height * 4);
memcpy(out.data(), pb.data(), out.size());
return true;
}

111
core/rend/boxart/scraper.cpp
View File

@ -19,6 +19,12 @@
#include "scraper.h"
#include "http_client.h"
#include "stdclass.h"
#include "emulator.h"
#include "imgread/common.h"
#include "imgread/isofs.h"
#include "reios/reios.h"
#include "pvrparser.h"
#include <stb_image_write.h>
bool Scraper::downloadImage(const std::string& url, const std::string& localName)
{
@ -56,3 +62,108 @@ std::string Scraper::makeUniqueFilename(const std::string& url)
} while (file_exists(path));
return path;
}
void OfflineScraper::scrape(GameBoxart& item)
{
if (item.parsed)
return;
item.parsed = true;
int platform = getGamePlatform(item.gamePath.c_str());
if (platform == DC_PLATFORM_DREAMCAST)
{
if (item.gamePath.empty())
{
// Dreamcast BIOS
item.uniqueId.clear();
item.searchName.clear();
return;
}
Disc *disc;
try {
disc = OpenDisc(item.gamePath.c_str());
} catch (const std::exception& e) {
WARN_LOG(COMMON, "Can't open disk %s: %s", item.gamePath.c_str(), e.what());
// No need to retry if the disk is invalid/corrupted
item.scraped = true;
item.uniqueId.clear();
item.searchName.clear();
return;
}
u8 sector[2048];
disc->ReadSectors(disc->GetBaseFAD(), 1, sector, sizeof(sector));
ip_meta_t diskId;
memcpy(&diskId, sector, sizeof(diskId));
if (item.boxartPath.empty())
{
IsoFs isofs(disc);
std::unique_ptr<IsoFs::Directory> root(isofs.getRoot());
if (root != nullptr)
{
std::unique_ptr<IsoFs::Entry> entry(root->getEntry("0GDTEX.PVR"));
if (entry != nullptr && !entry->isDirectory())
{
IsoFs::File *gdtexFile = (IsoFs::File *)entry.get();
std::vector<u8> data(gdtexFile->getSize());
gdtexFile->read(data.data(), data.size());
std::vector<u8> out;
u32 w, h;
if (pvrParse(data.data(), data.size(), w, h, out))
{
stbi_flip_vertically_on_write(0);
item.setBoxartPath(makeUniqueFilename("gdtex.png"));
stbi_write_png(item.boxartPath.c_str(), w, h, 4, out.data(), 0);
}
}
}
}
delete disc;
item.uniqueId = trim_trailing_ws(std::string(diskId.product_number, sizeof(diskId.product_number)));
item.searchName = trim_trailing_ws(std::string(diskId.software_name, sizeof(diskId.software_name)));
if (item.searchName.empty())
item.searchName = item.name;
else
{
for (char& c : item.searchName)
if (!std::isprint(c))
c = ' ';
}
if (diskId.area_symbols[0] != '\0')
{
item.region = 0;
if (diskId.area_symbols[0] == 'J')
item.region |= GameBoxart::JAPAN;
if (diskId.area_symbols[1] == 'U')
item.region |= GameBoxart::USA;
if (diskId.area_symbols[2] == 'E')
item.region |= GameBoxart::EUROPE;
}
else
item.region = GameBoxart::JAPAN | GameBoxart::USA | GameBoxart::EUROPE;
}
else
{
item.uniqueId.clear();
// Use first one in case of alternate names (Virtua Tennis / Power Smash)
size_t spos = item.searchName.find('/');
if (spos != std::string::npos)
item.searchName = trim_trailing_ws(item.searchName.substr(0, spos));
// Delete trailing (...) and [...]
while (!item.searchName.empty())
{
size_t pos{ std::string::npos };
if (item.searchName.back() == ')')
pos = item.searchName.find_last_of('(');
else if (item.searchName.back() == ']')
pos = item.searchName.find_last_of('[');
if (pos == std::string::npos)
break;
item.searchName = trim_trailing_ws(item.searchName.substr(0, pos));
}
}
}

27
core/rend/boxart/scraper.h
View File

@ -37,12 +37,11 @@ struct GameBoxart
std::string overview;
std::string gamePath;
std::string screenshotPath;
std::string fanartPath;
std::string boxartPath;
bool parsed = false;
bool scraped = false;
bool found = false;
bool busy = false;
enum Region { JAPAN = 1, USA = 2, EUROPE = 4 };
@ -52,14 +51,13 @@ struct GameBoxart
{ "file_name", fileName },
{ "name", name },
{ "unique_id", uniqueId },
{ "search_name", searchName },
{ "region", region },
{ "release_date", releaseDate },
{ "overview", overview },
{ "screenshot_path", screenshotPath },
{ "fanart_path", fanartPath },
{ "boxart_path", boxartPath },
{ "parsed", parsed },
{ "scraped", scraped },
{ "found", found },
};
return j;
}
@ -85,14 +83,19 @@ struct GameBoxart
loadProperty(fileName, j, "file_name");
loadProperty(name, j, "name");
loadProperty(uniqueId, j, "unique_id");
loadProperty(searchName, j, "search_name");
loadProperty(region, j, "region");
loadProperty(releaseDate, j, "release_date");
loadProperty(overview, j, "overview");
loadProperty(screenshotPath, j, "screenshot_path");
loadProperty(fanartPath, j, "fanart_path");
loadProperty(boxartPath, j, "boxart_path");
loadProperty(parsed, j, "parsed");
loadProperty(scraped, j, "scraped");
loadProperty(found, j, "found");
}
void setBoxartPath(const std::string& path) {
if (!boxartPath.empty())
nowide::remove(boxartPath.c_str());
boxartPath = path;
}
};
@ -120,3 +123,9 @@ protected:
private:
std::string saveDirectory;
};
class OfflineScraper : public Scraper
{
public:
void scrape(GameBoxart& item) override;
};