From 3014dadfa8c2065f81009895ecf81396de5120bc Mon Sep 17 00:00:00 2001 From: JosJuice Date: Thu, 21 Mar 2019 22:20:23 +0100 Subject: [PATCH 01/10] DiscIO: Add way of checking whether blob data size is accurate --- Source/Core/DiscIO/Blob.h | 1 + Source/Core/DiscIO/CISOBlob.h | 1 + Source/Core/DiscIO/CompressedBlob.h | 3 ++- Source/Core/DiscIO/DirectoryBlob.h | 1 + Source/Core/DiscIO/DriveBlob.h | 3 ++- Source/Core/DiscIO/FileBlob.h | 3 ++- Source/Core/DiscIO/TGCBlob.h | 3 ++- Source/Core/DiscIO/Volume.h | 1 + Source/Core/DiscIO/VolumeFileBlobReader.h | 3 ++- Source/Core/DiscIO/VolumeGC.cpp | 5 +++++ Source/Core/DiscIO/VolumeGC.h | 1 + Source/Core/DiscIO/VolumeWad.cpp | 5 +++++ Source/Core/DiscIO/VolumeWad.h | 1 + Source/Core/DiscIO/VolumeWii.cpp | 5 +++++ Source/Core/DiscIO/VolumeWii.h | 1 + Source/Core/DiscIO/WbfsBlob.h | 3 ++- 16 files changed, 34 insertions(+), 6 deletions(-) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index 703cf1199b..8c13354963 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -44,6 +44,7 @@ public: virtual BlobType GetBlobType() const = 0; virtual u64 GetRawSize() const = 0; virtual u64 GetDataSize() const = 0; + virtual bool IsDataSizeAccurate() const = 0; // NOT thread-safe - can't call this from multiple threads. virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0; diff --git a/Source/Core/DiscIO/CISOBlob.h b/Source/Core/DiscIO/CISOBlob.h index 07a340fe1d..c2de890495 100644 --- a/Source/Core/DiscIO/CISOBlob.h +++ b/Source/Core/DiscIO/CISOBlob.h @@ -40,6 +40,7 @@ public: // The CISO format does not save the original file size. // This function returns an upper bound. u64 GetDataSize() const override; + bool IsDataSizeAccurate() const override { return false; } u64 GetRawSize() const override; bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; diff --git a/Source/Core/DiscIO/CompressedBlob.h b/Source/Core/DiscIO/CompressedBlob.h index ecf8695c53..1d53353ba8 100644 --- a/Source/Core/DiscIO/CompressedBlob.h +++ b/Source/Core/DiscIO/CompressedBlob.h @@ -49,8 +49,9 @@ public: ~CompressedBlobReader(); const CompressedBlobHeader& GetHeader() const { return m_header; } BlobType GetBlobType() const override { return BlobType::GCZ; } - u64 GetDataSize() const override { return m_header.data_size; } u64 GetRawSize() const override { return m_file_size; } + u64 GetDataSize() const override { return m_header.data_size; } + bool IsDataSizeAccurate() const override { return true; } u64 GetBlockCompressedSize(u64 block_num) const; bool GetBlock(u64 block_num, u8* out_ptr) override; diff --git a/Source/Core/DiscIO/DirectoryBlob.h b/Source/Core/DiscIO/DirectoryBlob.h index 9da34cc0f0..614afe77ac 100644 --- a/Source/Core/DiscIO/DirectoryBlob.h +++ b/Source/Core/DiscIO/DirectoryBlob.h @@ -146,6 +146,7 @@ public: BlobType GetBlobType() const override; u64 GetRawSize() const override; u64 GetDataSize() const override; + bool IsDataSizeAccurate() const override { return true; } private: struct PartitionWithType diff --git a/Source/Core/DiscIO/DriveBlob.h b/Source/Core/DiscIO/DriveBlob.h index 5580d30179..3d3997af1f 100644 --- a/Source/Core/DiscIO/DriveBlob.h +++ b/Source/Core/DiscIO/DriveBlob.h @@ -24,8 +24,9 @@ public: static std::unique_ptr Create(const std::string& drive); ~DriveReader(); BlobType GetBlobType() const override { return BlobType::DRIVE; } - u64 GetDataSize() const override { return m_size; } u64 GetRawSize() const override { return m_size; } + u64 GetDataSize() const override { return m_size; } + bool IsDataSizeAccurate() const override { return true; } private: DriveReader(const std::string& drive); diff --git a/Source/Core/DiscIO/FileBlob.h b/Source/Core/DiscIO/FileBlob.h index 6543a2c218..30a707f746 100644 --- a/Source/Core/DiscIO/FileBlob.h +++ b/Source/Core/DiscIO/FileBlob.h @@ -20,8 +20,9 @@ public: static std::unique_ptr Create(File::IOFile file); BlobType GetBlobType() const override { return BlobType::PLAIN; } - u64 GetDataSize() const override { return m_size; } u64 GetRawSize() const override { return m_size; } + u64 GetDataSize() const override { return m_size; } + bool IsDataSizeAccurate() const override { return true; } bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DiscIO/TGCBlob.h b/Source/Core/DiscIO/TGCBlob.h index 31a46fca00..47d3492744 100644 --- a/Source/Core/DiscIO/TGCBlob.h +++ b/Source/Core/DiscIO/TGCBlob.h @@ -43,8 +43,9 @@ public: static std::unique_ptr Create(File::IOFile file); BlobType GetBlobType() const override { return BlobType::TGC; } - u64 GetDataSize() const override; u64 GetRawSize() const override { return m_size; } + u64 GetDataSize() const override; + bool IsDataSizeAccurate() const override { return true; } bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: diff --git a/Source/Core/DiscIO/Volume.h b/Source/Core/DiscIO/Volume.h index 3b70f765b7..c7c8e5849a 100644 --- a/Source/Core/DiscIO/Volume.h +++ b/Source/Core/DiscIO/Volume.h @@ -101,6 +101,7 @@ public: virtual BlobType GetBlobType() const = 0; // Size of virtual disc (may be inaccurate depending on the blob type) virtual u64 GetSize() const = 0; + virtual bool IsSizeAccurate() const = 0; // Size on disc (compressed size) virtual u64 GetRawSize() const = 0; diff --git a/Source/Core/DiscIO/VolumeFileBlobReader.h b/Source/Core/DiscIO/VolumeFileBlobReader.h index 46a988d367..b3a8c2c3aa 100644 --- a/Source/Core/DiscIO/VolumeFileBlobReader.h +++ b/Source/Core/DiscIO/VolumeFileBlobReader.h @@ -23,8 +23,9 @@ public: Create(const Volume& volume, const Partition& partition, const std::string& file_path); BlobType GetBlobType() const override { return BlobType::PLAIN; } - u64 GetDataSize() const override; u64 GetRawSize() const override; + u64 GetDataSize() const override; + bool IsDataSizeAccurate() const override { return true; } bool Read(u64 offset, u64 length, u8* out_ptr) override; private: diff --git a/Source/Core/DiscIO/VolumeGC.cpp b/Source/Core/DiscIO/VolumeGC.cpp index e3db29c1f5..67cf71072e 100644 --- a/Source/Core/DiscIO/VolumeGC.cpp +++ b/Source/Core/DiscIO/VolumeGC.cpp @@ -179,6 +179,11 @@ u64 VolumeGC::GetSize() const return m_reader->GetDataSize(); } +bool VolumeGC::IsSizeAccurate() const +{ + return m_reader->IsDataSizeAccurate(); +} + u64 VolumeGC::GetRawSize() const { return m_reader->GetRawSize(); diff --git a/Source/Core/DiscIO/VolumeGC.h b/Source/Core/DiscIO/VolumeGC.h index 1bf6139768..fbbdad56e5 100644 --- a/Source/Core/DiscIO/VolumeGC.h +++ b/Source/Core/DiscIO/VolumeGC.h @@ -52,6 +52,7 @@ public: Country GetCountry(const Partition& partition = PARTITION_NONE) const override; BlobType GetBlobType() const override; u64 GetSize() const override; + bool IsSizeAccurate() const override; u64 GetRawSize() const override; private: diff --git a/Source/Core/DiscIO/VolumeWad.cpp b/Source/Core/DiscIO/VolumeWad.cpp index a49488506b..a7316ffde9 100644 --- a/Source/Core/DiscIO/VolumeWad.cpp +++ b/Source/Core/DiscIO/VolumeWad.cpp @@ -175,6 +175,11 @@ u64 VolumeWAD::GetSize() const return m_reader->GetDataSize(); } +bool VolumeWAD::IsSizeAccurate() const +{ + return m_reader->IsDataSizeAccurate(); +} + u64 VolumeWAD::GetRawSize() const { return m_reader->GetRawSize(); diff --git a/Source/Core/DiscIO/VolumeWad.h b/Source/Core/DiscIO/VolumeWad.h index 32d8bfe3a6..fce0a10d30 100644 --- a/Source/Core/DiscIO/VolumeWad.h +++ b/Source/Core/DiscIO/VolumeWad.h @@ -54,6 +54,7 @@ public: BlobType GetBlobType() const override; u64 GetSize() const override; + bool IsSizeAccurate() const override; u64 GetRawSize() const override; private: diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index 777d774d82..f7261b0a61 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -379,6 +379,11 @@ u64 VolumeWii::GetSize() const return m_reader->GetDataSize(); } +bool VolumeWii::IsSizeAccurate() const +{ + return m_reader->IsDataSizeAccurate(); +} + u64 VolumeWii::GetRawSize() const { return m_reader->GetRawSize(); diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index 48010d5b37..e2b82c79f2 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -62,6 +62,7 @@ public: Country GetCountry(const Partition& partition) const override; BlobType GetBlobType() const override; u64 GetSize() const override; + bool IsSizeAccurate() const override; u64 GetRawSize() const override; static constexpr unsigned int BLOCK_HEADER_SIZE = 0x0400; diff --git a/Source/Core/DiscIO/WbfsBlob.h b/Source/Core/DiscIO/WbfsBlob.h index 181e32b6b3..dacdb43b50 100644 --- a/Source/Core/DiscIO/WbfsBlob.h +++ b/Source/Core/DiscIO/WbfsBlob.h @@ -24,12 +24,13 @@ public: static std::unique_ptr Create(File::IOFile file, const std::string& path); BlobType GetBlobType() const override { return BlobType::WBFS; } + u64 GetRawSize() const override { return m_size; } // The WBFS format does not save the original file size. // This function returns a constant upper bound // (the size of a double-layer Wii disc). u64 GetDataSize() const override; + bool IsDataSizeAccurate() const override { return false; } - u64 GetRawSize() const override { return m_size; } bool Read(u64 offset, u64 nbytes, u8* out_ptr) override; private: From cf9ab6ddcc93cd279aaf62897095ac5b8929bab9 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Thu, 21 Mar 2019 22:23:12 +0100 Subject: [PATCH 02/10] DiscExtractor: Make P prefix for partition names optional Also added constants for common partition types. --- Source/Core/DiscIO/DiscExtractor.cpp | 12 ++++++------ Source/Core/DiscIO/DiscExtractor.h | 6 +++++- Source/Core/DolphinQt/Config/FilesystemWidget.cpp | 3 +-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Source/Core/DiscIO/DiscExtractor.cpp b/Source/Core/DiscIO/DiscExtractor.cpp index c9a26f5bdb..34c55a1d73 100644 --- a/Source/Core/DiscIO/DiscExtractor.cpp +++ b/Source/Core/DiscIO/DiscExtractor.cpp @@ -19,15 +19,15 @@ namespace DiscIO { -std::string DirectoryNameForPartitionType(u32 partition_type) +std::string NameForPartitionType(u32 partition_type, bool include_prefix) { switch (partition_type) { - case 0: + case PARTITION_DATA: return "DATA"; - case 1: + case PARTITION_UPDATE: return "UPDATE"; - case 2: + case PARTITION_CHANNEL: return "CHANNEL"; default: const std::string type_as_game_id{static_cast((partition_type >> 24) & 0xFF), @@ -37,10 +37,10 @@ std::string DirectoryNameForPartitionType(u32 partition_type) if (std::all_of(type_as_game_id.cbegin(), type_as_game_id.cend(), [](char c) { return std::isalnum(c, std::locale::classic()); })) { - return "P-" + type_as_game_id; + return include_prefix ? "P-" + type_as_game_id : type_as_game_id; } - return StringFromFormat("P%u", partition_type); + return StringFromFormat(include_prefix ? "P%u" : "%u", partition_type); } } diff --git a/Source/Core/DiscIO/DiscExtractor.h b/Source/Core/DiscIO/DiscExtractor.h index 3471dc700b..39af10d1a3 100644 --- a/Source/Core/DiscIO/DiscExtractor.h +++ b/Source/Core/DiscIO/DiscExtractor.h @@ -15,7 +15,11 @@ class FileInfo; struct Partition; class Volume; -std::string DirectoryNameForPartitionType(u32 partition_type); +constexpr u32 PARTITION_DATA = 0; +constexpr u32 PARTITION_UPDATE = 1; +constexpr u32 PARTITION_CHANNEL = 2; + +std::string NameForPartitionType(u32 partition_type, bool include_prefix); u64 ReadFile(const Volume& volume, const Partition& partition, const FileInfo* file_info, u8* buffer, u64 max_buffer_size, u64 offset_in_file = 0); diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp index 3ff72d8e06..662091581c 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp @@ -225,8 +225,7 @@ void FilesystemWidget::ShowContextMenu(const QPoint&) { if (const std::optional partition_type = m_volume->GetPartitionType(p)) { - const std::string partition_name = - DiscIO::DirectoryNameForPartitionType(*partition_type); + const std::string partition_name = DiscIO::NameForPartitionType(*partition_type, true); ExtractPartition(p, folder + QChar(u'/') + QString::fromStdString(partition_name)); } } From abb3c5bccd0a678fb5c70417064854b006ed110c Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 23 Mar 2019 13:05:42 +0100 Subject: [PATCH 03/10] VolumeWad: Implement GetTicket --- Source/Core/DiscIO/VolumeWad.cpp | 17 +++++++++++++---- Source/Core/DiscIO/VolumeWad.h | 7 +++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/Source/Core/DiscIO/VolumeWad.cpp b/Source/Core/DiscIO/VolumeWad.cpp index a7316ffde9..48a315331b 100644 --- a/Source/Core/DiscIO/VolumeWad.cpp +++ b/Source/Core/DiscIO/VolumeWad.cpp @@ -33,16 +33,20 @@ VolumeWAD::VolumeWAD(std::unique_ptr reader) : m_reader(std::move(re // Source: http://wiibrew.org/wiki/WAD_files m_hdr_size = m_reader->ReadSwapped(0x00).value_or(0); m_cert_size = m_reader->ReadSwapped(0x08).value_or(0); - m_tick_size = m_reader->ReadSwapped(0x10).value_or(0); + m_ticket_size = m_reader->ReadSwapped(0x10).value_or(0); m_tmd_size = m_reader->ReadSwapped(0x14).value_or(0); m_data_size = m_reader->ReadSwapped(0x18).value_or(0); - m_offset = Common::AlignUp(m_hdr_size, 0x40) + Common::AlignUp(m_cert_size, 0x40); + m_ticket_offset = Common::AlignUp(m_hdr_size, 0x40) + Common::AlignUp(m_cert_size, 0x40); m_tmd_offset = Common::AlignUp(m_hdr_size, 0x40) + Common::AlignUp(m_cert_size, 0x40) + - Common::AlignUp(m_tick_size, 0x40); + Common::AlignUp(m_ticket_size, 0x40); m_opening_bnr_offset = m_tmd_offset + Common::AlignUp(m_tmd_size, 0x40) + Common::AlignUp(m_data_size, 0x40); + std::vector ticket_buffer(m_ticket_size); + Read(m_ticket_offset, m_ticket_size, ticket_buffer.data()); + m_ticket.SetBytes(std::move(ticket_buffer)); + if (!IOS::ES::IsValidTMDSize(m_tmd_size)) { ERROR_LOG(DISCIO, "TMD is too large: %u bytes", m_tmd_size); @@ -95,6 +99,11 @@ Country VolumeWAD::GetCountry(const Partition& partition) const return CountryCodeToCountry(country_byte, Platform::WiiWAD, region); } +const IOS::ES::TicketReader& VolumeWAD::GetTicket(const Partition& partition) const +{ + return m_ticket; +} + const IOS::ES::TMDReader& VolumeWAD::GetTMD(const Partition& partition) const { return m_tmd; @@ -126,7 +135,7 @@ std::string VolumeWAD::GetMakerID(const Partition& partition) const std::optional VolumeWAD::GetTitleID(const Partition& partition) const { - return ReadSwapped(m_offset + 0x01DC, partition); + return ReadSwapped(m_ticket_offset + 0x01DC, partition); } std::optional VolumeWAD::GetRevision(const Partition& partition) const diff --git a/Source/Core/DiscIO/VolumeWad.h b/Source/Core/DiscIO/VolumeWad.h index fce0a10d30..02276f7797 100644 --- a/Source/Core/DiscIO/VolumeWad.h +++ b/Source/Core/DiscIO/VolumeWad.h @@ -33,6 +33,8 @@ public: const Partition& partition = PARTITION_NONE) const override; const FileSystem* GetFileSystem(const Partition& partition = PARTITION_NONE) const override; std::optional GetTitleID(const Partition& partition = PARTITION_NONE) const override; + const IOS::ES::TicketReader& + GetTicket(const Partition& partition = PARTITION_NONE) const override; const IOS::ES::TMDReader& GetTMD(const Partition& partition = PARTITION_NONE) const override; std::string GetGameID(const Partition& partition = PARTITION_NONE) const override; std::string GetGameTDBID(const Partition& partition = PARTITION_NONE) const override; @@ -59,13 +61,14 @@ public: private: std::unique_ptr m_reader; + IOS::ES::TicketReader m_ticket; IOS::ES::TMDReader m_tmd; - u32 m_offset = 0; + u32 m_ticket_offset = 0; u32 m_tmd_offset = 0; u32 m_opening_bnr_offset = 0; u32 m_hdr_size = 0; u32 m_cert_size = 0; - u32 m_tick_size = 0; + u32 m_ticket_size = 0; u32 m_tmd_size = 0; u32 m_data_size = 0; }; From c028a84531ce08bb1f24f88ebdcc8885561a89e1 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Thu, 21 Mar 2019 23:04:44 +0100 Subject: [PATCH 04/10] Volume: Add a GetCertificateChain function --- Source/Core/DiscIO/Volume.cpp | 1 + Source/Core/DiscIO/Volume.h | 7 ++++++- Source/Core/DiscIO/VolumeWad.cpp | 16 ++++++++++++---- Source/Core/DiscIO/VolumeWad.h | 6 +++++- Source/Core/DiscIO/VolumeWii.cpp | 21 ++++++++++++++++++++- Source/Core/DiscIO/VolumeWii.h | 2 ++ 6 files changed, 46 insertions(+), 7 deletions(-) diff --git a/Source/Core/DiscIO/Volume.cpp b/Source/Core/DiscIO/Volume.cpp index e95a2b746f..8aba2c981b 100644 --- a/Source/Core/DiscIO/Volume.cpp +++ b/Source/Core/DiscIO/Volume.cpp @@ -25,6 +25,7 @@ namespace DiscIO { const IOS::ES::TicketReader Volume::INVALID_TICKET{}; const IOS::ES::TMDReader Volume::INVALID_TMD{}; +const std::vector Volume::INVALID_CERT_CHAIN{}; std::map Volume::ReadWiiNames(const std::vector& data) { diff --git a/Source/Core/DiscIO/Volume.h b/Source/Core/DiscIO/Volume.h index c7c8e5849a..e5e77d4479 100644 --- a/Source/Core/DiscIO/Volume.h +++ b/Source/Core/DiscIO/Volume.h @@ -69,6 +69,10 @@ public: return INVALID_TICKET; } virtual const IOS::ES::TMDReader& GetTMD(const Partition& partition) const { return INVALID_TMD; } + virtual const std::vector& GetCertificateChain(const Partition& partition) const + { + return INVALID_CERT_CHAIN; + } // Returns a non-owning pointer. Returns nullptr if the file system couldn't be read. virtual const FileSystem* GetFileSystem(const Partition& partition) const = 0; virtual u64 PartitionOffsetToRawOffset(u64 offset, const Partition& partition) const @@ -129,8 +133,9 @@ protected: static const IOS::ES::TicketReader INVALID_TICKET; static const IOS::ES::TMDReader INVALID_TMD; + static const std::vector INVALID_CERT_CHAIN; }; std::unique_ptr CreateVolumeFromFilename(const std::string& filename); -} // namespace +} // namespace DiscIO diff --git a/Source/Core/DiscIO/VolumeWad.cpp b/Source/Core/DiscIO/VolumeWad.cpp index 48a315331b..94191c05c7 100644 --- a/Source/Core/DiscIO/VolumeWad.cpp +++ b/Source/Core/DiscIO/VolumeWad.cpp @@ -32,14 +32,14 @@ VolumeWAD::VolumeWAD(std::unique_ptr reader) : m_reader(std::move(re // Source: http://wiibrew.org/wiki/WAD_files m_hdr_size = m_reader->ReadSwapped(0x00).value_or(0); - m_cert_size = m_reader->ReadSwapped(0x08).value_or(0); + m_cert_chain_size = m_reader->ReadSwapped(0x08).value_or(0); m_ticket_size = m_reader->ReadSwapped(0x10).value_or(0); m_tmd_size = m_reader->ReadSwapped(0x14).value_or(0); m_data_size = m_reader->ReadSwapped(0x18).value_or(0); - m_ticket_offset = Common::AlignUp(m_hdr_size, 0x40) + Common::AlignUp(m_cert_size, 0x40); - m_tmd_offset = Common::AlignUp(m_hdr_size, 0x40) + Common::AlignUp(m_cert_size, 0x40) + - Common::AlignUp(m_ticket_size, 0x40); + m_cert_chain_offset = Common::AlignUp(m_hdr_size, 0x40); + m_ticket_offset = m_cert_chain_offset + Common::AlignUp(m_cert_chain_size, 0x40); + m_tmd_offset = m_ticket_offset + Common::AlignUp(m_ticket_size, 0x40); m_opening_bnr_offset = m_tmd_offset + Common::AlignUp(m_tmd_size, 0x40) + Common::AlignUp(m_data_size, 0x40); @@ -56,6 +56,9 @@ VolumeWAD::VolumeWAD(std::unique_ptr reader) : m_reader(std::move(re std::vector tmd_buffer(m_tmd_size); Read(m_tmd_offset, m_tmd_size, tmd_buffer.data()); m_tmd.SetBytes(std::move(tmd_buffer)); + + m_cert_chain.resize(m_cert_chain_size); + Read(m_cert_chain_offset, m_cert_chain_size, m_cert_chain.data()); } VolumeWAD::~VolumeWAD() @@ -109,6 +112,11 @@ const IOS::ES::TMDReader& VolumeWAD::GetTMD(const Partition& partition) const return m_tmd; } +const std::vector& VolumeWAD::GetCertificateChain(const Partition& partition) const +{ + return m_cert_chain; +} + std::string VolumeWAD::GetGameID(const Partition& partition) const { return m_tmd.GetGameID(); diff --git a/Source/Core/DiscIO/VolumeWad.h b/Source/Core/DiscIO/VolumeWad.h index 02276f7797..42da715b9e 100644 --- a/Source/Core/DiscIO/VolumeWad.h +++ b/Source/Core/DiscIO/VolumeWad.h @@ -36,6 +36,8 @@ public: const IOS::ES::TicketReader& GetTicket(const Partition& partition = PARTITION_NONE) const override; const IOS::ES::TMDReader& GetTMD(const Partition& partition = PARTITION_NONE) const override; + const std::vector& + GetCertificateChain(const Partition& partition = PARTITION_NONE) const override; std::string GetGameID(const Partition& partition = PARTITION_NONE) const override; std::string GetGameTDBID(const Partition& partition = PARTITION_NONE) const override; std::string GetMakerID(const Partition& partition = PARTITION_NONE) const override; @@ -63,11 +65,13 @@ private: std::unique_ptr m_reader; IOS::ES::TicketReader m_ticket; IOS::ES::TMDReader m_tmd; + std::vector m_cert_chain; + u32 m_cert_chain_offset = 0; u32 m_ticket_offset = 0; u32 m_tmd_offset = 0; u32 m_opening_bnr_offset = 0; u32 m_hdr_size = 0; - u32 m_cert_size = 0; + u32 m_cert_chain_size = 0; u32 m_ticket_size = 0; u32 m_tmd_size = 0; u32 m_data_size = 0; diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index f7261b0a61..f5b0a253cc 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -97,6 +97,18 @@ VolumeWii::VolumeWii(std::unique_ptr reader) return IOS::ES::TMDReader{std::move(tmd_buffer)}; }; + auto get_cert_chain = [this, partition]() -> std::vector { + const std::optional size = m_reader->ReadSwapped(partition.offset + 0x2ac); + const std::optional address = + ReadSwappedAndShifted(partition.offset + 0x2b0, PARTITION_NONE); + if (!size || !address) + return {}; + std::vector cert_chain(*size); + if (!m_reader->Read(partition.offset + *address, *size, cert_chain.data())) + return {}; + return cert_chain; + }; + auto get_key = [this, partition]() -> std::unique_ptr { const IOS::ES::TicketReader& ticket = *m_partitions[partition].ticket; if (!ticket.IsValid()) @@ -120,6 +132,7 @@ VolumeWii::VolumeWii(std::unique_ptr reader) partition, PartitionDetails{Common::Lazy>(get_key), Common::Lazy(get_ticket), Common::Lazy(get_tmd), + Common::Lazy>(get_cert_chain), Common::Lazy>(get_file_system), Common::Lazy(get_data_offset), *partition_type}); } @@ -239,6 +252,12 @@ const IOS::ES::TMDReader& VolumeWii::GetTMD(const Partition& partition) const return it != m_partitions.end() ? *it->second.tmd : INVALID_TMD; } +const std::vector& VolumeWii::GetCertificateChain(const Partition& partition) const +{ + auto it = m_partitions.find(partition); + return it != m_partitions.end() ? *it->second.cert_chain : INVALID_CERT_CHAIN; +} + const FileSystem* VolumeWii::GetFileSystem(const Partition& partition) const { auto it = m_partitions.find(partition); @@ -468,4 +487,4 @@ bool VolumeWii::CheckIntegrity(const Partition& partition) const return true; } -} // namespace +} // namespace DiscIO diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index e2b82c79f2..e6b7e4117d 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -40,6 +40,7 @@ public: std::optional GetTitleID(const Partition& partition) const override; const IOS::ES::TicketReader& GetTicket(const Partition& partition) const override; const IOS::ES::TMDReader& GetTMD(const Partition& partition) const override; + const std::vector& GetCertificateChain(const Partition& partition) const override; const FileSystem* GetFileSystem(const Partition& partition) const override; static u64 EncryptedPartitionOffsetToRawOffset(u64 offset, const Partition& partition, u64 partition_data_offset); @@ -78,6 +79,7 @@ private: Common::Lazy> key; Common::Lazy ticket; Common::Lazy tmd; + Common::Lazy> cert_chain; Common::Lazy> file_system; Common::Lazy data_offset; u32 type; From c885fed9da6f6b0c5862b99904bbb5edbba144fe Mon Sep 17 00:00:00 2001 From: JosJuice Date: Fri, 22 Mar 2019 20:47:05 +0100 Subject: [PATCH 05/10] DiscIO: Make Korean GC checks in Enums.cpp less fragile --- Source/Core/DiscIO/Enums.cpp | 50 +++++++++++++++++++++++++------- Source/Core/DiscIO/Enums.h | 7 +++-- Source/Core/DiscIO/VolumeGC.cpp | 5 ++-- Source/Core/DiscIO/VolumeWad.cpp | 5 ++-- Source/Core/DiscIO/VolumeWii.cpp | 5 ++-- Source/Core/DiscIO/VolumeWii.h | 14 ++++----- 6 files changed, 60 insertions(+), 26 deletions(-) diff --git a/Source/Core/DiscIO/Enums.cpp b/Source/Core/DiscIO/Enums.cpp index e46ca238d0..f3f1f3bf71 100644 --- a/Source/Core/DiscIO/Enums.cpp +++ b/Source/Core/DiscIO/Enums.cpp @@ -145,7 +145,8 @@ Country TypicalCountryForRegion(Region region) } } -Region CountryCodeToRegion(u8 country_code, Platform platform, Region expected_region) +Region CountryCodeToRegion(u8 country_code, Platform platform, Region expected_region, + std::optional revision) { switch (country_code) { @@ -159,11 +160,24 @@ Region CountryCodeToRegion(u8 country_code, Platform platform, Region expected_r return Region::NTSC_J; // Korean GC games in English or Taiwanese Wii games case 'E': - if (expected_region == Region::NTSC_J) - return Region::NTSC_J; // Korean GC games in English - else + if (platform != Platform::GameCubeDisc) return Region::NTSC_U; // The most common country code for NTSC-U + if (revision) + { + if (*revision >= 0x30) + return Region::NTSC_J; // Korean GC games in English + else + return Region::NTSC_U; // The most common country code for NTSC-U + } + else + { + if (expected_region == Region::NTSC_J) + return Region::NTSC_J; // Korean GC games in English + else + return Region::NTSC_U; // The most common country code for NTSC-U + } + case 'B': case 'N': return Region::NTSC_U; @@ -198,7 +212,8 @@ Region CountryCodeToRegion(u8 country_code, Platform platform, Region expected_r } } -Country CountryCodeToCountry(u8 country_code, Platform platform, Region region) +Country CountryCodeToCountry(u8 country_code, Platform platform, Region region, + std::optional revision) { switch (country_code) { @@ -214,10 +229,10 @@ Country CountryCodeToCountry(u8 country_code, Platform platform, Region region) return region == Region::NTSC_U ? Country::USA : Country::Europe; case 'W': - if (region == Region::PAL) - return Country::Europe; // Only the Nordic version of Ratatouille (Wii) - else if (platform == Platform::GameCubeDisc) + if (platform == Platform::GameCubeDisc) return Country::Korea; // GC games in English released in Korea + else if (region == Region::PAL) + return Country::Europe; // Only the Nordic version of Ratatouille (Wii) else return Country::Taiwan; // Wii games in traditional Chinese released in Taiwan @@ -251,11 +266,24 @@ Country CountryCodeToCountry(u8 country_code, Platform platform, Region region) // NTSC case 'E': - if (region == Region::NTSC_J) - return Country::Korea; // GC games in English released in Korea - else + if (platform != Platform::GameCubeDisc) return Country::USA; // The most common country code for NTSC-U + if (revision) + { + if (*revision >= 0x30) + return Country::Korea; // GC games in English released in Korea + else + return Country::USA; // The most common country code for NTSC-U + } + else + { + if (region == Region::NTSC_J) + return Country::Korea; // GC games in English released in Korea + else + return Country::USA; // The most common country code for NTSC-U + } + case 'B': // PAL games released on NTSC-U VC case 'N': // NTSC-J games released on NTSC-U VC return Country::USA; diff --git a/Source/Core/DiscIO/Enums.h b/Source/Core/DiscIO/Enums.h index e44de11ee7..020f608bfb 100644 --- a/Source/Core/DiscIO/Enums.h +++ b/Source/Core/DiscIO/Enums.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include "Common/CommonTypes.h" @@ -78,8 +79,10 @@ bool IsNTSC(Region region); Country TypicalCountryForRegion(Region region); // Avoid using this function if you can. Country codes aren't always reliable region indicators. Region CountryCodeToRegion(u8 country_code, Platform platform, - Region expected_region = Region::Unknown); -Country CountryCodeToCountry(u8 country_code, Platform platform, Region region = Region::Unknown); + Region expected_region = Region::Unknown, + std::optional revision = {}); +Country CountryCodeToCountry(u8 country_code, Platform platform, Region region = Region::Unknown, + std::optional revision = {}); Region GetSysMenuRegion(u16 title_version); std::string GetSysMenuVersionString(u16 title_version); diff --git a/Source/Core/DiscIO/VolumeGC.cpp b/Source/Core/DiscIO/VolumeGC.cpp index 67cf71072e..6d9a48cd0e 100644 --- a/Source/Core/DiscIO/VolumeGC.cpp +++ b/Source/Core/DiscIO/VolumeGC.cpp @@ -97,11 +97,12 @@ Country VolumeGC::GetCountry(const Partition& partition) const // The 0 that we use as a default value is mapped to Country::Unknown and Region::Unknown const u8 country = ReadSwapped(3, partition).value_or(0); const Region region = GetRegion(); + const std::optional revision = GetRevision(); - if (CountryCodeToRegion(country, Platform::GameCubeDisc, region) != region) + if (CountryCodeToRegion(country, Platform::GameCubeDisc, region, revision) != region) return TypicalCountryForRegion(region); - return CountryCodeToCountry(country, Platform::GameCubeDisc, region); + return CountryCodeToCountry(country, Platform::GameCubeDisc, region, revision); } std::string VolumeGC::GetMakerID(const Partition& partition) const diff --git a/Source/Core/DiscIO/VolumeWad.cpp b/Source/Core/DiscIO/VolumeWad.cpp index 94191c05c7..19f907e6f7 100644 --- a/Source/Core/DiscIO/VolumeWad.cpp +++ b/Source/Core/DiscIO/VolumeWad.cpp @@ -96,10 +96,11 @@ Country VolumeWAD::GetCountry(const Partition& partition) const return TypicalCountryForRegion(GetSysMenuRegion(m_tmd.GetTitleVersion())); const Region region = GetRegion(); - if (CountryCodeToRegion(country_byte, Platform::WiiWAD, region) != region) + const std::optional revision = GetRevision(); + if (CountryCodeToRegion(country_byte, Platform::WiiWAD, region, revision) != region) return TypicalCountryForRegion(region); - return CountryCodeToCountry(country_byte, Platform::WiiWAD, region); + return CountryCodeToCountry(country_byte, Platform::WiiWAD, region, revision); } const IOS::ES::TicketReader& VolumeWAD::GetTicket(const Partition& partition) const diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index f5b0a253cc..f589c69921 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -316,11 +316,12 @@ Country VolumeWii::GetCountry(const Partition& partition) const // The 0 that we use as a default value is mapped to Country::Unknown and Region::Unknown const u8 country_byte = ReadSwapped(3, partition).value_or(0); const Region region = GetRegion(); + const std::optional revision = GetRevision(); - if (CountryCodeToRegion(country_byte, Platform::WiiDisc, region) != region) + if (CountryCodeToRegion(country_byte, Platform::WiiDisc, region, revision) != region) return TypicalCountryForRegion(region); - return CountryCodeToCountry(country_byte, Platform::WiiDisc, region); + return CountryCodeToCountry(country_byte, Platform::WiiDisc, region, revision); } std::string VolumeWii::GetMakerID(const Partition& partition) const diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index e6b7e4117d..01df4f053f 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -45,22 +45,22 @@ public: static u64 EncryptedPartitionOffsetToRawOffset(u64 offset, const Partition& partition, u64 partition_data_offset); u64 PartitionOffsetToRawOffset(u64 offset, const Partition& partition) const override; - std::string GetGameID(const Partition& partition) const override; - std::string GetGameTDBID(const Partition& partition) const override; - std::string GetMakerID(const Partition& partition) const override; - std::optional GetRevision(const Partition& partition) const override; - std::string GetInternalName(const Partition& partition) const override; + std::string GetGameID(const Partition& partition = PARTITION_NONE) const override; + std::string GetGameTDBID(const Partition& partition = PARTITION_NONE) const override; + std::string GetMakerID(const Partition& partition = PARTITION_NONE) const override; + std::optional GetRevision(const Partition& partition = PARTITION_NONE) const override; + std::string GetInternalName(const Partition& partition = PARTITION_NONE) const override; std::map GetLongNames() const override; std::vector GetBanner(u32* width, u32* height) const override; std::string GetApploaderDate(const Partition& partition) const override; - std::optional GetDiscNumber(const Partition& partition) const override; + std::optional GetDiscNumber(const Partition& partition = PARTITION_NONE) const override; Platform GetVolumeType() const override; bool SupportsIntegrityCheck() const override { return true; } bool CheckIntegrity(const Partition& partition) const override; Region GetRegion() const override; - Country GetCountry(const Partition& partition) const override; + Country GetCountry(const Partition& partition = PARTITION_NONE) const override; BlobType GetBlobType() const override; u64 GetSize() const override; bool IsSizeAccurate() const override; From 84cbd5150f62bd5dd4a8d621fd0ccf7623ca4e2c Mon Sep 17 00:00:00 2001 From: JosJuice Date: Thu, 21 Mar 2019 23:04:56 +0100 Subject: [PATCH 06/10] Add a Verify tab to game properties --- Source/Core/Core/IOS/ES/ES.h | 41 +- Source/Core/Core/IOS/ES/Formats.cpp | 5 + Source/Core/Core/IOS/ES/Formats.h | 1 + Source/Core/DiscIO/CMakeLists.txt | 1 + Source/Core/DiscIO/DiscExtractor.cpp | 5 + Source/Core/DiscIO/DiscExtractor.h | 3 +- Source/Core/DiscIO/DiscIO.vcxproj | 2 + Source/Core/DiscIO/DiscIO.vcxproj.filters | 6 + Source/Core/DiscIO/VolumeVerifier.cpp | 689 ++++++++++++++++++ Source/Core/DiscIO/VolumeVerifier.h | 94 +++ Source/Core/DolphinQt/CMakeLists.txt | 1 + .../DolphinQt/Config/FilesystemWidget.cpp | 4 +- .../Core/DolphinQt/Config/FilesystemWidget.h | 7 +- .../DolphinQt/Config/PropertiesDialog.cpp | 23 +- Source/Core/DolphinQt/Config/VerifyWidget.cpp | 116 +++ Source/Core/DolphinQt/Config/VerifyWidget.h | 37 + Source/Core/DolphinQt/DolphinQt.vcxproj | 3 + 17 files changed, 1006 insertions(+), 32 deletions(-) create mode 100644 Source/Core/DiscIO/VolumeVerifier.cpp create mode 100644 Source/Core/DiscIO/VolumeVerifier.h create mode 100644 Source/Core/DolphinQt/Config/VerifyWidget.cpp create mode 100644 Source/Core/DolphinQt/Config/VerifyWidget.h diff --git a/Source/Core/Core/IOS/ES/ES.h b/Source/Core/Core/IOS/ES/ES.h index a6a5736937..89c47a90c7 100644 --- a/Source/Core/Core/IOS/ES/ES.h +++ b/Source/Core/Core/IOS/ES/ES.h @@ -140,6 +140,27 @@ public: bool CreateTitleDirectories(u64 title_id, u16 group_id) const; + enum class VerifyContainerType + { + TMD, + Ticket, + Device, + }; + enum class VerifyMode + { + // Whether or not new certificates should be added to the certificate store (/sys/cert.sys). + DoNotUpdateCertStore, + UpdateCertStore, + }; + // On success, if issuer_handle is non-null, the IOSC object for the issuer will be written to it. + // The caller is responsible for using IOSC_DeleteObject. + ReturnCode VerifyContainer(VerifyContainerType type, VerifyMode mode, + const IOS::ES::SignedBlobReader& signed_blob, + const std::vector& cert_chain, u32* issuer_handle = nullptr); + ReturnCode VerifyContainer(VerifyContainerType type, VerifyMode mode, + const IOS::ES::CertReader& certificate, + const std::vector& cert_chain, u32 certificate_iosc_handle); + private: enum { @@ -308,29 +329,9 @@ private: ReturnCode CheckStreamKeyPermissions(u32 uid, const u8* ticket_view, const IOS::ES::TMDReader& tmd) const; - enum class VerifyContainerType - { - TMD, - Ticket, - Device, - }; - enum class VerifyMode - { - // Whether or not new certificates should be added to the certificate store (/sys/cert.sys). - DoNotUpdateCertStore, - UpdateCertStore, - }; bool IsIssuerCorrect(VerifyContainerType type, const IOS::ES::CertReader& issuer_cert) const; ReturnCode ReadCertStore(std::vector* buffer) const; ReturnCode WriteNewCertToStore(const IOS::ES::CertReader& cert); - // On success, if issuer_handle is non-null, the IOSC object for the issuer will be written to it. - // The caller is responsible for using IOSC_DeleteObject. - ReturnCode VerifyContainer(VerifyContainerType type, VerifyMode mode, - const IOS::ES::SignedBlobReader& signed_blob, - const std::vector& cert_chain, u32* issuer_handle = nullptr); - ReturnCode VerifyContainer(VerifyContainerType type, VerifyMode mode, - const IOS::ES::CertReader& certificate, - const std::vector& cert_chain, u32 certificate_iosc_handle); // Start a title import. bool InitImport(const IOS::ES::TMDReader& tmd); diff --git a/Source/Core/Core/IOS/ES/Formats.cpp b/Source/Core/Core/IOS/ES/Formats.cpp index c4100a2fc6..15c042075b 100644 --- a/Source/Core/Core/IOS/ES/Formats.cpp +++ b/Source/Core/Core/IOS/ES/Formats.cpp @@ -441,6 +441,11 @@ u64 TicketReader::GetTitleId() const return Common::swap64(m_bytes.data() + offsetof(Ticket, title_id)); } +u8 TicketReader::GetCommonKeyIndex() const +{ + return m_bytes[offsetof(Ticket, common_key_index)]; +} + std::array TicketReader::GetTitleKey(const HLE::IOSC& iosc) const { u8 iv[16] = {}; diff --git a/Source/Core/Core/IOS/ES/Formats.h b/Source/Core/Core/IOS/ES/Formats.h index 21950c072b..c9ad6a142e 100644 --- a/Source/Core/Core/IOS/ES/Formats.h +++ b/Source/Core/Core/IOS/ES/Formats.h @@ -240,6 +240,7 @@ public: u32 GetDeviceId() const; u64 GetTitleId() const; + u8 GetCommonKeyIndex() const; // Get the decrypted title key. std::array GetTitleKey(const HLE::IOSC& iosc) const; // Same as the above version, but guesses the console type depending on the issuer diff --git a/Source/Core/DiscIO/CMakeLists.txt b/Source/Core/DiscIO/CMakeLists.txt index 5dfbc5fd59..78d603acbc 100644 --- a/Source/Core/DiscIO/CMakeLists.txt +++ b/Source/Core/DiscIO/CMakeLists.txt @@ -16,6 +16,7 @@ add_library(discio Volume.cpp VolumeFileBlobReader.cpp VolumeGC.cpp + VolumeVerifier.cpp VolumeWad.cpp VolumeWii.cpp WiiSaveBanner.cpp diff --git a/Source/Core/DiscIO/DiscExtractor.cpp b/Source/Core/DiscIO/DiscExtractor.cpp index 34c55a1d73..057a22378d 100644 --- a/Source/Core/DiscIO/DiscExtractor.cpp +++ b/Source/Core/DiscIO/DiscExtractor.cpp @@ -29,6 +29,11 @@ std::string NameForPartitionType(u32 partition_type, bool include_prefix) return "UPDATE"; case PARTITION_CHANNEL: return "CHANNEL"; + case PARTITION_INSTALL: + // wit doesn't recognize the name "INSTALL", so we can't use it when naming partition folders + if (!include_prefix) + return "INSTALL"; + // [[fallthrough]] default: const std::string type_as_game_id{static_cast((partition_type >> 24) & 0xFF), static_cast((partition_type >> 16) & 0xFF), diff --git a/Source/Core/DiscIO/DiscExtractor.h b/Source/Core/DiscIO/DiscExtractor.h index 39af10d1a3..16b8f932fd 100644 --- a/Source/Core/DiscIO/DiscExtractor.h +++ b/Source/Core/DiscIO/DiscExtractor.h @@ -17,7 +17,8 @@ class Volume; constexpr u32 PARTITION_DATA = 0; constexpr u32 PARTITION_UPDATE = 1; -constexpr u32 PARTITION_CHANNEL = 2; +constexpr u32 PARTITION_CHANNEL = 2; // Mario Kart Wii, Wii Fit, Wii Fit Plus, Rabbids Go Home +constexpr u32 PARTITION_INSTALL = 3; // Dragon Quest X only std::string NameForPartitionType(u32 partition_type, bool include_prefix); diff --git a/Source/Core/DiscIO/DiscIO.vcxproj b/Source/Core/DiscIO/DiscIO.vcxproj index 47f34130a1..830834bd48 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj +++ b/Source/Core/DiscIO/DiscIO.vcxproj @@ -52,6 +52,7 @@ + @@ -75,6 +76,7 @@ + diff --git a/Source/Core/DiscIO/DiscIO.vcxproj.filters b/Source/Core/DiscIO/DiscIO.vcxproj.filters index 4f64f4e33b..c6240a5b6e 100644 --- a/Source/Core/DiscIO/DiscIO.vcxproj.filters +++ b/Source/Core/DiscIO/DiscIO.vcxproj.filters @@ -84,6 +84,9 @@ NAND + + Volume + @@ -149,6 +152,9 @@ NAND + + Volume + diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp new file mode 100644 index 0000000000..7d3eba4c8f --- /dev/null +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -0,0 +1,689 @@ +// Copyright 2019 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DiscIO/VolumeVerifier.h" + +#include +#include +#include +#include +#include +#include + +#include "Common/Align.h" +#include "Common/Assert.h" +#include "Common/CommonTypes.h" +#include "Common/MsgHandler.h" +#include "Common/StringUtil.h" +#include "Common/Swap.h" +#include "Core/IOS/Device.h" +#include "Core/IOS/ES/ES.h" +#include "Core/IOS/ES/Formats.h" +#include "Core/IOS/IOS.h" +#include "Core/IOS/IOSC.h" +#include "DiscIO/Blob.h" +#include "DiscIO/DiscExtractor.h" +#include "DiscIO/Enums.h" +#include "DiscIO/Filesystem.h" +#include "DiscIO/Volume.h" +#include "DiscIO/VolumeWii.h" + +namespace DiscIO +{ +constexpr u64 MINI_DVD_SIZE = 1459978240; // GameCube +constexpr u64 SL_DVD_SIZE = 4699979776; // Wii retail +constexpr u64 SL_DVD_R_SIZE = 4707319808; // Wii RVT-R +constexpr u64 DL_DVD_SIZE = 8511160320; // Wii retail +constexpr u64 DL_DVD_R_SIZE = 8543666176; // Wii RVT-R + +constexpr u64 BLOCK_SIZE = 0x20000; + +VolumeVerifier::VolumeVerifier(const Volume& volume) + : m_volume(volume), m_started(false), m_done(false), m_progress(0), + m_max_progress(volume.GetSize()) +{ +} + +void VolumeVerifier::Start() +{ + ASSERT(!m_started); + m_started = true; + + m_is_tgc = m_volume.GetBlobType() == BlobType::TGC; + m_is_datel = IsDisc(m_volume.GetVolumeType()) && + !GetBootDOLOffset(m_volume, m_volume.GetGamePartition()).has_value(); + m_is_not_retail = + (m_volume.GetVolumeType() == Platform::WiiDisc && !m_volume.IsEncryptedAndHashed()) || + IsDebugSigned(); + + CheckPartitions(); + if (m_volume.GetVolumeType() == Platform::WiiWAD) + CheckCorrectlySigned(PARTITION_NONE, GetStringT("This title is not correctly signed.")); + CheckDiscSize(); + CheckMisc(); +} + +void VolumeVerifier::CheckPartitions() +{ + const std::vector partitions = m_volume.GetPartitions(); + if (partitions.empty()) + { + if (m_volume.GetVolumeType() != Platform::WiiWAD && + !m_volume.GetFileSystem(m_volume.GetGamePartition())) + { + AddProblem(Severity::High, GetStringT("The filesystem is invalid or could not be read.")); + } + return; + } + + std::optional partitions_in_first_table = m_volume.ReadSwapped(0x40000, PARTITION_NONE); + if (partitions_in_first_table && *partitions_in_first_table > 8) + { + // Not sure if 8 actually is the limit, but there certainly aren't any discs + // released that have as many partitions as 8 in the first partition table. + // The only game that has that many partitions in total is Super Smash Bros. Brawl, + // and that game places all partitions other than UPDATE and DATA in the second table. + AddProblem(Severity::Low, + GetStringT("There are too many partitions in the first partition table.")); + } + + std::vector types; + for (const Partition& partition : partitions) + { + const std::optional type = m_volume.GetPartitionType(partition); + if (type) + types.emplace_back(*type); + } + + if (std::find(types.cbegin(), types.cend(), PARTITION_UPDATE) == types.cend()) + AddProblem(Severity::Low, GetStringT("The update partition is missing.")); + + if (std::find(types.cbegin(), types.cend(), PARTITION_DATA) == types.cend()) + AddProblem(Severity::High, GetStringT("The data partition is missing.")); + + const bool has_channel_partition = + std::find(types.cbegin(), types.cend(), PARTITION_CHANNEL) != types.cend(); + if (ShouldHaveChannelPartition() && !has_channel_partition) + AddProblem(Severity::Medium, GetStringT("The channel partition is missing.")); + + const bool has_install_partition = + std::find(types.cbegin(), types.cend(), PARTITION_INSTALL) != types.cend(); + if (ShouldHaveInstallPartition() && !has_install_partition) + AddProblem(Severity::High, GetStringT("The install partition is missing.")); + + if (ShouldHaveMasterpiecePartitions() && + types.cend() == + std::find_if(types.cbegin(), types.cend(), [](u32 type) { return type >= 0xFF; })) + { + // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces + // where you play demos of NES/SNES/N64 games. Official translations: + // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre + // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean). + // If your language is not one of the languages above, consider leaving the string untranslated + // so that people will recognize it as the name of the game mode. + AddProblem(Severity::Medium, GetStringT("The Masterpiece partitions are missing.")); + } + + for (const Partition& partition : partitions) + { + if (m_volume.GetPartitionType(partition) == PARTITION_UPDATE && partition.offset != 0x50000) + { + AddProblem(Severity::Low, GetStringT("The update partition is not at its normal position.")); + } + + const u64 normal_data_offset = m_volume.IsEncryptedAndHashed() ? 0xF800000 : 0x838000; + if (m_volume.GetPartitionType(partition) == PARTITION_DATA && + partition.offset != normal_data_offset && !has_channel_partition && !has_install_partition) + { + AddProblem( + Severity::Low, + GetStringT("The data partition is not at its normal position. This will affect the " + "emulated loading times. When using NetPlay or sending input recordings to " + "other people, you will experience desyncs if anyone is using a good dump.")); + } + } + + for (const Partition& partition : partitions) + CheckPartition(partition); +} + +bool VolumeVerifier::CheckPartition(const Partition& partition) +{ + std::optional type = m_volume.GetPartitionType(partition); + if (!type) + { + // Not sure if this can happen in practice + AddProblem(Severity::Medium, GetStringT("The type of a partition could not be read.")); + return false; + } + + Severity severity = Severity::Medium; + if (*type == PARTITION_DATA || *type == PARTITION_INSTALL) + severity = Severity::High; + else if (*type == PARTITION_UPDATE) + severity = Severity::Low; + + std::string name = NameForPartitionType(*type, false); + if (ShouldHaveMasterpiecePartitions() && *type > 0xFF) + { + // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces + // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo + // rather than the game mode as a whole, so please use the singular form. Official translations: + // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre + // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean). + // If your language is not one of the languages above, consider leaving the string untranslated + // so that people will recognize it as the name of the game mode. + name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str()); + } + + if (partition.offset % VolumeWii::BLOCK_TOTAL_SIZE != 0 || + m_volume.PartitionOffsetToRawOffset(0, partition) % VolumeWii::BLOCK_TOTAL_SIZE != 0) + { + AddProblem(Severity::Medium, + StringFromFormat(GetStringT("The %s partition is not properly aligned.").c_str(), + name.c_str())); + } + + CheckCorrectlySigned( + partition, StringFromFormat(GetStringT("The %s partition is not correctly signed.").c_str(), + name.c_str())); + + bool invalid_disc_header = false; + std::vector disc_header(0x80); + constexpr u32 WII_MAGIC = 0x5D1C9EA3; + if (!m_volume.Read(0, disc_header.size(), disc_header.data(), partition)) + { + invalid_disc_header = true; + } + else if (Common::swap32(disc_header.data() + 0x18) != WII_MAGIC) + { + for (size_t i = 0; i < disc_header.size(); i += 4) + { + if (Common::swap32(disc_header.data() + i) != i) + { + invalid_disc_header = true; + break; + } + } + + // The loop above ends without breaking for discs that legitimately lack updates. + // No such discs have been released to end users. Most such discs are debug signed, + // but there is apparently at least one that is retail signed, the Movie-Ch Install Disc. + return false; + } + if (invalid_disc_header) + { + // This can happen when certain programs that create WBFS files scrub the entirety of + // the Masterpiece partitions in Super Smash Bros. Brawl without removing them from + // the partition table. https://bugs.dolphin-emu.org/issues/8733 + const std::string text = StringFromFormat( + GetStringT("The %s partition does not seem to contain valid data.").c_str(), name.c_str()); + AddProblem(severity, text); + return false; + } + + const DiscIO::FileSystem* filesystem = m_volume.GetFileSystem(partition); + if (!filesystem) + { + const std::string text = StringFromFormat( + GetStringT("The %s partition does not have a valid file system.").c_str(), name.c_str()); + AddProblem(severity, text); + return false; + } + + if (type == PARTITION_UPDATE) + { + std::unique_ptr file_info = filesystem->FindFileInfo("_sys"); + bool has_correct_ios = false; + if (file_info) + { + const IOS::ES::TMDReader& tmd = m_volume.GetTMD(m_volume.GetGamePartition()); + if (tmd.IsValid()) + { + const std::string correct_ios = "IOS" + std::to_string(tmd.GetIOSId() & 0xFF) + "-"; + for (const FileInfo& f : *file_info) + { + if (StringBeginsWith(f.GetName(), correct_ios)) + { + has_correct_ios = true; + break; + } + } + } + } + + if (!has_correct_ios) + { + // This is reached for hacked dumps where the update partition has been replaced with + // a very old update partition so that no updates will be installed. + AddProblem(Severity::Low, + GetStringT("The update partition does not contain the IOS used by this title.")); + } + } + + return true; +} + +void VolumeVerifier::CheckCorrectlySigned(const Partition& partition, const std::string& error_text) +{ + IOS::HLE::Kernel ios; + const auto es = ios.GetES(); + const std::vector cert_chain = m_volume.GetCertificateChain(partition); + + if (IOS::HLE::IPC_SUCCESS != + es->VerifyContainer(IOS::HLE::Device::ES::VerifyContainerType::Ticket, + IOS::HLE::Device::ES::VerifyMode::DoNotUpdateCertStore, + m_volume.GetTicket(partition), cert_chain) || + IOS::HLE::IPC_SUCCESS != + es->VerifyContainer(IOS::HLE::Device::ES::VerifyContainerType::TMD, + IOS::HLE::Device::ES::VerifyMode::DoNotUpdateCertStore, + m_volume.GetTMD(partition), cert_chain)) + { + AddProblem(Severity::Low, error_text); + } +} + +bool VolumeVerifier::IsDebugSigned() const +{ + const IOS::ES::TicketReader& ticket = m_volume.GetTicket(m_volume.GetGamePartition()); + return ticket.IsValid() ? ticket.GetConsoleType() == IOS::HLE::IOSC::ConsoleType::RVT : false; +} + +bool VolumeVerifier::ShouldHaveChannelPartition() const +{ + const std::unordered_set channel_discs{ + "RFNE01", "RFNJ01", "RFNK01", "RFNP01", "RFNW01", "RFPE01", "RFPJ01", "RFPK01", "RFPP01", + "RFPW01", "RGWE41", "RGWJ41", "RGWP41", "RGWX41", "RMCE01", "RMCJ01", "RMCK01", "RMCP01", + }; + + return channel_discs.find(m_volume.GetGameID()) != channel_discs.end(); +} + +bool VolumeVerifier::ShouldHaveInstallPartition() const +{ + const std::unordered_set dragon_quest_x{"S4MJGD", "S4SJGD", "S6TJGD", "SDQJGD"}; + return dragon_quest_x.find(m_volume.GetGameID()) != dragon_quest_x.end(); +} + +bool VolumeVerifier::ShouldHaveMasterpiecePartitions() const +{ + const std::unordered_set ssbb{"RSBE01", "RSBJ01", "RSBK01", "RSBP01"}; + return ssbb.find(m_volume.GetGameID()) != ssbb.end(); +} + +bool VolumeVerifier::ShouldBeDualLayer() const +{ + // The Japanese versions of Xenoblade and The Last Story are single-layer + // (unlike the other versions) and must not be added to this list. + const std::unordered_set dual_layer_discs{ + "R3ME01", "R3MP01", "R3OE01", "R3OJ01", "R3OP01", "RSBE01", "RSBJ01", "RSBK01", "RSBP01", + "RXMJ8P", "S59E01", "S59JC8", "S59P01", "S5QJC8", "SK8X52", "SAKENS", "SAKPNS", "SK8V52", + "SK8X52", "SLSEXJ", "SLSP01", "SQIE4Q", "SQIP4Q", "SQIY4Q", "SR5E41", "SR5P41", "SUOE41", + "SUOP41", "SVXX52", "SVXY52", "SX4E01", "SX4P01", "SZ3EGT", "SZ3PGT", + }; + + return dual_layer_discs.find(m_volume.GetGameID()) != dual_layer_discs.end(); +} + +void VolumeVerifier::CheckDiscSize() +{ + if (!IsDisc(m_volume.GetVolumeType())) + return; + + const u64 biggest_offset = GetBiggestUsedOffset(); + if (biggest_offset > m_volume.GetSize()) + { + const bool second_layer_missing = + biggest_offset > SL_DVD_SIZE && m_volume.GetSize() >= SL_DVD_SIZE; + const std::string text = + second_layer_missing ? + GetStringT( + "This disc image is too small and lacks some data. The problem is most likely that " + "this is a dual-layer disc that has been dumped as a single-layer disc.") : + GetStringT( + "This disc image is too small and lacks some data. If your dumping program saved " + "the disc image as several parts, you need to merge them into one file."); + AddProblem(Severity::High, text); + return; + } + + if (ShouldBeDualLayer() && biggest_offset <= SL_DVD_R_SIZE) + { + AddProblem( + Severity::Medium, + GetStringT("This game has been hacked to fit on a single-layer DVD. Some content such as " + "pre-rendered videos, extra languages or entire game modes will be broken. " + "This problem generally only exists in illegal copies of games.")); + } + + if (!m_volume.IsSizeAccurate()) + { + AddProblem(Severity::Low, GetStringT("The format that the disc image is saved in does not " + "store the size of the disc image.")); + } + else if (!m_is_tgc) + { + const Platform platform = m_volume.GetVolumeType(); + const u64 size = m_volume.GetSize(); + + const bool valid_gamecube = size == MINI_DVD_SIZE; + const bool valid_retail_wii = size == SL_DVD_SIZE || size == DL_DVD_SIZE; + const bool valid_debug_wii = size == SL_DVD_R_SIZE || size == DL_DVD_R_SIZE; + + const bool debug = IsDebugSigned(); + if ((platform == Platform::GameCubeDisc && !valid_gamecube) || + (platform == Platform::WiiDisc && (debug ? !valid_debug_wii : !valid_retail_wii))) + { + if (debug && valid_retail_wii) + { + AddProblem(Severity::Low, + GetStringT("This debug disc image has the size of a retail disc image.")); + } + else + { + const bool small = + (m_volume.GetVolumeType() == Platform::GameCubeDisc && size < MINI_DVD_SIZE) || + (m_volume.GetVolumeType() == Platform::WiiDisc && size < SL_DVD_SIZE); + + if (small) + { + AddProblem(Severity::Low, + GetStringT("This disc image has an unusual size. This will likely make the " + "emulated loading times longer. When using NetPlay or sending " + "input recordings to other people, you will likely experience " + "desyncs if anyone is using a good dump.")); + } + else + { + AddProblem(Severity::Low, GetStringT("This disc image has an unusual size.")); + } + } + } + } +} + +u64 VolumeVerifier::GetBiggestUsedOffset() +{ + std::vector partitions = m_volume.GetPartitions(); + if (partitions.empty()) + partitions.emplace_back(m_volume.GetGamePartition()); + + const u64 disc_header_size = m_volume.GetVolumeType() == Platform::GameCubeDisc ? 0x460 : 0x50000; + u64 biggest_offset = disc_header_size; + for (const Partition& partition : partitions) + { + if (partition != PARTITION_NONE) + { + const u64 offset = m_volume.PartitionOffsetToRawOffset(0x440, partition); + biggest_offset = std::max(biggest_offset, offset); + } + + const std::optional dol_offset = GetBootDOLOffset(m_volume, partition); + if (dol_offset) + { + const std::optional dol_size = GetBootDOLSize(m_volume, partition, *dol_offset); + if (dol_size) + { + const u64 offset = m_volume.PartitionOffsetToRawOffset(*dol_offset + *dol_size, partition); + biggest_offset = std::max(biggest_offset, offset); + } + } + + const std::optional fst_offset = GetFSTOffset(m_volume, partition); + const std::optional fst_size = GetFSTSize(m_volume, partition); + if (fst_offset && fst_size) + { + const u64 offset = m_volume.PartitionOffsetToRawOffset(*fst_offset + *fst_size, partition); + biggest_offset = std::max(biggest_offset, offset); + } + + const FileSystem* fs = m_volume.GetFileSystem(partition); + if (fs) + { + const u64 offset = + m_volume.PartitionOffsetToRawOffset(GetBiggestUsedOffset(fs->GetRoot()), partition); + biggest_offset = std::max(biggest_offset, offset); + } + } + return biggest_offset; +} + +u64 VolumeVerifier::GetBiggestUsedOffset(const FileInfo& file_info) const +{ + if (file_info.IsDirectory()) + { + u64 biggest_offset = 0; + for (const FileInfo& f : file_info) + biggest_offset = std::max(biggest_offset, GetBiggestUsedOffset(f)); + return biggest_offset; + } + else + { + return file_info.GetOffset() + file_info.GetSize(); + } +} + +void VolumeVerifier::CheckMisc() +{ + const std::string game_id_unencrypted = m_volume.GetGameID(PARTITION_NONE); + const std::string game_id_encrypted = m_volume.GetGameID(m_volume.GetGamePartition()); + + if (game_id_unencrypted != game_id_encrypted) + { + bool inconsistent_game_id = true; + if (game_id_encrypted == "RELSAB") + { + if (StringBeginsWith(game_id_unencrypted, "410")) + { + // This is the Wii Backup Disc (aka "pinkfish" disc), + // which legitimately has an inconsistent game ID. + inconsistent_game_id = false; + } + else if (StringBeginsWith(game_id_unencrypted, "010")) + { + // Hacked version of the Wii Backup Disc (aka "pinkfish" disc). + std::string proper_game_id = game_id_unencrypted; + proper_game_id[0] = '4'; + AddProblem(Severity::Low, + StringFromFormat(GetStringT("The game ID is %s but should be %s.").c_str(), + game_id_unencrypted.c_str(), proper_game_id.c_str())); + inconsistent_game_id = false; + } + } + + if (inconsistent_game_id) + { + AddProblem(Severity::Low, GetStringT("The game ID is inconsistent.")); + } + } + + const Region region = m_volume.GetRegion(); + const Platform platform = m_volume.GetVolumeType(); + + if (game_id_encrypted.size() < 4) + { + AddProblem(Severity::Low, GetStringT("The game ID is unusually short.")); + } + else + { + const char country_code = game_id_encrypted[3]; + if (CountryCodeToRegion(country_code, platform, region) != region) + { + AddProblem( + Severity::Medium, + GetStringT("The region code does not match the game ID. If this is because the " + "region code has been modified, the game might run at the wrong speed, " + "graphical elements might be offset, or the game might not run at all.")); + } + } + + const IOS::ES::TMDReader& tmd = m_volume.GetTMD(m_volume.GetGamePartition()); + if (tmd.IsValid()) + { + const u64 ios_id = tmd.GetIOSId() & 0xFF; + + // List of launch day Korean IOSes obtained from https://hackmii.com/2008/09/korean-wii/. + // More IOSes were released later that were used in Korean games, but they're all over 40. + // Also, old IOSes like IOS36 did eventually get released for Korean Wiis as part of system + // updates, but there are likely no Korean games using them since those IOSes were old by then. + if (region == Region::NTSC_K && ios_id < 40 && ios_id != 4 && ios_id != 9 && ios_id != 21 && + ios_id != 37) + { + // This is intended to catch pirated Korean games that have had the IOS slot set to 36 + // as a side effect of having to fakesign after changing the common key slot to 0. + // (IOS36 was the last IOS to have the Trucha bug.) https://bugs.dolphin-emu.org/issues/10319 + AddProblem(Severity::High, + // i18n: You may want to leave the term "ERROR #002" untranslated, + // since the emulated software always displays it in English. + GetStringT("This Korean title is set to use an IOS that typically isn't used on " + "Korean consoles. This is likely to lead to ERROR #002.")); + } + + if (ios_id >= 0x80) + { + // This is also intended to catch fakesigned pirated Korean games, + // but this time with the IOS slot set to cIOS instead of IOS36. + AddProblem(Severity::High, GetStringT("This title is set to use an invalid IOS.")); + } + } + + const IOS::ES::TicketReader& ticket = m_volume.GetTicket(m_volume.GetGamePartition()); + if (ticket.IsValid()) + { + const u8 common_key = ticket.GetCommonKeyIndex(); + + if (common_key > 1) + { + // Many fakesigned WADs have the common key index set to a (random?) bogus value. + // For WADs, Dolphin will detect this and use common key 0 instead, making this low severity. + const Severity severity = + m_volume.GetVolumeType() == Platform::WiiWAD ? Severity::Low : Severity::High; + // i18n: This is "common" as in "shared", not the opposite of "uncommon" + AddProblem(Severity::Low, GetStringT("This title is set to use an invalid common key.")); + } + + if (common_key == 1 && region != Region::NTSC_K) + { + // Apparently a certain pirate WAD of Chronos Twins DX unluckily got an index of 1, + // which Dolphin does not change to 0 because 1 is valid on Korean Wiis. + // https://forums.dolphin-emu.org/Thread-wiiware-chronos-twins-dx + AddProblem(Severity::High, + // i18n: This is "common" as in "shared", not the opposite of "uncommon" + GetStringT("This non-Korean title is set to use the Korean common key.")); + } + } + + if (IsDisc(m_volume.GetVolumeType())) + { + constexpr u32 NKIT_MAGIC = 0x4E4B4954; // "NKIT" + if (m_volume.ReadSwapped(0x200, PARTITION_NONE) == NKIT_MAGIC) + { + AddProblem(Severity::Low, + GetStringT("This disc image is in the NKit format. It is not a good dump in its " + "current form, but it might become a good dump if converted back. " + "The CRC32 of this file might match the CRC32 of a good dump even " + "though the files are not identical.")); + } + } +} + +void VolumeVerifier::Process() +{ + ASSERT(m_started); + ASSERT(!m_done); + + if (m_progress == m_max_progress) + return; + + m_progress += std::min(m_max_progress - m_progress, BLOCK_SIZE); +} + +u64 VolumeVerifier::GetBytesProcessed() const +{ + return m_progress; +} + +u64 VolumeVerifier::GetTotalBytes() const +{ + return m_max_progress; +} + +void VolumeVerifier::Finish() +{ + if (m_done) + return; + m_done = true; + + // Show the most serious problems at the top + std::stable_sort(m_result.problems.begin(), m_result.problems.end(), + [](const Problem& p1, const Problem& p2) { return p1.severity > p2.severity; }); + const Severity highest_severity = + m_result.problems.empty() ? Severity::None : m_result.problems[0].severity; + + if (m_is_datel) + { + m_result.summary_text = GetStringT("Dolphin is unable to verify unlicensed discs."); + return; + } + + if (m_is_tgc) + { + m_result.summary_text = GetStringT("Dolphin is unable to verify typical TGC files properly, " + "since they are not dumps of actual discs."); + return; + } + + switch (highest_severity) + { + case Severity::None: + if (IsWii(m_volume.GetVolumeType()) && !m_is_not_retail) + { + m_result.summary_text = + GetStringT("No problems were found. This does not guarantee that this is a good dump, " + "but since Wii titles contain a lot of verification data, it does mean that " + "there most likely are no problems that will affect emulation."); + } + else + { + m_result.summary_text = GetStringT("No problems were found."); + } + break; + case Severity::Low: + m_result.summary_text = GetStringT("Problems with low severity were found. They will most " + "likely not prevent the game from running."); + break; + case Severity::Medium: + m_result.summary_text = GetStringT("Problems with medium severity were found. The whole game " + "or certain parts of the game might not work correctly."); + break; + case Severity::High: + m_result.summary_text = GetStringT( + "Problems with high severity were found. The game will most likely not work at all."); + break; + } + + if (m_volume.GetVolumeType() == Platform::GameCubeDisc) + { + m_result.summary_text += + GetStringT("\n\nBecause GameCube disc images contain little verification data, " + "there may be problems that Dolphin is unable to detect."); + } + else if (m_is_not_retail) + { + m_result.summary_text += GetStringT("\n\nBecause this title is not for retail Wii consoles, " + "Dolphin cannot verify that it hasn't been tampered with."); + } +} + +const VolumeVerifier::Result& VolumeVerifier::GetResult() const +{ + return m_result; +} + +void VolumeVerifier::AddProblem(Severity severity, const std::string& text) +{ + m_result.problems.emplace_back(Problem{severity, text}); +} + +} // namespace DiscIO diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h new file mode 100644 index 0000000000..8503c1f89d --- /dev/null +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -0,0 +1,94 @@ +// Copyright 2019 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include "Common/CommonTypes.h" +#include "DiscIO/Volume.h" + +// To be used as follows: +// +// VolumeVerifier verifier(volume); +// verifier.Start(); +// while (verifier.GetBytesProcessed() != verifier.GetTotalBytes()) +// verifier.Process(); +// verifier.Finish(); +// auto result = verifier.GetResult(); +// +// Start, Process and Finish may take some time to run. +// +// GetResult() can be called before the processing is finished, but the result will be incomplete. + +namespace IOS::ES +{ +class SignedBlobReader; +} + +namespace DiscIO +{ +class FileInfo; + +class VolumeVerifier final +{ +public: + enum class Severity + { + None, // Only used internally + Low, + Medium, + High, + }; + + struct Problem + { + Severity severity; + std::string text; + }; + + struct Result + { + std::string summary_text; + std::vector problems; + }; + + VolumeVerifier(const Volume& volume); + void Start(); + void Process(); + u64 GetBytesProcessed() const; + u64 GetTotalBytes() const; + void Finish(); + const Result& GetResult() const; + +private: + void CheckPartitions(); + bool CheckPartition(const Partition& partition); // Returns false if partition should be ignored + void CheckCorrectlySigned(const Partition& partition, const std::string& error_text); + bool IsDebugSigned() const; + bool ShouldHaveChannelPartition() const; + bool ShouldHaveInstallPartition() const; + bool ShouldHaveMasterpiecePartitions() const; + bool ShouldBeDualLayer() const; + void CheckDiscSize(); + u64 GetBiggestUsedOffset(); + u64 GetBiggestUsedOffset(const FileInfo& file_info) const; + void CheckMisc(); + + void AddProblem(Severity severity, const std::string& text); + + const Volume& m_volume; + Result m_result; + bool m_is_tgc; + bool m_is_datel; + bool m_is_not_retail; + + bool m_started; + bool m_done; + u64 m_progress; + u64 m_max_progress; +}; + +} // namespace DiscIO diff --git a/Source/Core/DolphinQt/CMakeLists.txt b/Source/Core/DolphinQt/CMakeLists.txt index 85bc526ae6..062626a72c 100644 --- a/Source/Core/DolphinQt/CMakeLists.txt +++ b/Source/Core/DolphinQt/CMakeLists.txt @@ -78,6 +78,7 @@ add_executable(dolphin-emu Config/PatchesWidget.cpp Config/PropertiesDialog.cpp Config/SettingsWindow.cpp + Config/VerifyWidget.cpp Debugger/BreakpointWidget.cpp Debugger/CodeViewWidget.cpp Debugger/CodeWidget.cpp diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp index 662091581c..e407ed9945 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp @@ -40,8 +40,8 @@ enum class EntryType }; Q_DECLARE_METATYPE(EntryType); -FilesystemWidget::FilesystemWidget(const UICommon::GameFile& game) - : m_game(game), m_volume(DiscIO::CreateVolumeFromFilename(game.GetFilePath())) +FilesystemWidget::FilesystemWidget(std::shared_ptr volume) + : m_volume(std::move(volume)) { CreateWidgets(); ConnectWidgets(); diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.h b/Source/Core/DolphinQt/Config/FilesystemWidget.h index b5b740be78..c5b14adad9 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.h +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.h @@ -8,8 +8,6 @@ #include #include -#include "UICommon/GameFile.h" - class QStandardItem; class QStandardItemModel; class QTreeView; @@ -26,7 +24,7 @@ class FilesystemWidget final : public QWidget { Q_OBJECT public: - explicit FilesystemWidget(const UICommon::GameFile& game); + explicit FilesystemWidget(std::shared_ptr volume); ~FilesystemWidget() override; private: @@ -52,8 +50,7 @@ private: QStandardItemModel* m_tree_model; QTreeView* m_tree_view; - UICommon::GameFile m_game; - std::unique_ptr m_volume; + std::shared_ptr m_volume; QIcon m_folder_icon; QIcon m_file_icon; diff --git a/Source/Core/DolphinQt/Config/PropertiesDialog.cpp b/Source/Core/DolphinQt/Config/PropertiesDialog.cpp index 6a29c9ffee..6aae9bef5d 100644 --- a/Source/Core/DolphinQt/Config/PropertiesDialog.cpp +++ b/Source/Core/DolphinQt/Config/PropertiesDialog.cpp @@ -2,12 +2,15 @@ // Licensed under GPLv2+ // Refer to the license.txt file included. +#include + #include #include #include #include #include "DiscIO/Enums.h" +#include "DiscIO/Volume.h" #include "DolphinQt/Config/ARCodeWidget.h" #include "DolphinQt/Config/FilesystemWidget.h" @@ -16,6 +19,7 @@ #include "DolphinQt/Config/InfoWidget.h" #include "DolphinQt/Config/PatchesWidget.h" #include "DolphinQt/Config/PropertiesDialog.h" +#include "DolphinQt/Config/VerifyWidget.h" #include "DolphinQt/QtUtils/WrapInScrollArea.h" #include "UICommon/GameFile.h" @@ -54,11 +58,22 @@ PropertiesDialog::PropertiesDialog(QWidget* parent, const UICommon::GameFile& ga tr("Gecko Codes")); tab_widget->addTab(GetWrappedWidget(info, this, padding_width, padding_height), tr("Info")); - if (DiscIO::IsDisc(game.GetPlatform())) + if (game.GetPlatform() != DiscIO::Platform::ELFOrDOL) { - FilesystemWidget* filesystem = new FilesystemWidget(game); - tab_widget->addTab(GetWrappedWidget(filesystem, this, padding_width, padding_height), - tr("Filesystem")); + std::shared_ptr volume = DiscIO::CreateVolumeFromFilename(game.GetFilePath()); + if (volume) + { + VerifyWidget* verify = new VerifyWidget(volume); + tab_widget->addTab(GetWrappedWidget(verify, this, padding_width, padding_height), + tr("Verify")); + + if (DiscIO::IsDisc(game.GetPlatform())) + { + FilesystemWidget* filesystem = new FilesystemWidget(volume); + tab_widget->addTab(GetWrappedWidget(filesystem, this, padding_width, padding_height), + tr("Filesystem")); + } + } } layout->addWidget(tab_widget); diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.cpp b/Source/Core/DolphinQt/Config/VerifyWidget.cpp new file mode 100644 index 0000000000..b857d2e88c --- /dev/null +++ b/Source/Core/DolphinQt/Config/VerifyWidget.cpp @@ -0,0 +1,116 @@ +// Copyright 2019 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#include "DolphinQt/Config/VerifyWidget.h" + +#include + +#include +#include +#include +#include + +#include "DiscIO/Volume.h" +#include "DiscIO/VolumeVerifier.h" + +VerifyWidget::VerifyWidget(std::shared_ptr volume) : m_volume(std::move(volume)) +{ + QVBoxLayout* layout = new QVBoxLayout(this); + + CreateWidgets(); + ConnectWidgets(); + + layout->addWidget(m_problems); + layout->addWidget(m_summary_text); + layout->addWidget(m_verify_button); + + layout->setStretchFactor(m_problems, 5); + layout->setStretchFactor(m_summary_text, 2); + + setLayout(layout); +} + +void VerifyWidget::CreateWidgets() +{ + m_problems = new QTableWidget(0, 2, this); + m_problems->setHorizontalHeaderLabels({tr("Problem"), tr("Severity")}); + m_problems->horizontalHeader()->setSectionResizeMode(0, QHeaderView::Stretch); + m_problems->horizontalHeader()->setSectionResizeMode(1, QHeaderView::ResizeToContents); + m_problems->horizontalHeader()->setHighlightSections(false); + m_problems->verticalHeader()->setSectionResizeMode(QHeaderView::ResizeToContents); + m_problems->verticalHeader()->hide(); + + m_summary_text = new QTextEdit(this); + m_summary_text->setReadOnly(true); + + m_verify_button = new QPushButton(tr("Verify Integrity"), this); +} + +void VerifyWidget::ConnectWidgets() +{ + connect(m_verify_button, &QPushButton::clicked, this, &VerifyWidget::Verify); +} + +void VerifyWidget::Verify() +{ + DiscIO::VolumeVerifier verifier(*m_volume); + + // We have to divide the number of processed bytes with something so it won't make ints overflow + constexpr int DIVISOR = 0x100; + + QProgressDialog* progress = new QProgressDialog(tr("Verifying"), tr("Cancel"), 0, + verifier.GetTotalBytes() / DIVISOR, this); + progress->setWindowTitle(tr("Verifying")); + progress->setWindowFlags(progress->windowFlags() & ~Qt::WindowContextHelpButtonHint); + progress->setMinimumDuration(500); + progress->setWindowModality(Qt::WindowModal); + + verifier.Start(); + while (verifier.GetBytesProcessed() != verifier.GetTotalBytes()) + { + progress->setValue(verifier.GetBytesProcessed() / DIVISOR); + if (progress->wasCanceled()) + return; + + verifier.Process(); + } + verifier.Finish(); + + DiscIO::VolumeVerifier::Result result = verifier.GetResult(); + progress->setValue(verifier.GetBytesProcessed() / DIVISOR); + + m_summary_text->setText(QString::fromStdString(result.summary_text)); + + m_problems->setRowCount(static_cast(result.problems.size())); + for (int i = 0; i < m_problems->rowCount(); ++i) + { + const DiscIO::VolumeVerifier::Problem problem = result.problems[i]; + + QString severity; + switch (problem.severity) + { + case DiscIO::VolumeVerifier::Severity::Low: + severity = tr("Low"); + break; + case DiscIO::VolumeVerifier::Severity::Medium: + severity = tr("Medium"); + break; + case DiscIO::VolumeVerifier::Severity::High: + severity = tr("High"); + break; + } + + SetProblemCellText(i, 0, QString::fromStdString(problem.text)); + SetProblemCellText(i, 1, severity); + } +} + +void VerifyWidget::SetProblemCellText(int row, int column, QString text) +{ + QLabel* label = new QLabel(text); + label->setTextInteractionFlags(Qt::TextSelectableByMouse); + label->setWordWrap(true); + label->setMargin(4); + m_problems->setCellWidget(row, column, label); +} diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.h b/Source/Core/DolphinQt/Config/VerifyWidget.h new file mode 100644 index 0000000000..4a895c651c --- /dev/null +++ b/Source/Core/DolphinQt/Config/VerifyWidget.h @@ -0,0 +1,37 @@ +// Copyright 2019 Dolphin Emulator Project +// Licensed under GPLv2+ +// Refer to the license.txt file included. + +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace DiscIO +{ +class Volume; +} + +class VerifyWidget final : public QWidget +{ + Q_OBJECT +public: + explicit VerifyWidget(std::shared_ptr volume); + +private: + void CreateWidgets(); + void ConnectWidgets(); + + void Verify(); + void SetProblemCellText(int row, int column, QString text); + + std::shared_ptr m_volume; + QTableWidget* m_problems; + QTextEdit* m_summary_text; + QPushButton* m_verify_button; +}; diff --git a/Source/Core/DolphinQt/DolphinQt.vcxproj b/Source/Core/DolphinQt/DolphinQt.vcxproj index e2fefa340c..baf2408f79 100644 --- a/Source/Core/DolphinQt/DolphinQt.vcxproj +++ b/Source/Core/DolphinQt/DolphinQt.vcxproj @@ -109,6 +109,7 @@ + @@ -267,6 +268,7 @@ + @@ -329,6 +331,7 @@ + From 4fd2d8e8c4db14cf01576ab228964b80c8ec37ad Mon Sep 17 00:00:00 2001 From: JosJuice Date: Tue, 26 Mar 2019 17:22:18 +0100 Subject: [PATCH 07/10] VolumeVerifier: Check hashes in Wii partitions --- Source/Core/DiscIO/Volume.h | 6 +- Source/Core/DiscIO/VolumeVerifier.cpp | 95 ++++++++++-- Source/Core/DiscIO/VolumeVerifier.h | 14 ++ Source/Core/DiscIO/VolumeWii.cpp | 138 ++++++++++-------- Source/Core/DiscIO/VolumeWii.h | 8 +- .../DolphinQt/Config/FilesystemWidget.cpp | 38 ----- .../Core/DolphinQt/Config/FilesystemWidget.h | 1 - 7 files changed, 184 insertions(+), 116 deletions(-) diff --git a/Source/Core/DiscIO/Volume.h b/Source/Core/DiscIO/Volume.h index e5e77d4479..5cd91ce60e 100644 --- a/Source/Core/DiscIO/Volume.h +++ b/Source/Core/DiscIO/Volume.h @@ -99,7 +99,11 @@ public: } virtual Platform GetVolumeType() const = 0; virtual bool SupportsIntegrityCheck() const { return false; } - virtual bool CheckIntegrity(const Partition& partition) const { return false; } + virtual bool CheckH3TableIntegrity(const Partition& partition) const { return false; } + virtual bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const + { + return false; + } virtual Region GetRegion() const = 0; virtual Country GetCountry(const Partition& partition = PARTITION_NONE) const = 0; virtual BlobType GetBlobType() const = 0; diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 7d3eba4c8f..2201784d1c 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -14,6 +14,7 @@ #include "Common/Align.h" #include "Common/Assert.h" #include "Common/CommonTypes.h" +#include "Common/Logging/Log.h" #include "Common/MsgHandler.h" #include "Common/StringUtil.h" #include "Common/Swap.h" @@ -62,6 +63,9 @@ void VolumeVerifier::Start() CheckCorrectlySigned(PARTITION_NONE, GetStringT("This title is not correctly signed.")); CheckDiscSize(); CheckMisc(); + + std::sort(m_blocks.begin(), m_blocks.end(), + [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); } void VolumeVerifier::CheckPartitions() @@ -164,18 +168,7 @@ bool VolumeVerifier::CheckPartition(const Partition& partition) else if (*type == PARTITION_UPDATE) severity = Severity::Low; - std::string name = NameForPartitionType(*type, false); - if (ShouldHaveMasterpiecePartitions() && *type > 0xFF) - { - // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces - // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo - // rather than the game mode as a whole, so please use the singular form. Official translations: - // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre - // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean). - // If your language is not one of the languages above, consider leaving the string untranslated - // so that people will recognize it as the name of the game mode. - name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str()); - } + const std::string name = GetPartitionName(type); if (partition.offset % VolumeWii::BLOCK_TOTAL_SIZE != 0 || m_volume.PartitionOffsetToRawOffset(0, partition) % VolumeWii::BLOCK_TOTAL_SIZE != 0) @@ -189,6 +182,13 @@ bool VolumeVerifier::CheckPartition(const Partition& partition) partition, StringFromFormat(GetStringT("The %s partition is not correctly signed.").c_str(), name.c_str())); + if (m_volume.SupportsIntegrityCheck() && !m_volume.CheckH3TableIntegrity(partition)) + { + const std::string text = StringFromFormat( + GetStringT("The H3 hash table for the %s partition is not correct.").c_str(), name.c_str()); + AddProblem(Severity::Low, text); + } + bool invalid_disc_header = false; std::vector disc_header(0x80); constexpr u32 WII_MAGIC = 0x5D1C9EA3; @@ -262,9 +262,43 @@ bool VolumeVerifier::CheckPartition(const Partition& partition) } } + // Prepare for hash verification in the Process step + if (m_volume.SupportsIntegrityCheck()) + { + u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition); + const std::optional size = + m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE); + const u64 end_offset = offset + size.value_or(0); + + for (size_t i = 0; offset < end_offset; ++i, offset += VolumeWii::BLOCK_TOTAL_SIZE) + m_blocks.emplace_back(BlockToVerify{partition, offset, i}); + + m_block_errors.emplace(partition, 0); + } + return true; } +std::string VolumeVerifier::GetPartitionName(std::optional type) const +{ + if (!type) + return "???"; + + std::string name = NameForPartitionType(*type, false); + if (ShouldHaveMasterpiecePartitions() && *type > 0xFF) + { + // i18n: This string is referring to a game mode in Super Smash Bros. Brawl called Masterpieces + // where you play demos of NES/SNES/N64 games. This string is referring to a specific such demo + // rather than the game mode as a whole, so please use the singular form. Official translations: + // 名作トライアル (Japanese), Masterpieces (English), Meisterstücke (German), Chefs-d'œuvre + // (French), Clásicos (Spanish), Capolavori (Italian), 클래식 게임 체험판 (Korean). + // If your language is not one of the languages above, consider leaving the string untranslated + // so that people will recognize it as the name of the game mode. + name = StringFromFormat(GetStringT("%s (Masterpiece)").c_str(), name.c_str()); + } + return name; +} + void VolumeVerifier::CheckCorrectlySigned(const Partition& partition, const std::string& error_text) { IOS::HLE::Kernel ios; @@ -596,7 +630,30 @@ void VolumeVerifier::Process() if (m_progress == m_max_progress) return; - m_progress += std::min(m_max_progress - m_progress, BLOCK_SIZE); + u64 bytes_to_read = BLOCK_SIZE; + if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress) + { + bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE; + } + else if (m_block_index + 1 < m_blocks.size() && m_blocks[m_block_index + 1].offset > m_progress) + { + bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index + 1].offset - m_progress); + } + bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress); + + m_progress += bytes_to_read; + + while (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset < m_progress) + { + if (!m_volume.CheckBlockIntegrity(m_blocks[m_block_index].block_index, + m_blocks[m_block_index].partition)) + { + WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64, + m_blocks[m_block_index].offset); + m_block_errors[m_blocks[m_block_index].partition]++; + } + m_block_index++; + } } u64 VolumeVerifier::GetBytesProcessed() const @@ -615,6 +672,18 @@ void VolumeVerifier::Finish() return; m_done = true; + for (auto pair : m_block_errors) + { + if (pair.second > 0) + { + const std::string name = GetPartitionName(m_volume.GetPartitionType(pair.first)); + AddProblem(Severity::Medium, + StringFromFormat( + GetStringT("Errors were found in %zu blocks in the %s partition.").c_str(), + pair.second, name.c_str())); + } + } + // Show the most serious problems at the top std::stable_sort(m_result.problems.begin(), m_result.problems.end(), [](const Problem& p1, const Problem& p2) { return p1.severity > p2.severity; }); diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 8503c1f89d..22b86107c0 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -4,6 +4,8 @@ #pragma once +#include +#include #include #include @@ -64,8 +66,16 @@ public: const Result& GetResult() const; private: + struct BlockToVerify + { + Partition partition; + u64 offset; + u64 block_index; + }; + void CheckPartitions(); bool CheckPartition(const Partition& partition); // Returns false if partition should be ignored + std::string GetPartitionName(std::optional type) const; void CheckCorrectlySigned(const Partition& partition, const std::string& error_text); bool IsDebugSigned() const; bool ShouldHaveChannelPartition() const; @@ -85,6 +95,10 @@ private: bool m_is_datel; bool m_is_not_retail; + std::vector m_blocks; + size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition + std::map m_block_errors; + bool m_started; bool m_done; u64 m_progress; diff --git a/Source/Core/DiscIO/VolumeWii.cpp b/Source/Core/DiscIO/VolumeWii.cpp index f589c69921..7d44315bf1 100644 --- a/Source/Core/DiscIO/VolumeWii.cpp +++ b/Source/Core/DiscIO/VolumeWii.cpp @@ -109,6 +109,19 @@ VolumeWii::VolumeWii(std::unique_ptr reader) return cert_chain; }; + auto get_h3_table = [this, partition]() -> std::vector { + if (!m_encrypted) + return {}; + const std::optional h3_table_offset = + ReadSwappedAndShifted(partition.offset + 0x2b4, PARTITION_NONE); + if (!h3_table_offset) + return {}; + std::vector h3_table(H3_TABLE_SIZE); + if (!m_reader->Read(partition.offset + *h3_table_offset, H3_TABLE_SIZE, h3_table.data())) + return {}; + return h3_table; + }; + auto get_key = [this, partition]() -> std::unique_ptr { const IOS::ES::TicketReader& ticket = *m_partitions[partition].ticket; if (!ticket.IsValid()) @@ -133,6 +146,7 @@ VolumeWii::VolumeWii(std::unique_ptr reader) Common::Lazy(get_ticket), Common::Lazy(get_tmd), Common::Lazy>(get_cert_chain), + Common::Lazy>(get_h3_table), Common::Lazy>(get_file_system), Common::Lazy(get_data_offset), *partition_type}); } @@ -409,82 +423,84 @@ u64 VolumeWii::GetRawSize() const return m_reader->GetRawSize(); } -bool VolumeWii::CheckIntegrity(const Partition& partition) const +bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const { - if (!m_encrypted) - return false; - - // Get the decryption key for the partition auto it = m_partitions.find(partition); if (it == m_partitions.end()) return false; const PartitionDetails& partition_details = it->second; + + const std::vector& h3_table = *partition_details.h3_table; + if (h3_table.size() != H3_TABLE_SIZE) + return false; + + const IOS::ES::TMDReader& tmd = *partition_details.tmd; + if (!tmd.IsValid()) + return false; + + const std::vector contents = tmd.GetContents(); + if (contents.size() != 1) + return false; + + std::array h3_table_sha1; + mbedtls_sha1(h3_table.data(), h3_table.size(), h3_table_sha1.data()); + return h3_table_sha1 == contents[0].sha1; +} + +bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition) const +{ + auto it = m_partitions.find(partition); + if (it == m_partitions.end()) + return false; + const PartitionDetails& partition_details = it->second; + + constexpr size_t SHA1_SIZE = 20; + if (block_index / 64 * SHA1_SIZE >= partition_details.h3_table->size()) + return false; + mbedtls_aes_context* aes_context = partition_details.key->get(); if (!aes_context) return false; - // Get partition data size - const auto part_data_size = ReadSwappedAndShifted(partition.offset + 0x2BC, PARTITION_NONE); - if (!part_data_size) + const u64 cluster_offset = + partition.offset + *partition_details.data_offset + block_index * BLOCK_TOTAL_SIZE; + + // Read and decrypt the cluster metadata + u8 cluster_metadata_crypted[BLOCK_HEADER_SIZE]; + u8 cluster_metadata[BLOCK_HEADER_SIZE]; + u8 iv[16] = {0}; + if (!m_reader->Read(cluster_offset, BLOCK_HEADER_SIZE, cluster_metadata_crypted)) + return false; + mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, BLOCK_HEADER_SIZE, iv, + cluster_metadata_crypted, cluster_metadata); + + u8 cluster_data[BLOCK_DATA_SIZE]; + if (!Read(block_index * BLOCK_DATA_SIZE, BLOCK_DATA_SIZE, cluster_data, partition)) return false; - const u32 num_clusters = static_cast(part_data_size.value() / 0x8000); - for (u32 cluster_id = 0; cluster_id < num_clusters; ++cluster_id) + for (u32 hash_index = 0; hash_index < 31; ++hash_index) { - const u64 cluster_offset = - partition.offset + *partition_details.data_offset + static_cast(cluster_id) * 0x8000; - - // Read and decrypt the cluster metadata - u8 cluster_metadata_crypted[0x400]; - u8 cluster_metadata[0x400]; - u8 iv[16] = {0}; - if (!m_reader->Read(cluster_offset, sizeof(cluster_metadata_crypted), cluster_metadata_crypted)) - { - WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: could not read metadata", cluster_id); + u8 h0_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_data + hash_index * 0x400, 0x400, h0_hash); + if (memcmp(h0_hash, cluster_metadata + hash_index * SHA1_SIZE, SHA1_SIZE)) return false; - } - mbedtls_aes_crypt_cbc(aes_context, MBEDTLS_AES_DECRYPT, sizeof(cluster_metadata), iv, - cluster_metadata_crypted, cluster_metadata); - - // Some clusters have invalid data and metadata because they aren't - // meant to be read by the game (for example, holes between files). To - // try to avoid reporting errors because of these clusters, we check - // the 0x00 paddings in the metadata. - // - // This may cause some false negatives though: some bad clusters may be - // skipped because they are *too* bad and are not even recognized as - // valid clusters. To be improved. - const u8* pad_begin = cluster_metadata + 0x26C; - const u8* pad_end = pad_begin + 0x14; - const bool meaningless = std::any_of(pad_begin, pad_end, [](u8 val) { return val != 0; }); - - if (meaningless) - continue; - - u8 cluster_data[0x7C00]; - if (!Read(cluster_id * sizeof(cluster_data), sizeof(cluster_data), cluster_data, partition)) - { - WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: could not read data", cluster_id); - return false; - } - - for (u32 hash_id = 0; hash_id < 31; ++hash_id) - { - u8 hash[20]; - - mbedtls_sha1(cluster_data + hash_id * sizeof(cluster_metadata), sizeof(cluster_metadata), - hash); - - // Note that we do not use strncmp here - if (memcmp(hash, cluster_metadata + hash_id * sizeof(hash), sizeof(hash))) - { - WARN_LOG(DISCIO, "Integrity Check: fail at cluster %d: hash %d is invalid", cluster_id, - hash_id); - return false; - } - } } + u8 h1_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_metadata, SHA1_SIZE * 31, h1_hash); + if (memcmp(h1_hash, cluster_metadata + 0x280 + (block_index % 8) * SHA1_SIZE, SHA1_SIZE)) + return false; + + u8 h2_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_metadata + 0x280, SHA1_SIZE * 8, h2_hash); + if (memcmp(h2_hash, cluster_metadata + 0x340 + (block_index / 8 % 8) * SHA1_SIZE, SHA1_SIZE)) + return false; + + u8 h3_hash[SHA1_SIZE]; + mbedtls_sha1(cluster_metadata + 0x340, SHA1_SIZE * 8, h3_hash); + if (memcmp(h3_hash, partition_details.h3_table->data() + block_index / 64 * SHA1_SIZE, SHA1_SIZE)) + return false; + return true; } diff --git a/Source/Core/DiscIO/VolumeWii.h b/Source/Core/DiscIO/VolumeWii.h index 01df4f053f..edfa75d21e 100644 --- a/Source/Core/DiscIO/VolumeWii.h +++ b/Source/Core/DiscIO/VolumeWii.h @@ -56,8 +56,9 @@ public: std::optional GetDiscNumber(const Partition& partition = PARTITION_NONE) const override; Platform GetVolumeType() const override; - bool SupportsIntegrityCheck() const override { return true; } - bool CheckIntegrity(const Partition& partition) const override; + bool SupportsIntegrityCheck() const override { return m_encrypted; } + bool CheckH3TableIntegrity(const Partition& partition) const override; + bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const override; Region GetRegion() const override; Country GetCountry(const Partition& partition = PARTITION_NONE) const override; @@ -66,6 +67,8 @@ public: bool IsSizeAccurate() const override; u64 GetRawSize() const override; + static constexpr unsigned int H3_TABLE_SIZE = 0x18000; + static constexpr unsigned int BLOCK_HEADER_SIZE = 0x0400; static constexpr unsigned int BLOCK_DATA_SIZE = 0x7C00; static constexpr unsigned int BLOCK_TOTAL_SIZE = BLOCK_HEADER_SIZE + BLOCK_DATA_SIZE; @@ -80,6 +83,7 @@ private: Common::Lazy ticket; Common::Lazy tmd; Common::Lazy> cert_chain; + Common::Lazy> h3_table; Common::Lazy> file_system; Common::Lazy data_offset; u32 type; diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp index e407ed9945..65caa17bf9 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.cpp +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.cpp @@ -238,12 +238,6 @@ void FilesystemWidget::ShowContextMenu(const QPoint&) if (!folder.isEmpty()) ExtractPartition(partition, folder); }); - if (m_volume->IsEncryptedAndHashed()) - { - menu->addSeparator(); - menu->addAction(tr("Check Partition Integrity"), this, - [this, partition] { CheckIntegrity(partition); }); - } break; case EntryType::File: menu->addAction(tr("Extract File..."), this, [this, partition, path] { @@ -327,35 +321,3 @@ void FilesystemWidget::ExtractFile(const DiscIO::Partition& partition, const QSt else ModalMessageBox::critical(this, tr("Error"), tr("Failed to extract file.")); } - -void FilesystemWidget::CheckIntegrity(const DiscIO::Partition& partition) -{ - QProgressDialog* dialog = new QProgressDialog(this); - std::future is_valid = std::async( - std::launch::async, [this, partition] { return m_volume->CheckIntegrity(partition); }); - - dialog->setLabelText(tr("Verifying integrity of partition...")); - dialog->setWindowFlags(dialog->windowFlags() & ~Qt::WindowContextHelpButtonHint); - dialog->setWindowTitle(tr("Verifying partition")); - - dialog->setMinimum(0); - dialog->setMaximum(0); - dialog->show(); - - while (is_valid.wait_for(std::chrono::milliseconds(50)) != std::future_status::ready) - QCoreApplication::processEvents(); - - dialog->close(); - - if (is_valid.get()) - { - ModalMessageBox::information(this, tr("Success"), - tr("Integrity check completed. No errors have been found.")); - } - else - { - ModalMessageBox::critical(this, tr("Error"), - tr("Integrity check for partition failed. The disc image is most " - "likely corrupted or has been patched incorrectly.")); - } -} diff --git a/Source/Core/DolphinQt/Config/FilesystemWidget.h b/Source/Core/DolphinQt/Config/FilesystemWidget.h index c5b14adad9..99629444cf 100644 --- a/Source/Core/DolphinQt/Config/FilesystemWidget.h +++ b/Source/Core/DolphinQt/Config/FilesystemWidget.h @@ -43,7 +43,6 @@ private: const QString& out); void ExtractFile(const DiscIO::Partition& partition, const QString& path, const QString& out); bool ExtractSystemData(const DiscIO::Partition& partition, const QString& out); - void CheckIntegrity(const DiscIO::Partition& partition); DiscIO::Partition GetPartitionFromID(int id); From eced9d7c7e2b05df304e800e4cfb7eae3ee9d2bd Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 30 Mar 2019 12:45:00 +0100 Subject: [PATCH 08/10] VolumeVerifier: Calculate CRC32/MD5/SHA-1 --- Source/Core/DiscIO/VolumeVerifier.cpp | 83 +++++++++++++++++-- Source/Core/DiscIO/VolumeVerifier.h | 21 ++++- Source/Core/DolphinQt/Config/InfoWidget.cpp | 53 ------------ Source/Core/DolphinQt/Config/InfoWidget.h | 3 - Source/Core/DolphinQt/Config/VerifyWidget.cpp | 42 +++++++++- Source/Core/DolphinQt/Config/VerifyWidget.h | 12 +++ 6 files changed, 151 insertions(+), 63 deletions(-) diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 2201784d1c..ac2f06cc15 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -11,6 +11,10 @@ #include #include +#include +#include +#include + #include "Common/Align.h" #include "Common/Assert.h" #include "Common/CommonTypes.h" @@ -40,9 +44,11 @@ constexpr u64 DL_DVD_R_SIZE = 8543666176; // Wii RVT-R constexpr u64 BLOCK_SIZE = 0x20000; -VolumeVerifier::VolumeVerifier(const Volume& volume) - : m_volume(volume), m_started(false), m_done(false), m_progress(0), - m_max_progress(volume.GetSize()) +VolumeVerifier::VolumeVerifier(const Volume& volume, Hashes hashes_to_calculate) + : m_volume(volume), m_hashes_to_calculate(hashes_to_calculate), + m_calculating_any_hash(hashes_to_calculate.crc32 || hashes_to_calculate.md5 || + hashes_to_calculate.sha1), + m_started(false), m_done(false), m_progress(0), m_max_progress(volume.GetSize()) { } @@ -64,8 +70,7 @@ void VolumeVerifier::Start() CheckDiscSize(); CheckMisc(); - std::sort(m_blocks.begin(), m_blocks.end(), - [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); + SetUpHashing(); } void VolumeVerifier::CheckPartitions() @@ -622,6 +627,27 @@ void VolumeVerifier::CheckMisc() } } +void VolumeVerifier::SetUpHashing() +{ + std::sort(m_blocks.begin(), m_blocks.end(), + [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); + + if (m_hashes_to_calculate.crc32) + m_crc32_context = crc32(0, nullptr, 0); + + if (m_hashes_to_calculate.md5) + { + mbedtls_md5_init(&m_md5_context); + mbedtls_md5_starts(&m_md5_context); + } + + if (m_hashes_to_calculate.sha1) + { + mbedtls_sha1_init(&m_sha1_context); + mbedtls_sha1_starts(&m_sha1_context); + } +} + void VolumeVerifier::Process() { ASSERT(m_started); @@ -641,6 +667,30 @@ void VolumeVerifier::Process() } bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress); + if (m_calculating_any_hash) + { + std::vector data(bytes_to_read); + if (!m_volume.Read(m_progress, bytes_to_read, data.data(), PARTITION_NONE)) + { + m_calculating_any_hash = false; + } + else + { + if (m_hashes_to_calculate.crc32) + { + // It would be nice to use crc32_z here instead of crc32, but it isn't available on Android + m_crc32_context = + crc32(m_crc32_context, data.data(), static_cast(bytes_to_read)); + } + + if (m_hashes_to_calculate.md5) + mbedtls_md5_update(&m_md5_context, data.data(), bytes_to_read); + + if (m_hashes_to_calculate.sha1) + mbedtls_sha1_update(&m_sha1_context, data.data(), bytes_to_read); + } + } + m_progress += bytes_to_read; while (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset < m_progress) @@ -672,6 +722,29 @@ void VolumeVerifier::Finish() return; m_done = true; + if (m_calculating_any_hash) + { + if (m_hashes_to_calculate.crc32) + { + m_result.hashes.crc32 = std::vector(4); + const u32 crc32_be = Common::swap32(m_crc32_context); + const u8* crc32_be_ptr = reinterpret_cast(&crc32_be); + std::copy(crc32_be_ptr, crc32_be_ptr + 4, m_result.hashes.crc32.begin()); + } + + if (m_hashes_to_calculate.md5) + { + m_result.hashes.md5 = std::vector(16); + mbedtls_md5_finish(&m_md5_context, m_result.hashes.md5.data()); + } + + if (m_hashes_to_calculate.sha1) + { + m_result.hashes.sha1 = std::vector(20); + mbedtls_sha1_finish(&m_sha1_context, m_result.hashes.sha1.data()); + } + } + for (auto pair : m_block_errors) { if (pair.second > 0) diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 22b86107c0..4337fdce46 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -9,6 +9,9 @@ #include #include +#include +#include + #include "Common/CommonTypes.h" #include "DiscIO/Volume.h" @@ -51,13 +54,22 @@ public: std::string text; }; + template + struct Hashes + { + T crc32; + T md5; + T sha1; + }; + struct Result { + Hashes> hashes; std::string summary_text; std::vector problems; }; - VolumeVerifier(const Volume& volume); + VolumeVerifier(const Volume& volume, Hashes hashes_to_calculate); void Start(); void Process(); u64 GetBytesProcessed() const; @@ -86,6 +98,7 @@ private: u64 GetBiggestUsedOffset(); u64 GetBiggestUsedOffset(const FileInfo& file_info) const; void CheckMisc(); + void SetUpHashing(); void AddProblem(Severity severity, const std::string& text); @@ -95,6 +108,12 @@ private: bool m_is_datel; bool m_is_not_retail; + Hashes m_hashes_to_calculate; + bool m_calculating_any_hash; + unsigned long m_crc32_context; + mbedtls_md5_context m_md5_context; + mbedtls_sha1_context m_sha1_context; + std::vector m_blocks; size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition std::map m_block_errors; diff --git a/Source/Core/DolphinQt/Config/InfoWidget.cpp b/Source/Core/DolphinQt/Config/InfoWidget.cpp index 79b980e52a..9ac8460bb1 100644 --- a/Source/Core/DolphinQt/Config/InfoWidget.cpp +++ b/Source/Core/DolphinQt/Config/InfoWidget.cpp @@ -78,7 +78,6 @@ QGroupBox* InfoWidget::CreateISODetails() QLineEdit* maker = CreateValueDisplay((game_maker.empty() ? UNKNOWN_NAME.toStdString() : game_maker) + " (" + m_game.GetMakerID() + ")"); - QWidget* checksum = CreateChecksumComputer(); layout->addRow(tr("Name:"), internal_name); layout->addRow(tr("File:"), file_path); @@ -89,8 +88,6 @@ QGroupBox* InfoWidget::CreateISODetails() if (!m_game.GetApploaderDate().empty()) layout->addRow(tr("Apploader Date:"), CreateValueDisplay(m_game.GetApploaderDate())); - layout->addRow(tr("MD5 Checksum:"), checksum); - group->setLayout(layout); return group; } @@ -194,53 +191,3 @@ void InfoWidget::ChangeLanguage() m_maker->setText(QString::fromStdString(m_game.GetLongMaker(language))); m_description->setText(QString::fromStdString(m_game.GetDescription(language))); } - -QWidget* InfoWidget::CreateChecksumComputer() -{ - QWidget* widget = new QWidget(); - QHBoxLayout* layout = new QHBoxLayout(); - layout->setContentsMargins(0, 0, 0, 0); - - m_checksum_result = new QLineEdit(); - m_checksum_result->setReadOnly(true); - QPushButton* calculate = new QPushButton(tr("Compute")); - connect(calculate, &QPushButton::clicked, this, &InfoWidget::ComputeChecksum); - layout->addWidget(m_checksum_result); - layout->addWidget(calculate); - - widget->setLayout(layout); - return widget; -} - -void InfoWidget::ComputeChecksum() -{ - QCryptographicHash hash(QCryptographicHash::Md5); - hash.reset(); - std::unique_ptr file(DiscIO::CreateBlobReader(m_game.GetFilePath())); - std::vector file_data(8 * 1080 * 1080); // read 1MB at a time - u64 game_size = file->GetDataSize(); - u64 read_offset = 0; - - // a maximum of 1000 is used instead of game_size because otherwise 8GB games overflow the int - // typed maximum parameter - QProgressDialog* progress = - new QProgressDialog(tr("Computing MD5 Checksum"), tr("Cancel"), 0, 1000, this); - progress->setWindowTitle(tr("Computing MD5 Checksum")); - progress->setWindowFlags(progress->windowFlags() & ~Qt::WindowContextHelpButtonHint); - progress->setMinimumDuration(500); - progress->setWindowModality(Qt::WindowModal); - while (read_offset < game_size) - { - progress->setValue(static_cast(read_offset) / static_cast(game_size) * 1000); - if (progress->wasCanceled()) - return; - - u64 read_size = std::min(file_data.size(), game_size - read_offset); - file->Read(read_offset, read_size, file_data.data()); - hash.addData(reinterpret_cast(file_data.data()), read_size); - read_offset += read_size; - } - m_checksum_result->setText(QString::fromUtf8(hash.result().toHex())); - Q_ASSERT(read_offset == game_size); - progress->setValue(1000); -} diff --git a/Source/Core/DolphinQt/Config/InfoWidget.h b/Source/Core/DolphinQt/Config/InfoWidget.h index b6fd934290..226d8e3c36 100644 --- a/Source/Core/DolphinQt/Config/InfoWidget.h +++ b/Source/Core/DolphinQt/Config/InfoWidget.h @@ -23,7 +23,6 @@ public: explicit InfoWidget(const UICommon::GameFile& game); private: - void ComputeChecksum(); void ChangeLanguage(); void SaveBanner(); @@ -31,12 +30,10 @@ private: QGroupBox* CreateISODetails(); QLineEdit* CreateValueDisplay(const QString& value); QLineEdit* CreateValueDisplay(const std::string& value = ""); - QWidget* CreateChecksumComputer(); void CreateLanguageSelector(); QWidget* CreateBannerGraphic(const QPixmap& image); UICommon::GameFile m_game; - QLineEdit* m_checksum_result; QComboBox* m_language_selector; QLineEdit* m_name; QLineEdit* m_maker; diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.cpp b/Source/Core/DolphinQt/Config/VerifyWidget.cpp index b857d2e88c..f3872df211 100644 --- a/Source/Core/DolphinQt/Config/VerifyWidget.cpp +++ b/Source/Core/DolphinQt/Config/VerifyWidget.cpp @@ -5,12 +5,17 @@ #include "DolphinQt/Config/VerifyWidget.h" #include +#include +#include +#include +#include #include #include #include #include +#include "Common/CommonTypes.h" #include "DiscIO/Volume.h" #include "DiscIO/VolumeVerifier.h" @@ -23,6 +28,7 @@ VerifyWidget::VerifyWidget(std::shared_ptr volume) : m_volume(st layout->addWidget(m_problems); layout->addWidget(m_summary_text); + layout->addLayout(m_hash_layout); layout->addWidget(m_verify_button); layout->setStretchFactor(m_problems, 5); @@ -44,17 +50,47 @@ void VerifyWidget::CreateWidgets() m_summary_text = new QTextEdit(this); m_summary_text->setReadOnly(true); + m_hash_layout = new QFormLayout(this); + std::tie(m_crc32_checkbox, m_crc32_line_edit) = AddHashLine(m_hash_layout, tr("CRC32:")); + std::tie(m_md5_checkbox, m_md5_line_edit) = AddHashLine(m_hash_layout, tr("MD5:")); + std::tie(m_sha1_checkbox, m_sha1_line_edit) = AddHashLine(m_hash_layout, tr("SHA-1:")); + m_verify_button = new QPushButton(tr("Verify Integrity"), this); } +std::pair VerifyWidget::AddHashLine(QFormLayout* layout, QString text) +{ + QLineEdit* line_edit = new QLineEdit(this); + line_edit->setReadOnly(true); + QCheckBox* checkbox = new QCheckBox(tr("Calculate"), this); + checkbox->setChecked(true); + + QHBoxLayout* hbox_layout = new QHBoxLayout(this); + hbox_layout->addWidget(line_edit); + hbox_layout->addWidget(checkbox); + + layout->addRow(text, hbox_layout); + + return std::pair(checkbox, line_edit); +} + void VerifyWidget::ConnectWidgets() { connect(m_verify_button, &QPushButton::clicked, this, &VerifyWidget::Verify); } +static void SetHash(QLineEdit* line_edit, const std::vector& hash) +{ + const QByteArray byte_array = QByteArray::fromRawData(reinterpret_cast(hash.data()), + static_cast(hash.size())); + line_edit->setText(QString::fromLatin1(byte_array.toHex())); +} + void VerifyWidget::Verify() { - DiscIO::VolumeVerifier verifier(*m_volume); + DiscIO::VolumeVerifier verifier( + *m_volume, + {m_crc32_checkbox->isChecked(), m_md5_checkbox->isChecked(), m_sha1_checkbox->isChecked()}); // We have to divide the number of processed bytes with something so it won't make ints overflow constexpr int DIVISOR = 0x100; @@ -104,6 +140,10 @@ void VerifyWidget::Verify() SetProblemCellText(i, 0, QString::fromStdString(problem.text)); SetProblemCellText(i, 1, severity); } + + SetHash(m_crc32_line_edit, result.hashes.crc32); + SetHash(m_md5_line_edit, result.hashes.md5); + SetHash(m_sha1_line_edit, result.hashes.sha1); } void VerifyWidget::SetProblemCellText(int row, int column, QString text) diff --git a/Source/Core/DolphinQt/Config/VerifyWidget.h b/Source/Core/DolphinQt/Config/VerifyWidget.h index 4a895c651c..c2433cbd49 100644 --- a/Source/Core/DolphinQt/Config/VerifyWidget.h +++ b/Source/Core/DolphinQt/Config/VerifyWidget.h @@ -6,7 +6,11 @@ #include #include +#include +#include +#include +#include #include #include #include @@ -25,6 +29,7 @@ public: private: void CreateWidgets(); + std::pair AddHashLine(QFormLayout* layout, QString text); void ConnectWidgets(); void Verify(); @@ -33,5 +38,12 @@ private: std::shared_ptr m_volume; QTableWidget* m_problems; QTextEdit* m_summary_text; + QFormLayout* m_hash_layout; + QCheckBox* m_crc32_checkbox; + QCheckBox* m_md5_checkbox; + QCheckBox* m_sha1_checkbox; + QLineEdit* m_crc32_line_edit; + QLineEdit* m_md5_line_edit; + QLineEdit* m_sha1_line_edit; QPushButton* m_verify_button; }; From a469fb315037d455884f25acb39bad57bc76e4ff Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 30 Mar 2019 16:20:45 +0100 Subject: [PATCH 09/10] VolumeVerifier: Check whether invalid blocks are unused --- Source/Core/DiscIO/CompressedBlob.cpp | 5 +++- Source/Core/DiscIO/DiscScrubber.cpp | 25 +++++++--------- Source/Core/DiscIO/DiscScrubber.h | 7 ++--- Source/Core/DiscIO/VolumeVerifier.cpp | 41 ++++++++++++++++++++++----- Source/Core/DiscIO/VolumeVerifier.h | 3 ++ 5 files changed, 55 insertions(+), 26 deletions(-) diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 916773b90a..3dc9f86709 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -27,6 +27,7 @@ #include "DiscIO/Blob.h" #include "DiscIO/CompressedBlob.h" #include "DiscIO/DiscScrubber.h" +#include "DiscIO/Volume.h" namespace DiscIO { @@ -181,9 +182,11 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi } DiscScrubber disc_scrubber; + std::unique_ptr volume; if (sub_type == 1) { - if (!disc_scrubber.SetupScrub(infile_path, block_size)) + volume = CreateVolumeFromFilename(infile_path); + if (!volume || !disc_scrubber.SetupScrub(volume.get(), block_size)) { PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.", infile_path.c_str()); diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index dc3c9bf590..0c944a782a 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -28,9 +28,11 @@ constexpr size_t CLUSTER_SIZE = 0x8000; DiscScrubber::DiscScrubber() = default; DiscScrubber::~DiscScrubber() = default; -bool DiscScrubber::SetupScrub(const std::string& filename, int block_size) +bool DiscScrubber::SetupScrub(const Volume* disc, int block_size) { - m_filename = filename; + if (!disc) + return false; + m_disc = disc; m_block_size = block_size; if (CLUSTER_SIZE % m_block_size != 0) @@ -40,20 +42,13 @@ bool DiscScrubber::SetupScrub(const std::string& filename, int block_size) return false; } - m_disc = CreateVolumeFromFilename(filename); - if (!m_disc) - return false; - m_file_size = m_disc->GetSize(); const size_t num_clusters = static_cast(m_file_size / CLUSTER_SIZE); // Warn if not DVD5 or DVD9 size if (num_clusters != 0x23048 && num_clusters != 0x46090) - { - WARN_LOG(DISCIO, "%s is not a standard sized Wii disc! (%zx blocks)", filename.c_str(), - num_clusters); - } + WARN_LOG(DISCIO, "Not a standard sized Wii disc! (%zx blocks)", num_clusters); // Table of free blocks m_free_table.resize(num_clusters, 1); @@ -61,8 +56,6 @@ bool DiscScrubber::SetupScrub(const std::string& filename, int block_size) // Fill out table of free blocks const bool success = ParseDisc(); - // Done with it; need it closed for the next part - m_disc.reset(); m_block_count = 0; m_is_scrubbing = success; @@ -72,10 +65,9 @@ bool DiscScrubber::SetupScrub(const std::string& filename, int block_size) size_t DiscScrubber::GetNextBlock(File::IOFile& in, u8* buffer) { const u64 current_offset = m_block_count * m_block_size; - const u64 i = current_offset / CLUSTER_SIZE; size_t read_bytes = 0; - if (m_is_scrubbing && m_free_table[i]) + if (CanBlockBeScrubbed(current_offset)) { DEBUG_LOG(DISCIO, "Freeing 0x%016" PRIx64, current_offset); std::fill(buffer, buffer + m_block_size, 0x00); @@ -92,6 +84,11 @@ size_t DiscScrubber::GetNextBlock(File::IOFile& in, u8* buffer) return read_bytes; } +bool DiscScrubber::CanBlockBeScrubbed(u64 offset) const +{ + return m_is_scrubbing && m_free_table[offset / CLUSTER_SIZE]; +} + void DiscScrubber::MarkAsUsed(u64 offset, u64 size) { u64 current_offset = offset; diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index 29af40c73f..15e0cda7f7 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -13,7 +13,6 @@ #pragma once #include -#include #include #include #include "Common/CommonTypes.h" @@ -35,8 +34,9 @@ public: DiscScrubber(); ~DiscScrubber(); - bool SetupScrub(const std::string& filename, int block_size); + bool SetupScrub(const Volume* disc, int block_size); size_t GetNextBlock(File::IOFile& in, u8* buffer); + bool CanBlockBeScrubbed(u64 offset) const; private: struct PartitionHeader final @@ -68,8 +68,7 @@ private: bool ParsePartitionData(const Partition& partition, PartitionHeader* header); void ParseFileSystemData(u64 partition_data_offset, const FileInfo& directory); - std::string m_filename; - std::unique_ptr m_disc; + const Volume* m_disc; std::vector m_free_table; u64 m_file_size = 0; diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index ac2f06cc15..74612656fa 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -29,6 +29,7 @@ #include "Core/IOS/IOSC.h" #include "DiscIO/Blob.h" #include "DiscIO/DiscExtractor.h" +#include "DiscIO/DiscScrubber.h" #include "DiscIO/Enums.h" #include "DiscIO/Filesystem.h" #include "DiscIO/Volume.h" @@ -629,6 +630,12 @@ void VolumeVerifier::CheckMisc() void VolumeVerifier::SetUpHashing() { + if (m_volume.GetVolumeType() == Platform::WiiDisc) + { + // Set up a DiscScrubber for checking whether blocks with errors are unused + m_scrubber.SetupScrub(&m_volume, VolumeWii::BLOCK_TOTAL_SIZE); + } + std::sort(m_blocks.begin(), m_blocks.end(), [](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; }); @@ -698,9 +705,17 @@ void VolumeVerifier::Process() if (!m_volume.CheckBlockIntegrity(m_blocks[m_block_index].block_index, m_blocks[m_block_index].partition)) { - WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64, - m_blocks[m_block_index].offset); - m_block_errors[m_blocks[m_block_index].partition]++; + const u64 offset = m_blocks[m_block_index].offset; + if (m_scrubber.CanBlockBeScrubbed(offset)) + { + WARN_LOG(DISCIO, "Integrity check failed for unused block at 0x%" PRIx64, offset); + m_unused_block_errors[m_blocks[m_block_index].partition]++; + } + else + { + WARN_LOG(DISCIO, "Integrity check failed for block at 0x%" PRIx64, offset); + m_block_errors[m_blocks[m_block_index].partition]++; + } } m_block_index++; } @@ -750,10 +765,22 @@ void VolumeVerifier::Finish() if (pair.second > 0) { const std::string name = GetPartitionName(m_volume.GetPartitionType(pair.first)); - AddProblem(Severity::Medium, - StringFromFormat( - GetStringT("Errors were found in %zu blocks in the %s partition.").c_str(), - pair.second, name.c_str())); + const std::string text = StringFromFormat( + GetStringT("Errors were found in %zu blocks in the %s partition.").c_str(), pair.second, + name.c_str()); + AddProblem(Severity::Medium, text); + } + } + + for (auto pair : m_unused_block_errors) + { + if (pair.second > 0) + { + const std::string name = GetPartitionName(m_volume.GetPartitionType(pair.first)); + const std::string text = StringFromFormat( + GetStringT("Errors were found in %zu unused blocks in the %s partition.").c_str(), + pair.second, name.c_str()); + AddProblem(Severity::Low, text); } } diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 4337fdce46..263ef65ff3 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -13,6 +13,7 @@ #include #include "Common/CommonTypes.h" +#include "DiscIO/DiscScrubber.h" #include "DiscIO/Volume.h" // To be used as follows: @@ -114,9 +115,11 @@ private: mbedtls_md5_context m_md5_context; mbedtls_sha1_context m_sha1_context; + DiscScrubber m_scrubber; std::vector m_blocks; size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition std::map m_block_errors; + std::map m_unused_block_errors; bool m_started; bool m_done; From 8709b21ac30f5da5c32ef056b6562a37345b5f82 Mon Sep 17 00:00:00 2001 From: JosJuice Date: Sat, 30 Mar 2019 17:42:31 +0100 Subject: [PATCH 10/10] VolumeVerifier: Verify WAD contents --- Source/Core/DiscIO/Volume.h | 1 + Source/Core/DiscIO/VolumeVerifier.cpp | 62 +++++++++++++++++++++++++-- Source/Core/DiscIO/VolumeVerifier.h | 4 ++ Source/Core/DiscIO/VolumeWad.cpp | 19 +++++++- Source/Core/DiscIO/VolumeWad.h | 2 + 5 files changed, 82 insertions(+), 6 deletions(-) diff --git a/Source/Core/DiscIO/Volume.h b/Source/Core/DiscIO/Volume.h index 5cd91ce60e..d331a3d881 100644 --- a/Source/Core/DiscIO/Volume.h +++ b/Source/Core/DiscIO/Volume.h @@ -73,6 +73,7 @@ public: { return INVALID_CERT_CHAIN; } + virtual std::vector GetContentOffsets() const { return {}; } // Returns a non-owning pointer. Returns nullptr if the file system couldn't be read. virtual const FileSystem* GetFileSystem(const Partition& partition) const = 0; virtual u64 PartitionOffsetToRawOffset(u64 offset, const Partition& partition) const diff --git a/Source/Core/DiscIO/VolumeVerifier.cpp b/Source/Core/DiscIO/VolumeVerifier.cpp index 74612656fa..6b70382044 100644 --- a/Source/Core/DiscIO/VolumeVerifier.cpp +++ b/Source/Core/DiscIO/VolumeVerifier.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -630,7 +631,11 @@ void VolumeVerifier::CheckMisc() void VolumeVerifier::SetUpHashing() { - if (m_volume.GetVolumeType() == Platform::WiiDisc) + if (m_volume.GetVolumeType() == Platform::WiiWAD) + { + m_content_offsets = m_volume.GetContentOffsets(); + } + else if (m_volume.GetVolumeType() == Platform::WiiDisc) { // Set up a DiscScrubber for checking whether blocks with errors are unused m_scrubber.SetupScrub(&m_volume, VolumeWii::BLOCK_TOTAL_SIZE); @@ -663,14 +668,28 @@ void VolumeVerifier::Process() if (m_progress == m_max_progress) return; + IOS::ES::Content content; + bool content_read = false; u64 bytes_to_read = BLOCK_SIZE; - if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress) + if (m_content_index < m_content_offsets.size() && + m_content_offsets[m_content_index] == m_progress) + { + m_volume.GetTMD(PARTITION_NONE).GetContent(m_content_index, &content); + bytes_to_read = Common::AlignUp(content.size, 0x40); + content_read = true; + } + else if (m_content_index < m_content_offsets.size() && + m_content_offsets[m_content_index] > m_progress) + { + bytes_to_read = std::min(bytes_to_read, m_content_offsets[m_content_index] - m_progress); + } + else if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress) { bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE; } - else if (m_block_index + 1 < m_blocks.size() && m_blocks[m_block_index + 1].offset > m_progress) + else if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset > m_progress) { - bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index + 1].offset - m_progress); + bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index].offset - m_progress); } bytes_to_read = std::min(bytes_to_read, m_max_progress - m_progress); @@ -700,6 +719,17 @@ void VolumeVerifier::Process() m_progress += bytes_to_read; + if (content_read) + { + if (!CheckContentIntegrity(content)) + { + AddProblem(Severity::High, + StringFromFormat(GetStringT("Content %08x is corrupt.").c_str(), content.id)); + } + + m_content_index++; + } + while (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset < m_progress) { if (!m_volume.CheckBlockIntegrity(m_blocks[m_block_index].block_index, @@ -721,6 +751,30 @@ void VolumeVerifier::Process() } } +bool VolumeVerifier::CheckContentIntegrity(const IOS::ES::Content& content) +{ + const u64 padded_size = Common::AlignUp(content.size, 0x40); + std::vector encrypted_data(padded_size); + m_volume.Read(m_content_offsets[m_content_index], padded_size, encrypted_data.data(), + PARTITION_NONE); + + mbedtls_aes_context context; + const std::array key = m_volume.GetTicket(PARTITION_NONE).GetTitleKey(); + mbedtls_aes_setkey_dec(&context, key.data(), 128); + + std::array iv{}; + iv[0] = static_cast(content.index >> 8); + iv[1] = static_cast(content.index & 0xFF); + + std::vector decrypted_data(padded_size); + mbedtls_aes_crypt_cbc(&context, MBEDTLS_AES_DECRYPT, padded_size, iv.data(), + encrypted_data.data(), decrypted_data.data()); + + std::array sha1; + mbedtls_sha1(decrypted_data.data(), content.size, sha1.data()); + return sha1 == content.sha1; +} + u64 VolumeVerifier::GetBytesProcessed() const { return m_progress; diff --git a/Source/Core/DiscIO/VolumeVerifier.h b/Source/Core/DiscIO/VolumeVerifier.h index 263ef65ff3..7e70ecec32 100644 --- a/Source/Core/DiscIO/VolumeVerifier.h +++ b/Source/Core/DiscIO/VolumeVerifier.h @@ -31,6 +31,7 @@ namespace IOS::ES { +struct Content; class SignedBlobReader; } @@ -100,6 +101,7 @@ private: u64 GetBiggestUsedOffset(const FileInfo& file_info) const; void CheckMisc(); void SetUpHashing(); + bool CheckContentIntegrity(const IOS::ES::Content& content); void AddProblem(Severity severity, const std::string& text); @@ -116,6 +118,8 @@ private: mbedtls_sha1_context m_sha1_context; DiscScrubber m_scrubber; + std::vector m_content_offsets; + u16 m_content_index = 0; std::vector m_blocks; size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition std::map m_block_errors; diff --git a/Source/Core/DiscIO/VolumeWad.cpp b/Source/Core/DiscIO/VolumeWad.cpp index 19f907e6f7..40516df6b6 100644 --- a/Source/Core/DiscIO/VolumeWad.cpp +++ b/Source/Core/DiscIO/VolumeWad.cpp @@ -40,8 +40,8 @@ VolumeWAD::VolumeWAD(std::unique_ptr reader) : m_reader(std::move(re m_cert_chain_offset = Common::AlignUp(m_hdr_size, 0x40); m_ticket_offset = m_cert_chain_offset + Common::AlignUp(m_cert_chain_size, 0x40); m_tmd_offset = m_ticket_offset + Common::AlignUp(m_ticket_size, 0x40); - m_opening_bnr_offset = - m_tmd_offset + Common::AlignUp(m_tmd_size, 0x40) + Common::AlignUp(m_data_size, 0x40); + m_data_offset = m_tmd_offset + Common::AlignUp(m_tmd_size, 0x40); + m_opening_bnr_offset = m_data_offset + Common::AlignUp(m_data_size, 0x40); std::vector ticket_buffer(m_ticket_size); Read(m_ticket_offset, m_ticket_size, ticket_buffer.data()); @@ -118,6 +118,21 @@ const std::vector& VolumeWAD::GetCertificateChain(const Partition& partition return m_cert_chain; } +std::vector VolumeWAD::GetContentOffsets() const +{ + const std::vector contents = m_tmd.GetContents(); + std::vector content_offsets; + content_offsets.reserve(contents.size()); + u64 offset = m_data_offset; + for (const IOS::ES::Content& content : contents) + { + content_offsets.emplace_back(offset); + offset += Common::AlignUp(content.size, 0x40); + } + + return content_offsets; +} + std::string VolumeWAD::GetGameID(const Partition& partition) const { return m_tmd.GetGameID(); diff --git a/Source/Core/DiscIO/VolumeWad.h b/Source/Core/DiscIO/VolumeWad.h index 42da715b9e..4971e813f0 100644 --- a/Source/Core/DiscIO/VolumeWad.h +++ b/Source/Core/DiscIO/VolumeWad.h @@ -38,6 +38,7 @@ public: const IOS::ES::TMDReader& GetTMD(const Partition& partition = PARTITION_NONE) const override; const std::vector& GetCertificateChain(const Partition& partition = PARTITION_NONE) const override; + std::vector GetContentOffsets() const override; std::string GetGameID(const Partition& partition = PARTITION_NONE) const override; std::string GetGameTDBID(const Partition& partition = PARTITION_NONE) const override; std::string GetMakerID(const Partition& partition = PARTITION_NONE) const override; @@ -69,6 +70,7 @@ private: u32 m_cert_chain_offset = 0; u32 m_ticket_offset = 0; u32 m_tmd_offset = 0; + u32 m_data_offset = 0; u32 m_opening_bnr_offset = 0; u32 m_hdr_size = 0; u32 m_cert_chain_size = 0;