ShuriZma
/
dolphin
mirror of https://github.com/dolphin-emu/dolphin.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

VolumeVerifier: Align partition reads to groups 

This improves the speed of verifying Wii WIA/RVZ files.
For me, the verification speed for LZMA2-compressed files
has gone from 11-12 MiB/s to 13-14 MiB/s.

One thing VolumeVerifier does to achieve parallelism is to
compute hashes for one chunk of data while reading the next
chunk of data. In master, when reading data from a Wii
partition, each such chunk is 32 KiB. This is normally fine,
but with WIA and RVZ it leads to rather lopsided read times
(without the compute times being lopsided): The first 32 KiB
of each 2 MiB takes a long time to read, and the remaining
part of the 2 MiB can be read nearly instantly. (The WIA/RVZ
code has to read the entire 2 MiB in order to compute hashes
which appear at the beginning of the 2 MiB, and then caches
the result afterwards.) This leads to us at times not doing
much reading and at other times not doing much computation.
To improve this, this change makes us use 2 MiB chunks
instead of 32 KiB chunks when reading from Wii partitions.

(block = 32 KiB, group = 2 MiB)
This commit is contained in:
JosJuice 2021-03-07 13:48:51 +01:00
parent 57c9c9eb31
commit c0eb95481f
5 changed files with 73 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Source/Core/DiscIO/Volume.h
View File

@ -126,7 +126,7 @@ public:
virtual bool IsNKit() const = 0;
virtual bool SupportsIntegrityCheck() const { return false; }
virtual bool CheckH3TableIntegrity(const Partition& partition) const { return false; }
virtual bool CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encrypted_data,
virtual bool CheckBlockIntegrity(u64 block_index, const u8* encrypted_data,
const Partition& partition) const
{
return false;

93
Source/Core/DiscIO/VolumeVerifier.cpp
View File

@ -357,7 +357,7 @@ RedumpVerifier::Result RedumpVerifier::Finish(const Hashes<std::vector<u8>>& has
return {Status::Unknown, Common::GetStringT("Unknown disc")};
}
constexpr u64 BLOCK_SIZE = 0x20000;
constexpr u64 DEFAULT_READ_SIZE = 0x20000; // Arbitrary value
VolumeVerifier::VolumeVerifier(const Volume& volume, bool redump_verification,
Hashes<bool> hashes_to_calculate)
@ -585,13 +585,25 @@ bool VolumeVerifier::CheckPartition(const Partition& partition)
// Prepare for hash verification in the Process step
if (m_volume.SupportsIntegrityCheck())
{
u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition);
const std::optional<u64> size =
m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE);
const u64 end_offset = offset + size.value_or(0);
const u64 data_size =
m_volume.ReadSwappedAndShifted(partition.offset + 0x2bc, PARTITION_NONE).value_or(0);
const size_t blocks = static_cast<size_t>(data_size / VolumeWii::BLOCK_TOTAL_SIZE);
for (size_t i = 0; offset < end_offset; ++i, offset += VolumeWii::BLOCK_TOTAL_SIZE)
m_blocks.emplace_back(BlockToVerify{partition, offset, i});
if (data_size % VolumeWii::BLOCK_TOTAL_SIZE != 0)
{
std::string text = Common::FmtFormatT(
"The data size for the {0} partition is not evenly divisible by the block size.", name);
AddProblem(Severity::Low, std::move(text));
}
u64 offset = m_volume.PartitionOffsetToRawOffset(0, partition);
for (size_t block_index = 0; block_index < blocks;
block_index += VolumeWii::BLOCKS_PER_GROUP, offset += VolumeWii::GROUP_TOTAL_SIZE)
{
m_groups.emplace_back(
GroupToVerify{partition, offset, block_index,
std::min(block_index + VolumeWii::BLOCKS_PER_GROUP, blocks)});
}
m_block_errors.emplace(partition, 0);
}
@ -753,7 +765,7 @@ void VolumeVerifier::CheckVolumeSize()
}
}
if (m_content_index != m_content_offsets.size() || m_block_index != m_blocks.size() ||
if (m_content_index != m_content_offsets.size() || m_group_index != m_groups.size() ||
(volume_size_roughly_known && m_biggest_referenced_offset > volume_size))
{
const bool second_layer_missing = is_disc && volume_size_roughly_known &&
@ -1020,8 +1032,8 @@ void VolumeVerifier::SetUpHashing()
m_scrubber.SetupScrub(&m_volume);
}
std::sort(m_blocks.begin(), m_blocks.end(),
[](const BlockToVerify& b1, const BlockToVerify& b2) { return b1.offset < b2.offset; });
std::sort(m_groups.begin(), m_groups.end(),
[](const GroupToVerify& a, const GroupToVerify& b) { return a.offset < b.offset; });
if (m_hashes_to_calculate.crc32)
m_crc32_context = crc32(0, nullptr, 0);
@ -1049,8 +1061,8 @@ void VolumeVerifier::WaitForAsyncOperations() const
m_sha1_future.wait();
if (m_content_future.valid())
m_content_future.wait();
if (m_block_future.valid())
m_block_future.wait();
if (m_group_future.valid())
m_group_future.wait();
}
bool VolumeVerifier::ReadChunkAndWaitForAsyncOperations(u64 bytes_to_read)
@ -1086,8 +1098,8 @@ void VolumeVerifier::Process()
IOS::ES::Content content{};
bool content_read = false;
bool block_read = false;
u64 bytes_to_read = BLOCK_SIZE;
bool group_read = false;
u64 bytes_to_read = DEFAULT_READ_SIZE;
u64 excess_bytes = 0;
if (m_content_index < m_content_offsets.size() &&
m_content_offsets[m_content_index] == m_progress)
@ -1107,20 +1119,22 @@ void VolumeVerifier::Process()
{
bytes_to_read = std::min(bytes_to_read, m_content_offsets[m_content_index] - m_progress);
}
else if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset == m_progress)
else if (m_group_index < m_groups.size() && m_groups[m_group_index].offset == m_progress)
{
bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE;
block_read = true;
const size_t blocks =
m_groups[m_group_index].block_index_end - m_groups[m_group_index].block_index_start;
bytes_to_read = VolumeWii::BLOCK_TOTAL_SIZE * blocks;
group_read = true;
if (m_block_index + 1 < m_blocks.size() &&
m_blocks[m_block_index + 1].offset < m_progress + bytes_to_read)
if (m_group_index + 1 < m_groups.size() &&
m_groups[m_group_index + 1].offset < m_progress + bytes_to_read)
{
excess_bytes = m_progress + bytes_to_read - m_blocks[m_block_index + 1].offset;
excess_bytes = m_progress + bytes_to_read - m_groups[m_group_index + 1].offset;
}
}
else if (m_block_index < m_blocks.size() && m_blocks[m_block_index].offset > m_progress)
else if (m_group_index < m_groups.size() && m_groups[m_group_index].offset > m_progress)
{
bytes_to_read = std::min(bytes_to_read, m_blocks[m_block_index].offset - m_progress);
bytes_to_read = std::min(bytes_to_read, m_groups[m_group_index].offset - m_progress);
}
if (m_progress + bytes_to_read > m_max_progress)
@ -1133,7 +1147,7 @@ void VolumeVerifier::Process()
excess_bytes -= bytes_over_max;
}
const bool is_data_needed = m_calculating_any_hash || content_read || block_read;
const bool is_data_needed = m_calculating_any_hash || content_read || group_read;
const bool read_succeeded = is_data_needed && ReadChunkAndWaitForAsyncOperations(bytes_to_read);
if (!read_succeeded)
@ -1185,33 +1199,40 @@ void VolumeVerifier::Process()
m_content_index++;
}
if (block_read)
if (group_read)
{
m_block_future = std::async(std::launch::async, [this, read_succeeded,
block_index = m_block_index] {
const BlockToVerify& block = m_blocks[block_index];
if (read_succeeded &&
m_volume.CheckBlockIntegrity(block.block_index, m_data, block.partition))
m_group_future = std::async(std::launch::async, [this, read_succeeded,
group_index = m_group_index] {
const GroupToVerify& group = m_groups[group_index];
u64 offset_in_group = 0;
for (u64 block_index = group.block_index_start; block_index < group.block_index_end;
++block_index, offset_in_group += VolumeWii::BLOCK_TOTAL_SIZE)
{
const u64 block_offset = group.offset + offset_in_group;
if (read_succeeded && m_volume.CheckBlockIntegrity(
block_index, m_data.data() + offset_in_group, group.partition))
{
m_biggest_verified_offset =
std::max(m_biggest_verified_offset, block.offset + VolumeWii::BLOCK_TOTAL_SIZE);
std::max(m_biggest_verified_offset, block_offset + VolumeWii::BLOCK_TOTAL_SIZE);
}
else
{
if (m_scrubber.CanBlockBeScrubbed(block.offset))
if (m_scrubber.CanBlockBeScrubbed(block_offset))
{
WARN_LOG_FMT(DISCIO, "Integrity check failed for unused block at {:#x}", block.offset);
m_unused_block_errors[block.partition]++;
WARN_LOG_FMT(DISCIO, "Integrity check failed for unused block at {:#x}", block_offset);
m_unused_block_errors[group.partition]++;
}
else
{
WARN_LOG_FMT(DISCIO, "Integrity check failed for block at {:#x}", block.offset);
m_block_errors[block.partition]++;
WARN_LOG_FMT(DISCIO, "Integrity check failed for block at {:#x}", block_offset);
m_block_errors[group.partition]++;
}
}
}
});
m_block_index++;
m_group_index++;
}
m_progress += byte_increment;

11
Source/Core/DiscIO/VolumeVerifier.h
View File

@ -135,11 +135,12 @@ public:
const Result& GetResult() const;
private:
struct BlockToVerify
struct GroupToVerify
{
Partition partition;
u64 offset;
u64 block_index;
size_t block_index_start;
size_t block_index_end;
};
std::vector<Partition> CheckPartitions();
@ -182,14 +183,14 @@ private:
std::future<void> m_md5_future;
std::future<void> m_sha1_future;
std::future<void> m_content_future;
std::future<void> m_block_future;
std::future<void> m_group_future;
DiscScrubber m_scrubber;
IOS::ES::TicketReader m_ticket;
std::vector<u64> m_content_offsets;
u16 m_content_index = 0;
std::vector<BlockToVerify> m_blocks;
size_t m_block_index = 0; // Index in m_blocks, not index in a specific partition
std::vector<GroupToVerify> m_groups;
size_t m_group_index = 0; // Index in m_groups, not index in a specific partition
std::map<Partition, size_t> m_block_errors;
std::map<Partition, size_t> m_unused_block_errors;

11
Source/Core/DiscIO/VolumeWii.cpp
View File

@ -412,12 +412,9 @@ bool VolumeWii::CheckH3TableIntegrity(const Partition& partition) const
return h3_table_sha1 == contents[0].sha1;
}
bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encrypted_data,
bool VolumeWii::CheckBlockIntegrity(u64 block_index, const u8* encrypted_data,
const Partition& partition) const
{
if (encrypted_data.size() != BLOCK_TOTAL_SIZE)
return false;
auto it = m_partitions.find(partition);
if (it == m_partitions.end())
return false;
@ -431,10 +428,10 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encr
return false;
HashBlock hashes;
DecryptBlockHashes(encrypted_data.data(), &hashes, aes_context);
DecryptBlockHashes(encrypted_data, &hashes, aes_context);
u8 cluster_data[BLOCK_DATA_SIZE];
DecryptBlockData(encrypted_data.data(), cluster_data, aes_context);
DecryptBlockData(encrypted_data, cluster_data, aes_context);
for (u32 hash_index = 0; hash_index < 31; ++hash_index)
{
@ -474,7 +471,7 @@ bool VolumeWii::CheckBlockIntegrity(u64 block_index, const Partition& partition)
std::vector<u8> cluster(BLOCK_TOTAL_SIZE);
if (!m_reader->Read(cluster_offset, cluster.size(), cluster.data()))
return false;
return CheckBlockIntegrity(block_index, cluster, partition);
return CheckBlockIntegrity(block_index, cluster.data(), partition);
}
bool VolumeWii::HashGroup(const std::array<u8, BLOCK_DATA_SIZE> in[BLOCKS_PER_GROUP],

2
Source/Core/DiscIO/VolumeWii.h
View File

@ -82,7 +82,7 @@ public:
bool IsDatelDisc() const override;
bool SupportsIntegrityCheck() const override { return m_encrypted; }
bool CheckH3TableIntegrity(const Partition& partition) const override;
bool CheckBlockIntegrity(u64 block_index, const std::vector<u8>& encrypted_data,
bool CheckBlockIntegrity(u64 block_index, const u8* encrypted_data,
const Partition& partition) const override;
bool CheckBlockIntegrity(u64 block_index, const Partition& partition) const override;