From f2f83a0c608f51fe4533624bddade0886e1febe7 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Thu, 27 Nov 2014 07:53:28 -0800 Subject: [PATCH 1/4] DolphinWX: Allow cancelling gcz compression. --- Source/Core/DiscIO/Blob.h | 2 +- Source/Core/DiscIO/CompressedBlob.cpp | 40 ++++++++++++++++++++------ Source/Core/DolphinWX/GameListCtrl.cpp | 10 ++++--- Source/Core/DolphinWX/GameListCtrl.h | 4 +-- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/Source/Core/DiscIO/Blob.h b/Source/Core/DiscIO/Blob.h index b9e5e7fda9..ed9bd3c7d0 100644 --- a/Source/Core/DiscIO/Blob.h +++ b/Source/Core/DiscIO/Blob.h @@ -66,7 +66,7 @@ private: // Factory function - examines the path to choose the right type of IBlobReader, and returns one. IBlobReader* CreateBlobReader(const std::string& filename); -typedef void (*CompressCB)(const std::string& text, float percent, void* arg); +typedef bool (*CompressCB)(const std::string& text, float percent, void* arg); bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u32 sub_type = 0, int sector_size = 16384, CompressCB callback = nullptr, void *arg = nullptr); diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index be5647ef11..660d5c8c98 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -195,6 +195,7 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u int num_compressed = 0; int num_stored = 0; int progress_monitor = std::max(1, header.num_blocks / 1000); + bool was_cancelled = false; for (u32 i = 0; i < header.num_blocks; i++) { @@ -206,7 +207,9 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u ratio = (int)(100 * position / inpos); std::string temp = StringFromFormat("%i of %i blocks. Compression ratio %i%%", i, header.num_blocks, ratio); - callback(temp, (float)i / (float)header.num_blocks, arg); + was_cancelled = !callback(temp, (float)i / (float)header.num_blocks, arg); + if (was_cancelled) + break; } offsets[i] = position; @@ -261,11 +264,20 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u header.compressed_data_size = position; - // Okay, go back and fill in headers - f.Seek(0, SEEK_SET); - f.WriteArray(&header, 1); - f.WriteArray(offsets, header.num_blocks); - f.WriteArray(hashes, header.num_blocks); + if (was_cancelled) + { + // Remove the incomplete output file. + f.Close(); + File::Delete(outfile); + } + else + { + // Okay, go back and fill in headers + f.Seek(0, SEEK_SET); + f.WriteArray(&header, 1); + f.WriteArray(offsets, header.num_blocks); + f.WriteArray(hashes, header.num_blocks); + } cleanup: // Cleanup @@ -301,12 +313,15 @@ bool DecompressBlobToFile(const std::string& infile, const std::string& outfile, const CompressedBlobHeader &header = reader->GetHeader(); u8* buffer = new u8[header.block_size]; int progress_monitor = std::max(1, header.num_blocks / 100); + bool was_cancelled = false; for (u64 i = 0; i < header.num_blocks; i++) { if (i % progress_monitor == 0) { - callback("Unpacking", (float)i / (float)header.num_blocks, arg); + was_cancelled = !callback("Unpacking", (float)i / (float)header.num_blocks, arg); + if (was_cancelled) + break; } reader->Read(i * header.block_size, header.block_size, buffer); f.WriteBytes(buffer, header.block_size); @@ -314,7 +329,16 @@ bool DecompressBlobToFile(const std::string& infile, const std::string& outfile, delete[] buffer; - f.Resize(header.data_size); + if (was_cancelled) + { + // Remove the incomplete output file. + f.Close(); + File::Delete(outfile); + } + else + { + f.Resize(header.data_size); + } delete reader; diff --git a/Source/Core/DolphinWX/GameListCtrl.cpp b/Source/Core/DolphinWX/GameListCtrl.cpp index 7efe9fe54e..124c68b8f4 100644 --- a/Source/Core/DolphinWX/GameListCtrl.cpp +++ b/Source/Core/DolphinWX/GameListCtrl.cpp @@ -1123,14 +1123,14 @@ void CGameListCtrl::OnWiki(wxCommandEvent& WXUNUSED (event)) WxUtils::Launch(wikiUrl); } -void CGameListCtrl::MultiCompressCB(const std::string& text, float percent, void* arg) +bool CGameListCtrl::MultiCompressCB(const std::string& text, float percent, void* arg) { percent = (((float)m_currentItem) + percent) / (float)m_numberItem; wxString textString(StrToWxStr(StringFromFormat("%s (%i/%i) - %s", m_currentFilename.c_str(), (int)m_currentItem+1, (int)m_numberItem, text.c_str()))); - ((wxProgressDialog*)arg)->Update((int)(percent*1000), textString); + return ((wxProgressDialog*)arg)->Update((int)(percent*1000), textString); } void CGameListCtrl::OnMultiCompressISO(wxCommandEvent& /*event*/) @@ -1162,6 +1162,7 @@ void CGameListCtrl::CompressSelection(bool _compress) 1000, this, wxPD_APP_MODAL | + wxPD_CAN_ABORT | wxPD_ELAPSED_TIME | wxPD_ESTIMATED_TIME | wxPD_REMAINING_TIME | wxPD_SMOOTH ); @@ -1235,9 +1236,9 @@ void CGameListCtrl::CompressSelection(bool _compress) Update(); } -void CGameListCtrl::CompressCB(const std::string& text, float percent, void* arg) +bool CGameListCtrl::CompressCB(const std::string& text, float percent, void* arg) { - ((wxProgressDialog*)arg)-> + return ((wxProgressDialog*)arg)-> Update((int)(percent*1000), StrToWxStr(text)); } @@ -1300,6 +1301,7 @@ void CGameListCtrl::OnCompressISO(wxCommandEvent& WXUNUSED (event)) 1000, this, wxPD_APP_MODAL | + wxPD_CAN_ABORT | wxPD_ELAPSED_TIME | wxPD_ESTIMATED_TIME | wxPD_REMAINING_TIME | wxPD_SMOOTH ); diff --git a/Source/Core/DolphinWX/GameListCtrl.h b/Source/Core/DolphinWX/GameListCtrl.h index 38af1451f1..15f89b2944 100644 --- a/Source/Core/DolphinWX/GameListCtrl.h +++ b/Source/Core/DolphinWX/GameListCtrl.h @@ -113,6 +113,6 @@ private: static size_t m_currentItem; static std::string m_currentFilename; static size_t m_numberItem; - static void CompressCB(const std::string& text, float percent, void* arg); - static void MultiCompressCB(const std::string& text, float percent, void* arg); + static bool CompressCB(const std::string& text, float percent, void* arg); + static bool MultiCompressCB(const std::string& text, float percent, void* arg); }; From 2635e7d9eaf6f96463de0a75a0b5a07c274b2cb4 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Thu, 27 Nov 2014 08:34:44 -0800 Subject: [PATCH 2/4] DiscIO: Decompress to file using a larger buffer. This improves performance by around 20% for me, and the memory use impact is negligible considering Dolphin is otherwise unusable. --- Source/Core/DiscIO/CompressedBlob.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 660d5c8c98..6170c5ad07 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -311,20 +311,23 @@ bool DecompressBlobToFile(const std::string& infile, const std::string& outfile, } const CompressedBlobHeader &header = reader->GetHeader(); - u8* buffer = new u8[header.block_size]; - int progress_monitor = std::max(1, header.num_blocks / 100); + static const size_t BUFFER_BLOCKS = 32; + size_t buffer_size = header.block_size * BUFFER_BLOCKS; + u8* buffer = new u8[buffer_size]; + u32 num_buffers = header.num_blocks / BUFFER_BLOCKS; + int progress_monitor = std::max(1, num_buffers / 100); bool was_cancelled = false; - for (u64 i = 0; i < header.num_blocks; i++) + for (u64 i = 0; i < num_buffers; i++) { if (i % progress_monitor == 0) { - was_cancelled = !callback("Unpacking", (float)i / (float)header.num_blocks, arg); + was_cancelled = !callback("Unpacking", (float)i / (float)num_buffers, arg); if (was_cancelled) break; } - reader->Read(i * header.block_size, header.block_size, buffer); - f.WriteBytes(buffer, header.block_size); + reader->Read(i * buffer_size, buffer_size, buffer); + f.WriteBytes(buffer, buffer_size); } delete[] buffer; From f54bf815208e61977392753b984e20abb8086c5f Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Thu, 27 Nov 2014 08:57:49 -0800 Subject: [PATCH 3/4] DiscIO: Avoid zeroing buffer when compressing gcz. This saves 6% time. --- Source/Core/DiscIO/CompressedBlob.cpp | 31 ++++++++++++++++----------- Source/Core/DiscIO/DiscScrubber.cpp | 7 ++++-- Source/Core/DiscIO/DiscScrubber.h | 2 +- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 6170c5ad07..61549e5bda 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -163,11 +163,19 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u scrubbing = true; } + z_stream z; + memset(&z, 0, sizeof(z)); + if (deflateInit(&z, 9) != Z_OK) + return false; + File::IOFile inf(infile, "rb"); File::IOFile f(outfile, "wb"); if (!f || !inf) + { + deflateEnd(&z); return false; + } callback("Files opened, ready to compress.", 0, arg); @@ -213,23 +221,20 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u } offsets[i] = position; - // u64 start = i * header.block_size; - // u64 size = header.block_size; - std::fill(in_buf, in_buf + header.block_size, 0); + + size_t read_bytes; if (scrubbing) - DiscScrubber::GetNextBlock(inf, in_buf); + read_bytes = DiscScrubber::GetNextBlock(inf, in_buf); else - inf.ReadBytes(in_buf, header.block_size); - z_stream z; - memset(&z, 0, sizeof(z)); - z.zalloc = Z_NULL; - z.zfree = Z_NULL; - z.opaque = Z_NULL; + inf.ReadArray(in_buf, header.block_size, &read_bytes); + if (read_bytes < header.block_size) + std::fill(in_buf + read_bytes, in_buf + header.block_size, 0); + + int retval = deflateReset(&z); z.next_in = in_buf; z.avail_in = header.block_size; z.next_out = out_buf; z.avail_out = block_size; - int retval = deflateInit(&z, 9); if (retval != Z_OK) { @@ -258,8 +263,6 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u position += comp_size; num_compressed++; } - - deflateEnd(&z); } header.compressed_data_size = position; @@ -286,6 +289,8 @@ cleanup: delete[] offsets; delete[] hashes; + deflateEnd(&z); + DiscScrubber::Cleanup(); callback("Done compressing disc image.", 1.0f, arg); return true; diff --git a/Source/Core/DiscIO/DiscScrubber.cpp b/Source/Core/DiscIO/DiscScrubber.cpp index 8cdc314ea4..03a45616a4 100644 --- a/Source/Core/DiscIO/DiscScrubber.cpp +++ b/Source/Core/DiscIO/DiscScrubber.cpp @@ -124,24 +124,27 @@ bool SetupScrub(const std::string& filename, int block_size) return success; } -void GetNextBlock(File::IOFile& in, u8* buffer) +size_t GetNextBlock(File::IOFile& in, u8* buffer) { u64 CurrentOffset = m_BlockCount * m_BlockSize; u64 i = CurrentOffset / CLUSTER_SIZE; + size_t ReadBytes = 0; if (m_isScrubbing && m_FreeTable[i]) { DEBUG_LOG(DISCIO, "Freeing 0x%016" PRIx64, CurrentOffset); std::fill(buffer, buffer + m_BlockSize, 0xFF); in.Seek(m_BlockSize, SEEK_CUR); + ReadBytes = m_BlockSize; } else { DEBUG_LOG(DISCIO, "Used 0x%016" PRIx64, CurrentOffset); - in.ReadBytes(buffer, m_BlockSize); + in.ReadArray(buffer, m_BlockSize, &ReadBytes); } m_BlockCount++; + return ReadBytes; } void Cleanup() diff --git a/Source/Core/DiscIO/DiscScrubber.h b/Source/Core/DiscIO/DiscScrubber.h index 0e5943ae2e..2311db4f7b 100644 --- a/Source/Core/DiscIO/DiscScrubber.h +++ b/Source/Core/DiscIO/DiscScrubber.h @@ -25,7 +25,7 @@ namespace DiscScrubber { bool SetupScrub(const std::string& filename, int block_size); -void GetNextBlock(File::IOFile& in, u8* buffer); +size_t GetNextBlock(File::IOFile& in, u8* buffer); void Cleanup(); } // namespace DiscScrubber From 815b7bec967c773a3c8605dfe12a7cfefa844d31 Mon Sep 17 00:00:00 2001 From: "Unknown W. Brackets" Date: Thu, 27 Nov 2014 11:12:17 -0800 Subject: [PATCH 4/4] DiscIO: Change a memset() to zero initialization. --- Source/Core/DiscIO/CompressedBlob.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/Core/DiscIO/CompressedBlob.cpp b/Source/Core/DiscIO/CompressedBlob.cpp index 61549e5bda..c3b79baf1c 100644 --- a/Source/Core/DiscIO/CompressedBlob.cpp +++ b/Source/Core/DiscIO/CompressedBlob.cpp @@ -163,8 +163,7 @@ bool CompressFileToBlob(const std::string& infile, const std::string& outfile, u scrubbing = true; } - z_stream z; - memset(&z, 0, sizeof(z)); + z_stream z = {}; if (deflateInit(&z, 9) != Z_OK) return false;