DiscIO: Allow converting from formats other than ISO and GCZ

The constant DESIRED_BUFFER_SIZE was determined by multiplying the
old hardcoded value 32 with the default GCZ block size 16 KiB.
Not sure if it actually is the best value, but it seems fine.

Source/Core/DiscIO/Blob.h

@@ -41,11 +41,16 @@ class BlobReader
 {
 public:
   virtual ~BlobReader() {}
+
   virtual BlobType GetBlobType() const = 0;
+
   virtual u64 GetRawSize() const = 0;
   virtual u64 GetDataSize() const = 0;
   virtual bool IsDataSizeAccurate() const = 0;
 
+  // Returns 0 if the format does not use blocks
+  virtual u64 GetBlockSize() const { return 0; }
+
   // NOT thread-safe - can't call this from multiple threads.
   virtual bool Read(u64 offset, u64 size, u8* out_ptr) = 0;
   template <typename T>
@@ -160,10 +165,9 @@ std::unique_ptr<BlobReader> CreateBlobReader(const std::string& filename);
 
 typedef bool (*CompressCB)(const std::string& text, float percent, void* arg);
 
-bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path,
+bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type = 0,
-                        u32 sub_type = 0, int sector_size = 16384, CompressCB callback = nullptr,
+                  int sector_size = 16384, CompressCB callback = nullptr, void* arg = nullptr);
-                        void* arg = nullptr);
+bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path,
-bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path,
+                    CompressCB callback = nullptr, void* arg = nullptr);
-                          CompressCB callback = nullptr, void* arg = nullptr);
 
 }  // namespace DiscIO

Source/Core/DiscIO/CISOBlob.h

@@ -37,12 +37,15 @@ public:
   static std::unique_ptr<CISOFileReader> Create(File::IOFile file);
 
   BlobType GetBlobType() const override { return BlobType::CISO; }
+
+  u64 GetRawSize() const override;
   // The CISO format does not save the original file size.
   // This function returns an upper bound.
   u64 GetDataSize() const override;
   bool IsDataSizeAccurate() const override { return false; }
 
-  u64 GetRawSize() const override;
+  u64 GetBlockSize() const override { return m_block_size; }
+
   bool Read(u64 offset, u64 nbytes, u8* out_ptr) override;
 
 private:

Source/Core/DiscIO/CompressedBlob.cpp

@@ -17,6 +17,7 @@
 #include <vector>
 #include <zlib.h>
 
+#include "Common/Assert.h"
 #include "Common/CommonTypes.h"
 #include "Common/File.h"
 #include "Common/FileUtil.h"
@@ -153,24 +154,20 @@ bool CompressedBlobReader::GetBlock(u64 block_num, u8* out_ptr)
   return true;
 }
 
-bool CompressFileToBlob(const std::string& infile_path, const std::string& outfile_path,
+bool ConvertToGCZ(const std::string& infile_path, const std::string& outfile_path, u32 sub_type,
-                        u32 sub_type, int block_size, CompressCB callback, void* arg)
+                  int block_size, CompressCB callback, void* arg)
 {
   bool scrubbing = false;
 
-  File::IOFile infile(infile_path, "rb");
+  std::unique_ptr<VolumeDisc> infile = CreateDisc(infile_path);
-  if (IsGCZBlob(infile))
-  {
-    PanicAlertT("\"%s\" is already compressed! Cannot compress it further.", infile_path.c_str());
-    return false;
-  }
-
   if (!infile)
   {
     PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str());
     return false;
   }
 
+  ASSERT(infile->IsSizeAccurate());
+
   File::IOFile outfile(outfile_path, "wb");
   if (!outfile)
   {
@@ -182,11 +179,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
   }
 
   DiscScrubber disc_scrubber;
-  std::unique_ptr<VolumeDisc> volume;
   if (sub_type == 1)
   {
-    volume = CreateDisc(infile_path);
+    if (!disc_scrubber.SetupScrub(infile.get()))
-    if (!volume || !disc_scrubber.SetupScrub(volume.get(), block_size))
     {
       PanicAlertT("\"%s\" failed to be scrubbed. Probably the image is corrupt.",
                   infile_path.c_str());
@@ -206,7 +201,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
   header.magic_cookie = GCZ_MAGIC;
   header.sub_type = sub_type;
   header.block_size = block_size;
-  header.data_size = infile.GetSize();
+  header.data_size = infile->GetSize();
 
   // round upwards!
   header.num_blocks = (u32)((header.data_size + (block_size - 1)) / block_size);
@@ -220,10 +215,9 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
   outfile.Seek(sizeof(CompressedBlobHeader), SEEK_CUR);
   // seek past the offset and hash tables (we will write them at the end)
   outfile.Seek((sizeof(u64) + sizeof(u32)) * header.num_blocks, SEEK_CUR);
-  // seek to the start of the input file to make sure we get everything
-  infile.Seek(0, SEEK_SET);
 
   // Now we are ready to write compressed data!
+  u64 inpos = 0;
   u64 position = 0;
   int num_compressed = 0;
   int num_stored = 0;
@@ -234,7 +228,6 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
   {
     if (i % progress_monitor == 0)
     {
-      const u64 inpos = infile.Tell();
       int ratio = 0;
       if (inpos != 0)
         ratio = (int)(100 * position / inpos);
@@ -252,13 +245,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
 
     offsets[i] = position;
 
-    size_t read_bytes;
+    const u64 bytes_to_read = scrubbing && disc_scrubber.CanBlockBeScrubbed(inpos) ?
-    if (scrubbing)
+                                  0 :
-      read_bytes = disc_scrubber.GetNextBlock(infile, in_buf.data());
+                                  std::min<u64>(block_size, header.data_size - inpos);
-    else
+
-      infile.ReadArray(in_buf.data(), header.block_size, &read_bytes);
+    success = infile->Read(inpos, bytes_to_read, in_buf.data(), PARTITION_NONE);
-    if (read_bytes < header.block_size)
+    if (!success)
-      std::fill(in_buf.begin() + read_bytes, in_buf.begin() + header.block_size, 0);
+    {
+      PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str());
+      break;
+    }
+
+    std::fill(in_buf.begin() + bytes_to_read, in_buf.begin() + header.block_size, 0);
 
     int retval = deflateReset(&z);
     z.next_in = in_buf.data();
@@ -305,6 +303,7 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
       break;
     }
 
+    inpos += block_size;
     position += write_size;
 
     hashes[i] = Common::HashAdler32(write_buf, write_size);
@@ -337,27 +336,18 @@ bool CompressFileToBlob(const std::string& infile_path, const std::string& outfi
   return success;
 }
 
-bool DecompressBlobToFile(const std::string& infile_path, const std::string& outfile_path,
+bool ConvertToPlain(const std::string& infile_path, const std::string& outfile_path,
-                          CompressCB callback, void* arg)
+                    CompressCB callback, void* arg)
 {
-  std::unique_ptr<CompressedBlobReader> reader;
+  std::unique_ptr<BlobReader> reader = CreateBlobReader(infile_path);
-  {
-    File::IOFile infile(infile_path, "rb");
-    if (!IsGCZBlob(infile))
-    {
-      PanicAlertT("File not compressed");
-      return false;
-    }
-
-    reader = CompressedBlobReader::Create(std::move(infile), infile_path);
-  }
-
   if (!reader)
   {
     PanicAlertT("Failed to open the input file \"%s\".", infile_path.c_str());
     return false;
   }
 
+  ASSERT(reader->IsDataSizeAccurate());
+
   File::IOFile outfile(outfile_path, "wb");
   if (!outfile)
   {
@@ -368,11 +358,20 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out
     return false;
   }
 
-  const CompressedBlobHeader& header = reader->GetHeader();
+  constexpr size_t DESIRED_BUFFER_SIZE = 0x80000;
-  static const size_t BUFFER_BLOCKS = 32;
+  u64 buffer_size = reader->GetBlockSize();
-  size_t buffer_size = header.block_size * BUFFER_BLOCKS;
+  if (buffer_size == 0)
+  {
+    buffer_size = DESIRED_BUFFER_SIZE;
+  }
+  else
+  {
+    while (buffer_size < DESIRED_BUFFER_SIZE)
+      buffer_size *= 2;
+  }
+
   std::vector<u8> buffer(buffer_size);
-  u32 num_buffers = (header.num_blocks + BUFFER_BLOCKS - 1) / BUFFER_BLOCKS;
+  const u64 num_buffers = (reader->GetDataSize() + buffer_size - 1) / buffer_size;
   int progress_monitor = std::max<int>(1, num_buffers / 100);
   bool success = true;
 
@@ -389,8 +388,13 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out
       }
     }
     const u64 inpos = i * buffer_size;
-    const u64 sz = std::min<u64>(buffer_size, header.data_size - inpos);
+    const u64 sz = std::min(buffer_size, reader->GetDataSize() - inpos);
-    reader->Read(inpos, sz, buffer.data());
+    if (!reader->Read(inpos, sz, buffer.data()))
+    {
+      PanicAlertT("Failed to read from the input file \"%s\".", infile_path.c_str());
+      success = false;
+      break;
+    }
     if (!outfile.WriteBytes(buffer.data(), sz))
     {
       PanicAlertT("Failed to write the output file \"%s\".\n"
@@ -407,10 +411,6 @@ bool DecompressBlobToFile(const std::string& infile_path, const std::string& out
     outfile.Close();
     File::Delete(outfile_path);
   }
-  else
-  {
-    outfile.Resize(header.data_size);
-  }
 
   return success;
 }

Source/Core/DiscIO/CompressedBlob.h

@@ -52,6 +52,7 @@ public:
   u64 GetRawSize() const override { return m_file_size; }
   u64 GetDataSize() const override { return m_header.data_size; }
   bool IsDataSizeAccurate() const override { return true; }
+  u64 GetBlockSize() const override { return m_header.block_size; }
   u64 GetBlockCompressedSize(u64 block_num) const;
   bool GetBlock(u64 block_num, u8* out_ptr) override;
 

Source/Core/DiscIO/DiscScrubber.cpp

@@ -14,6 +14,7 @@
 #include <vector>
 
 #include "Common/Align.h"
+#include "Common/Assert.h"
 #include "Common/CommonTypes.h"
 #include "Common/File.h"
 #include "Common/Logging/Log.h"
@@ -29,19 +30,11 @@ constexpr size_t CLUSTER_SIZE = 0x8000;
 DiscScrubber::DiscScrubber() = default;
 DiscScrubber::~DiscScrubber() = default;
 
-bool DiscScrubber::SetupScrub(const Volume* disc, int block_size)
+bool DiscScrubber::SetupScrub(const Volume* disc)
 {
   if (!disc)
     return false;
   m_disc = disc;
-  m_block_size = block_size;
-
-  if (CLUSTER_SIZE % m_block_size != 0)
-  {
-    ERROR_LOG(DISCIO, "Block size %u is not a factor of 0x8000, scrubbing not possible",
-              m_block_size);
-    return false;
-  }
 
   m_file_size = m_disc->GetSize();
 
@@ -54,34 +47,10 @@ bool DiscScrubber::SetupScrub(const Volume* disc, int block_size)
   // Fill out table of free blocks
   const bool success = ParseDisc();
 
-  m_block_count = 0;
-
   m_is_scrubbing = success;
   return success;
 }
 
-size_t DiscScrubber::GetNextBlock(File::IOFile& in, u8* buffer)
-{
-  const u64 current_offset = m_block_count * m_block_size;
-
-  size_t read_bytes = 0;
-  if (CanBlockBeScrubbed(current_offset))
-  {
-    DEBUG_LOG(DISCIO, "Freeing 0x%016" PRIx64, current_offset);
-    std::fill(buffer, buffer + m_block_size, 0x00);
-    in.Seek(m_block_size, SEEK_CUR);
-    read_bytes = m_block_size;
-  }
-  else
-  {
-    DEBUG_LOG(DISCIO, "Used    0x%016" PRIx64, current_offset);
-    in.ReadArray(buffer, m_block_size, &read_bytes);
-  }
-
-  m_block_count++;
-  return read_bytes;
-}
-
 bool DiscScrubber::CanBlockBeScrubbed(u64 offset) const
 {
   return m_is_scrubbing && m_free_table[offset / CLUSTER_SIZE];

Source/Core/DiscIO/DiscScrubber.h

@@ -34,8 +34,9 @@ public:
   DiscScrubber();
   ~DiscScrubber();
 
-  bool SetupScrub(const Volume* disc, int block_size);
+  bool SetupScrub(const Volume* disc);
-  size_t GetNextBlock(File::IOFile& in, u8* buffer);
+
+  // Returns true if the specified 32 KiB block only contains unused data
   bool CanBlockBeScrubbed(u64 offset) const;
 
 private:
@@ -72,8 +73,6 @@ private:
 
   std::vector<u8> m_free_table;
   u64 m_file_size = 0;
-  u64 m_block_count = 0;
-  u32 m_block_size = 0;
   bool m_is_scrubbing = false;
 };
 

Source/Core/DiscIO/DriveBlob.h

@@ -23,11 +23,15 @@ class DriveReader : public SectorReader
 public:
   static std::unique_ptr<DriveReader> Create(const std::string& drive);
   ~DriveReader();
+
   BlobType GetBlobType() const override { return BlobType::DRIVE; }
+
   u64 GetRawSize() const override { return m_size; }
   u64 GetDataSize() const override { return m_size; }
   bool IsDataSizeAccurate() const override { return true; }
 
+  u64 GetBlockSize() const override { return ECC_BLOCK_SIZE; }
+
 private:
   DriveReader(const std::string& drive);
   bool GetBlock(u64 block_num, u8* out_ptr) override;
@@ -41,6 +45,7 @@ private:
   File::IOFile m_file;
   bool IsOK() const { return m_file.IsOpen() && m_file.IsGood(); }
 #endif
+  static constexpr u64 ECC_BLOCK_SIZE = 0x8000;
   u64 m_size = 0;
 };
 

Source/Core/DiscIO/VolumeVerifier.cpp

@@ -1031,7 +1031,7 @@ void VolumeVerifier::SetUpHashing()
   else if (m_volume.GetVolumeType() == Platform::WiiDisc)
   {
     // Set up a DiscScrubber for checking whether blocks with errors are unused
-    m_scrubber.SetupScrub(&m_volume, VolumeWii::BLOCK_TOTAL_SIZE);
+    m_scrubber.SetupScrub(&m_volume);
   }
 
   std::sort(m_blocks.begin(), m_blocks.end(),

Source/Core/DiscIO/WbfsBlob.h

@@ -24,6 +24,7 @@ public:
   static std::unique_ptr<WbfsFileReader> Create(File::IOFile file, const std::string& path);
 
   BlobType GetBlobType() const override { return BlobType::WBFS; }
+
   u64 GetRawSize() const override { return m_size; }
   // The WBFS format does not save the original file size.
   // This function returns a constant upper bound
@@ -31,6 +32,8 @@ public:
   u64 GetDataSize() const override;
   bool IsDataSizeAccurate() const override { return false; }
 
+  u64 GetBlockSize() const override { return m_wbfs_sector_size; }
+
   bool Read(u64 offset, u64 nbytes, u8* out_ptr) override;
 
 private:

Source/Core/DolphinQt/GameList/GameList.cpp

@@ -595,8 +595,8 @@ void GameList::CompressISO(bool decompress)
       }
 
       good = std::async(std::launch::async, [&] {
-        const bool good = DiscIO::DecompressBlobToFile(original_path, dst_path.toStdString(),
+        const bool good = DiscIO::ConvertToPlain(original_path, dst_path.toStdString(), &CompressCB,
-                                                       &CompressCB, &progress_dialog);
+                                                 &progress_dialog);
         progress_dialog.Reset();
         return good;
       });
@@ -612,9 +612,9 @@ void GameList::CompressISO(bool decompress)
 
       good = std::async(std::launch::async, [&] {
         const bool good =
-            DiscIO::CompressFileToBlob(original_path, dst_path.toStdString(),
+            DiscIO::ConvertToGCZ(original_path, dst_path.toStdString(),
-                                       file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0,
+                                 file->GetPlatform() == DiscIO::Platform::WiiDisc ? 1 : 0, 16384,
-                                       16384, &CompressCB, &progress_dialog);
+                                 &CompressCB, &progress_dialog);
         progress_dialog.Reset();
         return good;
       });