RVZ: Store pseudorandom junk data efficiently

Source/Core/DiscIO/CMakeLists.txt

@@ -21,6 +21,8 @@ add_library(discio
   FileSystemGCWii.h
   Filesystem.cpp
   Filesystem.h
+  LaggedFibonacciGenerator.cpp
+  LaggedFibonacciGenerator.h
   MultithreadedCompressor.h
   NANDImporter.cpp
   NANDImporter.h

Source/Core/DiscIO/DiscIO.vcxproj

@@ -55,6 +55,7 @@
     <ClCompile Include="FileBlob.cpp" />
     <ClCompile Include="Filesystem.cpp" />
     <ClCompile Include="FileSystemGCWii.cpp" />
+    <ClCompile Include="LaggedFibonacciGenerator.cpp" />
     <ClCompile Include="NANDImporter.cpp" />
     <ClCompile Include="ScrubbedBlob.cpp" />
     <ClCompile Include="TGCBlob.cpp" />
@@ -81,6 +82,7 @@
     <ClInclude Include="FileBlob.h" />
     <ClInclude Include="Filesystem.h" />
     <ClInclude Include="FileSystemGCWii.h" />
+    <ClInclude Include="LaggedFibonacciGenerator.h" />
     <ClInclude Include="MultithreadedCompressor.h" />
     <ClInclude Include="NANDImporter.h" />
     <ClInclude Include="ScrubbedBlob.h" />

Source/Core/DiscIO/DiscIO.vcxproj.filters

@@ -93,6 +93,9 @@
     <ClCompile Include="WIABlob.cpp">
       <Filter>Volume\Blob</Filter>
     </ClCompile>
+    <ClCompile Include="LaggedFibonacciGenerator.cpp">
+      <Filter>Volume\Blob</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="DiscScrubber.h">
@@ -170,6 +173,9 @@
     <ClInclude Include="WIABlob.h">
       <Filter>Volume\Blob</Filter>
     </ClInclude>
+    <ClInclude Include="LaggedFibonacciGenerator.h">
+      <Filter>Volume\Blob</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <Text Include="CMakeLists.txt" />

Source/Core/DiscIO/LaggedFibonacciGenerator.cpp

@@ -0,0 +1,212 @@
+// This file is under the public domain.
+
+#include "DiscIO/LaggedFibonacciGenerator.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstring>
+
+#include "Common/Align.h"
+#include "Common/Assert.h"
+#include "Common/CommonTypes.h"
+#include "Common/Swap.h"
+
+namespace DiscIO
+{
+void LaggedFibonacciGenerator::SetSeed(const u32 seed[SEED_SIZE])
+{
+  SetSeed(reinterpret_cast<const u8*>(seed));
+}
+
+void LaggedFibonacciGenerator::SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)])
+{
+  m_position_bytes = 0;
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    m_buffer[i] = Common::swap32(seed + i * sizeof(u32));
+
+  Initialize(false);
+}
+
+size_t LaggedFibonacciGenerator::GetSeed(const u8* data, size_t size, size_t data_offset,
+                                         u32 seed_out[SEED_SIZE])
+{
+  if ((reinterpret_cast<uintptr_t>(data) - data_offset) % alignof(u32) != 0)
+  {
+    ASSERT(false);
+    return 0;
+  }
+
+  // For code simplicity, only include whole u32 words when regenerating the seed. It would be
+  // possible to get rid of this restriction and use a few additional bytes, but it's probably more
+  // effort than it's worth considering that junk data often starts or ends on 4-byte offsets.
+  const size_t bytes_to_skip = Common::AlignUp(data_offset, sizeof(u32)) - data_offset;
+  const u32* u32_data = reinterpret_cast<const u32*>(data + bytes_to_skip);
+  const size_t u32_size = (size - bytes_to_skip) / sizeof(u32);
+  const size_t u32_data_offset = (data_offset + bytes_to_skip) / sizeof(u32);
+
+  LaggedFibonacciGenerator lfg;
+  if (!GetSeed(u32_data, u32_size, u32_data_offset, &lfg, seed_out))
+    return false;
+
+  lfg.m_position_bytes = data_offset % (LFG_K * sizeof(u32));
+
+  const u8* end = data + size;
+  size_t reconstructed_bytes = 0;
+  while (data < end && lfg.GetByte() == *data)
+  {
+    ++reconstructed_bytes;
+    ++data;
+  }
+  return reconstructed_bytes;
+}
+
+bool LaggedFibonacciGenerator::GetSeed(const u32* data, size_t size, size_t data_offset,
+                                       LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE])
+{
+  if (size < LFG_K)
+    return false;
+
+  // If the data doesn't look like something we can regenerate, return early to save time
+  if (!std::all_of(data, data + LFG_K, [](u32 x) {
+        return (Common::swap32(x) & 0x00C00000) == (Common::swap32(x) >> 2 & 0x00C00000);
+      }))
+  {
+    return false;
+  }
+
+  const size_t data_offset_mod_k = data_offset % LFG_K;
+  const size_t data_offset_div_k = data_offset / LFG_K;
+
+  std::copy(data, data + LFG_K - data_offset_mod_k, lfg->m_buffer.data() + data_offset_mod_k);
+  std::copy(data + LFG_K - data_offset_mod_k, data + LFG_K, lfg->m_buffer.data());
+
+  lfg->Backward(0, data_offset_mod_k);
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Backward();
+
+  if (!lfg->Reinitialize(seed_out))
+    return false;
+
+  for (size_t i = 0; i < data_offset_div_k; ++i)
+    lfg->Forward();
+
+  return true;
+}
+
+void LaggedFibonacciGenerator::GetBytes(size_t count, u8* out)
+{
+  while (count > 0)
+  {
+    const size_t length = std::min(count, LFG_K * sizeof(u32) - m_position_bytes);
+
+    std::memcpy(out, reinterpret_cast<u8*>(m_buffer.data()) + m_position_bytes, length);
+
+    m_position_bytes += length;
+    count -= length;
+    out += length;
+
+    if (m_position_bytes == LFG_K * sizeof(u32))
+    {
+      Forward();
+      m_position_bytes = 0;
+    }
+  }
+}
+
+u8 LaggedFibonacciGenerator::GetByte()
+{
+  const u8 result = reinterpret_cast<u8*>(m_buffer.data())[m_position_bytes];
+
+  ++m_position_bytes;
+
+  if (m_position_bytes == LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes = 0;
+  }
+
+  return result;
+}
+
+void LaggedFibonacciGenerator::Forward(size_t count)
+{
+  m_position_bytes += count;
+  while (m_position_bytes >= LFG_K * sizeof(u32))
+  {
+    Forward();
+    m_position_bytes -= LFG_K * sizeof(u32);
+  }
+}
+
+void LaggedFibonacciGenerator::Forward()
+{
+  for (size_t i = 0; i < LFG_J; ++i)
+    m_buffer[i] ^= m_buffer[i + LFG_K - LFG_J];
+
+  for (size_t i = LFG_J; i < LFG_K; ++i)
+    m_buffer[i] ^= m_buffer[i - LFG_J];
+}
+
+void LaggedFibonacciGenerator::Backward(size_t start_word, size_t end_word)
+{
+  const size_t loop_end = std::max(LFG_J, start_word);
+  for (size_t i = std::min(end_word, LFG_K); i > loop_end; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 - LFG_J];
+
+  for (size_t i = std::min(end_word, LFG_J); i > start_word; --i)
+    m_buffer[i - 1] ^= m_buffer[i - 1 + LFG_K - LFG_J];
+}
+
+bool LaggedFibonacciGenerator::Reinitialize(u32 seed_out[SEED_SIZE])
+{
+  for (size_t i = 0; i < 4; ++i)
+    Backward();
+
+  for (u32& x : m_buffer)
+    x = Common::swap32(x);
+
+  // Reconstruct the bits which are missing due to the output code shifting by 18 instead of 16.
+  // Unfortunately we can't reconstruct bits 16 and 17 (counting LSB as 0) for the first word,
+  // but the observable result (when shifting by 18 instead of 16) is not affected by this.
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+  {
+    m_buffer[i] = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000) |
+                  ((m_buffer[i + 16] ^ m_buffer[i + 15]) << 9 & 0x00030000);
+  }
+
+  for (size_t i = 0; i < SEED_SIZE; ++i)
+    seed_out[i] = Common::swap32(m_buffer[i]);
+
+  return Initialize(true);
+}
+
+bool LaggedFibonacciGenerator::Initialize(bool check_existing_data)
+{
+  for (size_t i = SEED_SIZE; i < LFG_K; ++i)
+  {
+    const u32 calculated = (m_buffer[i - 17] << 23) ^ (m_buffer[i - 16] >> 9) ^ m_buffer[i - 1];
+
+    if (check_existing_data)
+    {
+      const u32 actual = (m_buffer[i] & 0xFF00FFFF) | (m_buffer[i] << 2 & 0x00FC0000);
+      if ((calculated & 0xFFFCFFFF) != actual)
+        return false;
+    }
+
+    m_buffer[i] = calculated;
+  }
+
+  // Instead of doing the "shift by 18 instead of 16" oddity when actually outputting the data,
+  // we can do the shifting (and byteswapping) at this point to make the output code simpler.
+  for (u32& x : m_buffer)
+    x = Common::swap32((x & 0xFF00FFFF) | ((x >> 2) & 0x00FF0000));
+
+  for (size_t i = 0; i < 4; ++i)
+    Forward();
+
+  return true;
+}
+
+}  // namespace DiscIO

Source/Core/DiscIO/LaggedFibonacciGenerator.h

@@ -0,0 +1,51 @@
+// This file is under the public domain.
+
+#pragma once
+
+#include <array>
+#include <cstddef>
+
+#include "Common/CommonTypes.h"
+
+namespace DiscIO
+{
+class LaggedFibonacciGenerator
+{
+public:
+  static constexpr size_t SEED_SIZE = 17;
+
+  // Reconstructs a seed and writes it to seed_out, then returns the number of bytes which can
+  // be reconstructed using that seed. Can return any number between 0 and size, inclusive.
+  // data - data_offset must be 4-byte aligned.
+  static size_t GetSeed(const u8* data, size_t size, size_t data_offset, u32 seed_out[SEED_SIZE]);
+
+  // SetSeed must be called before using the functions below
+  void SetSeed(const u32 seed[SEED_SIZE]);
+  void SetSeed(const u8 seed[SEED_SIZE * sizeof(u32)]);
+
+  // Outputs a number of bytes and advances the internal state by the same amount.
+  void GetBytes(size_t count, u8* out);
+  u8 GetByte();
+
+  // Advances the internal state like GetBytes, but without outputting data. O(N), like GetBytes.
+  void Forward(size_t count);
+
+private:
+  static bool GetSeed(const u32* data, size_t size, size_t data_offset,
+                      LaggedFibonacciGenerator* lfg, u32 seed_out[SEED_SIZE]);
+
+  void Forward();
+  void Backward(size_t start_word = 0, size_t end_word = LFG_K);
+
+  bool Reinitialize(u32 seed_out[SEED_SIZE]);
+  bool Initialize(bool check_existing_data);
+
+  static constexpr size_t LFG_K = 521;
+  static constexpr size_t LFG_J = 32;
+
+  std::array<u32, LFG_K> m_buffer;
+
+  size_t m_position_bytes = 0;
+};
+
+}  // namespace DiscIO

Source/Core/DiscIO/WIABlob.cpp

@@ -34,6 +34,7 @@
 
 #include "DiscIO/Blob.h"
 #include "DiscIO/DiscExtractor.h"
+#include "DiscIO/LaggedFibonacciGenerator.h"
 #include "DiscIO/MultithreadedCompressor.h"
 #include "DiscIO/Volume.h"
 #include "DiscIO/VolumeWii.h"
@@ -192,10 +193,9 @@ bool WIAFileReader::Initialize(const std::string& path)
 
   const u32 number_of_raw_data_entries = Common::swap32(m_header_2.number_of_raw_data_entries);
   m_raw_data_entries.resize(number_of_raw_data_entries);
-  Chunk& raw_data_entries =
+  Chunk& raw_data_entries = ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset),
-      ReadCompressedData(Common::swap64(m_header_2.raw_data_entries_offset),
                                                Common::swap32(m_header_2.raw_data_entries_size),
-                         number_of_raw_data_entries * sizeof(RawDataEntry), false);
+                                               number_of_raw_data_entries * sizeof(RawDataEntry));
   if (!raw_data_entries.ReadAll(&m_raw_data_entries))
     return false;
 
@@ -211,7 +211,7 @@ bool WIAFileReader::Initialize(const std::string& path)
   m_group_entries.resize(number_of_group_entries);
   Chunk& group_entries = ReadCompressedData(Common::swap64(m_header_2.group_entries_offset),
                                             Common::swap32(m_header_2.group_entries_size),
-                                            number_of_group_entries * sizeof(GroupEntry), false);
+                                            number_of_group_entries * sizeof(GroupEntry));
   if (!group_entries.ReadAll(&m_group_entries))
     return false;
 
@@ -444,8 +444,8 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu
     else
     {
       const u64 group_offset_in_file = static_cast<u64>(Common::swap32(group.data_offset)) << 2;
-      Chunk& chunk =
+      Chunk& chunk = ReadCompressedData(group_offset_in_file, group_data_size, chunk_size,
-          ReadCompressedData(group_offset_in_file, group_data_size, chunk_size, exception_lists);
+                                        exception_lists, m_rvz, group_offset_in_data);
       if (!chunk.Read(offset_in_group, bytes_to_read, *out_ptr))
       {
         m_cached_chunk_offset = std::numeric_limits<u64>::max();  // Invalidate the cache
@@ -472,7 +472,8 @@ bool WIAFileReader::ReadFromGroups(u64* offset, u64* size, u8** out_ptr, u64 chu
 }
 
 WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64 compressed_size,
-                                                        u64 decompressed_size, u32 exception_lists)
+                                                        u64 decompressed_size, u32 exception_lists,
+                                                        bool rvz_pack, u64 data_offset)
 {
   if (offset_in_file == m_cached_chunk_offset)
     return m_cached_chunk;
@@ -504,8 +505,9 @@ WIAFileReader::Chunk& WIAFileReader::ReadCompressedData(u64 offset_in_file, u64
 
   const bool compressed_exception_lists = m_compression_type > WIACompressionType::Purge;
 
-  m_cached_chunk = Chunk(&m_file, offset_in_file, compressed_size, decompressed_size,
+  m_cached_chunk =
-                         exception_lists, compressed_exception_lists, std::move(decompressor));
+      Chunk(&m_file, offset_in_file, compressed_size, decompressed_size, exception_lists,
+            compressed_exception_lists, rvz_pack, data_offset, std::move(decompressor));
   m_cached_chunk_offset = offset_in_file;
   return m_cached_chunk;
 }
@@ -793,6 +795,135 @@ bool WIAFileReader::ZstdDecompressor::Decompress(const DecompressionBuffer& in,
   return !ZSTD_isError(result);
 }
 
+WIAFileReader::RVZPackDecompressor::RVZPackDecompressor(std::unique_ptr<Decompressor> decompressor,
+                                                        DecompressionBuffer decompressed,
+                                                        u64 data_offset)
+    : m_decompressor(std::move(decompressor)), m_decompressed(std::move(decompressed)),
+      m_data_offset(data_offset)
+{
+}
+
+std::optional<bool> WIAFileReader::RVZPackDecompressor::ReadToDecompressed(
+    const DecompressionBuffer& in, size_t* in_bytes_read, size_t decompressed_bytes_read,
+    size_t bytes_to_read)
+{
+  if (m_decompressed.data.size() < decompressed_bytes_read + bytes_to_read)
+    m_decompressed.data.resize(decompressed_bytes_read + bytes_to_read);
+
+  if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read)
+  {
+    if (!m_decompressor->Decompress(in, &m_decompressed, in_bytes_read))
+      return false;
+
+    if (m_decompressed.bytes_written < decompressed_bytes_read + bytes_to_read)
+      return true;
+  }
+
+  return std::nullopt;
+}
+
+bool WIAFileReader::RVZPackDecompressor::Decompress(const DecompressionBuffer& in,
+                                                    DecompressionBuffer* out, size_t* in_bytes_read)
+{
+  while (out->data.size() != out->bytes_written && !Done())
+  {
+    if (m_size == 0)
+    {
+      if (m_decompressed.bytes_written == m_decompressed_bytes_read)
+      {
+        m_decompressed.data.resize(sizeof(u32));
+        m_decompressed.bytes_written = 0;
+        m_decompressed_bytes_read = 0;
+      }
+
+      std::optional<bool> result =
+          ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read, sizeof(u32));
+      if (result)
+        return *result;
+
+      m_size = Common::swap32(m_decompressed.data.data() + m_decompressed_bytes_read);
+
+      m_junk = m_size & 0x80000000;
+      if (m_junk)
+      {
+        m_size &= 0x7FFFFFFF;
+
+        constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32);
+
+        result = ReadToDecompressed(in, in_bytes_read, m_decompressed_bytes_read + sizeof(u32),
+                                    SEED_SIZE);
+        if (result)
+          return *result;
+
+        m_lfg.SetSeed(m_decompressed.data.data() + m_decompressed_bytes_read + sizeof(u32));
+        m_lfg.Forward(m_data_offset % VolumeWii::BLOCK_TOTAL_SIZE);
+
+        m_decompressed_bytes_read += SEED_SIZE;
+      }
+
+      m_decompressed_bytes_read += sizeof(u32);
+    }
+
+    size_t bytes_to_write = std::min<size_t>(m_size, out->data.size() - out->bytes_written);
+    if (m_junk)
+    {
+      m_lfg.GetBytes(bytes_to_write, out->data.data() + out->bytes_written);
+      out->bytes_written += bytes_to_write;
+    }
+    else
+    {
+      if (m_decompressed.bytes_written != m_decompressed_bytes_read)
+      {
+        bytes_to_write =
+            std::min(bytes_to_write, m_decompressed.bytes_written - m_decompressed_bytes_read);
+
+        std::memcpy(out->data.data() + out->bytes_written,
+                    m_decompressed.data.data() + m_decompressed_bytes_read, bytes_to_write);
+
+        m_decompressed_bytes_read += bytes_to_write;
+        out->bytes_written += bytes_to_write;
+      }
+      else
+      {
+        const size_t prev_out_bytes_written = out->bytes_written;
+        const size_t old_out_size = out->data.size();
+        const size_t new_out_size = out->bytes_written + bytes_to_write;
+
+        if (new_out_size < old_out_size)
+          out->data.resize(new_out_size);
+
+        if (!m_decompressor->Decompress(in, out, in_bytes_read))
+          return false;
+
+        out->data.resize(old_out_size);
+
+        bytes_to_write = out->bytes_written - prev_out_bytes_written;
+        if (bytes_to_write == 0)
+          return true;
+      }
+    }
+
+    m_data_offset += bytes_to_write;
+    m_size -= static_cast<u32>(bytes_to_write);
+  }
+
+  // If out is full but not all data has been read from in, give the decompressor a chance to read
+  // from in anyway. This is needed for the case where zstd has read everything except the checksum.
+  if (out->data.size() == out->bytes_written && in.bytes_written != *in_bytes_read)
+  {
+    if (!m_decompressor->Decompress(in, out, in_bytes_read))
+      return false;
+  }
+
+  return true;
+}
+
+bool WIAFileReader::RVZPackDecompressor::Done() const
+{
+  return m_size == 0 && m_decompressed.bytes_written == m_decompressed_bytes_read &&
+         m_decompressor->Done();
+}
+
 WIAFileReader::Compressor::~Compressor() = default;
 
 WIAFileReader::PurgeCompressor::PurgeCompressor()
@@ -1169,11 +1300,11 @@ WIAFileReader::Chunk::Chunk() = default;
 
 WIAFileReader::Chunk::Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size,
                             u64 decompressed_size, u32 exception_lists,
-                            bool compressed_exception_lists,
+                            bool compressed_exception_lists, bool rvz_pack, u64 data_offset,
                             std::unique_ptr<Decompressor> decompressor)
     : m_file(file), m_offset_in_file(offset_in_file), m_exception_lists(exception_lists),
-      m_compressed_exception_lists(compressed_exception_lists),
+      m_compressed_exception_lists(compressed_exception_lists), m_rvz_pack(rvz_pack),
-      m_decompressor(std::move(decompressor))
+      m_data_offset(data_offset), m_decompressor(std::move(decompressor))
 {
   constexpr size_t MAX_SIZE_PER_EXCEPTION_LIST =
       Common::AlignUp(VolumeWii::BLOCK_HEADER_SIZE, sizeof(SHA1)) / sizeof(SHA1) *
@@ -1250,7 +1381,7 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr)
 
     if (m_exception_lists == 0 || m_compressed_exception_lists)
     {
-      if (!m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read))
+      if (!Decompress())
         return false;
     }
 
@@ -1261,6 +1392,12 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr)
       {
         return false;
       }
+
+      if (m_rvz_pack && m_exception_lists == 0)
+      {
+        if (!Decompress())
+          return false;
+      }
     }
 
     if (m_exception_lists == 0)
@@ -1289,6 +1426,26 @@ bool WIAFileReader::Chunk::Read(u64 offset, u64 size, u8* out_ptr)
   return true;
 }
 
+bool WIAFileReader::Chunk::Decompress()
+{
+  if (m_rvz_pack && m_exception_lists == 0)
+  {
+    m_rvz_pack = false;
+
+    const size_t bytes_to_move = m_out.bytes_written - m_out_bytes_used_for_exceptions;
+
+    DecompressionBuffer in{std::vector<u8>(bytes_to_move), bytes_to_move};
+    std::memcpy(in.data.data(), m_out.data.data() + m_out_bytes_used_for_exceptions, bytes_to_move);
+
+    m_out.bytes_written = m_out_bytes_used_for_exceptions;
+
+    m_decompressor = std::make_unique<RVZPackDecompressor>(std::move(m_decompressor), std::move(in),
+                                                           m_data_offset);
+  }
+
+  return m_decompressor->Decompress(m_in, &m_out, &m_in_bytes_read);
+}
+
 bool WIAFileReader::Chunk::HandleExceptions(const u8* data, size_t bytes_allocated,
                                             size_t bytes_written, size_t* bytes_used, bool align)
 {
@@ -1633,23 +1790,120 @@ static bool AllSame(const u8* begin, const u8* end)
   return AllAre(begin, end, *begin);
 };
 
-ConversionResult<WIAFileReader::OutputParameters>
+void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk,
-WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters parameters,
+                            size_t chunks, u64 total_size, u64 data_offset, u64 in_offset,
+                            bool allow_junk_reuse)
+{
+  using Seed = std::array<u32, LaggedFibonacciGenerator::SEED_SIZE>;
+  struct JunkInfo
+  {
+    size_t start_offset;
+    Seed seed;
+  };
+
+  // Maps end_offset -> (start_offset, seed)
+  std::map<size_t, JunkInfo> junk_info;
+
+  size_t position = 0;
+  while (position < total_size)
+  {
+    const size_t bytes_to_read =
+        std::min(Common::AlignUp(data_offset + 1, VolumeWii::BLOCK_TOTAL_SIZE) - data_offset,
+                 total_size - position);
+
+    const size_t data_offset_mod = static_cast<size_t>(data_offset % VolumeWii::BLOCK_TOTAL_SIZE);
+
+    Seed seed;
+    const size_t bytes_reconstructed = LaggedFibonacciGenerator::GetSeed(
+        in + in_offset + position, bytes_to_read, data_offset_mod, seed.data());
+
+    if (bytes_reconstructed > 0)
+      junk_info.emplace(position + bytes_reconstructed, JunkInfo{position, seed});
+
+    position += bytes_to_read;
+    data_offset += bytes_to_read;
+  }
+
+  for (size_t i = 0; i < chunks; ++i)
+  {
+    OutputParametersEntry& entry = out[i];
+    if (entry.reused_group)
+      continue;
+
+    u64 current_offset = i * bytes_per_chunk;
+    const u64 end_offset = std::min(current_offset + bytes_per_chunk, total_size);
+
+    const bool store_junk_efficiently = allow_junk_reuse || !entry.reuse_id;
+
+    while (current_offset < end_offset)
+    {
+      constexpr size_t SEED_SIZE = LaggedFibonacciGenerator::SEED_SIZE * sizeof(u32);
+
+      u64 next_junk_start = end_offset;
+      u64 next_junk_end = end_offset;
+      Seed* seed = nullptr;
+      if (store_junk_efficiently && end_offset - current_offset > SEED_SIZE)
+      {
+        const auto next_junk_it = junk_info.upper_bound(current_offset + SEED_SIZE);
+        if (next_junk_it != junk_info.end() &&
+            next_junk_it->second.start_offset + SEED_SIZE < end_offset)
+        {
+          next_junk_start = std::max<u64>(current_offset, next_junk_it->second.start_offset);
+          next_junk_end = std::min<u64>(end_offset, next_junk_it->first);
+          seed = &next_junk_it->second.seed;
+        }
+      }
+
+      const u64 non_junk_bytes = next_junk_start - current_offset;
+      if (non_junk_bytes > 0)
+      {
+        const u8* ptr = in + in_offset + current_offset;
+
+        PushBack(&entry.main_data, Common::swap32(static_cast<u32>(non_junk_bytes)));
+        PushBack(&entry.main_data, ptr, ptr + non_junk_bytes);
+
+        current_offset += non_junk_bytes;
+      }
+
+      const u64 junk_bytes = next_junk_end - current_offset;
+      if (junk_bytes > 0)
+      {
+        PushBack(&entry.main_data, Common::swap32(static_cast<u32>(junk_bytes) | 0x80000000));
+        PushBack(&entry.main_data, *seed);
+
+        current_offset += junk_bytes;
+      }
+    }
+  }
+}
+
+void WIAFileReader::RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset,
+                            bool allow_junk_reuse)
+{
+  RVZPack(in, out, size, 1, size, data_offset, 0, allow_junk_reuse);
+}
+
+ConversionResult<WIAFileReader::OutputParameters> WIAFileReader::ProcessAndCompress(
+    CompressThreadState* state, CompressParameters parameters,
     const std::vector<PartitionEntry>& partition_entries,
-                                  const std::vector<DataEntry>& data_entries,
+    const std::vector<DataEntry>& data_entries, std::map<ReuseID, GroupEntry>* reusable_groups,
-                                  std::map<ReuseID, GroupEntry>* reusable_groups,
+    std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group, u64 exception_lists_per_chunk,
-                                  std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group,
+    bool compressed_exception_lists, bool rvz)
-                                  u64 exception_lists_per_chunk, bool compressed_exception_lists)
 {
   std::vector<OutputParametersEntry> output_entries;
 
   if (!parameters.data_entry->is_partition)
   {
     OutputParametersEntry& entry = output_entries.emplace_back();
-    entry.main_data = std::move(parameters.data);
+    std::vector<u8>& data = parameters.data;
 
-    if (AllSame(entry.main_data))
+    if (AllSame(data))
-      entry.reuse_id = ReuseID{nullptr, entry.main_data.size(), false, entry.main_data.front()};
+      entry.reuse_id = ReuseID{nullptr, data.size(), false, data.front()};
+
+    if (rvz)
+      RVZPack(data.data(), output_entries.data(), data.size(), parameters.data_offset, true);
+    else
+      entry.main_data = std::move(data);
   }
   else
   {
@@ -1676,9 +1930,9 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
     const size_t first_chunk = output_entries.size();
 
     const auto create_reuse_id = [&partition_entry, blocks,
-                                  blocks_per_chunk](u8 value, bool decrypted, u64 block) {
+                                  blocks_per_chunk](u8 value, bool encrypted, u64 block) {
       const u64 size = std::min(blocks - block, blocks_per_chunk) * VolumeWii::BLOCK_DATA_SIZE;
-      return ReuseID{&partition_entry.partition_key, size, decrypted, value};
+      return ReuseID{&partition_entry.partition_key, size, encrypted, value};
     };
 
     const u8* parameters_data_end = parameters.data.data() + parameters.data.size();
@@ -1692,7 +1946,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
       // Set this chunk as reusable if the encrypted data is AllSame
       const u8* data = parameters.data.data() + block_index * VolumeWii::BLOCK_TOTAL_SIZE;
       if (AllSame(data, std::min(parameters_data_end, data + in_data_per_chunk)))
-        reuse_id = create_reuse_id(parameters.data.front(), false, i * blocks_per_chunk);
+        reuse_id = create_reuse_id(parameters.data.front(), true, i * blocks_per_chunk);
 
       TryReuse(reusable_groups, reusable_groups_mutex, &entry);
       if (!entry.reused_group && reuse_id)
@@ -1794,6 +2048,22 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
 
         static_assert(std::is_trivially_copyable_v<decltype(
                           CompressThreadState::decryption_buffer)::value_type>);
+        if (rvz)
+        {
+          // We must not store junk efficiently for chunks that may get reused at a position
+          // which has a different value of data_offset % VolumeWii::BLOCK_TOTAL_SIZE
+          const bool allow_junk_reuse = chunks_per_wii_group == 1;
+
+          const u64 bytes_per_chunk = std::min(out_data_per_chunk, VolumeWii::GROUP_DATA_SIZE);
+          const u64 total_size = blocks_in_this_group * VolumeWii::BLOCK_DATA_SIZE;
+          const u64 data_offset = parameters.data_offset + write_offset_of_group;
+
+          RVZPack(state->decryption_buffer[0].data(), output_entries.data() + first_chunk,
+                  bytes_per_chunk, chunks, total_size, data_offset, write_offset_of_group,
+                  allow_junk_reuse);
+        }
+        else
+        {
           const u8* in_ptr = state->decryption_buffer[0].data();
           for (u64 j = 0; j < chunks; ++j)
           {
@@ -1816,11 +2086,11 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
               if (i == 0 && !entry.reuse_id)
               {
                 if (AllSame(in_ptr, in_ptr + bytes_to_write))
-                entry.reuse_id = create_reuse_id(*in_ptr, true, j * blocks_per_chunk);
+                  entry.reuse_id = create_reuse_id(*in_ptr, false, j * blocks_per_chunk);
               }
               else
               {
-              if (entry.reuse_id && entry.reuse_id->decrypted &&
+                if (entry.reuse_id && !entry.reuse_id->encrypted &&
                     (!AllSame(in_ptr, in_ptr + bytes_to_write) || entry.reuse_id->value != *in_ptr))
                 {
                   entry.reuse_id.reset();
@@ -1831,6 +2101,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
             in_ptr += out_data_per_chunk;
           }
         }
+      }
 
       for (size_t i = 0; i < exception_lists.size(); ++i)
       {
@@ -1853,7 +2124,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
 
         // If this chunk was set as reusable because the decrypted data is AllSame,
         // but it has exceptions, unmark it as reusable
-        if (entry.reuse_id && entry.reuse_id->decrypted && !AllZero(entry.exception_lists))
+        if (entry.reuse_id && !entry.reuse_id->encrypted && !AllZero(entry.exception_lists))
           entry.reuse_id.reset();
       }
     }
@@ -1866,7 +2137,7 @@ WIAFileReader::ProcessAndCompress(CompressThreadState* state, CompressParameters
       continue;
 
     // Special case - a compressed size of zero is treated by WIA as meaning the data is all zeroes
-    if (AllZero(entry.exception_lists) && AllZero(entry.main_data))
+    if (entry.reuse_id && !entry.reuse_id->encrypted && entry.reuse_id->value == 0)
     {
       entry.exception_lists.clear();
       entry.main_data.clear();
@@ -2084,7 +2355,7 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume,
   const auto process_and_compress = [&](CompressThreadState* state, CompressParameters parameters) {
     return ProcessAndCompress(state, std::move(parameters), partition_entries, data_entries,
                               &reusable_groups, &reusable_groups_mutex, chunks_per_wii_group,
-                              exception_lists_per_chunk, compressed_exception_lists);
+                              exception_lists_per_chunk, compressed_exception_lists, rvz);
   };
 
   const auto output = [&](OutputParameters parameters) {
@@ -2110,6 +2381,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume,
     u64 data_offset;
     u64 data_size;
 
+    u64 data_offset_in_partition;
+
     if (data_entry.is_partition)
     {
       const PartitionEntry& partition_entry = partition_entries[data_entry.index];
@@ -2119,9 +2392,14 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume,
       first_group = Common::swap32(partition_data_entry.group_index);
       last_group = first_group + Common::swap32(partition_data_entry.number_of_groups);
 
-      data_offset = Common::swap32(partition_data_entry.first_sector) * VolumeWii::BLOCK_TOTAL_SIZE;
+      const u32 first_sector = Common::swap32(partition_data_entry.first_sector);
+      data_offset = first_sector * VolumeWii::BLOCK_TOTAL_SIZE;
       data_size =
           Common::swap32(partition_data_entry.number_of_sectors) * VolumeWii::BLOCK_TOTAL_SIZE;
+
+      const u32 block_in_partition =
+          first_sector - Common::swap32(partition_entry.data_entries[0].first_sector);
+      data_offset_in_partition = block_in_partition * VolumeWii::BLOCK_DATA_SIZE;
     }
     else
     {
@@ -2136,6 +2414,8 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume,
       const u64 skipped_data = data_offset % VolumeWii::BLOCK_TOTAL_SIZE;
       data_offset -= skipped_data;
       data_size += skipped_data;
+
+      data_offset_in_partition = data_offset;
     }
 
     ASSERT(groups_processed == first_group);
@@ -2157,11 +2437,26 @@ WIAFileReader::ConvertToWIA(BlobReader* infile, const VolumeDisc* infile_volume,
         return ConversionResultCode::ReadFailed;
       bytes_read += bytes_to_read;
 
-      mt_compressor.CompressAndWrite(
+      mt_compressor.CompressAndWrite(CompressParameters{
-          CompressParameters{buffer, &data_entry, bytes_read, groups_processed});
+          buffer, &data_entry, data_offset_in_partition, bytes_read, groups_processed});
+
+      data_offset += bytes_to_read;
+      data_size -= bytes_to_read;
+
+      if (data_entry.is_partition)
+      {
+        data_offset_in_partition +=
+            bytes_to_read / VolumeWii::BLOCK_TOTAL_SIZE * VolumeWii::BLOCK_DATA_SIZE;
+      }
+      else
+      {
+        data_offset_in_partition += bytes_to_read;
+      }
 
       groups_processed += Common::AlignUp(bytes_to_read, chunk_size) / chunk_size;
     }
+
+    ASSERT(data_size == 0);
   }
 
   ASSERT(groups_processed == total_groups);

Source/Core/DiscIO/WIABlob.h

@@ -10,6 +10,7 @@
 #include <memory>
 #include <mutex>
 #include <optional>
+#include <type_traits>
 #include <utility>
 
 #include <bzlib.h>
@@ -21,6 +22,7 @@
 #include "Common/File.h"
 #include "Common/Swap.h"
 #include "DiscIO/Blob.h"
+#include "DiscIO/LaggedFibonacciGenerator.h"
 #include "DiscIO/MultithreadedCompressor.h"
 #include "DiscIO/WiiEncryptionCache.h"
 
@@ -267,6 +269,31 @@ private:
     ZSTD_DStream* m_stream;
   };
 
+  class RVZPackDecompressor final : public Decompressor
+  {
+  public:
+    RVZPackDecompressor(std::unique_ptr<Decompressor> decompressor,
+                        DecompressionBuffer decompressed, u64 data_offset);
+
+    bool Decompress(const DecompressionBuffer& in, DecompressionBuffer* out,
+                    size_t* in_bytes_read) override;
+
+    bool Done() const override;
+
+  private:
+    std::optional<bool> ReadToDecompressed(const DecompressionBuffer& in, size_t* in_bytes_read,
+                                           size_t decompressed_bytes_read, size_t bytes_to_read);
+
+    std::unique_ptr<Decompressor> m_decompressor;
+    DecompressionBuffer m_decompressed;
+    size_t m_decompressed_bytes_read = 0;
+    u64 m_data_offset;
+
+    u32 m_size = 0;
+    bool m_junk;
+    LaggedFibonacciGenerator m_lfg;
+  };
+
   class Compressor
   {
   public:
@@ -375,7 +402,7 @@ private:
   public:
     Chunk();
     Chunk(File::IOFile* file, u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
-          u32 exception_lists, bool compressed_exception_lists,
+          u32 exception_lists, bool compressed_exception_lists, bool rvz_pack, u64 data_offset,
           std::unique_ptr<Decompressor> decompressor);
 
     bool Read(u64 offset, u64 size, u8* out_ptr);
@@ -391,6 +418,7 @@ private:
     }
 
   private:
+    bool Decompress();
     bool HandleExceptions(const u8* data, size_t bytes_allocated, size_t bytes_written,
                           size_t* bytes_used, bool align);
 
@@ -407,6 +435,8 @@ private:
     size_t m_in_bytes_used_for_exceptions = 0;
     u32 m_exception_lists = 0;
     bool m_compressed_exception_lists = false;
+    bool m_rvz_pack = false;
+    u64 m_data_offset = 0;
   };
 
   explicit WIAFileReader(File::IOFile file, const std::string& path);
@@ -417,7 +447,7 @@ private:
                       u64 data_offset, u64 data_size, u32 group_index, u32 number_of_groups,
                       u32 exception_lists);
   Chunk& ReadCompressedData(u64 offset_in_file, u64 compressed_size, u64 decompressed_size,
-                            u32 exception_lists);
+                            u32 exception_lists = 0, bool rvz_pack = false, u64 data_offset = 0);
 
   static bool ApplyHashExceptions(const std::vector<HashExceptionEntry>& exception_list,
                                   VolumeWii::HashBlock hash_blocks[VolumeWii::BLOCKS_PER_GROUP]);
@@ -430,18 +460,18 @@ private:
   {
     bool operator==(const ReuseID& other) const
     {
-      return std::tie(partition_key, data_size, decrypted, value) ==
+      return std::tie(partition_key, data_size, encrypted, value) ==
-             std::tie(other.partition_key, other.data_size, other.decrypted, other.value);
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
     }
     bool operator<(const ReuseID& other) const
     {
-      return std::tie(partition_key, data_size, decrypted, value) <
+      return std::tie(partition_key, data_size, encrypted, value) <
-             std::tie(other.partition_key, other.data_size, other.decrypted, other.value);
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
     }
     bool operator>(const ReuseID& other) const
     {
-      return std::tie(partition_key, data_size, decrypted, value) >
+      return std::tie(partition_key, data_size, encrypted, value) >
-             std::tie(other.partition_key, other.data_size, other.decrypted, other.value);
+             std::tie(other.partition_key, other.data_size, other.encrypted, other.value);
     }
     bool operator!=(const ReuseID& other) const { return !operator==(other); }
     bool operator>=(const ReuseID& other) const { return !operator<(other); }
@@ -449,7 +479,7 @@ private:
 
     const WiiKey* partition_key;
     u64 data_size;
-    bool decrypted;
+    bool encrypted;
     u8 value;
   };
 
@@ -470,6 +500,7 @@ private:
   {
     std::vector<u8> data;
     const DataEntry* data_entry;
+    u64 data_offset;
     u64 bytes_read;
     size_t group_index;
   };
@@ -512,13 +543,17 @@ private:
                               WIAHeader2* header_2);
   static bool TryReuse(std::map<ReuseID, GroupEntry>* reusable_groups,
                        std::mutex* reusable_groups_mutex, OutputParametersEntry* entry);
+  static void RVZPack(const u8* in, OutputParametersEntry* out, u64 bytes_per_chunk, size_t chunks,
+                      u64 total_size, u64 data_offset, u64 in_offset, bool allow_junk_reuse);
+  static void RVZPack(const u8* in, OutputParametersEntry* out, u64 size, u64 data_offset,
+                      bool allow_junk_reuse);
   static ConversionResult<OutputParameters>
   ProcessAndCompress(CompressThreadState* state, CompressParameters parameters,
                      const std::vector<PartitionEntry>& partition_entries,
                      const std::vector<DataEntry>& data_entries,
                      std::map<ReuseID, GroupEntry>* reusable_groups,
                      std::mutex* reusable_groups_mutex, u64 chunks_per_wii_group,
-                     u64 exception_lists_per_chunk, bool compressed_exception_lists);
+                     u64 exception_lists_per_chunk, bool compressed_exception_lists, bool rvz);
   static ConversionResultCode Output(std::vector<OutputParametersEntry>* entries,
                                      File::IOFile* outfile,
                                      std::map<ReuseID, GroupEntry>* reusable_groups,
@@ -528,13 +563,20 @@ private:
                                           u32 total_groups, u64 iso_size, CompressCB callback,
                                           void* arg);
 
+  static void PushBack(std::vector<u8>* vector, const u8* begin, const u8* end)
+  {
+    const size_t offset_in_vector = vector->size();
+    vector->resize(offset_in_vector + (end - begin));
+    std::copy(begin, end, vector->data() + offset_in_vector);
+  }
+
   template <typename T>
   static void PushBack(std::vector<u8>* vector, const T& x)
   {
-    const size_t offset_in_vector = vector->size();
+    static_assert(std::is_trivially_copyable_v<T>);
-    vector->resize(offset_in_vector + sizeof(T));
+
     const u8* x_ptr = reinterpret_cast<const u8*>(&x);
-    std::copy(x_ptr, x_ptr + sizeof(T), vector->data() + offset_in_vector);
+    PushBack(vector, x_ptr, x_ptr + sizeof(T));
   }
 
   bool m_valid;
@@ -566,9 +608,9 @@ private:
   static constexpr u32 WIA_VERSION_WRITE_COMPATIBLE = 0x01000000;
   static constexpr u32 WIA_VERSION_READ_COMPATIBLE = 0x00080000;
 
-  static constexpr u32 RVZ_VERSION = 0x00010000;
+  static constexpr u32 RVZ_VERSION = 0x00020000;
-  static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00010000;
+  static constexpr u32 RVZ_VERSION_WRITE_COMPATIBLE = 0x00020000;
-  static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00010000;
+  static constexpr u32 RVZ_VERSION_READ_COMPATIBLE = 0x00020000;
 };
 
 }  // namespace DiscIO

Source/Core/DolphinQt/ConvertDialog.cpp

@@ -252,6 +252,10 @@ void ConvertDialog::OnFormatChanged()
 
   m_block_size->setEnabled(m_block_size->count() > 1);
   m_compression->setEnabled(m_compression->count() > 1);
+
+  m_scrub->setEnabled(format != DiscIO::BlobType::RVZ);
+  if (format == DiscIO::BlobType::RVZ)
+    m_scrub->setChecked(false);
 }
 
 void ConvertDialog::OnCompressionChanged()