DiscIO: Add support for the NFS format

For a few years now, I've been thinking it would be nice to make Dolphin
support reading Wii games in the format they come in when you download
them from the Wii U eShop. The Wii U eShop has some good deals on Wii
games (Metroid Prime Trilogy especially is rather expensive if you try
to buy it physically!), and it's the only place right now where you can
buy Wii games digitally.

Of course, Nintendo being Nintendo, next year they're going to shut down
this only place where you can buy Wii games digitally. I kind of wish I
had implemented this feature earlier so that people would've had ample
time to buy the games they want, but... better late than never, right?

I used MIT-licensed code from the NOD library as a reference when
implementing this. None of the code has been directly copied, but
you may notice that the names of the struct members are very similar.
c1635245b8/lib/DiscIONFS.cpp

Source/Android/app/src/main/java/org/dolphinemu/dolphinemu/utils/FileBrowserHelper.java

@@ -30,7 +30,8 @@ import java.util.Set;
 public final class FileBrowserHelper
 {
   public static final HashSet<String> GAME_EXTENSIONS = new HashSet<>(Arrays.asList(
-          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "wad", "dol", "elf", "json"));
+          "gcm", "tgc", "iso", "ciso", "gcz", "wbfs", "wia", "rvz", "nfs", "wad", "dol", "elf",
+          "json"));
 
   public static final HashSet<String> GAME_LIKE_EXTENSIONS = new HashSet<>(GAME_EXTENSIONS);
 

Source/Core/Core/Boot/Boot.cpp

@@ -231,7 +231,7 @@ std::unique_ptr<BootParameters> BootParameters::GenerateFromFile(std::vector<std
 #endif
 
   static const std::unordered_set<std::string> disc_image_extensions = {
-      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".dol", ".elf"}};
+      {".gcm", ".iso", ".tgc", ".wbfs", ".ciso", ".gcz", ".wia", ".rvz", ".nfs", ".dol", ".elf"}};
   if (disc_image_extensions.find(extension) != disc_image_extensions.end() || is_drive)
   {
     std::unique_ptr<DiscIO::VolumeDisc> disc = DiscIO::CreateDisc(path);

Source/Core/DiscIO/Blob.cpp

@@ -20,6 +20,7 @@
 #include "DiscIO/DirectoryBlob.h"
 #include "DiscIO/DriveBlob.h"
 #include "DiscIO/FileBlob.h"
+#include "DiscIO/NFSBlob.h"
 #include "DiscIO/TGCBlob.h"
 #include "DiscIO/WIABlob.h"
 #include "DiscIO/WbfsBlob.h"
@@ -52,6 +53,8 @@ std::string GetName(BlobType blob_type, bool translate)
     return "RVZ";
   case BlobType::MOD_DESCRIPTOR:
     return translate_str("Mod");
+  case BlobType::NFS:
+    return "NFS";
   default:
     return "";
   }
@@ -242,6 +245,8 @@ std::unique_ptr<BlobReader> CreateBlobReader(const std::string& filename)
     return WIAFileReader::Create(std::move(file), filename);
   case RVZ_MAGIC:
     return RVZFileReader::Create(std::move(file), filename);
+  case NFS_MAGIC:
+    return NFSFileReader::Create(std::move(file), filename);
   default:
     if (auto directory_blob = DirectoryBlobReader::Create(filename))
       return std::move(directory_blob);

Source/Core/DiscIO/Blob.h

@@ -40,6 +40,7 @@ enum class BlobType
   WIA,
   RVZ,
   MOD_DESCRIPTOR,
+  NFS,
 };
 
 std::string GetName(BlobType blob_type, bool translate);

Source/Core/DiscIO/CMakeLists.txt

@@ -30,6 +30,8 @@ add_library(discio
   MultithreadedCompressor.h
   NANDImporter.cpp
   NANDImporter.h
+  NFSBlob.cpp
+  NFSBlob.h
   RiivolutionParser.cpp
   RiivolutionParser.h
   RiivolutionPatcher.cpp

Source/Core/DiscIO/NFSBlob.cpp

@@ -0,0 +1,306 @@
+// Copyright 2022 Dolphin Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#include "DiscIO/NFSBlob.h"
+
+#include <algorithm>
+#include <array>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include <fmt/format.h>
+
+#include "Common/Align.h"
+#include "Common/CommonTypes.h"
+#include "Common/Crypto/AES.h"
+#include "Common/IOFile.h"
+#include "Common/Logging/Log.h"
+#include "Common/StringUtil.h"
+#include "Common/Swap.h"
+
+namespace DiscIO
+{
+bool NFSFileReader::ReadKey(const std::string& path, const std::string& directory, Key* key_out)
+{
+  const std::string_view directory_without_trailing_slash =
+      std::string_view(directory).substr(0, directory.size() - 1);
+
+  std::string parent, parent_name, parent_extension;
+  SplitPath(directory_without_trailing_slash, &parent, &parent_name, &parent_extension);
+
+  if (parent_name + parent_extension != "content")
+  {
+    ERROR_LOG_FMT(DISCIO, "hif_000000.nfs is not in a directory named 'content': {}", path);
+    return false;
+  }
+
+  const std::string key_path = parent + "code/htk.bin";
+  File::IOFile key_file(key_path, "rb");
+  if (!key_file.ReadBytes(key_out->data(), key_out->size()))
+  {
+    ERROR_LOG_FMT(DISCIO, "Failed to read from {}", key_path);
+    return false;
+  }
+
+  return true;
+}
+
+std::vector<NFSLBARange> NFSFileReader::GetLBARanges(const NFSHeader& header)
+{
+  const size_t lba_range_count =
+      std::min<size_t>(Common::swap32(header.lba_range_count), header.lba_ranges.size());
+
+  std::vector<NFSLBARange> lba_ranges;
+  lba_ranges.reserve(lba_range_count);
+
+  for (size_t i = 0; i < lba_range_count; ++i)
+  {
+    const NFSLBARange& unswapped_lba_range = header.lba_ranges[i];
+    lba_ranges.push_back(NFSLBARange{Common::swap32(unswapped_lba_range.start_block),
+                                     Common::swap32(unswapped_lba_range.num_blocks)});
+  }
+
+  return lba_ranges;
+}
+
+std::vector<File::IOFile> NFSFileReader::OpenFiles(const std::string& directory,
+                                                   File::IOFile first_file, u64 expected_raw_size,
+                                                   u64* raw_size_out)
+{
+  const u64 file_count = Common::AlignUp(expected_raw_size, MAX_FILE_SIZE) / MAX_FILE_SIZE;
+
+  std::vector<File::IOFile> files;
+  files.reserve(file_count);
+
+  u64 raw_size = first_file.GetSize();
+  files.emplace_back(std::move(first_file));
+
+  for (u64 i = 1; i < file_count; ++i)
+  {
+    const std::string child_path = fmt::format("{}hif_{:06}.nfs", directory, i);
+    File::IOFile child(child_path, "rb");
+    if (!child)
+    {
+      ERROR_LOG_FMT(DISCIO, "Failed to open {}", child_path);
+      return {};
+    }
+
+    raw_size += child.GetSize();
+    files.emplace_back(std::move(child));
+  }
+
+  if (raw_size < expected_raw_size)
+  {
+    ERROR_LOG_FMT(
+        DISCIO,
+        "Expected sum of NFS file sizes for {} to be at least {} bytes, but it was {} bytes",
+        directory, expected_raw_size, raw_size);
+    return {};
+  }
+
+  return files;
+}
+
+u64 NFSFileReader::CalculateExpectedRawSize(const std::vector<NFSLBARange>& lba_ranges)
+{
+  u64 total_blocks = 0;
+  for (const NFSLBARange& range : lba_ranges)
+    total_blocks += range.num_blocks;
+
+  return sizeof(NFSHeader) + total_blocks * BLOCK_SIZE;
+}
+
+u64 NFSFileReader::CalculateExpectedDataSize(const std::vector<NFSLBARange>& lba_ranges)
+{
+  u32 greatest_block_index = 0;
+  for (const NFSLBARange& range : lba_ranges)
+    greatest_block_index = std::max(greatest_block_index, range.start_block + range.num_blocks);
+
+  return u64(greatest_block_index) * BLOCK_SIZE;
+}
+
+std::unique_ptr<NFSFileReader> NFSFileReader::Create(File::IOFile first_file,
+                                                     const std::string& path)
+{
+  std::string directory, filename, extension;
+  SplitPath(path, &directory, &filename, &extension);
+  if (filename + extension != "hif_000000.nfs")
+    return nullptr;
+
+  std::array<u8, 16> key;
+  if (!ReadKey(path, directory, &key))
+    return nullptr;
+
+  NFSHeader header;
+  if (!first_file.Seek(0, File::SeekOrigin::Begin) ||
+      !first_file.ReadArray(&header, 1) && header.magic != NFS_MAGIC)
+  {
+    return nullptr;
+  }
+
+  std::vector<NFSLBARange> lba_ranges = GetLBARanges(header);
+
+  const u64 expected_raw_size = CalculateExpectedRawSize(lba_ranges);
+
+  u64 raw_size;
+  std::vector<File::IOFile> files =
+      OpenFiles(directory, std::move(first_file), expected_raw_size, &raw_size);
+
+  if (files.empty())
+    return nullptr;
+
+  return std::unique_ptr<NFSFileReader>(
+      new NFSFileReader(std::move(lba_ranges), std::move(files), key, raw_size));
+}
+
+NFSFileReader::NFSFileReader(std::vector<NFSLBARange> lba_ranges, std::vector<File::IOFile> files,
+                             Key key, u64 raw_size)
+    : m_lba_ranges(std::move(lba_ranges)), m_files(std::move(files)),
+      m_aes_context(Common::AES::CreateContextDecrypt(key.data())), m_raw_size(raw_size)
+{
+  m_data_size = CalculateExpectedDataSize(m_lba_ranges);
+}
+
+u64 NFSFileReader::GetDataSize() const
+{
+  return m_data_size;
+}
+
+u64 NFSFileReader::GetRawSize() const
+{
+  return m_raw_size;
+}
+
+u64 NFSFileReader::ToPhysicalBlockIndex(u64 logical_block_index)
+{
+  u64 physical_blocks_so_far = 0;
+
+  for (const NFSLBARange& range : m_lba_ranges)
+  {
+    if (logical_block_index >= range.start_block &&
+        logical_block_index < range.start_block + range.num_blocks)
+    {
+      return physical_blocks_so_far + (logical_block_index - range.start_block);
+    }
+
+    physical_blocks_so_far += range.num_blocks;
+  }
+
+  return std::numeric_limits<u64>::max();
+}
+
+bool NFSFileReader::ReadEncryptedBlock(u64 physical_block_index)
+{
+  constexpr u64 BLOCKS_PER_FILE = MAX_FILE_SIZE / BLOCK_SIZE;
+
+  const u64 file_index = physical_block_index / BLOCKS_PER_FILE;
+  const u64 block_in_file = physical_block_index % BLOCKS_PER_FILE;
+
+  if (block_in_file == BLOCKS_PER_FILE - 1)
+  {
+    // Special case. Because of the 0x200 byte header at the very beginning,
+    // the last block of each file has its last 0x200 bytes stored in the next file.
+
+    constexpr size_t PART_1_SIZE = BLOCK_SIZE - sizeof(NFSHeader);
+    constexpr size_t PART_2_SIZE = sizeof(NFSHeader);
+
+    File::IOFile& file_1 = m_files[file_index];
+    File::IOFile& file_2 = m_files[file_index + 1];
+
+    if (!file_1.Seek(sizeof(NFSHeader) + block_in_file * BLOCK_SIZE, File::SeekOrigin::Begin) ||
+        !file_1.ReadBytes(m_current_block_encrypted.data(), PART_1_SIZE))
+    {
+      file_1.ClearError();
+      return false;
+    }
+
+    if (!file_2.Seek(0, File::SeekOrigin::Begin) ||
+        !file_2.ReadBytes(m_current_block_encrypted.data() + PART_1_SIZE, PART_2_SIZE))
+    {
+      file_2.ClearError();
+      return false;
+    }
+  }
+  else
+  {
+    // Normal case. The read is offset by 0x200 bytes, but it's all within one file.
+
+    File::IOFile& file = m_files[file_index];
+
+    if (!file.Seek(sizeof(NFSHeader) + block_in_file * BLOCK_SIZE, File::SeekOrigin::Begin) ||
+        !file.ReadBytes(m_current_block_encrypted.data(), BLOCK_SIZE))
+    {
+      file.ClearError();
+      return false;
+    }
+  }
+
+  return true;
+}
+
+void NFSFileReader::DecryptBlock(u64 logical_block_index)
+{
+  std::array<u8, 16> iv{};
+  const u64 swapped_block_index = Common::swap64(logical_block_index);
+  std::memcpy(iv.data() + iv.size() - sizeof(swapped_block_index), &swapped_block_index,
+              sizeof(swapped_block_index));
+
+  m_aes_context->Crypt(iv.data(), m_current_block_encrypted.data(),
+                       m_current_block_decrypted.data(), BLOCK_SIZE);
+}
+
+bool NFSFileReader::ReadAndDecryptBlock(u64 logical_block_index)
+{
+  const u64 physical_block_index = ToPhysicalBlockIndex(logical_block_index);
+
+  if (physical_block_index == std::numeric_limits<u64>::max())
+  {
+    // The block isn't physically present. Treat its contents as all zeroes.
+    m_current_block_decrypted.fill(0);
+  }
+  else
+  {
+    if (!ReadEncryptedBlock(physical_block_index))
+      return false;
+
+    DecryptBlock(logical_block_index);
+  }
+
+  // Small hack: Set 0x61 of the header to 1 so that VolumeWii realizes that the disc is unencrypted
+  if (logical_block_index == 0)
+    m_current_block_decrypted[0x61] = 1;
+
+  return true;
+}
+
+bool NFSFileReader::Read(u64 offset, u64 nbytes, u8* out_ptr)
+{
+  while (nbytes != 0)
+  {
+    const u64 logical_block_index = offset / BLOCK_SIZE;
+    const u64 offset_in_block = offset % BLOCK_SIZE;
+
+    if (logical_block_index != m_current_logical_block_index)
+    {
+      if (!ReadAndDecryptBlock(logical_block_index))
+        return false;
+
+      m_current_logical_block_index = logical_block_index;
+    }
+
+    const u64 bytes_to_copy = std::min(nbytes, BLOCK_SIZE - offset_in_block);
+    std::memcpy(out_ptr, m_current_block_decrypted.data() + offset_in_block, bytes_to_copy);
+
+    offset += bytes_to_copy;
+    nbytes -= bytes_to_copy;
+    out_ptr += bytes_to_copy;
+  }
+
+  return true;
+}
+
+}  // namespace DiscIO

Source/Core/DiscIO/NFSBlob.h

@@ -0,0 +1,91 @@
+// Copyright 2022 Dolphin Emulator Project
+// SPDX-License-Identifier: GPL-2.0-or-later
+
+#pragma once
+
+#include <array>
+#include <limits>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "Common/CommonTypes.h"
+#include "Common/Crypto/AES.h"
+#include "Common/IOFile.h"
+#include "DiscIO/Blob.h"
+
+// This is the file format used for Wii games released on the Wii U eShop.
+
+namespace DiscIO
+{
+static constexpr u32 NFS_MAGIC = 0x53474745;  // "EGGS" (byteswapped to little endian)
+
+struct NFSLBARange
+{
+  u32 start_block;
+  u32 num_blocks;
+};
+
+struct NFSHeader
+{
+  u32 magic;  // EGGS
+  u32 version;
+  u32 unknown_1;
+  u32 unknown_2;
+  u32 lba_range_count;
+  std::array<NFSLBARange, 61> lba_ranges;
+  u32 end_magic;  // SGGE
+};
+static_assert(sizeof(NFSHeader) == 0x200);
+
+class NFSFileReader : public BlobReader
+{
+public:
+  static std::unique_ptr<NFSFileReader> Create(File::IOFile first_file,
+                                               const std::string& directory_path);
+
+  BlobType GetBlobType() const override { return BlobType::NFS; }
+
+  u64 GetRawSize() const override;
+  u64 GetDataSize() const override;
+  bool IsDataSizeAccurate() const override { return false; }
+
+  u64 GetBlockSize() const override { return BLOCK_SIZE; }
+  bool HasFastRandomAccessInBlock() const override { return false; }
+  std::string GetCompressionMethod() const override { return {}; }
+  std::optional<int> GetCompressionLevel() const override { return std::nullopt; }
+
+  bool Read(u64 offset, u64 nbytes, u8* out_ptr) override;
+
+private:
+  using Key = std::array<u8, Common::AES::Context::KEY_SIZE>;
+  static constexpr u32 BLOCK_SIZE = 0x8000;
+  static constexpr u32 MAX_FILE_SIZE = 0xFA00000;
+
+  static bool ReadKey(const std::string& path, const std::string& directory, Key* key_out);
+  static std::vector<NFSLBARange> GetLBARanges(const NFSHeader& header);
+  static std::vector<File::IOFile> OpenFiles(const std::string& directory, File::IOFile first_file,
+                                             u64 expected_raw_size, u64* raw_size_out);
+  static u64 CalculateExpectedRawSize(const std::vector<NFSLBARange>& lba_ranges);
+  static u64 CalculateExpectedDataSize(const std::vector<NFSLBARange>& lba_ranges);
+
+  NFSFileReader(std::vector<NFSLBARange> lba_ranges, std::vector<File::IOFile> files, Key key,
+                u64 raw_size);
+
+  u64 ToPhysicalBlockIndex(u64 logical_block_index);
+  bool ReadEncryptedBlock(u64 physical_block_index);
+  void DecryptBlock(u64 logical_block_index);
+  bool ReadAndDecryptBlock(u64 logical_block_index);
+
+  std::array<u8, BLOCK_SIZE> m_current_block_encrypted;
+  std::array<u8, BLOCK_SIZE> m_current_block_decrypted;
+  u64 m_current_logical_block_index = std::numeric_limits<u64>::max();
+
+  std::vector<NFSLBARange> m_lba_ranges;
+  std::vector<File::IOFile> m_files;
+  std::unique_ptr<Common::AES::Context> m_aes_context;
+  u64 m_raw_size;
+  u64 m_data_size;
+};
+
+}  // namespace DiscIO

Source/Core/DolphinLib.props

@@ -442,6 +442,7 @@
     <ClInclude Include="DiscIO\LaggedFibonacciGenerator.h" />
     <ClInclude Include="DiscIO\MultithreadedCompressor.h" />
     <ClInclude Include="DiscIO\NANDImporter.h" />
+    <ClInclude Include="DiscIO\NFSBlob.h" />
     <ClInclude Include="DiscIO\RiivolutionParser.h" />
     <ClInclude Include="DiscIO\RiivolutionPatcher.h" />
     <ClInclude Include="DiscIO\ScrubbedBlob.h" />
@@ -1056,6 +1057,7 @@
     <ClCompile Include="DiscIO\GameModDescriptor.cpp" />
     <ClCompile Include="DiscIO\LaggedFibonacciGenerator.cpp" />
     <ClCompile Include="DiscIO\NANDImporter.cpp" />
+    <ClCompile Include="DiscIO\NFSBlob.cpp" />
     <ClCompile Include="DiscIO\RiivolutionParser.cpp" />
     <ClCompile Include="DiscIO\RiivolutionPatcher.cpp" />
     <ClCompile Include="DiscIO\ScrubbedBlob.cpp" />

Source/Core/DolphinQt/GameList/GameTracker.cpp

@@ -22,12 +22,13 @@
 
 // NOTE: Qt likes to be case-sensitive here even though it shouldn't be thus this ugly regex hack
 static const QStringList game_filters{
-    QStringLiteral("*.[gG][cC][mM]"), QStringLiteral("*.[iI][sS][oO]"),
-    QStringLiteral("*.[tT][gG][cC]"), QStringLiteral("*.[cC][iI][sS][oO]"),
-    QStringLiteral("*.[gG][cC][zZ]"), QStringLiteral("*.[wW][bB][fF][sS]"),
-    QStringLiteral("*.[wW][iI][aA]"), QStringLiteral("*.[rR][vV][zZ]"),
-    QStringLiteral("*.[wW][aA][dD]"), QStringLiteral("*.[eE][lL][fF]"),
-    QStringLiteral("*.[dD][oO][lL]"), QStringLiteral("*.[jJ][sS][oO][nN]")};
+    QStringLiteral("*.[gG][cC][mM]"),    QStringLiteral("*.[iI][sS][oO]"),
+    QStringLiteral("*.[tT][gG][cC]"),    QStringLiteral("*.[cC][iI][sS][oO]"),
+    QStringLiteral("*.[gG][cC][zZ]"),    QStringLiteral("*.[wW][bB][fF][sS]"),
+    QStringLiteral("*.[wW][iI][aA]"),    QStringLiteral("*.[rR][vV][zZ]"),
+    QStringLiteral("hif_000000.nfs"),    QStringLiteral("*.[wW][aA][dD]"),
+    QStringLiteral("*.[eE][lL][fF]"),    QStringLiteral("*.[dD][oO][lL]"),
+    QStringLiteral("*.[jJ][sS][oO][nN]")};
 
 GameTracker::GameTracker(QObject* parent) : QFileSystemWatcher(parent)
 {

Source/Core/DolphinQt/Info.plist.in

@@ -14,6 +14,7 @@
                 <string>gcz</string>
                 <string>iso</string>
                 <string>m3u</string>
+                <string>nfs</string>
                 <string>rvz</string>
                 <string>tgc</string>
                 <string>wad</string>

Source/Core/DolphinQt/MainWindow.cpp

@@ -725,8 +725,8 @@ QStringList MainWindow::PromptFileNames()
   QStringList paths = DolphinFileDialog::getOpenFileNames(
       this, tr("Select a File"),
       settings.value(QStringLiteral("mainwindow/lastdir"), QString{}).toString(),
-      QStringLiteral("%1 (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz *.wad "
-                     "*.dff *.m3u *.json);;%2 (*)")
+      QStringLiteral("%1 (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz "
+                     "hif_000000.nfs *.wad *.dff *.m3u *.json);;%2 (*)")
           .arg(tr("All GC/Wii files"))
           .arg(tr("All Files")));
 

Source/Core/DolphinQt/Settings/PathPane.cpp

@@ -45,8 +45,8 @@ void PathPane::BrowseDefaultGame()
 {
   QString file = QDir::toNativeSeparators(DolphinFileDialog::getOpenFileName(
       this, tr("Select a Game"), Settings::Instance().GetDefaultGame(),
-      QStringLiteral("%1 (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz *.wad "
-                     "*.m3u *.json);;%2 (*)")
+      QStringLiteral("%1 (*.elf *.dol *.gcm *.iso *.tgc *.wbfs *.ciso *.gcz *.wia *.rvz "
+                     "hif_000000.nfs *.wad *.m3u *.json);;%2 (*)")
           .arg(tr("All GC/Wii files"))
           .arg(tr("All Files"))));
 

Source/Core/UICommon/GameFileCache.cpp

@@ -27,14 +27,14 @@
 
 namespace UICommon
 {
-static constexpr u32 CACHE_REVISION = 21;  // Last changed in PR 10187
+static constexpr u32 CACHE_REVISION = 22;  // Last changed in PR 10932
 
 std::vector<std::string> FindAllGamePaths(const std::vector<std::string>& directories_to_scan,
                                           bool recursive_scan)
 {
-  static const std::vector<std::string> search_extensions = {".gcm", ".tgc",  ".iso", ".ciso",
-                                                             ".gcz", ".wbfs", ".wia", ".rvz",
-                                                             ".wad", ".dol",  ".elf", ".json"};
+  static const std::vector<std::string> search_extensions = {
+      ".gcm", ".tgc", ".iso", ".ciso", ".gcz", ".wbfs", ".wia",
+      ".rvz", ".nfs", ".wad", ".dol",  ".elf", ".json"};
 
   // TODO: We could process paths iteratively as they are found
   return Common::DoFileSearch(directories_to_scan, search_extensions, recursive_scan);