Support loading Zstandard-compressed ROMs (#1667)

This is different from the archive support in that the compressed ROMs
are standalone files, rather than archives, making it possible to use
them exactly as if they were regular ROMs, while saving a bunch of space
on disk. This is supported both for DS and GBA ROMs, though given GBA
ROMs' generally small size it's mostly useful for the former.

.github/workflows/build-ubuntu-aarch64.yml

@@ -33,7 +33,7 @@ jobs:
         rm /etc/apt/sources.list
         mv /etc/apt/sources.list{.new,}
         apt update
-        DEBIAN_FRONTEND=noninteractive apt install -y {gcc-10,g++-10,pkg-config}-aarch64-linux-gnu {libsdl2,qtbase5,qtbase5-private,qtmultimedia5,libslirp,libarchive}-dev:arm64 cmake extra-cmake-modules dpkg-dev
+        DEBIAN_FRONTEND=noninteractive apt install -y {gcc-10,g++-10,pkg-config}-aarch64-linux-gnu {libsdl2,qtbase5,qtbase5-private,qtmultimedia5,libslirp,libarchive,libzstd}-dev:arm64 zstd:arm64 cmake extra-cmake-modules dpkg-dev
     - name: Configure
       shell: bash
       run: |

.github/workflows/build-ubuntu.yml

@@ -19,7 +19,7 @@ jobs:
       run: |
         sudo rm -f /etc/apt/sources.list.d/dotnetdev.list /etc/apt/sources.list.d/microsoft-prod.list
         sudo apt update
-        sudo apt install cmake extra-cmake-modules libcurl4-gnutls-dev libpcap0.8-dev libsdl2-dev qt5-default qtbase5-private-dev qtmultimedia5-dev libslirp0 libslirp-dev libarchive-dev --allow-downgrades
+        sudo apt install cmake extra-cmake-modules libcurl4-gnutls-dev libpcap0.8-dev libsdl2-dev qt5-default qtbase5-private-dev qtmultimedia5-dev libslirp0 libslirp-dev libarchive-dev zstd libzstd-dev --allow-downgrades
     - name: Create build environment
       run: mkdir ${{runner.workspace}}/build
     - name: Configure

README.md

@@ -35,9 +35,9 @@ As for the rest, the interface should be pretty straightforward. If you have a q
 
 ### Linux
 1. Install dependencies:
-   * Ubuntu 22.04: `sudo apt install cmake extra-cmake-modules libcurl4-gnutls-dev libpcap0.8-dev libsdl2-dev qtbase5-dev qtbase5-private-dev qtmultimedia5-dev libslirp-dev libarchive-dev`
+   * Ubuntu 22.04: `sudo apt install cmake extra-cmake-modules libcurl4-gnutls-dev libpcap0.8-dev libsdl2-dev qtbase5-dev qtbase5-private-dev qtmultimedia5-dev libslirp-dev libarchive-dev libzstd-dev`
-   * Older Ubuntu: `sudo apt install cmake extra-cmake-modules libcurl4-gnutls-dev libpcap0.8-dev libsdl2-dev qt5-default qtbase5-private-dev qtmultimedia5-dev libslirp-dev libarchive-dev`
+   * Older Ubuntu: `sudo apt install cmake extra-cmake-modules libcurl4-gnutls-dev libpcap0.8-dev libsdl2-dev qt5-default qtbase5-private-dev qtmultimedia5-dev libslirp-dev libarchive-dev libzstd-dev`
-   * Arch Linux: `sudo pacman -S base-devel cmake extra-cmake-modules git libpcap sdl2 qt5-base qt5-multimedia libslirp libarchive`
+   * Arch Linux: `sudo pacman -S base-devel cmake extra-cmake-modules git libpcap sdl2 qt5-base qt5-multimedia libslirp libarchive zstd`
 3. Download the melonDS repository and prepare:
    ```bash
    git clone https://github.com/melonDS-emu/melonDS
@@ -64,7 +64,7 @@ As for the rest, the interface should be pretty straightforward. If you have a q
    cd melonDS
    ```
 #### Dynamic builds (with DLLs)
-5. Install dependencies: `pacman -S mingw-w64-x86_64-{cmake,SDL2,toolchain,qt5-base,qt5-svg,qt5-multimedia,libslirp,libarchive}`
+5. Install dependencies: `pacman -S mingw-w64-x86_64-{cmake,SDL2,toolchain,qt5-base,qt5-svg,qt5-multimedia,libslirp,libarchive,zstd}`
 6. Compile:
    ```bash
    cmake -B build
@@ -75,7 +75,7 @@ As for the rest, the interface should be pretty straightforward. If you have a q
 If everything went well, melonDS and the libraries it needs should now be in the `dist` folder.
 
 #### Static builds (without DLLs, standalone executable)
-5. Install dependencies: `pacman -S mingw-w64-x86_64-{cmake,SDL2,toolchain,qt5-static,libslirp,libarchive}`
+5. Install dependencies: `pacman -S mingw-w64-x86_64-{cmake,SDL2,toolchain,qt5-static,libslirp,libarchive,zstd}`
 6. Compile:
    ```bash
    cmake -B build -DBUILD_STATIC=ON -DCMAKE_PREFIX_PATH=/mingw64/qt5-static
@@ -85,7 +85,7 @@ If everything went well, melonDS should now be in the `build` folder.
 
 ### macOS
 1. Install the [Homebrew Package Manager](https://brew.sh)
-2. Install dependencies: `brew install git pkg-config cmake sdl2 qt@6 libslirp libarchive`
+2. Install dependencies: `brew install git pkg-config cmake sdl2 qt@6 libslirp libarchive zstd`
 3. Download the melonDS repository and prepare:
    ```zsh
    git clone https://github.com/melonDS-emu/melonDS

res/melon.plist.in

@@ -39,6 +39,10 @@
                 <string>srl</string>
                 <string>dsi</string>
                 <string>ids</string>
+                <string>nds.zst</string>
+                <string>srl.zst</string>
+                <string>dsi.zst</string>
+                <string>ids.zst</string>
             </array>
             <key>CFBundleTypeRole</key>
             <string>Viewer</string>
@@ -50,6 +54,8 @@
             <array>
                 <string>gba</string>
                 <string>agb</string>
+                <string>gba.zst</string>
+                <string>agb.zst</string>
             </array>
             <key>CFBundleTypeRole</key>
             <string>Viewer</string>

src/frontend/qt_sdl/CMakeLists.txt

@@ -82,6 +82,7 @@ endif()
 pkg_check_modules(SDL2 REQUIRED IMPORTED_TARGET sdl2)
 pkg_check_modules(Slirp REQUIRED IMPORTED_TARGET slirp)
 pkg_check_modules(LibArchive REQUIRED IMPORTED_TARGET libarchive)
+pkg_check_modules(Zstd REQUIRED IMPORTED_TARGET libzstd)
 
 fix_interface_includes(PkgConfig::SDL2 PkgConfig::Slirp PkgConfig::LibArchive)
 
@@ -154,7 +155,7 @@ else()
     target_include_directories(melonDS PUBLIC ${Qt5Gui_PRIVATE_INCLUDE_DIRS})
 endif()
 target_link_libraries(melonDS PRIVATE core)
-target_link_libraries(melonDS PRIVATE PkgConfig::SDL2 PkgConfig::Slirp PkgConfig::LibArchive)
+target_link_libraries(melonDS PRIVATE PkgConfig::SDL2 PkgConfig::Slirp PkgConfig::LibArchive PkgConfig::Zstd)
 target_link_libraries(melonDS PRIVATE ${QT_LINK_LIBS} ${CMAKE_DL_LIBS})
 
 if (UNIX)

src/frontend/qt_sdl/ROMManager.cpp

@@ -22,6 +22,7 @@
 #include <string>
 #include <utility>
 
+#include <zstd.h>
 #ifdef ARCHIVE_SUPPORT_ENABLED
 #include "ArchiveUtil.h"
 #endif
@@ -491,6 +492,27 @@ bool LoadBIOS()
     return true;
 }
 
+u32 DecompressROM(const u8* inContent, const u32 inSize, u8** outContent)
+{
+    u64 realSize = ZSTD_getFrameContentSize(inContent, inSize);
+
+    if (realSize == ZSTD_CONTENTSIZE_UNKNOWN || realSize == ZSTD_CONTENTSIZE_ERROR || realSize > 0x40000000)
+    {
+        return 0;
+    }
+
+    u8* realContent = new u8[realSize];
+    u64 decompressed = ZSTD_decompress(realContent, realSize, inContent, inSize);
+
+    if (ZSTD_isError(decompressed))
+    {
+        delete[] realContent;
+        return 0;
+    }
+
+    *outContent = realContent;
+    return realSize;
+}
 
 bool LoadROM(QStringList filepath, bool reset)
 {
@@ -533,6 +555,25 @@ bool LoadROM(QStringList filepath, bool reset)
         fclose(f);
         filelen = (u32)len;
 
+        if (filename.length() > 4 && filename.substr(filename.length() - 4) == ".zst")
+        {
+            u8* outContent = nullptr;
+            u32 decompressed = DecompressROM(filedata, len, &outContent);
+
+            if (decompressed > 0)
+            {
+                delete[] filedata;
+                filedata = outContent;
+                filelen = decompressed;
+                filename = filename.substr(0, filename.length() - 4);
+            }
+            else
+            {
+                delete[] filedata;
+                return false;
+            }
+        }
+
         int pos = LastSep(filename);
         if(pos != -1)
             basepath = filename.substr(0, pos);
@@ -543,14 +584,14 @@ bool LoadROM(QStringList filepath, bool reset)
     {
         // file inside archive
 
-            s32 lenread = Archive::ExtractFileFromArchive(filepath.at(0), filepath.at(1), &filedata, &filelen);
+        s32 lenread = Archive::ExtractFileFromArchive(filepath.at(0), filepath.at(1), &filedata, &filelen);
-            if (lenread < 0) return false;
+        if (lenread < 0) return false;
-            if (!filedata) return false;
+        if (!filedata) return false;
-            if (lenread != filelen)
+        if (lenread != filelen)
-            {
+        {
-                delete[] filedata;
+            delete[] filedata;
-                return false;
+            return false;
-            }
+        }
 
         std::string std_archivepath = filepath.at(0).toStdString();
         basepath = std_archivepath.substr(0, LastSep(std_archivepath));
@@ -695,6 +736,25 @@ bool LoadGBAROM(QStringList filepath)
         fclose(f);
         filelen = (u32)len;
 
+        if (filename.length() > 4 && filename.substr(filename.length() - 4) == ".zst")
+        {
+            u8* outContent = nullptr;
+            u32 decompressed = DecompressROM(filedata, len, &outContent);
+
+            if (decompressed > 0)
+            {
+                delete[] filedata;
+                filedata = outContent;
+                filelen = decompressed;
+                filename = filename.substr(0, filename.length() - 4);
+            }
+            else
+            {
+                delete[] filedata;
+                return false;
+            }
+        }
+
         int pos = LastSep(filename);
         basepath = filename.substr(0, pos);
         romname = filename.substr(pos+1);

src/frontend/qt_sdl/main.cpp

@@ -146,7 +146,7 @@ const QStringList ArchiveExtensions
     ".tar.lz",
     ".tar.lzma", ".tlz",
     ".tar.lrz", ".tlrz",
-    ".tar.lzo", ".tzo",
+    ".tar.lzo", ".tzo"
 #endif
 };
 
@@ -1568,9 +1568,23 @@ static bool SupportedArchiveByMimetype(const QMimeType& mimetype)
     return MimeTypeInList(mimetype, ArchiveMimeTypes);
 }
 
+static bool ZstdNdsRomByExtension(const QString& filename)
+{
+    if (filename.endsWith(".zst", Qt::CaseInsensitive))
+        return NdsRomByExtension(filename.left(filename.size() - 4));
+}
+
+static bool ZstdGbaRomByExtension(const QString& filename)
+{
+    if (filename.endsWith(".zst", Qt::CaseInsensitive))
+        return GbaRomByExtension(filename.left(filename.size() - 4));
+}
 
 static bool FileIsSupportedFiletype(const QString& filename, bool insideArchive = false)
 {
+    if (ZstdNdsRomByExtension(filename) || ZstdGbaRomByExtension(filename))
+        return true;
+
     if (NdsRomByExtension(filename) || GbaRomByExtension(filename) || SupportedArchiveByExtension(filename))
         return true;
 
@@ -2207,7 +2221,12 @@ void MainWindow::dropEvent(QDropEvent* event)
     const auto matchMode = romInsideArchive ? QMimeDatabase::MatchExtension : QMimeDatabase::MatchDefault;
     const QMimeType mimetype = QMimeDatabase().mimeTypeForFile(filename, matchMode);
 
-    if (NdsRomByExtension(filename) || NdsRomByMimetype(mimetype))
+    bool isNdsRom = NdsRomByExtension(filename) || NdsRomByMimetype(mimetype);
+    bool isGbaRom = GbaRomByExtension(filename) || GbaRomByMimetype(mimetype);
+    isNdsRom |= ZstdNdsRomByExtension(filename);
+    isGbaRom |= ZstdGbaRomByExtension(filename);
+
+    if (isNdsRom)
     {
         if (!ROMManager::LoadROM(file, true))
         {
@@ -2227,7 +2246,7 @@ void MainWindow::dropEvent(QDropEvent* event)
 
         updateCartInserted(false);
     }
-    else if (GbaRomByExtension(filename) || GbaRomByMimetype(mimetype))
+    else if (isGbaRom)
     {
         if (!ROMManager::LoadGBAROM(file))
         {
@@ -2452,14 +2471,25 @@ QStringList MainWindow::pickROM(bool gba)
     const QString console = gba ? "GBA" : "DS";
     const QStringList& romexts = gba ? GbaRomExtensions : NdsRomExtensions;
 
-    static const QString filterSuffix = ArchiveExtensions.empty()
+    QString rawROMs = romexts.join(" *");
-        ? ");;Any file (*.*)"
+    QString extraFilters = ";;" + console + " ROMs (*" + rawROMs;
-        : " *" + ArchiveExtensions.join(" *") + ");;Any file (*.*)";
+    QString allROMs = rawROMs;
+
+    QString zstdROMs = "*" + romexts.join(".zst *") + ".zst";
+    extraFilters += ");;Zstandard-compressed " + console + " ROMs (" + zstdROMs + ")";
+    allROMs += " " + zstdROMs;
+
+#ifdef ARCHIVE_SUPPORT_ENABLED
+    QString archives = "*" + ArchiveExtensions.join(" *");
+    extraFilters += ";;Archives (" + archives + ")";
+    allROMs += " " + archives;
+#endif
+    extraFilters += ";;All files (*.*)";
 
     const QString filename = QFileDialog::getOpenFileName(
         this, "Open " + console + " ROM",
         QString::fromStdString(Config::LastROMFolder),
-        console + " ROMs (*" + romexts.join(" *") + filterSuffix
+        "All supported files (*" + allROMs + ")" + extraFilters
     );
 
     if (filename.isEmpty()) return {};