diff --git a/src/common/PaletteHandler.cxx b/src/common/PaletteHandler.cxx
index 3e416f08a..841c6adbf 100644
--- a/src/common/PaletteHandler.cxx
+++ b/src/common/PaletteHandler.cxx
@@ -26,8 +26,6 @@
 PaletteHandler::PaletteHandler(OSystem& system)
   : myOSystem{system}
 {
-  // Load user-defined palette for this ROM
-  loadUserPalette();
 }
 
 // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
@@ -315,6 +313,10 @@ void PaletteHandler::setPalette()
   {
     const string& name = myOSystem.settings().getString("palette");
 
+    // Load user-defined palette for this ROM
+    if(name == SETTING_USER)
+      loadUserPalette();
+
     // Look at all the palettes, since we don't know which one is
     // currently active
     static constexpr BSPF::array2D<const PaletteArray*, PaletteType::NumTypes, int(ConsoleTiming::numTimings)> palettes = {{