diff --git a/Source/Core/Common/Config/Config.h b/Source/Core/Common/Config/Config.h
index e0d110719e..b5e2df7b7e 100644
--- a/Source/Core/Common/Config/Config.h
+++ b/Source/Core/Common/Config/Config.h
@@ -94,8 +94,8 @@ LayerType GetActiveLayerForConfig(const Info<T>& info)
 template <typename T>
 void Set(LayerType layer, const Info<T>& info, const std::common_type_t<T>& value)
 {
-  GetLayer(layer)->Set(info, value);
-  OnConfigChanged();
+  if (GetLayer(layer)->Set(info, value))
+    OnConfigChanged();
 }
 
 template <typename T>
diff --git a/Source/Core/Common/Config/Layer.h b/Source/Core/Common/Config/Layer.h
index 5c4185cdfe..0fd80753ad 100644
--- a/Source/Core/Common/Config/Layer.h
+++ b/Source/Core/Common/Config/Layer.h
@@ -118,24 +118,25 @@ public:
   }
 
   template <typename T>
-  void Set(const Info<T>& config_info, const std::common_type_t<T>& value)
+  bool Set(const Info<T>& config_info, const std::common_type_t<T>& value)
   {
-    Set(config_info.GetLocation(), value);
+    return Set(config_info.GetLocation(), value);
   }
 
   template <typename T>
-  void Set(const Location& location, const T& value)
+  bool Set(const Location& location, const T& value)
   {
-    Set(location, ValueToString(value));
+    return Set(location, ValueToString(value));
   }
 
-  void Set(const Location& location, std::string new_value)
+  bool Set(const Location& location, std::string new_value)
   {
     const auto iter = m_map.find(location);
     if (iter != m_map.end() && iter->second == new_value)
-      return;
+      return false;
     m_is_dirty = true;
     m_map.insert_or_assign(location, std::move(new_value));
+    return true;
   }
 
   void MarkAsDirty() { m_is_dirty = true; }
diff --git a/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPClient.cpp b/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPClient.cpp
index 61ac7b1318..d5468e4e7b 100644
--- a/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPClient.cpp
+++ b/Source/Core/InputCommon/ControllerInterface/DualShockUDPClient/DualShockUDPClient.cpp
@@ -381,7 +381,9 @@ void Init()
     Config::SetBase(Settings::SERVER_PORT, 0);
   }
 
+  // It would be much better to unbind from this callback on DeInit but it's not possible as of now
   Config::AddConfigChangedCallback(ConfigChanged);
+  ConfigChanged();  // Call it immediately to load settings
 }
 
 void PopulateDevices()