diff --git a/dep/CMakeLists.txt b/dep/CMakeLists.txt
index 7da47d532..95493b8d1 100644
--- a/dep/CMakeLists.txt
+++ b/dep/CMakeLists.txt
@@ -9,15 +9,18 @@ add_subdirectory(xxhash)
 add_subdirectory(rapidjson)
 add_subdirectory(glslang)
 add_subdirectory(vulkan-loader)
-add_subdirectory(googletest)
-add_subdirectory(cubeb)
 add_subdirectory(imgui)
 add_subdirectory(simpleini)
-add_subdirectory(tinyxml2)
 
 set(LIBSAMPLERATE_ENABLE_SINC_BEST_CONVERTER OFF)
 add_subdirectory(libsamplerate)
 
+if(NOT BUILD_LIBRETRO_CORE)
+  add_subdirectory(tinyxml2)
+  add_subdirectory(cubeb)
+  add_subdirectory(googletest)
+endif()
+
 if(ENABLE_DISCORD_PRESENCE)
   add_subdirectory(discord-rpc)
 endif()
@@ -26,10 +29,6 @@ if(ENABLE_CHEEVOS)
   add_subdirectory(rcheevos)
 endif()
 
-if(BUILD_LIBRETRO_CORE)
-  add_subdirectory(libretro-common)
-endif()
-
 if(${CPU_ARCH} STREQUAL "aarch32" OR ${CPU_ARCH} STREQUAL "aarch64")
   add_subdirectory(vixl)
 endif()
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fbd8bb9ce..70d42a876 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -6,6 +6,10 @@ add_subdirectory(common)
 add_subdirectory(core)
 add_subdirectory(scmversion)
 
+if(BUILD_NOGUI_FRONTEND OR BUILD_QT_FRONTEND)
+  add_subdirectory(frontend-common)
+endif()
+
 if(NOT BUILD_LIBRETRO_CORE)
   add_subdirectory(common-tests)
   if(WIN32)
@@ -13,8 +17,6 @@ if(NOT BUILD_LIBRETRO_CORE)
   endif()
 endif()
 
-add_subdirectory(frontend-common)
-
 if(BUILD_NOGUI_FRONTEND)
   add_subdirectory(duckstation-nogui)
 endif()