diff --git a/pcsx2/CMakeLists.txt b/pcsx2/CMakeLists.txt
index 969ef71d92..92a34f7126 100644
--- a/pcsx2/CMakeLists.txt
+++ b/pcsx2/CMakeLists.txt
@@ -39,7 +39,11 @@ else()
     set(pcsx2FinalFlags "")
 endif()
 
-set(pcsx2FinalFlags ${pcsx2FinalFlags} "-DSPU2X_PORTAUDIO" ${CommonFlags})
+set(pcsx2FinalFlags ${pcsx2FinalFlags} ${CommonFlags})
+
+if(PORTAUDIO_FOUND)
+    set(pcsx2FinalFlags ${pcsx2FinalFlags} -DSPU2X_PORTAUDIO)
+endif()
 
 if(XDG_STD)
     set(pcsx2FinalFlags ${pcsx2FinalFlags} -DXDG_STD)
@@ -237,7 +241,6 @@ set(pcsx2SPU2Sources
 	SPU2/Reverb.cpp
 	SPU2/SndOut.cpp
 	SPU2/SndOut_SDL.cpp
-	SPU2/SndOut_Portaudio.cpp
 	SPU2/spu2freeze.cpp
 	SPU2/spu2replay.cpp
 	SPU2/spu2sys.cpp
@@ -253,6 +256,10 @@ set(pcsx2SPU2Sources
 	SPU2/wx/wxConfig.cpp
     )
 
+if(PORTAUDIO_FOUND)
+    set(pcsx2SPU2Sources ${pcsx2SPU2Sources} SPU2/SndOut_Portaudio.cpp)
+endif()
+
 # SPU2 headers
 set(pcsx2SPU2Headers
 	SPU2/Config.h
@@ -774,10 +781,13 @@ set(pcsx2FinalLibs
     ${ALSA_LIBRARIES}
     ${SOUNDTOUCH_LIBRARIES}
     ${SDL2_LIBRARIES}
-    ${PORTAUDIO_LIBRARIES}
     ${Platform_Libs}
 )
 
+if(PORTAUDIO_FOUND)
+    set(pcsx2FinalLibs ${pcsx2FinalLibs} ${PORTAUDIO_LIBRARIES})
+endif()
+
 if(BUILTIN_GS)
     set(pcsx2FinalLibs "${pcsx2FinalLibs} GSdx")
 endif()