diff --git a/appveyor.yml b/appveyor.yml
index 1590ec5955..9abb3226fa 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -6,6 +6,21 @@ version: 1.{build}-{branch}
 # Do not build feature branch with open Pull Requests
 skip_branch_with_pr: true
 
+# Skip building commits if only the following directories/files are changed
+skip_commits:
+  files:
+    - .gitignore
+    - .travis.yml
+    - .clang-format
+    - '**/*.md'
+    - debian-packager/
+    - bin/PCSX2_keys.ini.default
+    # Linux only plugins for now
+    - plugins/onepad/
+    - plugins/onepad_legacy/
+    # Tools
+    - tools/GSDumpGUI/
+
 environment:
   matrix:
   - platform: Win32