diff --git a/BizHawk.Client.EmuHawk/Input/Input.cs b/BizHawk.Client.EmuHawk/Input/Input.cs
index abd129b99e..4acea6455b 100644
--- a/BizHawk.Client.EmuHawk/Input/Input.cs
+++ b/BizHawk.Client.EmuHawk/Input/Input.cs
@@ -269,8 +269,8 @@ namespace BizHawk.Client.EmuHawk
 								Source = source
 							});
 					}
+					ModifierState.Remove(button);
 				}
-				ModifierState.Remove(button);
 			}
 
 			_NewEvents.Add(ie);