refactored ESC key handling

prevent exiting debugger from within Options Dialog (hack)

src/debugger/Debugger.cxx

@@ -64,7 +64,8 @@ Debugger::Debugger(OSystem& osystem, Console& console)
     mySystem(console.system()),
     myDialog(nullptr),
     myWidth(DebuggerDialog::kSmallFontMinW),
-    myHeight(DebuggerDialog::kSmallFontMinH)
+    myHeight(DebuggerDialog::kSmallFontMinH),
+    myMenuMode(false)
 {
   // Init parser
   myParser = make_unique<DebuggerParser>(*this, osystem.settings());

src/debugger/Debugger.hxx

@@ -240,6 +240,13 @@ class Debugger : public DialogContainer
     void lockBankswitchState();
     void unlockBankswitchState();
 
+    /**
+      Used to make sure that debugger cannot be exited while Options dialog is open.
+      (ugly hack!)
+    */
+    void setMenuMode(bool enable) { myMenuMode = enable; };
+    bool inMenuMode() { return myMenuMode; };
+
   private:
     /**
       Save state of each debugger subsystem and, by default, mark all
@@ -323,6 +330,8 @@ class Debugger : public DialogContainer
     static BuiltinFunction ourBuiltinFunctions[NUM_BUILTIN_FUNCS];
     static PseudoRegister ourPseudoRegisters[NUM_PSEUDO_REGS];
 
+    bool myMenuMode;
+
   private:
     // rewind/unwind n states
     uInt16 windStates(uInt16 numStates, bool unwind, string& message);

src/debugger/gui/DebuggerDialog.cxx

@@ -174,6 +174,7 @@ void DebuggerDialog::handleCommand(CommandSender* sender, int cmd,
       break;
 
     case kDDOptionsCmd:
+      instance().debugger().setMenuMode(true);
       myOptions->open();
       loadConfig();
       break;

src/emucore/EventHandler.cxx

@@ -614,11 +614,36 @@ void EventHandler::handleKeyEvent(StellaKey key, StellaMod mod, bool state)
   // Don't pass the key on if we've already taken care of it
   if(handled) return;
 
-  // Handle keys which switch eventhandler state
   // Arrange the logic to take advantage of short-circuit evaluation
-  if(!(StellaModTest::isControl(mod) || StellaModTest::isShift(mod) || StellaModTest::isAlt(mod)) &&
-      !state && eventStateChange(myKeyTable[key][kEmulationMode]))
-    return;
+  if(!(StellaModTest::isControl(mod) || StellaModTest::isShift(mod) || StellaModTest::isAlt(mod)))
+  {
+    // special handling for Escape key
+    if(state && key == KBDK_ESCAPE)
+    {
+      if(myState == EventHandlerState::PAUSE)
+      {
+        setEventState(EventHandlerState::EMULATION);
+        return;
+      }
+      else if(myState == EventHandlerState::CMDMENU ||
+              myState == EventHandlerState::TIMEMACHINE)
+      {
+        leaveMenuMode();
+        return;
+      }
+      // TODO: this currently does not work, because it exits the search dialog too
+      // How can we identify if the focus is in a different dialog?
+      /*else if(myState == EventHandlerState::DEBUGGER && !myOSystem.debugger().inMenuMode())
+      {
+        leaveDebugMode();
+        return;
+      }*/
+    }
+
+    // Handle keys which switch eventhandler state
+    if(!state && eventStateChange(myKeyTable[key][kEmulationMode]))
+      return;
+  }
 
   // Otherwise, let the event handler deal with it
   switch(myState)
@@ -1250,25 +1275,12 @@ bool EventHandler::eventStateChange(Event::Type type)
     case Event::DebuggerMode:
       if(myState == EventHandlerState::EMULATION || myState == EventHandlerState::PAUSE)
         enterDebugMode();
-      else if(myState == EventHandlerState::DEBUGGER)
+      else if(myState == EventHandlerState::DEBUGGER && !myOSystem.debugger().inMenuMode())
         leaveDebugMode();
       else
         handled = false;
       break;
 
-    case Event::LauncherMode:
-      if (myState == EventHandlerState::PAUSE ||
-          myState == EventHandlerState::TIMEMACHINE)
-        setEventState(EventHandlerState::EMULATION);
-      else if(myState == EventHandlerState::CMDMENU)
-        leaveMenuMode();
-      // TODO: this currently does not work, because it exits all open dialogs too
-      /*else if(myState == EventHandlerState::DEBUGGER)
-        leaveDebugMode();*/
-      else
-        handled = false;
-      break;
-
     default:
       handled = false;
   }

src/gui/InputTextDialog.cxx

@@ -216,7 +216,6 @@ void InputTextDialog::handleCommand(CommandSender* sender, int cmd,
       Dialog::handleCommand(sender, GuiObject::kCloseCmd, data, id);
       break;
 
-
     default:
       Dialog::handleCommand(sender, cmd, data, id);
       break;

src/gui/OptionsDialog.cxx

@@ -36,6 +36,7 @@
 #include "HelpDialog.hxx"
 #include "AboutDialog.hxx"
 #include "OptionsDialog.hxx"
+#include "Debugger.hxx"
 #include "Launcher.hxx"
 
 #ifdef CHEATCODE_SUPPORT
@@ -251,7 +252,10 @@ void OptionsDialog::handleCommand(CommandSender* sender, int cmd,
 
     case kExitCmd:
       if(myMode != emulator)
+      {
+        instance().debugger().setMenuMode(false);
         close();
+      }
       else
         instance().eventHandler().leaveMenuMode();
       break;