disable fullscreen GDI objects (menu & mode confirm dialog) for D3D & OpenGL

src/win32/Direct3D.cpp

@@ -152,11 +152,6 @@ void Direct3DDisplay::prepareDisplayMode()
 	dpp.PresentationInterval = theApp.vsync ? D3DPRESENT_INTERVAL_ONE : D3DPRESENT_INTERVAL_IMMEDIATE;
 	// D3DPRESENT_INTERVAL_ONE means VSync ON
 
-	if( theApp.vsync && ( dpp.Windowed == FALSE ) && theApp.menuToggle ) {
-		// HACK: VSync will be disabled when the menu is opened in full screen mode
-		dpp.PresentationInterval = D3DPRESENT_INTERVAL_IMMEDIATE;
-	}
-
 
 #ifdef _DEBUG
 	// make debugging full screen easier

src/win32/MainWndFile.cpp

@@ -697,6 +697,12 @@ void MainWnd::OnFileTogglemenu()
   if(theApp.videoOption <= VIDEO_4X)
     return;
 
+  if( theApp.renderMethod != DIRECT_DRAW ) {
+	  // display API does not support GDI objects in fullscreen
+	  theApp.updateWindowSize( theApp.lastWindowed );
+	  return;
+  }
+
   theApp.menuToggle = !theApp.menuToggle;
 
   if(theApp.menuToggle) {

src/win32/MainWndOptions.cpp

@@ -313,7 +313,9 @@ void MainWnd::OnUpdateOptionsVideoFullscreen1280x1024(CCmdUI *pCmdUI)
 BOOL MainWnd::OnOptionVideoSize(UINT nID)
 {
 	theApp.updateVideoSize(nID);
-	theApp.m_pMainWnd->PostMessage(VBA_CONFIRM_MODE);
+	if( theApp.renderMethod == DIRECT_DRAW ) {
+		theApp.m_pMainWnd->PostMessage(VBA_CONFIRM_MODE);
+	}
 	return TRUE;
 }
 
@@ -372,7 +374,9 @@ void MainWnd::OnOptionsVideoFullscreen()
         regSetDwordValue("defaultVideoDriver", TRUE);
       }
       theApp.updateVideoSize(ID_OPTIONS_VIDEO_FULLSCREEN);
-      theApp.m_pMainWnd->PostMessage(VBA_CONFIRM_MODE);
+	  if( theApp.renderMethod == DIRECT_DRAW ) {
+		  theApp.m_pMainWnd->PostMessage(VBA_CONFIRM_MODE);
+	  }
     }
   }
   theApp.winAccelMgr.UpdateMenu(theApp.menu);

src/win32/VBA.cpp

@@ -332,6 +332,7 @@ VBA::VBA()
   autoLoadMostRecent = false;
   fsMaxScale = 0;
   romSize = 0;
+  lastWindowed = VIDEO_3X;
 
   updateCount = 0;
 
@@ -932,6 +933,10 @@ void VBA::updateMenuBar()
     popup = NULL;
   }
 
+  if( ( renderMethod != DIRECT_DRAW ) && ( videoOption >= VIDEO_320x240 ) ) {
+	  return;
+  }
+
   m_menu.Attach(winResLoadMenu(MAKEINTRESOURCE(IDR_MENU)));
   menu = (HMENU)m_menu;
 
@@ -1794,6 +1799,9 @@ void VBA::updateWindowSize(int value)
      fsForceChange) {
     fsForceChange = false;
     changingVideoSize = true;
+	if( videoOption <= VIDEO_4X ) {
+		lastWindowed = (VIDEO_SIZE)videoOption; // save for when leaving full screen
+	}
     shutdownDisplay();
     if(input) {
       delete input;
@@ -1826,7 +1834,10 @@ void VBA::updateWindowSize(int value)
       return;
     }
     input->checkKeys();
-    updateMenuBar();
+	
+	if( renderMethod == DIRECT_DRAW ) {
+		updateMenuBar();
+	}
     changingVideoSize = FALSE;
     updateWindowSize(videoOption);
     return;

src/win32/VBA.h

@@ -45,7 +45,7 @@
 // See VBA.cpp for the implementation of this class
 //
 
-enum {
+enum VIDEO_SIZE{
   VIDEO_1X, VIDEO_2X, VIDEO_3X, VIDEO_4X,
   VIDEO_320x240, VIDEO_640x480, VIDEO_800x600, VIDEO_1024x768, VIDEO_1280x1024,
   VIDEO_OTHER
@@ -209,6 +209,7 @@ class VBA : public CWinApp
   bool autoLoadMostRecent;
   int fsMaxScale;
   int romSize;
+  VIDEO_SIZE lastWindowed;
 
   CList<IUpdateListener *, IUpdateListener*&> updateList;
   int updateCount;