Jit: Add more error checking to ProtectStack

Source/Core/Common/MemoryUtil.cpp

@@ -160,18 +160,25 @@ void* AllocateAlignedMemory(size_t size, size_t alignment)
   return ptr;
 }
 
-void FreeMemoryPages(void* ptr, size_t size)
+bool FreeMemoryPages(void* ptr, size_t size)
 {
   if (ptr)
   {
 #ifdef _WIN32
     if (!VirtualFree(ptr, 0, MEM_RELEASE))
+    {
       PanicAlertFmt("FreeMemoryPages failed!\nVirtualFree: {}", GetLastErrorString());
+      return false;
+    }
 #else
     if (munmap(ptr, size) != 0)
+    {
       PanicAlertFmt("FreeMemoryPages failed!\nmunmap: {}", LastStrerrorString());
+      return false;
+    }
 #endif
   }
+  return true;
 }
 
 void FreeAlignedMemory(void* ptr)
@@ -186,39 +193,56 @@ void FreeAlignedMemory(void* ptr)
   }
 }
 
-void ReadProtectMemory(void* ptr, size_t size)
+bool ReadProtectMemory(void* ptr, size_t size)
 {
 #ifdef _WIN32
   DWORD oldValue;
   if (!VirtualProtect(ptr, size, PAGE_NOACCESS, &oldValue))
+  {
     PanicAlertFmt("ReadProtectMemory failed!\nVirtualProtect: {}", GetLastErrorString());
+    return false;
+  }
 #else
   if (mprotect(ptr, size, PROT_NONE) != 0)
+  {
     PanicAlertFmt("ReadProtectMemory failed!\nmprotect: {}", LastStrerrorString());
+    return false;
+  }
 #endif
+  return true;
 }
 
-void WriteProtectMemory(void* ptr, size_t size, bool allowExecute)
+bool WriteProtectMemory(void* ptr, size_t size, bool allowExecute)
 {
 #ifdef _WIN32
   DWORD oldValue;
   if (!VirtualProtect(ptr, size, allowExecute ? PAGE_EXECUTE_READ : PAGE_READONLY, &oldValue))
+  {
     PanicAlertFmt("WriteProtectMemory failed!\nVirtualProtect: {}", GetLastErrorString());
+    return false;
+  }
 #elif !(defined(_M_ARM_64) && defined(__APPLE__))
   // MacOS 11.2 on ARM does not allow for changing the access permissions of pages
   // that were marked executable, instead it uses the protections offered by MAP_JIT
   // for write protection.
   if (mprotect(ptr, size, allowExecute ? (PROT_READ | PROT_EXEC) : PROT_READ) != 0)
+  {
     PanicAlertFmt("WriteProtectMemory failed!\nmprotect: {}", LastStrerrorString());
+    return false;
+  }
 #endif
+  return true;
 }
 
-void UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute)
+bool UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute)
 {
 #ifdef _WIN32
   DWORD oldValue;
   if (!VirtualProtect(ptr, size, allowExecute ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE, &oldValue))
+  {
     PanicAlertFmt("UnWriteProtectMemory failed!\nVirtualProtect: {}", GetLastErrorString());
+    return false;
+  }
 #elif !(defined(_M_ARM_64) && defined(__APPLE__))
   // MacOS 11.2 on ARM does not allow for changing the access permissions of pages
   // that were marked executable, instead it uses the protections offered by MAP_JIT
@@ -227,8 +251,10 @@ void UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute)
                allowExecute ? (PROT_READ | PROT_WRITE | PROT_EXEC) : PROT_WRITE | PROT_READ) != 0)
   {
     PanicAlertFmt("UnWriteProtectMemory failed!\nmprotect: {}", LastStrerrorString());
+    return false;
   }
 #endif
+  return true;
 }
 
 size_t MemPhysical()

Source/Core/Common/MemoryUtil.h

@@ -27,12 +27,12 @@ struct ScopedJITPageWriteAndNoExecute
   ~ScopedJITPageWriteAndNoExecute() { JITPageWriteDisableExecuteEnable(); }
 };
 void* AllocateMemoryPages(size_t size);
-void FreeMemoryPages(void* ptr, size_t size);
+bool FreeMemoryPages(void* ptr, size_t size);
 void* AllocateAlignedMemory(size_t size, size_t alignment);
 void FreeAlignedMemory(void* ptr);
-void ReadProtectMemory(void* ptr, size_t size);
-void WriteProtectMemory(void* ptr, size_t size, bool executable = false);
-void UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute = false);
+bool ReadProtectMemory(void* ptr, size_t size);
+bool WriteProtectMemory(void* ptr, size_t size, bool executable = false);
+bool UnWriteProtectMemory(void* ptr, size_t size, bool allowExecute = false);
 size_t MemPhysical();
 
 }  // namespace Common

Source/Core/Core/PowerPC/JitCommon/JitBase.cpp

@@ -151,7 +151,12 @@ void JitBase::ProtectStack()
 
 #ifdef _WIN32
   ULONG reserveSize = SAFE_STACK_SIZE;
-  SetThreadStackGuarantee(&reserveSize);
+  if (!SetThreadStackGuarantee(&reserveSize))
+  {
+    PanicAlertFmt("Failed to set thread stack guarantee");
+    m_enable_blr_optimization = false;
+    return;
+  }
 #else
   auto [stack_addr, stack_size] = Common::GetCurrentThreadStack();
 
@@ -184,7 +189,12 @@ void JitBase::ProtectStack()
   }
 
   m_stack_guard = reinterpret_cast<u8*>(stack_guard_addr);
-  Common::ReadProtectMemory(m_stack_guard, GUARD_SIZE);
+  if (!Common::ReadProtectMemory(m_stack_guard, GUARD_SIZE))
+  {
+    m_stack_guard = nullptr;
+    m_enable_blr_optimization = false;
+    return;
+  }
 #endif
 }