Merge pull request #5271 from JosJuice/allow-aslr

Allow (but don't force) ASLR

CMakeLists.txt

@@ -218,9 +218,6 @@ if(CMAKE_C_COMPILER_ID MATCHES "MSVC")
   dolphin_compile_definitions(_DEBUG DEBUG_ONLY)
 
   string(APPEND CMAKE_EXE_LINKER_FLAGS " /NXCOMPAT")
-  string(APPEND CMAKE_EXE_LINKER_FLAGS " /BASE:0x00400000")
-  string(APPEND CMAKE_EXE_LINKER_FLAGS " /DYNAMICBASE:NO")
-  string(APPEND CMAKE_EXE_LINKER_FLAGS " /FIXED")
 else()
   add_definitions(-D_DEFAULT_SOURCE)
   check_and_add_flag(HAVE_WALL -Wall)
@@ -252,15 +249,6 @@ else()
 
   dolphin_compile_definitions(_DEBUG DEBUG_ONLY)
   check_and_add_flag(GGDB -ggdb DEBUG_ONLY)
-
-  if(NOT ANDROID AND _M_X86_64)
-    # PIE is required on Android, but not supported with the x86_64 jit currently
-    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fno-pie")
-    check_c_compiler_flag("-no-pie" NO_PIE_UPSTREAM)
-    if(NO_PIE_UPSTREAM)
-      set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie")
-    endif()
-  endif()
 endif()
 
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
@@ -283,10 +271,6 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   # Linker flags.
   # Drop unreachable code and data.
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-dead_strip,-dead_strip_dylibs")
-  # Reserve the minimum size for the zero page.
-  # Our JIT requires virtual memory space below 2GB, while the default zero
-  # page on x86_64 is 4GB in size.
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-pagezero_size,0x1000")
 
   find_library(APPKIT_LIBRARY AppKit)
   find_library(APPSERV_LIBRARY ApplicationServices)

Source/Core/Common/CodeBlock.h

@@ -44,11 +44,11 @@ public:
   }
 
   // Call this before you generate any code.
-  void AllocCodeSpace(size_t size, bool need_low = true)
+  void AllocCodeSpace(size_t size)
   {
     region_size = size;
     total_region_size = size;
-    region = static_cast<u8*>(Common::AllocateExecutableMemory(total_region_size, need_low));
+    region = static_cast<u8*>(Common::AllocateExecutableMemory(total_region_size));
     T::SetCodePtr(region);
   }
 

Source/Core/Common/MemArena.cpp

@@ -7,6 +7,7 @@
 #include <set>
 #include <string>
 
+#include "Common/CommonFuncs.h"
 #include "Common/CommonTypes.h"
 #include "Common/Logging/Log.h"
 #include "Common/MemArena.h"
@@ -125,19 +126,22 @@ void MemArena::ReleaseView(void* view, size_t size)
 
 u8* MemArena::FindMemoryBase()
 {
-#if _ARCH_64
+#if _ARCH_32
+  const size_t memory_size = 0x31000000;
+#else
+  const size_t memory_size = 0x400000000;
+#endif
+
 #ifdef _WIN32
-  // 64 bit
+  u8* base = static_cast<u8*>(VirtualAlloc(nullptr, memory_size, MEM_RESERVE, PAGE_READWRITE));
-  u8* base = (u8*)VirtualAlloc(0, 0x400000000, MEM_RESERVE, PAGE_READWRITE);
+  if (!base)
+  {
+    PanicAlert("Failed to map enough memory space: %s", GetLastErrorMsg().c_str());
+    return nullptr;
+  }
   VirtualFree(base, 0, MEM_RELEASE);
   return base;
 #else
-  // Very precarious - mmap cannot return an error when trying to map already used pages.
-  // This makes the Windows approach above unusable on Linux, so we will simply pray...
-  return reinterpret_cast<u8*>(0x2300000000ULL);
-#endif
-
-#else  // 32 bit
 #ifdef ANDROID
   // Android 4.3 changed how mmap works.
   // if we map it private and then munmap it, we can't use the base returned.
@@ -146,14 +150,13 @@ u8* MemArena::FindMemoryBase()
 #else
   const int flags = MAP_ANON | MAP_PRIVATE;
 #endif
-  const u32 MemSize = 0x31000000;
+  void* base = mmap(nullptr, memory_size, PROT_NONE, flags, -1, 0);
-  void* base = mmap(0, MemSize, PROT_NONE, flags, -1, 0);
   if (base == MAP_FAILED)
   {
-    PanicAlert("Failed to map 1 GB of memory space: %s", strerror(errno));
+    PanicAlert("Failed to map enough memory space: %s", GetLastErrorMsg().c_str());
-    return 0;
+    return nullptr;
   }
-  munmap(base, MemSize);
+  munmap(base, memory_size);
   return static_cast<u8*>(base);
 #endif
 }

Source/Core/Common/MemoryUtil.cpp

@@ -29,74 +29,25 @@
 #endif
 #endif
 
-// Valgrind doesn't support MAP_32BIT.
-// Uncomment the following line to be able to run Dolphin in Valgrind.
-//#undef MAP_32BIT
-
 namespace Common
 {
-#if !defined(_WIN32) && defined(_M_X86_64) && !defined(MAP_32BIT)
-#include <unistd.h>
-static uintptr_t RoundPage(uintptr_t addr)
-{
-  uintptr_t mask = getpagesize() - 1;
-  return (addr + mask) & ~mask;
-}
-#endif
-
 // This is purposely not a full wrapper for virtualalloc/mmap, but it
 // provides exactly the primitive operations that Dolphin needs.
 
-void* AllocateExecutableMemory(size_t size, bool low)
+void* AllocateExecutableMemory(size_t size)
 {
 #if defined(_WIN32)
-  void* ptr = VirtualAlloc(0, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
+  void* ptr = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_EXECUTE_READWRITE);
 #else
-  static char* map_hint = nullptr;
+  void* ptr =
-#if defined(_M_X86_64) && !defined(MAP_32BIT)
+      mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE, -1, 0);
-  // This OS has no flag to enforce allocation below the 4 GB boundary,
-  // but if we hint that we want a low address it is very likely we will
-  // get one.
-  // An older version of this code used MAP_FIXED, but that has the side
-  // effect of discarding already mapped pages that happen to be in the
-  // requested virtual memory range (such as the emulated RAM, sometimes).
-  if (low && (!map_hint))
-    map_hint = (char*)RoundPage(512 * 1024 * 1024); /* 0.5 GB rounded up to the next page */
-#endif
-  void* ptr = mmap(map_hint, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_ANON | MAP_PRIVATE
-#if defined(_M_X86_64) && defined(MAP_32BIT)
-                                                                           | (low ? MAP_32BIT : 0)
-#endif
-                                                                           ,
-                   -1, 0);
-#endif /* defined(_WIN32) */
 
-#ifdef _WIN32
-  if (ptr == nullptr)
-  {
-#else
   if (ptr == MAP_FAILED)
-  {
     ptr = nullptr;
 #endif
-    PanicAlert("Failed to allocate executable memory. If you are running Dolphin in Valgrind, try "
-               "'#undef MAP_32BIT'.");
-  }
-#if !defined(_WIN32) && defined(_M_X86_64) && !defined(MAP_32BIT)
-  else
-  {
-    if (low)
-    {
-      map_hint += size;
-      map_hint = (char*)RoundPage((uintptr_t)map_hint); /* round up to the next page */
-    }
-  }
-#endif
 
-#if _M_X86_64
+  if (ptr == nullptr)
-  if ((u64)ptr >= 0x80000000 && low == true)
+    PanicAlert("Failed to allocate executable memory");
-    PanicAlert("Executable memory ended up above 2GB!");
-#endif
 
   return ptr;
 }
@@ -104,7 +55,7 @@ void* AllocateExecutableMemory(size_t size, bool low)
 void* AllocateMemoryPages(size_t size)
 {
 #ifdef _WIN32
-  void* ptr = VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE);
+  void* ptr = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE);
 #else
   void* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
 

Source/Core/Common/MemoryUtil.h

@@ -9,7 +9,7 @@
 
 namespace Common
 {
-void* AllocateExecutableMemory(size_t size, bool low = true);
+void* AllocateExecutableMemory(size_t size);
 void* AllocateMemoryPages(size_t size);
 void FreeMemoryPages(void* ptr, size_t size);
 void* AllocateAlignedMemory(size_t size, size_t alignment);

Source/Core/Core/PowerPC/Jit64/Jit.cpp

@@ -50,14 +50,6 @@ using namespace PowerPC;
 // * Does not recompile all instructions - sometimes falls back to inserting a CALL to the
 // corresponding Interpreter function.
 
-// Various notes below
-
-// IMPORTANT:
-// Make sure that all generated code and all emulator state sits under the 2GB boundary so that
-// RIP addressing can be used easily. Windows will always allocate static code under the 2GB
-// boundary.
-// Also make sure to use VirtualAlloc and specify EXECUTE permission.
-
 // Open questions
 // * Should there be any statically allocated registers? r3, r4, r5, r8, r0 come to mind.. maybe sp
 // * Does it make sense to finish off the remaining non-jitted instructions? Seems we are hitting

Source/Core/DolphinQt2/DolphinQt2.vcxproj

@@ -46,9 +46,6 @@
   </PropertyGroup>
   <ItemDefinitionGroup>
     <Link>
-      <BaseAddress>0x00400000</BaseAddress>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <FixedBaseAddress>true</FixedBaseAddress>
       <AdditionalLibraryDirectories>$(ExternalsDir)ffmpeg\lib;$(ExternalsDir)OpenAL\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>avrt.lib;iphlpapi.lib;winmm.lib;setupapi.lib;opengl32.lib;glu32.lib;rpcrt4.lib;comctl32.lib;avcodec.lib;avformat.lib;avutil.lib;swresample.lib;swscale.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem Condition="'$(Configuration)|$(Platform)'=='Release|x64'">Windows</SubSystem>

Source/Core/DolphinWX/DolphinWX.vcxproj

@@ -38,9 +38,6 @@
   <PropertyGroup Label="UserMacros" />
   <ItemDefinitionGroup>
     <Link>
-      <BaseAddress>0x00400000</BaseAddress>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <FixedBaseAddress>true</FixedBaseAddress>
       <AdditionalLibraryDirectories>$(ExternalsDir)ffmpeg\lib;$(ExternalsDir)OpenAL\lib;%(AdditionalLibraryDirectories)</AdditionalLibraryDirectories>
       <AdditionalDependencies>avrt.lib;iphlpapi.lib;winmm.lib;setupapi.lib;opengl32.lib;glu32.lib;rpcrt4.lib;comctl32.lib;avcodec.lib;avformat.lib;avutil.lib;swresample.lib;swscale.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <AdditionalOptions Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">/NODEFAULTLIB:libcmt %(AdditionalOptions)</AdditionalOptions>

Source/Core/VideoCommon/VertexLoaderX64.cpp

@@ -46,7 +46,7 @@ VertexLoaderX64::VertexLoaderX64(const TVtxDesc& vtx_desc, const VAT& vtx_att)
   if (!IsInitialized())
     return;
 
-  AllocCodeSpace(4096, false);
+  AllocCodeSpace(4096);
   ClearCodeSpace();
   GenerateVertexLoader();
   WriteProtect();

Source/UnitTests/UnitTests.vcxproj

@@ -41,10 +41,6 @@
       <AdditionalIncludeDirectories>$(ExternalsDir)gtest\include;$(ExternalsDir)gtest;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
     </ClCompile>
     <Link>
-      <!--This junk is needed for JIT to function correctly-->
-      <BaseAddress>0x00400000</BaseAddress>
-      <RandomizedBaseAddress>false</RandomizedBaseAddress>
-      <FixedBaseAddress>true</FixedBaseAddress>
       <!--
         The following libs are needed since we pull in pretty much the entire
         dolphin codebase.