From 51c7a723dbd845b55dfb3a76895f341d20591303 Mon Sep 17 00:00:00 2001
From: TellowKrinkle <tellowkrinkle@gmail.com>
Date: Wed, 21 Aug 2024 03:02:09 -0500
Subject: [PATCH] Common: Allow non-file mappings in SharedMemoryMappingArea

---
 common/HostSys.h              |  2 +-
 common/Linux/LnxHostSys.cpp   | 32 ++++++++++++++++++++++++--------
 common/Windows/WinHostSys.cpp | 21 ++++++++++++++++-----
 3 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/common/HostSys.h b/common/HostSys.h
index 23646284d9..7847a84bce 100644
--- a/common/HostSys.h
+++ b/common/HostSys.h
@@ -146,7 +146,7 @@ namespace PageFaultHandler
 class SharedMemoryMappingArea
 {
 public:
-	static std::unique_ptr<SharedMemoryMappingArea> Create(size_t size);
+	static std::unique_ptr<SharedMemoryMappingArea> Create(size_t size, bool jit = false);
 
 	~SharedMemoryMappingArea();
 
diff --git a/common/Linux/LnxHostSys.cpp b/common/Linux/LnxHostSys.cpp
index 7b3939810d..b5128c64e2 100644
--- a/common/Linux/LnxHostSys.cpp
+++ b/common/Linux/LnxHostSys.cpp
@@ -201,11 +201,16 @@ SharedMemoryMappingArea::~SharedMemoryMappingArea()
 }
 
 
-std::unique_ptr<SharedMemoryMappingArea> SharedMemoryMappingArea::Create(size_t size)
+std::unique_ptr<SharedMemoryMappingArea> SharedMemoryMappingArea::Create(size_t size, bool jit)
 {
 	pxAssertRel(Common::IsAlignedPow2(size, __pagesize), "Size is page aligned");
 
-	void* alloc = mmap(nullptr, size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+	uint flags = MAP_ANONYMOUS | MAP_PRIVATE;
+#ifdef __APPLE__
+	if (jit)
+		flags |= MAP_JIT;
+#endif
+	void* alloc = mmap(nullptr, size, PROT_NONE, flags, -1, 0);
 	if (alloc == MAP_FAILED)
 		return nullptr;
 
@@ -216,15 +221,26 @@ u8* SharedMemoryMappingArea::Map(void* file_handle, size_t file_offset, void* ma
 {
 	pxAssert(static_cast<u8*>(map_base) >= m_base_ptr && static_cast<u8*>(map_base) < (m_base_ptr + m_size));
 
-	// MAP_FIXED is okay here, since we've reserved the entire region, and *want* to overwrite the mapping.
 	const uint lnxmode = LinuxProt(mode);
-	void* const ptr = mmap(map_base, map_size, lnxmode, MAP_SHARED | MAP_FIXED,
-		static_cast<int>(reinterpret_cast<intptr_t>(file_handle)), static_cast<off_t>(file_offset));
-	if (ptr == MAP_FAILED)
-		return nullptr;
+	if (file_handle)
+	{
+		const int fd = static_cast<int>(reinterpret_cast<intptr_t>(file_handle));
+		// MAP_FIXED is okay here, since we've reserved the entire region, and *want* to overwrite the mapping.
+		void* const ptr = mmap(map_base, map_size, lnxmode, MAP_SHARED | MAP_FIXED, fd, static_cast<off_t>(file_offset));
+		if (ptr == MAP_FAILED)
+			return nullptr;
+	}
+	else
+	{
+		// macOS doesn't seem to allow MAP_JIT with MAP_FIXED
+		// So we do the MAP_JIT in the allocation, and just mprotect here
+		// Note that this will only work the first time for a given region
+		if (mprotect(map_base, map_size, lnxmode) < 0)
+			return nullptr;
+	}
 
 	m_num_mappings++;
-	return static_cast<u8*>(ptr);
+	return static_cast<u8*>(map_base);
 }
 
 bool SharedMemoryMappingArea::Unmap(void* map_base, size_t map_size)
diff --git a/common/Windows/WinHostSys.cpp b/common/Windows/WinHostSys.cpp
index 54e76fa6cc..7f05a8460e 100644
--- a/common/Windows/WinHostSys.cpp
+++ b/common/Windows/WinHostSys.cpp
@@ -183,7 +183,7 @@ SharedMemoryMappingArea::PlaceholderMap::iterator SharedMemoryMappingArea::FindP
 		return m_placeholder_ranges.end();
 }
 
-std::unique_ptr<SharedMemoryMappingArea> SharedMemoryMappingArea::Create(size_t size)
+std::unique_ptr<SharedMemoryMappingArea> SharedMemoryMappingArea::Create(size_t size, bool jit)
 {
 	pxAssertRel(Common::IsAlignedPow2(size, __pagesize), "Size is page aligned");
 
@@ -241,11 +241,22 @@ u8* SharedMemoryMappingArea::Map(void* file_handle, size_t file_offset, void* ma
 	}
 
 	// actually do the mapping, replacing the placeholder on the range
-	if (!MapViewOfFile3(static_cast<HANDLE>(file_handle), GetCurrentProcess(),
-			map_base, file_offset, map_size, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0))
+	if (file_handle)
 	{
-		Console.Error("(SharedMemoryMappingArea) MapViewOfFile3() failed: %u", GetLastError());
-		return nullptr;
+		if (!MapViewOfFile3(static_cast<HANDLE>(file_handle), GetCurrentProcess(),
+				map_base, file_offset, map_size, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0))
+		{
+			Console.Error("(SharedMemoryMappingArea) MapViewOfFile3() failed: %u", GetLastError());
+			return nullptr;
+		}
+	}
+	else
+	{
+		if (!VirtualAlloc2(GetCurrentProcess(), map_base, map_size, MEM_REPLACE_PLACEHOLDER, PAGE_READWRITE, nullptr, 0))
+		{
+			Console.Error("(SharedMemoryMappingArea) VirtualAlloc2() failed: %u", GetLastError());
+			return nullptr;
+		}
 	}
 
 	const DWORD prot = ConvertToWinApi(mode);