GS: Always allocate wrapped local memory

Simplifies things since we don't need to care about <Win10 support now.

Also gets rid of vmalloc() in favor of the common routines.

pcsx2/GS/GS.cpp

@@ -14,10 +14,15 @@
  */
 
 #include "PrecompiledHeader.h"
+
 #ifndef PCSX2_CORE
-// NOTE: The include order matters - GS.h includes windows.h
+#ifdef _WIN32
+// Need to ensure we include windows.h and set _WIN32_WINNT before wx does..
+#include "common/RedtapeWindows.h"
+#endif
 #include "GS/Window/GSwxDialog.h"
 #endif
+
 #include "GS.h"
 #include "GSGL.h"
 #include "GSUtil.h"
@@ -963,25 +968,9 @@ const std::string root_hw("/tmp/GS_HW_dump32/");
 
 #ifdef _WIN32
 
-void* vmalloc(size_t size, bool code)
-{
-	void* ptr = VirtualAlloc(NULL, size, MEM_COMMIT | MEM_RESERVE, code ? PAGE_EXECUTE_READWRITE : PAGE_READWRITE);
-	if (!ptr)
-		throw std::bad_alloc();
-	return ptr;
-}
-
-void vmfree(void* ptr, size_t size)
-{
-	VirtualFree(ptr, 0, MEM_RELEASE);
-}
-
-#ifdef PCSX2_CORE
-// Safe, placeholder-based mapping for Win10+ and Qt.
-
 static HANDLE s_fh = NULL;
 
-void* fifo_alloc(size_t size, size_t repeat)
+void* GSAllocateWrappedMemory(size_t size, size_t repeat)
 {
 	pxAssertRel(!s_fh, "Has no file mapping");
 
@@ -1030,7 +1019,7 @@ void* fifo_alloc(size_t size, size_t repeat)
 	return nullptr;
 }
 
-void fifo_free(void* ptr, size_t size, size_t repeat)
+void GSFreeWrappedMemory(void* ptr, size_t size, size_t repeat)
 {
 	pxAssertRel(s_fh, "Has a file mapping");
 
@@ -1046,130 +1035,14 @@ void fifo_free(void* ptr, size_t size, size_t repeat)
 
 #else
 
-// "Best effort" mapping for <Win10 and wx build.
-// This can be removed once the wx UI is dropped.
-
-static HANDLE s_fh = NULL;
-static u8* s_Next[8];
-
-void* fifo_alloc(size_t size, size_t repeat)
-{
-	ASSERT(s_fh == NULL);
-
-	if (repeat >= std::size(s_Next))
-	{
-		fprintf(stderr, "Memory mapping overflow (%zu >= %u)\n", repeat, static_cast<unsigned>(std::size(s_Next)));
-		return nullptr; // Fallback to default vmalloc
-	}
-
-	s_fh = CreateFileMapping(INVALID_HANDLE_VALUE, nullptr, PAGE_READWRITE, 0, size, nullptr);
-	DWORD errorID = ::GetLastError();
-	if (s_fh == NULL)
-	{
-		fprintf(stderr, "Failed to reserve memory. WIN API ERROR:%u\n", errorID);
-		return nullptr; // Fallback to default vmalloc
-	}
-
-	int mmap_segment_failed = 0;
-	void* fifo = MapViewOfFile(s_fh, FILE_MAP_ALL_ACCESS, 0, 0, size);
-	for (size_t i = 1; i < repeat; i++)
-	{
-		void* base = (u8*)fifo + size * i;
-		s_Next[i] = (u8*)MapViewOfFileEx(s_fh, FILE_MAP_ALL_ACCESS, 0, 0, size, base);
-		errorID = ::GetLastError();
-		if (s_Next[i] != base)
-		{
-			mmap_segment_failed++;
-			if (mmap_segment_failed > 4)
-			{
-				fprintf(stderr, "Memory mapping failed after %d attempts, aborting. WIN API ERROR:%u\n", mmap_segment_failed, errorID);
-				fifo_free(fifo, size, repeat);
-				return nullptr; // Fallback to default vmalloc
-			}
-			do
-			{
-				UnmapViewOfFile(s_Next[i]);
-				s_Next[i] = 0;
-			} while (--i > 0);
-
-			fifo = MapViewOfFile(s_fh, FILE_MAP_ALL_ACCESS, 0, 0, size);
-		}
-	}
-
-	return fifo;
-}
-
-void fifo_free(void* ptr, size_t size, size_t repeat)
-{
-	ASSERT(s_fh != NULL);
-
-	if (s_fh == NULL)
-	{
-		if (ptr != NULL)
-			vmfree(ptr, size);
-		return;
-	}
-
-	UnmapViewOfFile(ptr);
-
-	for (size_t i = 1; i < std::size(s_Next); i++)
-	{
-		if (s_Next[i] != 0)
-		{
-			UnmapViewOfFile(s_Next[i]);
-			s_Next[i] = 0;
-		}
-	}
-
-	CloseHandle(s_fh);
-	s_fh = NULL;
-}
-
-#endif // PCSX2_CORE
-
-#else
-
 #include <sys/mman.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
 
-void* vmalloc(size_t size, bool code)
-{
-	size_t mask = getpagesize() - 1;
-
-	size = (size + mask) & ~mask;
-
-	int prot = PROT_READ | PROT_WRITE;
-	int flags = MAP_PRIVATE | MAP_ANONYMOUS;
-
-	if (code)
-	{
-		prot |= PROT_EXEC;
-#if defined(_M_AMD64) && !defined(__APPLE__)
-		// macOS doesn't allow any mappings in the first 4GB of address space
-		flags |= MAP_32BIT;
-#endif
-	}
-
-	void* ptr = mmap(NULL, size, prot, flags, -1, 0);
-	if (ptr == MAP_FAILED)
-		throw std::bad_alloc();
-	return ptr;
-}
-
-void vmfree(void* ptr, size_t size)
-{
-	size_t mask = getpagesize() - 1;
-
-	size = (size + mask) & ~mask;
-
-	munmap(ptr, size);
-}
-
 static int s_shm_fd = -1;
 
-void* fifo_alloc(size_t size, size_t repeat)
+void* GSAllocateWrappedMemory(size_t size, size_t repeat)
 {
 	ASSERT(s_shm_fd == -1);
 
@@ -1201,7 +1074,7 @@ void* fifo_alloc(size_t size, size_t repeat)
 	return fifo;
 }
 
-void fifo_free(void* ptr, size_t size, size_t repeat)
+void GSFreeWrappedMemory(void* ptr, size_t size, size_t repeat)
 {
 	ASSERT(s_shm_fd >= 0);
 

pcsx2/GS/GSClut.cpp

@@ -17,17 +17,20 @@
 #include "GSClut.h"
 #include "GSLocalMemory.h"
 #include "GSGL.h"
-
+#include "common/AlignedMalloc.h"
-#define CLUT_ALLOC_SIZE (2 * 4096)
 
 GSClut::GSClut(GSLocalMemory* mem)
 	: m_mem(mem)
 {
-	u8* p = (u8*)vmalloc(CLUT_ALLOC_SIZE, false);
+	static constexpr u32 CLUT_ALLOC_SIZE = 4096 * 2;
 
-	m_clut = (u16*)&p[0];      // 1k + 1k for mirrored area simulating wrapping memory
+	// 1k + 1k for mirrored area simulating wrapping memory
-	m_buff32 = (u32*)&p[2048]; // 1k
+	m_clut = static_cast<u16*>(_aligned_malloc(CLUT_ALLOC_SIZE, 32));
-	m_buff64 = (u64*)&p[4096]; // 2k
+	if (!m_clut)
+		throw std::bad_alloc();
+
+	m_buff32 = reinterpret_cast<u32*>(reinterpret_cast<u8*>(m_clut) + 2048); // 1k
+	m_buff64 = reinterpret_cast<u64*>(reinterpret_cast<u8*>(m_clut) + 4096); // 2k
 	m_write.dirty = 1;
 	m_read.dirty = true;
 
@@ -100,7 +103,7 @@ GSClut::GSClut(GSLocalMemory* mem)
 
 GSClut::~GSClut()
 {
-	vmfree(m_clut, CLUT_ALLOC_SIZE);
+	_aligned_free(m_clut);
 }
 
 u8 GSClut::IsInvalid()

pcsx2/GS/GSCodeBuffer.cpp

@@ -16,6 +16,7 @@
 #include "PrecompiledHeader.h"
 #include "GSCodeBuffer.h"
 #include "GSExtra.h"
+#include "common/General.h"
 
 GSCodeBuffer::GSCodeBuffer(size_t blocksize)
 	: m_blocksize(blocksize)
@@ -29,7 +30,7 @@ GSCodeBuffer::~GSCodeBuffer()
 {
 	for (auto buffer : m_buffers)
 	{
-		vmfree(buffer, m_blocksize);
+		HostSys::Munmap(buffer, m_blocksize);
 	}
 }
 
@@ -42,7 +43,9 @@ void* GSCodeBuffer::GetBuffer(size_t size)
 
 	if (m_ptr == NULL || m_pos + size > m_blocksize)
 	{
-		m_ptr = (u8*)vmalloc(m_blocksize, true);
+		m_ptr = (u8*)HostSys::Mmap(nullptr, m_blocksize, PageProtectionMode().All());
+		if (!m_ptr)
+			pxFailRel("Failed to allocate GS code buffer");
 
 		m_pos = 0;
 

pcsx2/GS/GSExtra.h

@@ -118,11 +118,8 @@ static constexpr u32 MAX_SKIPPED_DUPLICATE_FRAMES = 3;
 extern const std::string root_sw;
 extern const std::string root_hw;
 
-extern void* vmalloc(size_t size, bool code);
+extern void* GSAllocateWrappedMemory(size_t size, size_t repeat);
-extern void vmfree(void* ptr, size_t size);
+extern void GSFreeWrappedMemory(void* ptr, size_t size, size_t repeat);
-
-extern void* fifo_alloc(size_t size, size_t repeat);
-extern void fifo_free(void* ptr, size_t size, size_t repeat);
 
 // clang-format off
 

pcsx2/GS/GSLocalMemory.cpp

@@ -62,17 +62,9 @@ GSLocalMemory::readImage GSLocalMemory::m_readImageX;
 GSLocalMemory::GSLocalMemory()
 	: m_clut(this)
 {
-	if (!GSConfig.UseHardwareRenderer())
+	m_vm8 = (u8*)GSAllocateWrappedMemory(m_vmsize, 4);
-		m_use_fifo_alloc = true;
+	if (!m_vm8)
-
+		throw std::bad_alloc();
-	m_vm8 = (u8*)fifo_alloc(m_vmsize, 4);
-
-	// Either we don't use fifo alloc or we get an error.
-	if (m_vm8 == nullptr)
-	{
-		m_vm8 = (u8*)vmalloc(m_vmsize * 4, false);
-		m_use_fifo_alloc = false;
-	}
 
 	memset(m_vm8, 0, m_vmsize);
 
@@ -255,10 +247,8 @@ GSLocalMemory::GSLocalMemory()
 
 GSLocalMemory::~GSLocalMemory()
 {
-	if (m_use_fifo_alloc)
+	if (m_vm8)
-		fifo_free(m_vm8, m_vmsize, 4);
+		GSFreeWrappedMemory(m_vm8, m_vmsize, 4);
-	else
-		vmfree(m_vm8, m_vmsize * 4);
 
 	for (auto& i : m_pomap)
 		_aligned_free(i.second);

pcsx2/GS/GSLocalMemory.h

@@ -487,9 +487,6 @@ public:
 
 	GSClut m_clut;
 
-protected:
-	bool m_use_fifo_alloc;
-
 public:
 	static constexpr GSSwizzleInfo swizzle32   {swizzleTables32};
 	static constexpr GSSwizzleInfo swizzle32Z  {swizzleTables32Z};

pcsx2/GS/GSRingHeap.cpp

@@ -17,6 +17,7 @@
 #include "GSRingHeap.h"
 #include "GS.h"
 #include "GSExtra.h"
+#include "common/AlignedMalloc.h"
 
 namespace
 {
@@ -120,7 +121,7 @@ struct GSRingHeap::Buffer
 		if (unlikely(m_amt_allocated.fetch_sub(amt, std::memory_order_release) == amt))
 		{
 			std::atomic_thread_fence(std::memory_order_acquire);
-			vmfree(this, m_size);
+			_aligned_free(this);
 		}
 	}
 
@@ -167,7 +168,7 @@ struct GSRingHeap::Buffer
 	static Buffer* make(int quadrant_shift)
 	{
 		size_t size = 4ull << quadrant_shift;
-		Buffer* buffer = reinterpret_cast<Buffer*>(vmalloc(size, false));
+		Buffer* buffer = reinterpret_cast<Buffer*>(_aligned_malloc(size, 32));
 		buffer->m_size = size;
 		buffer->m_quadrant_shift = quadrant_shift;
 		buffer->m_amt_allocated.store(1, std::memory_order_relaxed);

pcsx2/GS/Renderers/SW/GSRasterizer.cpp

@@ -19,6 +19,7 @@
 #include "GSRasterizer.h"
 #include "GS/GSExtra.h"
 #include "PerformanceMetrics.h"
+#include "common/AlignedMalloc.h"
 #include "common/StringUtil.h"
 
 #ifdef PCSX2_CORE
@@ -56,8 +57,10 @@ GSRasterizer::GSRasterizer(IDrawScanline* ds, int id, int threads)
 
 	m_thread_height = compute_best_thread_height(threads);
 
-	m_edge.buff = (GSVertexSW*)vmalloc(sizeof(GSVertexSW) * 2048, false);
+	m_edge.buff = static_cast<GSVertexSW*>(_aligned_malloc(sizeof(GSVertexSW) * 2048, 32));
 	m_edge.count = 0;
+	if (!m_edge.buff)
+		throw std::bad_alloc();
 
 	int rows = (2048 >> m_thread_height) + 16;
 	m_scanline = (u8*)_aligned_malloc(rows, 64);
@@ -71,10 +74,7 @@ GSRasterizer::GSRasterizer(IDrawScanline* ds, int id, int threads)
 GSRasterizer::~GSRasterizer()
 {
 	_aligned_free(m_scanline);
-
+	_aligned_free(m_edge.buff);
-	if (m_edge.buff != NULL)
-		vmfree(m_edge.buff, sizeof(GSVertexSW) * 2048);
-
 	delete m_ds;
 }