gsdx: implement a shared memory to emulate the wrapping of the gs memory

Code can be enabled with "wrap_gs_mem = 1". Code only allow a single shared memory but
I don't think we need more anyway.

Linux only, Kernel panic expected with the HW renderer.

Fix FMV on Silent Hill 3 with the SW renderer

plugins/GSdx/GSLocalMemory.cpp

@@ -26,6 +26,7 @@
 
 #include "stdafx.h"
 #include "GSLocalMemory.h"
+#include "GSdx.h"
 
 #define ASSERT_BLOCK(r, w, h) \
 	ASSERT((r).width() >= (w) && (r).height() >= (h) && !((r).left & ((w) - 1)) && !((r).top & ((h) - 1)) && !((r).right & ((w) - 1)) && !((r).bottom & ((h) - 1))); \
@@ -83,7 +84,11 @@ GSLocalMemory::psm_t GSLocalMemory::m_psm[64];
 GSLocalMemory::GSLocalMemory()
 	: m_clut(this)
 {
-	m_vm8 = (uint8*)vmalloc(m_vmsize * 4, false);
+	if (theApp.GetConfigB("wrap_gs_mem"))
+		m_vm8 = (uint8*)fifo_alloc(m_vmsize, 4);
+	else
+		m_vm8 = (uint8*)vmalloc(m_vmsize * 4, false);
+
 	m_vm16 = (uint16*)m_vm8;
 	m_vm32 = (uint32*)m_vm8;
 
@@ -467,7 +472,10 @@ GSLocalMemory::GSLocalMemory()
 
 GSLocalMemory::~GSLocalMemory()
 {
-	vmfree(m_vm8, m_vmsize * 4);
+	if (theApp.GetConfigB("wrap_gs_mem"))
+		fifo_free(m_vm8, m_vmsize, 4);
+	else
+		vmfree(m_vm8, m_vmsize * 4);
 
 	for_each(m_omap.begin(), m_omap.end(), aligned_free_second());
 	for_each(m_pomap.begin(), m_pomap.end(), aligned_free_second());

plugins/GSdx/GSRenderer.cpp

@@ -127,7 +127,7 @@ bool GSRenderer::Merge(int field)
 		return false;
 	}
 
-	GL_PUSH("Renderer Merge %d", s_n);
+	GL_PUSH("Renderer Merge %d (0: enabled %d 0x%x, 1: enabled %d 0x%x)", s_n, en[0], m_regs->DISP[0].DISPFB.Block(), en[1], m_regs->DISP[1].DISPFB.Block());
 
 	// try to avoid fullscreen blur, could be nice on tv but on a monitor it's like double vision, hurts my eyes (persona 4, guitar hero)
 	//

plugins/GSdx/GSdx.cpp

@@ -356,6 +356,7 @@ void GSdxApp::Init()
 	m_default_configuration["UserHacks_TCOffset"]                         = "0";
 	m_default_configuration["UserHacks_TextureInsideRt"]                  = "0";
 	m_default_configuration["UserHacks_WildHack"]                         = "0";
+	m_default_configuration["wrap_gs_mem"]                                = "0";
 	m_default_configuration["vsync"]                                      = "0";
 }
 

plugins/GSdx/stdafx.cpp

@@ -71,6 +71,18 @@ void vmfree(void* ptr, size_t size)
 	VirtualFree(ptr, 0, MEM_RELEASE);
 }
 
+void* fifo_alloc(size_t size, size_t repeat)
+{
+	// FIXME check linux code
+	return vmalloc(size * repeat, false);
+}
+
+void fifo_free(void* ptr, size_t size, size_t repeat)
+{
+	// FIXME check linux code
+	return vmfree(ptr, size * repeat);
+}
+
 #else
 
 #include <sys/mman.h>
@@ -101,6 +113,52 @@ void vmfree(void* ptr, size_t size)
 	munmap(ptr, size);
 }
 
+static int s_shm_fd = -1;
+
+void* fifo_alloc(size_t size, size_t repeat)
+{
+	fprintf(stderr, "FIFO ALLOC\n");
+	ASSERT(s_shm_fd == -1);
+
+	const char* file_name = "/GSDX.mem";
+	s_shm_fd = shm_open(file_name, O_RDWR | O_CREAT | O_EXCL, 0600);
+	if (s_shm_fd != -1)
+		shm_unlink(file_name); // file is deleted but descriptor is still open
+	else
+		fprintf(stderr, "Failed to open %s due to %s\n", file_name, strerror(errno));
+
+	if (ftruncate(s_shm_fd, repeat * size) < 0)
+		fprintf(stderr, "Failed to reserve memory due to %s\n", strerror(errno));
+
+	void* fifo = mmap(nullptr, size * repeat, PROT_READ | PROT_WRITE, MAP_SHARED, s_shm_fd, 0);
+
+	for (size_t i = 1; i < repeat; i++) {
+		void* base = (uint8*)fifo + size * i;
+		uint8* next = (uint8*)mmap(base, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, s_shm_fd, 0);
+		if (next != base)
+			fprintf(stderr, "Fail to mmap contiguous segment\n");
+		else
+			fprintf(stderr, "MMAP next %x\n", (uintptr_t)base);
+	}
+
+	return fifo;
+}
+
+void fifo_free(void* ptr, size_t size, size_t repeat)
+{
+	fprintf(stderr, "FIFO FREE\n");
+
+	ASSERT(s_shm_fd >= 0);
+
+	if (s_shm_fd < 0)
+		return;
+
+	munmap(ptr, size * repeat);
+
+	close(s_shm_fd);
+	s_shm_fd = -1;
+}
+
 #endif
 
 #if !defined(_MSC_VER)

plugins/GSdx/stdafx.h

@@ -45,6 +45,10 @@
 
 #define D3DCOLORWRITEENABLE_RGBA (D3DCOLORWRITEENABLE_RED | D3DCOLORWRITEENABLE_GREEN | D3DCOLORWRITEENABLE_BLUE | D3DCOLORWRITEENABLE_ALPHA)
 
+#else
+
+#include <fcntl.h>
+
 #endif
 
 
@@ -404,6 +408,9 @@ struct aligned_free_second {template<class T> void operator()(T& p) {_aligned_fr
 extern void* vmalloc(size_t size, bool code);
 extern void vmfree(void* ptr, size_t size);
 
+extern void* fifo_alloc(size_t size, size_t repeat);
+extern void fifo_free(void* ptr, size_t size, size_t repeat);
+
 #ifdef _WIN32
 
 	#ifdef ENABLE_VTUNE