gsdx: dump gsdump in xz format directly

Reduce disk space. Easy to share.

It would be nice to port the code to Windows.
libzma code was taken from https://git.tukaani.org/xz.git

Note: only short dumps are supported so far. Big dump will freeze the interface during the compression.
Or will suck all the RAM.
Note2: a multithreaded encoder would badly impact the compression ratio

Thanks to Turtleli for all review comments

plugins/GSdx/GSDump.cpp

@@ -22,74 +22,177 @@
 #include "stdafx.h"
 #include "GSDump.h"
 
-GSDump::GSDump()
+GSDumpBase::GSDumpBase(const std::string& fn)
-	: m_gs(NULL)
+	: m_frames(0)
-	, m_frames(0)
+	, m_extra_frames(2)
-	, m_extra_frames(0)
 {
+	m_gs = fopen(fn.c_str(), "wb");
+	if (!m_gs)
+		fprintf(stderr, "GSDump: Error failed to open %s\n", fn.c_str());
 }
 
-GSDump::~GSDump()
+GSDumpBase::~GSDumpBase()
-{
-	Close();
-}
-
-void GSDump::Open(const string& fn, uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs)
-{
-	m_gs = fopen((fn + ".gs").c_str(), "wb");
-
-	m_frames = 0;
-	m_extra_frames = 2;
-
-	if(m_gs)
-	{
-		fwrite(&crc, 4, 1, m_gs);
-		fwrite(&fd.size, 4, 1, m_gs);
-		fwrite(fd.data, fd.size, 1, m_gs);
-		fwrite(regs, sizeof(*regs), 1, m_gs);
-	}
-}
-
-void GSDump::Close()
-{
-	if(m_gs) {fclose(m_gs); m_gs = NULL;}
-}
-
-void GSDump::Transfer(int index, const uint8* mem, size_t size)
-{
-	if(m_gs && size > 0)
-	{
-		fputc(0, m_gs);
-		fputc(index, m_gs);
-		fwrite(&size, 4, 1, m_gs);
-		fwrite(mem, size, 1, m_gs);
-	}
-}
-
-void GSDump::ReadFIFO(uint32 size)
-{
-	if(m_gs && size > 0)
-	{
-		fputc(2, m_gs);
-		fwrite(&size, 4, 1, m_gs);
-	}
-}
-
-void GSDump::VSync(int field, bool last, const GSPrivRegSet* regs)
 {
 	if(m_gs)
-	{
+		fclose(m_gs);
-		fputc(3, m_gs);
+}
-		fwrite(regs, sizeof(*regs), 1, m_gs);
 
-		fputc(1, m_gs);
+void GSDumpBase::AddHeader(uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs)
-		fputc(field, m_gs);
+{
+	AppendRawData(&crc, 4);
+	AppendRawData(&fd.size, 4);
+	AppendRawData(fd.data, fd.size);
+	AppendRawData(regs, sizeof(*regs));
+}
 
-		if((++m_frames & 1) == 0 && last && (m_extra_frames <= 0))
+void GSDumpBase::Transfer(int index, const uint8* mem, size_t size)
-		{
+{
-			Close();
+	if (size == 0)
-		} else if (last) {
+		return;
-			m_extra_frames--;
+
+	AppendRawData(0);
+	AppendRawData(index);
+	AppendRawData(&size, 4);
+	AppendRawData(mem, size);
+}
+
+void GSDumpBase::ReadFIFO(uint32 size)
+{
+	if (size == 0)
+		return;
+
+	AppendRawData(2);
+	AppendRawData(&size, 4);
+}
+
+bool GSDumpBase::VSync(int field, bool last, const GSPrivRegSet* regs)
+{
+	// dump file is bad, return done to delete the object
+	if (!m_gs)
+		return true;
+
+	AppendRawData(3);
+	AppendRawData(regs, sizeof(*regs));
+
+	AppendRawData(1);
+	AppendRawData(field);
+
+	if (last)
+		m_extra_frames--;
+
+	return (++m_frames & 1) == 0 && last && (m_extra_frames < 0);
+}
+
+void GSDumpBase::Write(const void *data, size_t size)
+{
+	if (!m_gs || size == 0)
+		return;
+
+	size_t written = fwrite(data, 1, size, m_gs);
+	if (written != size)
+		fprintf(stderr, "GSDump: Error failed to write data\n");
+}
+
+//////////////////////////////////////////////////////////////////////
+// GSDump implementation
+//////////////////////////////////////////////////////////////////////
+
+GSDump::GSDump(const std::string& fn, uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs)
+	: GSDumpBase(fn + ".gs")
+{
+	AddHeader(crc, fd, regs);
+}
+
+void GSDump::AppendRawData(const void *data, size_t size)
+{
+	Write(data, size);
+}
+
+void GSDump::AppendRawData(uint8 c)
+{
+	Write(&c, 1);
+}
+
+//////////////////////////////////////////////////////////////////////
+// GSDumpXz implementation
+//////////////////////////////////////////////////////////////////////
+
+#ifdef LZMA_SUPPORTED
+
+GSDumpXz::GSDumpXz(const std::string& fn, uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs)
+	: GSDumpBase(fn + ".gs.xz")
+{
+	m_strm = LZMA_STREAM_INIT;
+	lzma_ret ret = lzma_easy_encoder(&m_strm, 6 /*level*/, LZMA_CHECK_CRC64);
+	if (ret != LZMA_OK) {
+		fprintf(stderr, "GSDumpXz: Error initializing LZMA encoder ! (error code %u)\n", ret);
+		return;
+	}
+
+	AddHeader(crc, fd, regs);
+}
+
+GSDumpXz::~GSDumpXz()
+{
+	Flush();
+
+	// Finish the stream
+	m_strm.avail_in = 0;
+	Compress(LZMA_FINISH, LZMA_STREAM_END);
+
+	lzma_end(&m_strm);
+}
+
+void GSDumpXz::AppendRawData(const void *data, size_t size)
+{
+	size_t old_size = m_in_buff.size();
+	m_in_buff.resize(old_size + size);
+	memcpy(&m_in_buff[old_size], data, size);
+
+	// Enough data was accumulated, time to write/compress it.  If compression
+	// is enabled, it will freeze PCSX2. 1GB should be enough for long dump.
+	//
+	// Note: long dumps are currently not supported so this path won't be executed
+	if (m_in_buff.size() > 1024*1024*1024)
+		Flush();
+}
+
+void GSDumpXz::AppendRawData(uint8 c)
+{
+	m_in_buff.push_back(c);
+}
+
+void GSDumpXz::Flush()
+{
+	if (m_in_buff.empty())
+		return;
+
+	m_strm.next_in = m_in_buff.data();
+	m_strm.avail_in = m_in_buff.size();
+
+	Compress(LZMA_RUN, LZMA_OK);
+
+	m_in_buff.clear();
+}
+
+void GSDumpXz::Compress(lzma_action action, lzma_ret expected_status)
+{
+	std::array<uint8, 1024*1024> out_buff;
+	do {
+		m_strm.next_out = out_buff.data();
+		m_strm.avail_out = out_buff.size();
+
+		lzma_ret ret = lzma_code(&m_strm, action);
+
+		if (ret != expected_status) {
+			fprintf (stderr, "GSDumpXz: Error %d\n", (int) ret);
+			return;
 		}
-	}
+
+		size_t write_size = out_buff.size() - m_strm.avail_out;
+		Write(out_buff.data(), write_size);
+
+	} while (m_strm.avail_out == 0);
 }
+
+#endif

plugins/GSdx/GSDump.h

@@ -43,20 +43,56 @@ Regs data (id == 3)
 
 */
 
-class GSDump
+class GSDumpBase
 {
-	FILE* m_gs;
 	int m_frames;
 	int m_extra_frames;
+	FILE* m_gs;
+
+protected:
+	void AddHeader(uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs);
+	void Write(const void *data, size_t size);
+
+	virtual void AppendRawData(const void *data, size_t size) = 0;
+	virtual void AppendRawData(uint8 c) = 0;
 
 public:
-	GSDump();
+	GSDumpBase(const std::string& fn);
-	virtual ~GSDump();
+	virtual ~GSDumpBase();
 
-	void Open(const string& fn, uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs);
-	void Close();
 	void ReadFIFO(uint32 size);
 	void Transfer(int index, const uint8* mem, size_t size);
-	void VSync(int field, bool last, const GSPrivRegSet* regs);
+	bool VSync(int field, bool last, const GSPrivRegSet* regs);
-	operator bool() {return m_gs != NULL;}
 };
+
+class GSDump final : public GSDumpBase
+{
+	void AppendRawData(const void *data, size_t size) final;
+	void AppendRawData(uint8 c) final;
+
+public:
+	GSDump(const std::string& fn, uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs);
+	virtual ~GSDump() = default;
+};
+
+#ifdef LZMA_SUPPORTED
+
+#include <lzma.h>
+
+class GSDumpXz final : public GSDumpBase
+{
+	lzma_stream m_strm;
+
+	std::vector<uint8> m_in_buff;
+
+	void Flush();
+	void Compress(lzma_action action, lzma_ret expected_status);
+	void AppendRawData(const void *data, size_t size);
+	void AppendRawData(uint8 c);
+
+public:
+	GSDumpXz(const std::string& fn, uint32 crc, const GSFreezeData& fd, const GSPrivRegSet* regs);
+	virtual ~GSDumpXz();
+};
+
+#endif

plugins/GSdx/GSRenderer.cpp

@@ -463,28 +463,21 @@ void GSRenderer::VSync(int field)
 
 	if(!m_snapshot.empty())
 	{
-		bool shift = false;
+		if(!m_dump && m_shift_key)
-
-		#ifdef _WIN32
-
-		shift = !!(::GetAsyncKeyState(VK_SHIFT) & 0x8000);
-
-		#else
-
-		shift = m_shift_key;
-
-		#endif
-
-		if(!m_dump && shift)
 		{
-			GSFreezeData fd;
+			GSFreezeData fd = {0, nullptr};
-			fd.size = 0;
-			fd.data = NULL;
 			Freeze(&fd, true);
 			fd.data = new uint8[fd.size];
 			Freeze(&fd, false);
 
-			m_dump.Open(m_snapshot, m_crc, fd, m_regs);
+#ifdef LZMA_SUPPORTED
+			if (m_control_key)
+				m_dump = std::unique_ptr<GSDumpBase>(new GSDump(m_snapshot, m_crc, fd, m_regs));
+			else
+				m_dump = std::unique_ptr<GSDumpBase>(new GSDumpXz(m_snapshot, m_crc, fd, m_regs));
+#else
+			m_dump = std::unique_ptr<GSDumpBase>(new GSDump(m_snapshot, m_crc, fd, m_regs));
+#endif
 
 			delete [] fd.data;
 		}
@@ -496,24 +489,10 @@ void GSRenderer::VSync(int field)
 
 		m_snapshot.clear();
 	}
-	else
+	else if(m_dump)
 	{
-		if(m_dump)
+		if(m_dump->VSync(field, !m_control_key, m_regs))
-		{
+			m_dump.reset();
-            bool control = false;
-
-            #ifdef _WIN32
-
-            control = !!(::GetAsyncKeyState(VK_CONTROL) & 0x8000);
-
-			#else
-
-			control = m_control_key;
-
-            #endif
-
-	    	m_dump.VSync(field, !control, m_regs);
-		}
 	}
 
 	// capture
@@ -587,14 +566,29 @@ void GSRenderer::EndCapture()
 
 void GSRenderer::KeyEvent(GSKeyEventData* e)
 {
+#ifdef _WIN32
+	m_shift_key = !!(::GetAsyncKeyState(VK_SHIFT) & 0x8000);
+	m_control_key = !!(::GetAsyncKeyState(VK_CONTROL) & 0x8000);
+#else
+	switch(e->key)
+	{
+		case XK_Shift_L:
+		case XK_Shift_R:
+			m_shift_key = (e->type == KEYPRESS);
+			return;
+		case XK_Control_L:
+		case XK_Control_R:
+			m_control_key = (e->type == KEYPRESS);
+			return;
+	}
+#endif
+
 	if(e->type == KEYPRESS)
 	{
 
-#ifdef _WIN32
-		int step = (::GetAsyncKeyState(VK_SHIFT) & 0x8000) ? -1 : 1;
-#elif defined(__unix__)
 		int step = m_shift_key ? -1 : 1;
 
+#if defined(__unix__)
 #define VK_F5 XK_F5
 #define VK_F6 XK_F6
 #define VK_F7 XK_F7
@@ -637,20 +631,6 @@ void GSRenderer::KeyEvent(GSKeyEventData* e)
 		}
 
 	}
-
-#if defined(__unix__)
-	switch(e->key)
-	{
-		case XK_Shift_L:
-		case XK_Shift_R:
-			m_shift_key = (e->type == KEYPRESS);
-			return;
-		case XK_Control_L:
-		case XK_Control_R:
-			m_control_key = (e->type == KEYPRESS);
-			return;
-	}
-#endif
 }
 
 void GSRenderer::PurgePool()

plugins/GSdx/GSRenderer.h

@@ -34,7 +34,6 @@ class GSRenderer : public GSState
 
 	bool Merge(int field);
 
-	// Only used on linux
 	bool m_shift_key;
 	bool m_control_key;
 

plugins/GSdx/GSState.cpp

@@ -2070,7 +2070,7 @@ void GSState::ReadFIFO(uint8* mem, int size)
 
 	if(m_dump)
 	{
-		m_dump.ReadFIFO(size);
+		m_dump->ReadFIFO(size);
 	}
 }
 
@@ -2289,7 +2289,7 @@ template<int index> void GSState::Transfer(const uint8* mem, uint32 size)
 
 	if(m_dump && mem > start)
 	{
-		m_dump.Transfer(index, start, mem - start);
+		m_dump->Transfer(index, start, mem - start);
 	}
 
 	if(index == 0)

plugins/GSdx/GSState.h

@@ -218,7 +218,7 @@ public:
 	GSPerfMon m_perfmon;
 	uint32 m_crc;
 	CRC::Game m_game;
-	GSDump m_dump;
+	std::unique_ptr<GSDumpBase> m_dump;
 	int m_options;
 	int m_frameskip;
 	bool m_framelimit;