D3D: Fix bugs in the shader cache, fixes crashes on x64. added some debugging stuff (only active in debug builds). assorted code cleanup.

git-svn-id: https://dolphin-emu.googlecode.com/svn/trunk@4145 8ced0084-cf51-0410-be5f-012b33b47a6e

Source/Core/VideoCommon/Src/IndexGenerator.cpp

@@ -26,7 +26,6 @@ QUAD simulator
 021231 243453 
 */
 
-
 void IndexGenerator::Start(unsigned short *startptr)
 {
 	ptr = startptr;

Source/Core/VideoCommon/Src/IndexGenerator.h

@@ -23,10 +23,6 @@
 
 class IndexGenerator
 {
-	unsigned short *ptr;
-	int numPrims;
-	int index;
-
 public:
 	void Start(unsigned short *startptr);
 	void AddList(int numVerts);
@@ -38,6 +34,10 @@ public:
 	void AddQuads(int numVerts);
 	int GetNumPrims() {return numPrims;} //returns numprimitives
 	int GetNumVerts() {return index;} //returns numprimitives
+private:
+	unsigned short *ptr;
+	int numPrims;
+	int index;
 };
 
 #endif  // _INDEXGENERATOR_H

Source/Core/VideoCommon/Src/PixelShaderGen.cpp

@@ -411,12 +411,13 @@ const char *GeneratePixelShader(u32 texture_mask, bool dstAlphaEnable, bool HLSL
 			WRITE(p, "uniform samplerRECT ");
         bool bfirst = true;
 		for (int i = 0; i < 8; ++i)
+		{
             if (texture_mask & (1<<i))
 			{
                 WRITE(p, "%s samp%d : register(s%d)", bfirst?"":",", i, i);
                 bfirst = false;
             }
-
+		}
         WRITE(p, ";\n");
 	}
 

Source/Core/VideoCommon/Src/VertexShaderGen.cpp

@@ -92,44 +92,17 @@ const char *GenerateVertexShader(u32 components, bool D3D)
 
     char *p = text;
     WRITE(p, "//Vertex Shader: comp:%x, \n", components);
-    WRITE(p, "typedef struct {\n"
+    WRITE(p, "typedef struct { float4 T0, T1, T2; float4 N0, N1, N2; } s_"I_POSNORMALMATRIX";\n"
-        "  float4 T0, T1, T2;\n"
+        "typedef struct { float4 t; } FLT4;\n"
-        "  float4 N0, N1, N2;\n"
+        "typedef struct { FLT4 T[24]; } s_"I_TEXMATRICES";\n"
-        "} s_"I_POSNORMALMATRIX";\n\n"
+        "typedef struct { FLT4 T[64]; } s_"I_TRANSFORMMATRICES";\n"
-        "typedef struct {\n"
+        "typedef struct { FLT4 T[32]; } s_"I_NORMALMATRICES";\n"
-        "  float4 t;\n"
+        "typedef struct { FLT4 T[64]; } s_"I_POSTTRANSFORMMATRICES";\n"
-        "} FLT4;\n"
+        "typedef struct { float4 col; float4 cosatt; float4 distatt; float4 pos; float4 dir; } Light;\n"
-        "typedef struct {\n"
+        "typedef struct { Light lights[8]; } s_"I_LIGHTS";\n"
-        "  FLT4 T[24];\n"
+        "typedef struct { float4 C0,C1,C2,C3; } s_"I_MATERIALS";\n"
-        "} s_"I_TEXMATRICES";\n\n"
+        "typedef struct { float4 T0,T1,T2,T3; } s_"I_PROJECTION";\n"
-        "typedef struct {\n"
+        "typedef struct { float4 params; } s_"I_FOGPARAMS";\n"); // a, b, c, b_shift
-        "  FLT4 T[64];\n"
-        "} s_"I_TRANSFORMMATRICES";\n\n"
-        "typedef struct {\n"
-        "  FLT4 T[32];\n"
-        "} s_"I_NORMALMATRICES";\n\n"
-        "typedef struct {\n"
-        "  FLT4 T[64];\n"
-        "} s_"I_POSTTRANSFORMMATRICES";\n\n"
-        "typedef struct {\n"
-        "  float4 col;\n"
-        "  float4 cosatt;\n"
-        "  float4 distatt;\n"
-        "  float4 pos;\n"
-        "  float4 dir;\n"
-        "} Light;\n\n"
-        "typedef struct {\n"
-        "  Light lights[8];\n"
-        "} s_"I_LIGHTS";\n\n"
-        "typedef struct {\n"
-        "  float4 C0, C1, C2, C3;\n"
-        "} s_"I_MATERIALS";\n\n"
-        "typedef struct {\n"
-        "  float4 T0,T1,T2,T3;\n"
-        "} s_"I_PROJECTION";\n"
-        "typedef struct {\n"
-        "  float4 params;\n" // a, b, c, b_shift
-        "} s_"I_FOGPARAMS";\n\n");
 
     WRITE(p, "struct VS_OUTPUT {\n");
     WRITE(p, "  float4 pos : POSITION;\n");
@@ -144,12 +117,9 @@ const char *GenerateVertexShader(u32 components, bool D3D)
 		for (int i = 0; i < xfregs.numTexGens; ++i)
 			WRITE(p, "  float%d tex%d : TEXCOORD%d;\n", i<4?4:3, i, i);
 	}
-
     WRITE(p, "};\n");
-    WRITE(p, "\n");
 
     // uniforms
-    
     // bool bTexMtx = ((components & VB_HAS_TEXMTXIDXALL)<<VB_HAS_UVTEXMTXSHIFT)!=0; unused TODO: keep?
 
     WRITE(p, "uniform s_"I_TRANSFORMMATRICES" "I_TRANSFORMMATRICES" : register(c%d);\n", C_TRANSFORMMATRICES);

Source/Plugins/Plugin_VideoDX9/Src/D3DTexture.cpp

@@ -80,7 +80,6 @@ LPDIRECT3DTEXTURE9 CreateTexture2D(const u8* buffer, const int width, const int
 		{
 			if (bExpand) { // I8
 				const u8 *pIn = buffer;
-
 				// TODO(XK): Find a better way that does not involve either unpacking
 				//           or downsampling (i.e. A4L4)
 				for (int y = 0; y < height; y++)
@@ -118,6 +117,8 @@ LPDIRECT3DTEXTURE9 CreateTexture2D(const u8* buffer, const int width, const int
 	case D3DFMT_DXT1:
 		memcpy(Lock.pBits,buffer,(width/4)*(height/4)*8);
 		break;
+	default:
+		PanicAlert("D3D: Invalid texture format %i", fmt);
 	}
 	pTexture->UnlockRect(level); 
 	return pTexture;

Source/Plugins/Plugin_VideoDX9/Src/D3DUtil.cpp

@@ -215,9 +215,8 @@ namespace D3D
 
 		dev->SetPixelShader(0);
 		dev->SetVertexShader(0);
-		dev->SetVertexDeclaration(0);
+		// dev->SetVertexDeclaration(0);
 		
-		// dev->SetFVF(D3DFVF_FONT2DVERTEX);
 		Renderer::SetFVF(D3DFVF_FONT2DVERTEX);
 
 		for (int i = 0; i < 6; i++) {
@@ -380,9 +379,8 @@ namespace D3D
 			{x2-0.5f, y2-0.5f, 0, 1, color, u2, v2},
 			{x1-0.5f, y2-0.5f, 0, 1, color, u1, v2},
 		};
-		dev->SetPixelShader(0);
+		dev->SetPixelShader(NULL);
-		dev->SetVertexShader(0);
+		dev->SetVertexShader(NULL);
-		dev->SetVertexDeclaration(0);
 		
 		Renderer::SetFVF(D3DFVF_XYZRHW | D3DFVF_DIFFUSE | D3DFVF_TEX1);
 		dev->DrawPrimitiveUP(D3DPT_TRIANGLEFAN, 2, coords, sizeof(Q2DVertex));

Source/Plugins/Plugin_VideoDX9/Src/NativeVertexFormat.cpp

@@ -163,5 +163,8 @@ void D3DVertexFormat::Initialize(const PortableVertexDeclaration &_vtx_decl)
 
 void D3DVertexFormat::SetupVertexPointers() const
 {
+	if (d3d_decl)
 		D3D::dev->SetVertexDeclaration(d3d_decl);
+	else
+		ERROR_LOG(VIDEO, "invalid d3d decl");
 }

Source/Plugins/Plugin_VideoDX9/Src/PixelShaderCache.cpp

@@ -32,6 +32,7 @@
 #include <Cg/cgD3D9.h>
 
 PixelShaderCache::PSCache PixelShaderCache::PixelShaders;
+const PixelShaderCache::PSCacheEntry *PixelShaderCache::last_entry;
 
 void SetPSConstant4f(int const_number, float f1, float f2, float f3, float f4)
 {
@@ -59,10 +60,7 @@ void PixelShaderCache::Shutdown()
 
 void PixelShaderCache::SetShader()
 {
-	if (D3D::GetShaderVersion() < 2)
+	static LPDIRECT3DPIXELSHADER9 last_shader = NULL;
-		return; // we are screwed
-
-	static LPDIRECT3DPIXELSHADER9 lastShader = NULL;
 
 	DVSTARTPROFILE();
 
@@ -75,11 +73,12 @@ void PixelShaderCache::SetShader()
 	if (iter != PixelShaders.end())
 	{
 		iter->second.frameCount = frameCount;
-		PSCacheEntry &entry = iter->second;
+		const PSCacheEntry &entry = iter->second;
-		if (!lastShader || entry.shader != lastShader)
+		last_entry = &entry;
+		if (!last_shader || entry.shader != last_shader)
 		{
 			D3D::dev->SetPixelShader(entry.shader);
-			lastShader = entry.shader;
+			last_shader = entry.shader;
 		}
 		return;
 	}
@@ -93,10 +92,15 @@ void PixelShaderCache::SetShader()
 		PSCacheEntry newentry;
 		newentry.shader = shader;
 		newentry.frameCount = frameCount;
+#ifdef _DEBUG
+		newentry.code = code;
+#endif
 		PixelShaders[uid] = newentry;
+		last_entry = &PixelShaders[uid];
 
 		Renderer::SetFVF(NULL);
 		D3D::dev->SetPixelShader(shader);
+		last_shader = shader;
 
 		INCSTAT(stats.numPixelShadersCreated);
 		SETSTAT(stats.numPixelShadersAlive, (int)PixelShaders.size());
@@ -146,3 +150,13 @@ void PixelShaderCache::Cleanup()
 	}
 	SETSTAT(stats.numPixelShadersAlive, (int)PixelShaders.size());
 }
+
+#ifdef _DEBUG
+std::string PixelShaderCache::GetCurrentShaderCode()
+{
+	if (last_entry)
+		return last_entry->code;
+	else
+		return "(no shader)\n";
+}
+#endif

Source/Plugins/Plugin_VideoDX9/Src/PixelShaderCache.h

@@ -31,11 +31,14 @@ tevhash GetCurrentTEV();
 
 class PixelShaderCache
 {
+private:
 	struct PSCacheEntry
 	{
 		LPDIRECT3DPIXELSHADER9 shader;
 		int frameCount;
-
+#ifdef _DEBUG
+		std::string code;
+#endif
 		PSCacheEntry() : shader(NULL), frameCount(0) {}
 		void Destroy()
 		{
@@ -47,12 +50,16 @@ class PixelShaderCache
 	typedef std::map<PIXELSHADERUID, PSCacheEntry> PSCache;
 
 	static PSCache PixelShaders;
+	static const PSCacheEntry *last_entry;
 
 public:
 	static void Init();
 	static void Cleanup();
 	static void Shutdown();
 	static void SetShader();
+#ifdef _DEBUG
+	static std::string GetCurrentShaderCode();
+#endif
 	static LPDIRECT3DPIXELSHADER9 CompileCgShader(const char *pstrprogram);
 };
 

Source/Plugins/Plugin_VideoDX9/Src/VertexManager.cpp

@@ -16,6 +16,7 @@
 // http://code.google.com/p/dolphin-emu/
 
 #include "Common.h"
+#include "FileUtil.h"
 
 #include "D3DBase.h"
 
@@ -81,6 +82,8 @@ bool Init()
 	collection = C_NOTHING;
 	fakeVBuffer = new u8[MAXVBUFFERSIZE];
 	fakeIBuffer = new u16[MAXIBUFFERSIZE];
+	memset(fakeVBuffer, 0, MAXVBUFFERSIZE);
+	memset(fakeIBuffer, 0, MAXIBUFFERSIZE * 2);
 	CreateDeviceObjects();
 	VertexManager::s_pCurBufferPointer = fakeVBuffer;
 	return true;
@@ -143,9 +146,6 @@ void AddVertices(int _primitive, int _numVertices)
 		//We are NOT collecting the right type.
 		Flush();
 
-		// Copy the extra verts that we lost.
-		memcpy(s_pCurBufferPointer, fakeVBuffer, _numVertices * g_nativeVertexFmt->GetVertexStride());
-
 		collection = type;
 		u16 *ptr = 0;
 		if (type != C_POINTS)
@@ -247,14 +247,23 @@ void Flush()
 			if (collection != C_POINTS)
 			{
 				int numPrimitives = indexGen.GetNumPrims();
-				D3D::dev->DrawIndexedPrimitiveUP(pts[(int)collection], 
+				if (FAILED(D3D::dev->DrawIndexedPrimitiveUP(
+					pts[(int)collection], 
 					0, 
 					numVertices, 
 					numPrimitives,
 					fakeIBuffer,
 					D3DFMT_INDEX16,
 					fakeVBuffer,
-												 stride);
+					stride))) {
+#ifdef _DEBUG
+					std::string error_shaders;
+					error_shaders.append(VertexShaderCache::GetCurrentShaderCode());
+					error_shaders.append(PixelShaderCache::GetCurrentShaderCode());
+					File::WriteStringToFile(true, error_shaders, "bad_shader_combo.txt");
+					PanicAlert("DrawIndexedPrimitiveUP failed. Shaders written to shaders.txt.");
+#endif
+				}
 			}
 			else
 			{

Source/Plugins/Plugin_VideoDX9/Src/VertexShaderCache.cpp

@@ -32,6 +32,7 @@
 #include <Cg/cgD3D9.h>
 
 VertexShaderCache::VSCache VertexShaderCache::vshaders;
+const VertexShaderCache::VSCacheEntry *VertexShaderCache::last_entry;
 
 void SetVSConstant4f(int const_number, float f1, float f2, float f3, float f4)
 {
@@ -49,7 +50,6 @@ void VertexShaderCache::Init()
 
 }
 
-
 void VertexShaderCache::Shutdown()
 {
 	VSCache::iterator iter = vshaders.begin();
@@ -61,10 +61,7 @@ void VertexShaderCache::Shutdown()
 
 void VertexShaderCache::SetShader(u32 components)
 {
-	if (D3D::GetShaderVersion() < 2)
+	static LPDIRECT3DVERTEXSHADER9 last_shader = NULL;
-		return; // we are screwed
-
-	static LPDIRECT3DVERTEXSHADER9 lastShader = NULL;
 
 	DVSTARTPROFILE();
 
@@ -76,11 +73,12 @@ void VertexShaderCache::SetShader(u32 components)
 	if (iter != vshaders.end())
 	{
 		iter->second.frameCount = frameCount;
-		VSCacheEntry &entry = iter->second;
+		const VSCacheEntry &entry = iter->second;
-		if (!lastShader || entry.shader != lastShader)
+		last_entry = &entry;
+		if (!last_shader || entry.shader != last_shader)
 		{
 			D3D::dev->SetVertexShader(entry.shader);
-			lastShader = entry.shader;
+			last_shader = entry.shader;
 		}
 		return;
 	}
@@ -94,7 +92,12 @@ void VertexShaderCache::SetShader(u32 components)
 		VSCacheEntry entry;
 		entry.shader = shader;
 		entry.frameCount = frameCount;
+#ifdef _DEBUG
+		entry.code = code;
+#endif
 		vshaders[uid] = entry;
+		last_entry = &vshaders[uid];
+		last_shader = entry.shader;
 
 		D3D::dev->SetVertexShader(shader);
 
@@ -146,3 +149,13 @@ void VertexShaderCache::Cleanup()
 	}
 	SETSTAT(stats.numVertexShadersAlive, (int)vshaders.size());
 }
+
+#ifdef _DEBUG
+std::string VertexShaderCache::GetCurrentShaderCode()
+{
+	if (last_entry)
+		return last_entry->code;
+	else
+		return "(no shader)\n";
+}
+#endif

Source/Plugins/Plugin_VideoDX9/Src/VertexShaderCache.h

@@ -21,16 +21,21 @@
 #include "D3DBase.h"
 
 #include <map>
+#include <string>
 
 #include "D3DBase.h"
 #include "VertexShaderGen.h"
 
 class VertexShaderCache
 {
+private:
 	struct VSCacheEntry
 	{ 
 		LPDIRECT3DVERTEXSHADER9 shader;
 		int frameCount;
+#ifdef _DEBUG
+		std::string code;
+#endif
 		VSCacheEntry() : shader(NULL), frameCount(0) {}
 		void Destroy()
 		{
@@ -42,12 +47,16 @@ class VertexShaderCache
 	typedef std::map<VERTEXSHADERUID, VSCacheEntry> VSCache;
 
 	static VSCache vshaders;
+	static const VSCacheEntry *last_entry;
 
 public:
 	static void Init();
 	static void Cleanup();
 	static void Shutdown();
 	static void SetShader(u32 components);
+#ifdef _DEBUG
+	static std::string GetCurrentShaderCode();
+#endif
 	static LPDIRECT3DVERTEXSHADER9 CompileCgShader(const char *pstrprogram);
 };