From 6c00a498063da5e99914030b16f41c777c6fa6eb Mon Sep 17 00:00:00 2001
From: zeromus <zeromus@users.sf.net>
Date: Sat, 18 Jul 2009 20:24:20 +0000
Subject: [PATCH] gpu: rework capture opaque flag handling a bit. this is a
 risky change and may break some other capture and layer blending scenarios,
 but the old way of doing it (writing 0x8000 to vram for opaque black pixels
 from capture blending) is simply wrong and unworkable. fixes hotel dusk.

---
 desmume/src/GPU.cpp | 213 ++++++++++++++++++++++++--------------------
 desmume/src/MMU.cpp |  83 ++++++-----------
 desmume/src/MMU.h   |  45 +++++++++-
 3 files changed, 183 insertions(+), 158 deletions(-)

diff --git a/desmume/src/GPU.cpp b/desmume/src/GPU.cpp
index c220e1c47..ec8c1dbdf 100644
--- a/desmume/src/GPU.cpp
+++ b/desmume/src/GPU.cpp
@@ -2427,6 +2427,11 @@ static void GPU_ligne_layer(NDS_Screen * screen, u16 l)
 	}
 }
 
+//NOTE : the 0x8000 flag is all jacked up. I think I want to get rid of it.
+//lately, this function does not respect the 0x8000 conventions because it
+//was necessary to fix hotel dusk
+//(i.e. do not write 0x8000 back for opaque black pixels because that data
+//buggily reused later as a BG clear tile and so should be 0x0000)
 template<bool SKIP> static void GPU_ligne_DispCapture(u16 l)
 {
 	//this macro takes advantage of the fact that there are only two possible values for capx
@@ -2454,11 +2459,13 @@ template<bool SKIP> static void GPU_ligne_DispCapture(u16 l)
 		}
 	}
 
+	bool skip = SKIP;
+
 	if (gpu->dispCapCnt.enabled)
 	{
 		//128-wide captures should write linearly into memory, with no gaps
 		//this is tested by hotel dusk
-		u32 ofsmul = gpu->dispCapCnt.capy==128?256:512;
+		u32 ofsmul = gpu->dispCapCnt.capx==DISPCAPCNT::_128?256:512;
 		u32 cap_src_adr = gpu->dispCapCnt.readOffset * 0x8000 + (l * 512);
 		u32 cap_dst_adr = gpu->dispCapCnt.writeOffset * 0x8000 + (l * ofsmul);
 
@@ -2473,111 +2480,123 @@ template<bool SKIP> static void GPU_ligne_DispCapture(u16 l)
 		u8* cap_src = ARM9Mem.ARM9_LCD + cap_src_adr;
 		u8* cap_dst = ARM9Mem.ARM9_LCD + cap_dst_adr;
 
-		if(!SKIP)
+		//we must block captures when the capture dest is not mapped to LCDC
+		if(vramConfiguration.banks[gpu->dispCapCnt.writeBlock].purpose != VramConfiguration::LCDC)
+			skip = true;
+
+		//we must return zero from reads from memory not mapped to lcdc
+		if(vramConfiguration.banks[gpu->dispCapCnt.readBlock].purpose != VramConfiguration::LCDC)
+			cap_src = ARM9Mem.blank_memory;
+
+		if(!skip)
 		if (l < gpu->dispCapCnt.capy)
+		{
+			switch (gpu->dispCapCnt.capSrc)
 			{
-				switch (gpu->dispCapCnt.capSrc)
-				{
-					case 0:		// Capture source is SourceA
+				case 0:		// Capture source is SourceA
+					{
+						//INFO("Capture source is SourceA\n");
+						switch (gpu->dispCapCnt.srcA)
 						{
-							//INFO("Capture source is SourceA\n");
-							switch (gpu->dispCapCnt.srcA)
-							{
-								case 0:			// Capture screen (BG + OBJ + 3D)
-									{
-										//INFO("Capture screen (BG + OBJ + 3D)\n");
-
-										u8 *src;
-										src = (u8*)(GPU_tempScanline);
-										CAPCOPY(src,cap_dst);
-									}
-								break;
-								case 1:			// Capture 3D
-									{
-										//INFO("Capture 3D\n");
-										u16* colorLine;
-										gfx3d_GetLineData(l, &colorLine, NULL);
-										CAPCOPY(((u8*)colorLine),cap_dst);
-									}
-								break;
-							}
-						}
-					break;
-					case 1:		// Capture source is SourceB
-						{
-							//INFO("Capture source is SourceB\n");
-							switch (gpu->dispCapCnt.srcB)
-							{
-								case 0:			// Capture VRAM
-									{
-										//INFO("Capture VRAM\n");
-										CAPCOPY(cap_src,cap_dst);
-									}
-									break;
-								case 1:			// Capture Main Memory Display FIFO
-									{
-										//INFO("Capture Main Memory Display FIFO\n");
-									}
-									break;
-							}
-						}
-					break;
-					default:	// Capture source is SourceA+B blended
-						{
-							//INFO("Capture source is SourceA+B blended\n");
-							u16 *srcA = NULL;
-							u16 *srcB = NULL;
-
-							if (gpu->dispCapCnt.srcA == 0)
-							{
-								// Capture screen (BG + OBJ + 3D)
-								srcA = (u16*)(GPU_tempScanline);
-							}
-							else
-							{
-								gfx3d_GetLineData(l, &srcA, NULL);
-							}
-
-							if (gpu->dispCapCnt.srcB == 0)			// VRAM screen
-								srcB = (u16 *)cap_src;
-							else
-								srcB = NULL; // DISP FIFOS
-
-							if ((srcA) && (srcB))
-							{
-								u16 a, r, g, b;
-								const int todo = (gpu->dispCapCnt.capx==DISPCAPCNT::_128?128:256);
-								for(u16 i = 0; i < todo; i++) 
+							case 0:			// Capture screen (BG + OBJ + 3D)
 								{
-									a = r = g = b =0;
+									//INFO("Capture screen (BG + OBJ + 3D)\n");
 
-									if (gpu->dispCapCnt.EVA && (srcA[i] & 0x8000))
-									{
-										a = 0x8000;
-										r = ((srcA[i] & 0x1F) * gpu->dispCapCnt.EVA);
-										g = (((srcA[i] >>  5) & 0x1F) * gpu->dispCapCnt.EVA);
-										b = (((srcA[i] >>  10) & 0x1F) * gpu->dispCapCnt.EVA);
-									}
-
-									if (gpu->dispCapCnt.EVB && (srcB[i] & 0x8000))
-									{
-										a = 0x8000;
-										r += ((srcB[i] & 0x1F) * gpu->dispCapCnt.EVB);
-										g += (((srcB[i] >>  5) & 0x1F) * gpu->dispCapCnt.EVB);
-										b += (((srcB[i] >> 10) & 0x1F) * gpu->dispCapCnt.EVB);
-									}
-
-									r >>= 4;
-									g >>= 4;
-									b >>= 4;
-
-									T2WriteWord(cap_dst, i << 1, (u16)(a | (b << 10) | (g << 5) | r));
+									u8 *src;
+									src = (u8*)(GPU_tempScanline);
+									CAPCOPY(src,cap_dst);
 								}
+							break;
+							case 1:			// Capture 3D
+								{
+									//INFO("Capture 3D\n");
+									u16* colorLine;
+									gfx3d_GetLineData(l, &colorLine, NULL);
+									CAPCOPY(((u8*)colorLine),cap_dst);
+								}
+							break;
+						}
+					}
+				break;
+				case 1:		// Capture source is SourceB
+					{
+						//INFO("Capture source is SourceB\n");
+						switch (gpu->dispCapCnt.srcB)
+						{
+							case 0:			// Capture VRAM
+								{
+									//INFO("Capture VRAM\n");
+									CAPCOPY(cap_src,cap_dst);
+								}
+								break;
+							case 1:			// Capture Main Memory Display FIFO
+								{
+									//INFO("Capture Main Memory Display FIFO\n");
+								}
+								break;
+						}
+					}
+				break;
+				default:	// Capture source is SourceA+B blended
+					{
+						//INFO("Capture source is SourceA+B blended\n");
+						u16 *srcA = NULL;
+						u16 *srcB = NULL;
+
+						if (gpu->dispCapCnt.srcA == 0)
+						{
+							// Capture screen (BG + OBJ + 3D)
+							srcA = (u16*)(GPU_tempScanline);
+						}
+						else
+						{
+							gfx3d_GetLineData(l, &srcA, NULL);
+						}
+
+						if (gpu->dispCapCnt.srcB == 0)			// VRAM screen
+							srcB = (u16 *)cap_src;
+						else
+							srcB = NULL; // DISP FIFOS
+
+						if ((srcA) && (srcB))
+						{
+							u16 a, r, g, b;
+							const int todo = (gpu->dispCapCnt.capx==DISPCAPCNT::_128?128:256);
+							for(u16 i = 0; i < todo; i++) 
+							{
+								a = r = g = b =0;
+
+								u16 a_alpha;
+								if(gpu->dispCapCnt.srcA == 0)
+									a_alpha = 1;
+								else a_alpha = srcA[i] & 0x8000;
+
+
+								if (gpu->dispCapCnt.EVA && a_alpha)
+								{
+									r = ((srcA[i] & 0x1F) * gpu->dispCapCnt.EVA);
+									g = (((srcA[i] >>  5) & 0x1F) * gpu->dispCapCnt.EVA);
+									b = (((srcA[i] >>  10) & 0x1F) * gpu->dispCapCnt.EVA);
+								}
+
+								if (gpu->dispCapCnt.EVB)
+								{
+									r += ((srcB[i] & 0x1F) * gpu->dispCapCnt.EVB);
+									g += (((srcB[i] >>  5) & 0x1F) * gpu->dispCapCnt.EVB);
+									b += (((srcB[i] >> 10) & 0x1F) * gpu->dispCapCnt.EVB);
+								}
+
+								r >>= 4;
+								g >>= 4;
+								b >>= 4;
+
+								T2WriteWord(cap_dst, i << 1, (b << 10) | (g << 5) | r);
 							}
 						}
-					break;
-				}
+					}
+				break;
 			}
+		}
 
 		if (l>=191)
 		{
diff --git a/desmume/src/MMU.cpp b/desmume/src/MMU.cpp
index ac049e792..285087d8d 100644
--- a/desmume/src/MMU.cpp
+++ b/desmume/src/MMU.cpp
@@ -295,21 +295,6 @@ u8 vram_arm7_map[2];
 
 //<---------
 
-#define VRAM_BANKS 9
-#define VRAM_BANK_A 0
-#define VRAM_BANK_B 1
-#define VRAM_BANK_C 2
-#define VRAM_BANK_D 3
-#define VRAM_BANK_E 4
-#define VRAM_BANK_F 5
-#define VRAM_BANK_G 6
-#define VRAM_BANK_H 7
-#define VRAM_BANK_I 8
-
-#define VRAM_PAGE_ABG 0
-#define VRAM_PAGE_BBG 128
-#define VRAM_PAGE_AOBJ 256
-#define VRAM_PAGE_BOBJ 384
 
 void MMU_VRAM_unmap_all();
 
@@ -398,53 +383,35 @@ static FORCEINLINE u32 MMU_LCDmap(u32 addr, bool& unmapped)
 
 #define LOG_VRAM_ERROR() LOG("No data for block %i MST %i\n", block, VRAMBankCnt & 0x07);
 
-struct VramConfiguration {
+VramConfiguration vramConfiguration;
 
-	enum Purpose {
-		OFF, INVALID, ABG, BBG, AOBJ, BOBJ, LCDC, ARM7, TEX, TEXPAL, ABGEXTPAL, BBGEXTPAL, AOBJEXTPAL, BOBJEXTPAL
-	};
-
-	struct BankInfo {
-		Purpose purpose;
-		int ofs;
-	} banks[VRAM_BANKS];
-	
-	void clear() {
-		for(int i=0;i<VRAM_BANKS;i++) {
-			banks[i].ofs = 0;
-			banks[i].purpose = OFF;
-		}
+std::string VramConfiguration::describePurpose(Purpose p) {
+	switch(p) {
+		case OFF: return "OFF";
+		case INVALID: return "INVALID";
+		case ABG: return "ABG";
+		case BBG: return "BBG";
+		case AOBJ: return "AOBJ";
+		case BOBJ: return "BOBJ";
+		case LCDC: return "LCDC";
+		case ARM7: return "ARM7";
+		case TEX: return "TEX";
+		case TEXPAL: return "TEXPAL";
+		case ABGEXTPAL: return "ABGEXTPAL";
+		case BBGEXTPAL: return "BBGEXTPAL";
+		case AOBJEXTPAL: return "AOBJEXTPAL";
+		case BOBJEXTPAL: return "BOBJEXTPAL";
+		default: return "UNHANDLED CASE";
 	}
+}
 
-	std::string describePurpose(Purpose p) {
-		switch(p) {
-			case OFF: return "OFF";
-			case INVALID: return "INVALID";
-			case ABG: return "ABG";
-			case BBG: return "BBG";
-			case AOBJ: return "AOBJ";
-			case BOBJ: return "BOBJ";
-			case LCDC: return "LCDC";
-			case ARM7: return "ARM7";
-			case TEX: return "TEX";
-			case TEXPAL: return "TEXPAL";
-			case ABGEXTPAL: return "ABGEXTPAL";
-			case BBGEXTPAL: return "BBGEXTPAL";
-			case AOBJEXTPAL: return "AOBJEXTPAL";
-			case BOBJEXTPAL: return "BOBJEXTPAL";
-			default: return "UNHANDLED CASE";
-		}
+std::string VramConfiguration::describe() {
+	std::stringstream ret;
+	for(int i=0;i<VRAM_BANKS;i++) {
+		ret << (char)(i+'A') << ": " << banks[i].ofs << " " << describePurpose(banks[i].purpose) << std::endl;
 	}
-
-	std::string describe() {
-		std::stringstream ret;
-		for(int i=0;i<VRAM_BANKS;i++) {
-			ret << (char)(i+'A') << ": " << banks[i].ofs << " " << describePurpose(banks[i].purpose) << std::endl;
-		}
-		return ret.str();
-	}
-} vramConfiguration;
-
+	return ret.str();
+}
 
 //maps the specified bank to LCDC
 static inline void MMU_vram_lcdc(const int bank)
diff --git a/desmume/src/MMU.h b/desmume/src/MMU.h
index 1a17373dd..dc2ef1649 100644
--- a/desmume/src/MMU.h
+++ b/desmume/src/MMU.h
@@ -190,13 +190,52 @@ extern struct armcpu_memory_iface arm9_base_memory_iface;
 extern struct armcpu_memory_iface arm7_base_memory_iface;
 extern struct armcpu_memory_iface arm9_direct_memory_iface;	
 
+#define VRAM_BANKS 9
+#define VRAM_BANK_A 0
+#define VRAM_BANK_B 1
+#define VRAM_BANK_C 2
+#define VRAM_BANK_D 3
+#define VRAM_BANK_E 4
+#define VRAM_BANK_F 5
+#define VRAM_BANK_G 6
+#define VRAM_BANK_H 7
+#define VRAM_BANK_I 8
+
+#define VRAM_PAGE_ABG 0
+#define VRAM_PAGE_BBG 128
+#define VRAM_PAGE_AOBJ 256
+#define VRAM_PAGE_BOBJ 384
+
+
+struct VramConfiguration {
+
+	enum Purpose {
+		OFF, INVALID, ABG, BBG, AOBJ, BOBJ, LCDC, ARM7, TEX, TEXPAL, ABGEXTPAL, BBGEXTPAL, AOBJEXTPAL, BOBJEXTPAL
+	};
+
+	struct BankInfo {
+		Purpose purpose;
+		int ofs;
+	} banks[VRAM_BANKS];
+	
+	inline void clear() {
+		for(int i=0;i<VRAM_BANKS;i++) {
+			banks[i].ofs = 0;
+			banks[i].purpose = OFF;
+		}
+	}
+
+	std::string describePurpose(Purpose p);
+	std::string describe();
+};
+
+extern VramConfiguration vramConfiguration;
+
 #define VRAM_ARM9_PAGES 512
 extern u8 vram_arm9_map[VRAM_ARM9_PAGES];
 FORCEINLINE void* MMU_gpu_map(u32 vram_addr)
 {
-	//THIS FUNCTION IS NOT AS DANGEROUS!
-	//as an alternative to the above, use this:
-	//it is supposed to map a single gpu vram address to emulator host memory
+	//this is supposed to map a single gpu vram address to emulator host memory
 	//but it returns a pointer to some zero memory in case of accesses to unmapped memory.
 	//this correctly handles the case with tile accesses to unmapped memory.
 	//it could also potentially go through a different LUT than vram_arm9_map in case we discover