From f734c2f6c7ac20f2715b612ee37a0c30f3b6f8dc Mon Sep 17 00:00:00 2001
From: Vicki Pfau <vi@endrift.com>
Date: Sun, 4 Oct 2020 01:02:28 -0700
Subject: [PATCH] GBA Video: Do not affect OBJ pixel priority when writing
 OBJWIN (fixes #1890)

---
 CHANGES                |  1 +
 src/gba/renderers/gl.c | 35 ++++++++++++++++++++++-------------
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/CHANGES b/CHANGES
index 265b7d56a..b2d6d55a7 100644
--- a/CHANGES
+++ b/CHANGES
@@ -10,6 +10,7 @@ Emulation fixes:
  - GBA Video: Invalidate map cache when modifying BGCNT (fixes mgba.io/i/1846)
  - GBA Video: Don't draw sprites using unmapped VRAM in GL renderer (fixes mgba.io/i/1865)
  - GBA Video: Fix rare regression blending semitransparent sprites (fixes mgba.io/i/1876)
+ - GBA Video: Do not affect OBJ pixel priority when writing OBJWIN (fixes mgba.io/i/1890)
 Other fixes:
  - 3DS: Redo video sync to be more precise
  - 3DS: Fix crash with libctru 2.0 when exiting
diff --git a/src/gba/renderers/gl.c b/src/gba/renderers/gl.c
index 7f5d96cb6..f01a67440 100644
--- a/src/gba/renderers/gl.c
+++ b/src/gba/renderers/gl.c
@@ -916,6 +916,7 @@ void GBAVideoGLRendererDeinit(struct GBAVideoRenderer* renderer) {
 	_deleteShader(&glRenderer->bgShader[3]);
 	_deleteShader(&glRenderer->objShader[0]);
 	_deleteShader(&glRenderer->objShader[1]);
+	_deleteShader(&glRenderer->objShader[2]);
 	_deleteShader(&glRenderer->finalizeShader);
 
 	int i;
@@ -1705,12 +1706,18 @@ void GBAVideoGLRendererDrawSprite(struct GBAVideoGLRenderer* renderer, struct GB
 	}
 	glUniform4i(uniforms[GBA_GL_OBJ_DIMS], width, height, totalWidth, totalHeight);
 	if (GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_OBJWIN) {
+		// OBJWIN writes do not affect pixel priority
 		glDisable(GL_DEPTH_TEST);
+		glDepthMask(GL_FALSE);
+		glStencilMask(GL_FALSE);
 		int window = renderer->objwin & 0x3F;
 		glUniform4i(uniforms[GBA_GL_OBJ_OBJWIN], 1, window, renderer->bldb, renderer->bldy);
 		glDrawBuffers(3, (GLenum[]) { GL_NONE, GL_NONE, GL_COLOR_ATTACHMENT2 });
 	} else {
 		glEnable(GL_DEPTH_TEST);
+		glDepthMask(GL_TRUE);
+		glStencilMask(GL_TRUE);
+		glStencilFunc(GL_ALWAYS, 1, 1);
 		glUniform4i(uniforms[GBA_GL_OBJ_OBJWIN], 0, 0, 0, 0);
 		glDrawBuffers(2, (GLenum[]) { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1 });
 	}
@@ -1723,23 +1730,25 @@ void GBAVideoGLRendererDrawSprite(struct GBAVideoGLRenderer* renderer, struct GB
 	} else {
 		glUniform4i(uniforms[GBA_GL_OBJ_MOSAIC], 0, 0, 0, 0);
 	}
-	glStencilFunc(GL_ALWAYS, 1, 1);
 	if (GBAObjAttributesAGetMode(sprite->a) != OBJ_MODE_OBJWIN || GBARegisterDISPCNTIsObjwinEnable(renderer->dispcnt)) {
 		glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
 	}
 
-	shader = &renderer->objShader[2];
-	uniforms = shader->uniforms;
-	glStencilFunc(GL_EQUAL, 1, 1);
-	glUseProgram(shader->program);
-	glDrawBuffers(2, (GLenum[]) { GL_NONE, GL_COLOR_ATTACHMENT1 });
-	glBindVertexArray(shader->vao);
-	glUniform2i(uniforms[GBA_GL_VS_LOC], totalHeight, 0);
-	glUniform2i(uniforms[GBA_GL_VS_MAXPOS], totalWidth, totalHeight);
-	glUniform4i(uniforms[GBA_GL_OBJ_INFLAGS], GBAObjAttributesCGetPriority(sprite->c),
-	                                          (renderer->target1Obj || GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_SEMITRANSPARENT) | (renderer->target2Obj * 2) | (renderer->blendEffect * 4),
-	                                          renderer->blda, GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_SEMITRANSPARENT);
-	glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+	if (GBAObjAttributesAGetMode(sprite->a) != OBJ_MODE_OBJWIN) {
+		// Update the pixel priority for already-written pixels
+		shader = &renderer->objShader[2];
+		uniforms = shader->uniforms;
+		glStencilFunc(GL_EQUAL, 1, 1);
+		glUseProgram(shader->program);
+		glDrawBuffers(2, (GLenum[]) { GL_NONE, GL_COLOR_ATTACHMENT1 });
+		glBindVertexArray(shader->vao);
+		glUniform2i(uniforms[GBA_GL_VS_LOC], totalHeight, 0);
+		glUniform2i(uniforms[GBA_GL_VS_MAXPOS], totalWidth, totalHeight);
+		glUniform4i(uniforms[GBA_GL_OBJ_INFLAGS], GBAObjAttributesCGetPriority(sprite->c),
+		                                          (renderer->target1Obj || GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_SEMITRANSPARENT) | (renderer->target2Obj * 2) | (renderer->blendEffect * 4),
+		                                          renderer->blda, GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_SEMITRANSPARENT);
+		glDrawArrays(GL_TRIANGLE_FAN, 0, 4);
+	}
 
 	glDrawBuffers(1, (GLenum[]) { GL_COLOR_ATTACHMENT0 });
 }