From 7d2020910d3603c8555e08dcfd50aeb8e85fd896 Mon Sep 17 00:00:00 2001
From: Vicki Pfau <vi@endrift.com>
Date: Sun, 22 Nov 2020 00:46:55 -0800
Subject: [PATCH] GBA Video: Fix OBJWIN erratic rendering in OpenGL renderer

---
 CHANGES                |  1 +
 src/gba/renderers/gl.c | 11 ++++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/CHANGES b/CHANGES
index e774851a4..8d9495084 100644
--- a/CHANGES
+++ b/CHANGES
@@ -4,6 +4,7 @@ Emulation fixes:
  - GBA I/O: Ignore high bits on IME
  - GBA Memory: Mark Famicom Mini games 22 through 28 as non-mirroring
  - GBA Memory: Return correct byte for odd ROM open bus addresses
+ - GBA Video: Fix OBJWIN erratic rendering in OpenGL renderer
 Other fixes:
  - CMake: Fix build with downstream minizip that exports incompatible symbols
  - Core: Fix threading improperly setting paused state while interrupted
diff --git a/src/gba/renderers/gl.c b/src/gba/renderers/gl.c
index 41efd4cc3..f82bdc079 100644
--- a/src/gba/renderers/gl.c
+++ b/src/gba/renderers/gl.c
@@ -445,7 +445,7 @@ static const char* const _renderObj =
 	"uniform ivec4 inflags;\n"
 	"uniform mat2x2 transform;\n"
 	"uniform ivec4 dims;\n"
-	"uniform ivec4 objwin;\n"
+	"uniform ivec3 objwin;\n"
 	"uniform ivec4 mosaic;\n"
 	"OUT(0) out vec4 color;\n"
 	"OUT(1) out ivec4 flags;\n"
@@ -474,7 +474,7 @@ static const char* const _renderObj =
 	"	color = pix;\n"
 	"	flags = inflags;\n"
 	"	gl_FragDepth = float(flags.x) / 16.;\n"
-	"	window = ivec4(objwin.yzw, 0);\n"
+	"	window = ivec4(objwin, 0);\n"
 	"}";
 
 static const struct GBAVideoGLUniform _uniformsObjPriority[] = {
@@ -1431,7 +1431,6 @@ void _drawScanlines(struct GBAVideoGLRenderer* glRenderer, int y) {
 	if (GBARegisterDISPCNTIsObjEnable(glRenderer->dispcnt) && !glRenderer->d.disableOBJ) {
 		int i;
 		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-		glEnable(GL_STENCIL_TEST);
 		glDepthFunc(GL_LESS);
 		for (i = 0; i < glRenderer->oamMax; ++i) {
 			struct GBAVideoRendererSprite* sprite = &glRenderer->sprites[i];
@@ -1708,20 +1707,21 @@ void GBAVideoGLRendererDrawSprite(struct GBAVideoGLRenderer* renderer, struct GB
 		glUniformMatrix2fv(uniforms[GBA_GL_OBJ_TRANSFORM], 1, GL_FALSE, (GLfloat[]) { flipX, 0, 0, flipY });
 	}
 	glUniform4i(uniforms[GBA_GL_OBJ_DIMS], width, height, totalWidth, totalHeight);
+	glDisable(GL_STENCIL_TEST);
 	if (GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_OBJWIN) {
 		// OBJWIN writes do not affect pixel priority
 		glDisable(GL_DEPTH_TEST);
 		glDepthMask(GL_FALSE);
 		glStencilMask(GL_FALSE);
 		int window = renderer->objwin & 0x3F;
-		glUniform4i(uniforms[GBA_GL_OBJ_OBJWIN], 1, window, renderer->bldb, renderer->bldy);
+		glUniform3i(uniforms[GBA_GL_OBJ_OBJWIN], window, renderer->bldb, renderer->bldy);
 		glDrawBuffers(3, (GLenum[]) { GL_NONE, GL_NONE, GL_COLOR_ATTACHMENT2 });
 	} else {
 		glEnable(GL_DEPTH_TEST);
 		glDepthMask(GL_TRUE);
 		glStencilMask(GL_TRUE);
 		glStencilFunc(GL_ALWAYS, 1, 1);
-		glUniform4i(uniforms[GBA_GL_OBJ_OBJWIN], 0, 0, 0, 0);
+		glUniform3i(uniforms[GBA_GL_OBJ_OBJWIN], 0, 0, 0);
 		glDrawBuffers(2, (GLenum[]) { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1 });
 	}
 	if (GBAObjAttributesAIsMosaic(sprite->a) && GBAObjAttributesAGetMode(sprite->a) != OBJ_MODE_OBJWIN) {
@@ -1741,6 +1741,7 @@ void GBAVideoGLRendererDrawSprite(struct GBAVideoGLRenderer* renderer, struct GB
 		// Update the pixel priority for already-written pixels
 		shader = &renderer->objShader[2];
 		uniforms = shader->uniforms;
+		glEnable(GL_STENCIL_TEST);
 		glStencilFunc(GL_EQUAL, 1, 1);
 		glUseProgram(shader->program);
 		glDrawBuffers(2, (GLenum[]) { GL_NONE, GL_COLOR_ATTACHMENT1 });