From 9902e2eeeed324eecf813b4dd93ceafc03fc6ae7 Mon Sep 17 00:00:00 2001
From: Vicki Pfau <vi@endrift.com>
Date: Sun, 22 Nov 2020 00:46:55 -0800
Subject: [PATCH] GBA Video: Fix OBJWIN erratic rendering in OpenGL renderer

---
 CHANGES                |  1 +
 src/gba/renderers/gl.c | 11 ++++++-----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/CHANGES b/CHANGES
index 5d1a88f04..87c409fed 100644
--- a/CHANGES
+++ b/CHANGES
@@ -45,6 +45,7 @@ Emulation fixes:
  - GBA Video: Fix Hblank timing
  - GBA Video: Implement green swap (fixes mgba.io/i/1609)
  - GBA Video: Emulate sprite cycle limits in OpenGL renderer (fixes mgba.io/i/1635)
+ - GBA Video: Fix OBJWIN erratic rendering in OpenGL renderer
  - SM83: Emulate HALT bug
 Other fixes:
  - 3DS: Fix thread cleanup
diff --git a/src/gba/renderers/gl.c b/src/gba/renderers/gl.c
index 31c16f5cd..07b034680 100644
--- a/src/gba/renderers/gl.c
+++ b/src/gba/renderers/gl.c
@@ -426,7 +426,7 @@ static const char* const _renderObj =
 	"uniform ivec4 inflags;\n"
 	"uniform mat2x2 transform;\n"
 	"uniform ivec4 dims;\n"
-	"uniform ivec4 objwin;\n"
+	"uniform ivec3 objwin;\n"
 	"uniform ivec4 mosaic;\n"
 	"uniform int cyclesRemaining[160];\n"
 	"OUT(0) out vec4 color;\n"
@@ -459,7 +459,7 @@ static const char* const _renderObj =
 	"	color = pix;\n"
 	"	flags = inflags;\n"
 	"	gl_FragDepth = float(flags.x) / 16.;\n"
-	"	window = ivec4(objwin.yzw, 0);\n"
+	"	window = ivec4(objwin, 0);\n"
 	"}";
 
 static const struct GBAVideoGLUniform _uniformsObjPriority[] = {
@@ -1428,7 +1428,6 @@ void _drawScanlines(struct GBAVideoGLRenderer* glRenderer, int y) {
 	GBAVideoGLRendererDrawWindow(glRenderer, y);
 	if (GBARegisterDISPCNTIsObjEnable(glRenderer->dispcnt) && !glRenderer->d.disableOBJ) {
 		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-		glEnable(GL_STENCIL_TEST);
 		glDepthFunc(GL_LESS);
 		for (i = 0; i < glRenderer->oamMax; ++i) {
 			struct GBAVideoRendererSprite* sprite = &glRenderer->sprites[i];
@@ -1716,20 +1715,21 @@ void GBAVideoGLRendererDrawSprite(struct GBAVideoGLRenderer* renderer, struct GB
 		glUniformMatrix2fv(uniforms[GBA_GL_OBJ_TRANSFORM], 1, GL_FALSE, (GLfloat[]) { flipX, 0, 0, flipY });
 	}
 	glUniform4i(uniforms[GBA_GL_OBJ_DIMS], width, height, totalWidth, totalHeight);
+	glDisable(GL_STENCIL_TEST);
 	if (GBAObjAttributesAGetMode(sprite->a) == OBJ_MODE_OBJWIN) {
 		// OBJWIN writes do not affect pixel priority
 		glDisable(GL_DEPTH_TEST);
 		glDepthMask(GL_FALSE);
 		glStencilMask(GL_FALSE);
 		int window = renderer->objwin & 0x3F;
-		glUniform4i(uniforms[GBA_GL_OBJ_OBJWIN], 1, window, renderer->bldb, renderer->bldy);
+		glUniform3i(uniforms[GBA_GL_OBJ_OBJWIN], window, renderer->bldb, renderer->bldy);
 		glDrawBuffers(3, (GLenum[]) { GL_NONE, GL_NONE, GL_COLOR_ATTACHMENT2 });
 	} else {
 		glEnable(GL_DEPTH_TEST);
 		glDepthMask(GL_TRUE);
 		glStencilMask(GL_TRUE);
 		glStencilFunc(GL_ALWAYS, 1, 1);
-		glUniform4i(uniforms[GBA_GL_OBJ_OBJWIN], 0, 0, 0, 0);
+		glUniform3i(uniforms[GBA_GL_OBJ_OBJWIN], 0, 0, 0);
 		glDrawBuffers(2, (GLenum[]) { GL_COLOR_ATTACHMENT0, GL_COLOR_ATTACHMENT1 });
 	}
 	if (GBAObjAttributesAIsMosaic(sprite->a) && GBAObjAttributesAGetMode(sprite->a) != OBJ_MODE_OBJWIN) {
@@ -1749,6 +1749,7 @@ void GBAVideoGLRendererDrawSprite(struct GBAVideoGLRenderer* renderer, struct GB
 		// Update the pixel priority for already-written pixels
 		shader = &renderer->objShader[2];
 		uniforms = shader->uniforms;
+		glEnable(GL_STENCIL_TEST);
 		glStencilFunc(GL_EQUAL, 1, 1);
 		glUseProgram(shader->program);
 		glDrawBuffers(2, (GLenum[]) { GL_NONE, GL_COLOR_ATTACHMENT1 });