GBA Video: Fix edge case with sprite blend modes and semitransparency

CHANGES

@@ -11,6 +11,7 @@ Bugfixes:
  - GBA Audio: Fix 8-bit writes to audio channel 3 and 4 registers
  - GBA Audio: Fix audio channels being silenced at the wrong time
  - VFS: Fix return values of VFileFILE.read and .write
+ - GBA Video: Fix edge case with sprite blend modes and semitransparency
 Misc:
  - Qt: Remove useless help icons in dialogs
  - GBA: Attempting to save a screenshot-style savestate should be allowed without libpng

src/gba/renderers/software-mode0.c

@@ -473,10 +473,6 @@ void GBAVideoSoftwareRendererDrawBackgroundMode0(struct GBAVideoSoftwareRenderer
 	int objwinFlags = FLAG_TARGET_1 * (background->target1 && renderer->blendEffect == BLEND_ALPHA && GBAWindowControlIsBlendEnable(renderer->objwin.packed));
 	objwinFlags |= flags;
 	flags |= FLAG_TARGET_1 * (background->target1 && renderer->blendEffect == BLEND_ALPHA && GBAWindowControlIsBlendEnable(renderer->currentWindow.packed));
-	if (renderer->blda == 0x10 && renderer->bldb == 0) {
-		flags &= ~(FLAG_TARGET_1 | FLAG_TARGET_2);
-		objwinFlags &= ~(FLAG_TARGET_1 | FLAG_TARGET_2); \
-	}
 
 	uint32_t screenBase;
 	uint32_t charBase;

src/gba/renderers/software-obj.c

@@ -126,7 +126,7 @@ int GBAVideoSoftwareRendererPreprocessSprite(struct GBAVideoSoftwareRenderer* re
 		target2 |= renderer->bg[1].target2 << (renderer->bg[1].priority);
 		target2 |= renderer->bg[2].target2 << (renderer->bg[2].priority);
 		target2 |= renderer->bg[3].target2 << (renderer->bg[3].priority);
-		if (GBAObjAttributesCGetPriority(sprite->c) < target2) {
+		if ((1 << GBAObjAttributesCGetPriority(sprite->c)) < target2) {
 			variant = 0;
 		}
 	}

src/gba/renderers/software-private.h

@@ -42,7 +42,7 @@ static inline void _compositeBlendObjwin(struct GBAVideoSoftwareRenderer* render
 		if (current & FLAG_TARGET_1 && color & FLAG_TARGET_2) {
 			color = _mix(renderer->blda, current, renderer->bldb, color);
 		} else {
-			color = current & 0x00FFFFFF;
+			color = current & (0x00FFFFFF | FLAG_TARGET_1);
 		}
 	} else {
 		color = (color & ~FLAG_TARGET_2) | (current & FLAG_OBJWIN);
@@ -55,7 +55,7 @@ static inline void _compositeBlendNoObjwin(struct GBAVideoSoftwareRenderer* rend
 		if (current & FLAG_TARGET_1 && color & FLAG_TARGET_2) {
 			color = _mix(renderer->blda, current, renderer->bldb, color);
 		} else {
-			color = current & 0x00FFFFFF;
+			color = current & (0x00FFFFFF | FLAG_TARGET_1);
 		}
 	} else {
 		color = color & ~FLAG_TARGET_2;
@@ -67,16 +67,20 @@ static inline void _compositeNoBlendObjwin(struct GBAVideoSoftwareRenderer* rend
                                            uint32_t current) {
 	UNUSED(renderer);
 	if (color < current) {
-		*pixel = color | (current & FLAG_OBJWIN);
+		color |= (current & FLAG_OBJWIN);
+	} else {
+		color = current & (0x00FFFFFF | FLAG_TARGET_1);
 	}
+	*pixel = color;
 }
 
 static inline void _compositeNoBlendNoObjwin(struct GBAVideoSoftwareRenderer* renderer, uint32_t* pixel, uint32_t color,
                                              uint32_t current) {
 	UNUSED(renderer);
-	if (color < current) {
+	if (color >= current) {
-		*pixel = color;
+		color = current & (0x00FFFFFF | FLAG_TARGET_1);
 	}
+	*pixel = color;
 }
 
 #define COMPOSITE_16_OBJWIN(BLEND)                                                                              \
@@ -180,10 +184,6 @@ static inline void _compositeNoBlendNoObjwin(struct GBAVideoSoftwareRenderer* re
 	objwinFlags |= flags;                                                                                             \
 	flags |= FLAG_TARGET_1 * (background->target1 && renderer->blendEffect == BLEND_ALPHA &&                          \
 	                          GBAWindowControlIsBlendEnable(renderer->currentWindow.packed));                         \
-	if (renderer->blda == 0x10 && renderer->bldb == 0) {                                                              \
-		flags &= ~(FLAG_TARGET_1 | FLAG_TARGET_2);                                                                    \
-		objwinFlags &= ~(FLAG_TARGET_1 | FLAG_TARGET_2);                                                              \
-	}                                                                                                                 \
 	int variant = background->target1 && GBAWindowControlIsBlendEnable(renderer->currentWindow.packed) &&             \
 	    (renderer->blendEffect == BLEND_BRIGHTEN || renderer->blendEffect == BLEND_DARKEN);                           \
 	color_t* palette = renderer->normalPalette;                                                                       \

src/gba/renderers/video-software.c

@@ -520,7 +520,7 @@ static void GBAVideoSoftwareRendererDrawScanline(struct GBAVideoRenderer* render
 	if (softwareRenderer->target2Bd) {
 		x = 0;
 		for (w = 0; w < softwareRenderer->nWindows; ++w) {
-		uint32_t backdrop = FLAG_UNWRITTEN;
+			uint32_t backdrop = FLAG_UNWRITTEN;
 			if (!softwareRenderer->target1Bd || softwareRenderer->blendEffect == BLEND_NONE || softwareRenderer->blendEffect == BLEND_ALPHA || !GBAWindowControlIsBlendEnable(softwareRenderer->windows[w].control.packed)) {
 				backdrop |= softwareRenderer->normalPalette[0];
 			} else {
@@ -529,12 +529,36 @@ static void GBAVideoSoftwareRendererDrawScanline(struct GBAVideoRenderer* render
 			int end = softwareRenderer->windows[w].endX;
 			for (; x < end; ++x) {
 				uint32_t color = softwareRenderer->row[x];
-				if (color & FLAG_TARGET_1) {
+				if (color & FLAG_TARGET_1 && color & FLAG_UNWRITTEN) {
 					softwareRenderer->row[x] = _mix(softwareRenderer->bldb, backdrop, softwareRenderer->blda, color);
 				}
 			}
 		}
 	}
+	if (softwareRenderer->target1Obj && (softwareRenderer->blendEffect == BLEND_DARKEN || softwareRenderer->blendEffect == BLEND_BRIGHTEN)) {
+		x = 0;
+		for (w = 0; w < softwareRenderer->nWindows; ++w) {
+			if (!GBAWindowControlIsBlendEnable(softwareRenderer->windows[w].control.packed)) {
+				continue;
+			}
+			int end = softwareRenderer->windows[w].endX;
+			if (softwareRenderer->blendEffect == BLEND_DARKEN) {
+				for (; x < end; ++x) {
+					uint32_t color = softwareRenderer->row[x];
+					if (color & FLAG_TARGET_1 && !(color & FLAG_UNWRITTEN)) {
+						softwareRenderer->row[x] = _darken(color, softwareRenderer->bldy);
+					}
+				}
+			} else if (softwareRenderer->blendEffect == BLEND_BRIGHTEN) {
+				for (; x < end; ++x) {
+					uint32_t color = softwareRenderer->row[x];
+					if (color & FLAG_TARGET_1 && !(color & FLAG_UNWRITTEN)) {
+						softwareRenderer->row[x] = _brighten(color, softwareRenderer->bldy);
+					}
+				}
+			}
+		}
+	}
 
 #ifdef COLOR_16_BIT
 #if defined(__ARM_NEON) && !defined(__APPLE__)