GBA Video: Disable BG target 1 blending when OBJ blending (fixes #2722)

CHANGES

@@ -6,6 +6,7 @@ Emulation fixes:
  - GB Serialize: Don't write BGP/OBP when loading SCGB state (fixes mgba.io/i/2694)
  - GBA: Fix resetting key IRQ state (fixes mgba.io/i/2716)
  - GBA Video: Ignore disabled backgrounds as OBJ blend target (fixes mgba.io/i/2489)
+ - GBA Video: Disable BG target 1 blending when OBJ blending (fixes mgba.io/i/2722)
 Other fixes:
  - Qt: Manually split filename to avoid overzealous splitting (fixes mgba.io/i/2681)
  - Qt: Expand criteria for tag branch names (fixes mgba.io/i/2679)

src/gba/renderers/software-mode0.c

@@ -17,7 +17,6 @@
 
 #define DRAW_BACKGROUND_MODE_0_TILE_SUFFIX_16(BLEND, OBJWIN) \
 	paletteData = GBA_TEXT_MAP_PALETTE(mapData) << 4; \
-	palette = &mainPalette[paletteData]; \
 	charBase = (background->charBase + (GBA_TEXT_MAP_TILE(mapData) << 5)) + (localY << 2); \
 	LOAD_32(tileData, charBase, vram); \
 	if (!GBA_TEXT_MAP_HFLIP(mapData)) { \
@@ -36,7 +35,6 @@
 	charBase = (background->charBase + (GBA_TEXT_MAP_TILE(mapData) << 5)) + (localY << 2); \
 	LOAD_32(tileData, charBase, vram); \
 	paletteData = GBA_TEXT_MAP_PALETTE(mapData) << 4; \
-	palette = &mainPalette[paletteData]; \
 	pixel = &renderer->row[outX]; \
 	if (!GBA_TEXT_MAP_HFLIP(mapData)) { \
 		if (outX < renderer->start) { \
@@ -93,7 +91,6 @@
 			carryData = 0; \
 		} else { \
 			paletteData = GBA_TEXT_MAP_PALETTE(mapData) << 4; \
-			palette = &mainPalette[paletteData]; \
 			LOAD_32(tileData, charBase, vram); \
 			if (!GBA_TEXT_MAP_HFLIP(mapData)) { \
 				tileData >>= 4 * baseX; \
@@ -123,7 +120,6 @@
 					carryData = 0; \
 				} else { \
 					paletteData = GBA_TEXT_MAP_PALETTE(mapData) << 4; \
-					palette = &mainPalette[paletteData]; \
 					LOAD_32(tileData, charBase, vram); \
 					if (!GBA_TEXT_MAP_HFLIP(mapData)) { \
 						tileData >>= x * 4; \
@@ -154,7 +150,6 @@
 			localY = 7 - localY; \
 		} \
 		paletteData = GBA_TEXT_MAP_PALETTE(mapData) << 4; \
-		palette = &mainPalette[paletteData]; \
 		charBase = (background->charBase + (GBA_TEXT_MAP_TILE(mapData) << 5)) + (localY << 2); \
 		if (UNLIKELY(charBase >= 0x10000)) { \
 			pixel += 8; \
@@ -517,17 +512,16 @@ void GBAVideoSoftwareRendererDrawBackgroundMode0(struct GBAVideoSoftwareRenderer
 
 	uint32_t screenBase;
 	uint32_t charBase;
-	color_t* mainPalette = renderer->normalPalette;
+	color_t* palette = renderer->normalPalette;
 	if (renderer->d.highlightAmount && background->highlight) {
-		mainPalette = renderer->highlightPalette;
+		palette = renderer->highlightPalette;
 	}
 	if (variant) {
-		mainPalette = renderer->variantPalette;
+		palette = renderer->variantPalette;
 		if (renderer->d.highlightAmount && background->highlight) {
-			mainPalette = renderer->highlightVariantPalette;
+			palette = renderer->highlightVariantPalette;
 		}
 	}
-	color_t* palette = mainPalette;
 	PREPARE_OBJWIN;
 
 	int outX = renderer->start;

src/gba/renderers/software-private.h

@@ -86,28 +86,55 @@ static inline void _compositeNoBlendNoObjwin(struct GBAVideoSoftwareRenderer* re
 
 #define COMPOSITE_16_OBJWIN(BLEND, IDX)  \
 	if (objwinForceEnable || (!(current & FLAG_OBJWIN)) == objwinOnly) {                                          \
-		unsigned color = (current & FLAG_OBJWIN) ? objwinPalette[paletteData | pixelData] : palette[pixelData]; \
+		unsigned color; \
 		unsigned mergedFlags = flags; \
 		if (current & FLAG_OBJWIN) { \
 			mergedFlags = objwinFlags; \
+			color = objwinPalette[paletteData | pixelData]; \
+		} else if ((current & (FLAG_IS_BACKGROUND | FLAG_REBLEND)) == FLAG_REBLEND) { \
+			color = renderer->normalPalette[paletteData | pixelData]; \
+		} else { \
+			color = palette[paletteData | pixelData]; \
 		} \
 		_composite ## BLEND ## Objwin(renderer, &pixel[IDX], color | mergedFlags, current); \
 	}
 
 #define COMPOSITE_16_NO_OBJWIN(BLEND, IDX) \
-	_composite ## BLEND ## NoObjwin(renderer, &pixel[IDX], palette[pixelData] | flags, current);
+	{ \
+		unsigned color; \
+		if ((current & (FLAG_IS_BACKGROUND | FLAG_REBLEND)) == FLAG_REBLEND) { \
+			color = renderer->normalPalette[paletteData | pixelData]; \
+		} else { \
+			color = palette[paletteData | pixelData]; \
+		} \
+		_composite ## BLEND ## NoObjwin(renderer, &pixel[IDX], color | flags, current); \
+	}
 
 #define COMPOSITE_256_OBJWIN(BLEND, IDX) \
 	if (objwinForceEnable || (!(current & FLAG_OBJWIN)) == objwinOnly) { \
-		unsigned color = (current & FLAG_OBJWIN) ? objwinPalette[pixelData] : palette[pixelData]; \
+		unsigned color; \
 		unsigned mergedFlags = flags; \
 		if (current & FLAG_OBJWIN) { \
 			mergedFlags = objwinFlags; \
+			color = objwinPalette[pixelData]; \
+		} else if ((current & (FLAG_IS_BACKGROUND | FLAG_REBLEND)) == FLAG_REBLEND) { \
+			color = renderer->normalPalette[pixelData]; \
+		} else { \
+			color = palette[pixelData]; \
 		} \
 		_composite ## BLEND ## Objwin(renderer, &pixel[IDX], color | mergedFlags, current); \
 	}
 
-#define COMPOSITE_256_NO_OBJWIN COMPOSITE_16_NO_OBJWIN
+#define COMPOSITE_256_NO_OBJWIN(BLEND, IDX) \
+	{ \
+		unsigned color; \
+		if ((current & (FLAG_IS_BACKGROUND | FLAG_REBLEND)) == FLAG_REBLEND) { \
+			color = renderer->normalPalette[pixelData]; \
+		} else { \
+			color = palette[pixelData]; \
+		} \
+		_composite ## BLEND ## NoObjwin(renderer, &pixel[IDX], color | flags, current); \
+	}
 
 #define BACKGROUND_DRAW_PIXEL_16(BLEND, OBJWIN, IDX) \
 	pixelData = tileData & 0xF; \