gen: work on sprite masking/overflow code. substantially improved. Fixes Sonic 1 title screen, Galaxy Force 2 level select, Landstalker sprite masking, and improved nemesis' test rom. Still something amiss in Sonic 2 title screen.

BizHawk.Emulation/Consoles/Sega/Genesis/Compat.txt

@@ -52,7 +52,6 @@ Insector X .. title screen gfx issue
 James Pond 3 crash in intro
 Jim Power - gfx glitches
 Jurassic Park 2 - crashes in intro
-Landstalker - Sprite masking on bottom is too high on the screen...
 Lemmings: Sound is royally effed... gfx glitches.
 Marvel Land .. holy shit thats psychadelic
 Mega Turrican some gfx glitches
@@ -67,6 +66,7 @@ Power Monger messed up
 RamboIII - intro gfx corrupted - MAYBE GOOD DEBUGGING TARGET
 Shining in the Darkness: Check out sprites in the tavern... very odd
 Sonic 2: Aside from lack of interlace mode, the shadows in the special stage are white....?
+Sonic 2: Something still amiss with title screen sprite masking
 Sonic 3 serious gfx glitches
 Star Control - Shit gets crazy
 Steel Empire - controls messed up. probably gfx issues also.
@@ -142,6 +142,13 @@ Games that require accurate VRAM fill emulation include Thunder Force IV,
 
 Sonic Spinball reads from VSRAM
 
+Games to test for sprite masking/overflow:
+ - Sonic 1 title screen (uses overflow to mask)
+ - Sonic 2 title screen (uses sprite mask modes)
+ - Galaxy Force 2 level select (uses sprite mask)
+ - Landstalker sprite masking (chests and such)
+ - Nemesis test rom
+
 Games known to use 2-cell vertical scroll mode:
  - Air Diver
  - Exo Squad

BizHawk.Emulation/Consoles/Sega/Genesis/GenVDP.Render.cs

@@ -55,7 +55,7 @@ namespace BizHawk.Emulation.Consoles.Sega
             int yTile = ((ScanLine + yScroll) / 8) % NameTableHeight;
             int nameTableWidth = NameTableWidth;
             if (window)
-                nameTableWidth = Display40Mode ? 64 : 32;
+                nameTableWidth = (DisplayWidth == 40) ? 64 : 32;
 
             // this is hellllla slow. but not optimizing until we implement & understand
             // all scrolling modes, shadow & hilight, etc.
@@ -237,6 +237,10 @@ namespace BizHawk.Emulation.Consoles.Sega
         {
             int scanLineBase = ScanLine * FrameWidth;
             int processedSprites = 0;
+            int processedSpritesThisLine = 0;
+            int processedDotsThisLine = 0;
+            bool spriteMaskPrecursor = false;
+
             // This is incredibly unoptimized. TODO...
 
             FetchSprite(0);
@@ -245,8 +249,14 @@ namespace BizHawk.Emulation.Consoles.Sega
                 if (sprite.Y > ScanLine || sprite.Y+sprite.HeightPixels <= ScanLine)
                     goto nextSprite;
 
-                if (sprite.X == -127) // masking code is not super tested
+                processedSpritesThisLine++;
-                    break; // TODO does masking mode 2 really exist?
+                processedDotsThisLine += sprite.WidthPixels;
+
+                if (sprite.X > -128)
+                    spriteMaskPrecursor = true;
+
+                if (sprite.X == -128 && spriteMaskPrecursor)
+                    break; // apply sprite mask
 
                 if (sprite.X + sprite.WidthPixels <= 0)
                     goto nextSprite;
@@ -300,7 +310,13 @@ namespace BizHawk.Emulation.Consoles.Sega
             nextSprite:
                 if (sprite.Link == 0)
                     break;
-                if (++processedSprites > 80)
+                if (++processedSprites >= SpriteLimit)
+                    break;
+                if (processedSpritesThisLine >= SpritePerLineLimit)
+                    break;
+                if (processedDotsThisLine >= DotsPerLineLimit)
+                    break;
+                if (DisplayWidth == 32 && sprite.Link >= 64)
                     break;
                 FetchSprite(sprite.Link);
             }

BizHawk.Emulation/Consoles/Sega/Genesis/GenVDP.cs

@@ -26,7 +26,6 @@ namespace BizHawk.Emulation.Consoles.Sega
         public bool     VInterruptEnabled    { get { return (Registers[1]  & 0x20) != 0; } }
         public bool     DmaEnabled           { get { return (Registers[1]  & 0x10) != 0; } }
         public bool     CellBasedVertScroll  { get { return (Registers[11] & 0x08) != 0; } }
-        public bool     Display40Mode        { get { return (Registers[12] & 0x81) != 0; } }
 
         public bool     InDisplayPeriod      { get { return ScanLine < 224 && DisplayEnabled; } }
 
@@ -38,6 +37,11 @@ namespace BizHawk.Emulation.Consoles.Sega
         int    NameTableWidth = 32;
         int    NameTableHeight = 32;
 
+        int    DisplayWidth;
+        int    SpriteLimit;
+        int    SpritePerLineLimit;
+        int    DotsPerLineLimit;
+
         bool   ControlWordPending;
         ushort VdpDataAddr;
         byte   VdpDataCode;
@@ -340,19 +344,25 @@ int orig_addr = VdpDataAddr;
                     if ((data & 0x81) == 0)
                     {
                         // Display is 32 cells wide
-                        if (FrameWidth != 256)
+                        if (DisplayWidth != 32)
                         {
                             FrameBuffer = new int[256*224];
                             FrameWidth = 256;
-                            //Log.Note("VDP", "SWITCH TO 32 CELL WIDE MODE");
+                            DisplayWidth = 32;
+                            SpriteLimit = 64;
+                            SpritePerLineLimit = 16;
+                            DotsPerLineLimit = 256;
                         }
                     } else {
                         // Display is 40 cells wide
-                        if (FrameWidth != 320)
+                        if (DisplayWidth != 40)
                         {
                             FrameBuffer = new int[320*224];
                             FrameWidth = 320;
-                            //Log.Note("VDP", "SWITCH TO 40 CELL WIDE MODE");
+                            DisplayWidth = 40;
+                            SpriteLimit = 80;
+                            SpritePerLineLimit = 20;
+                            DotsPerLineLimit = 320;
                         }
                     }
                     break;