undo everything

src/GPU3D.cpp

@@ -965,51 +965,50 @@ void GPU3D::SubmitPolygon() noexcept
     VertexSlotCounter = 1;
     VertexSlotsFree = 0b11110;
 
-    // determine polygon facing direction
+    // culling
+    // TODO: work out how it works on the real thing
+    // the normalization part is a wild guess
 
     Vertex *v0, *v1, *v2, *v3;
+    s64 normalX, normalY, normalZ;
+    s64 dot;
 
     v0 = &TempVertexBuffer[0];
     v1 = &TempVertexBuffer[1];
     v2 = &TempVertexBuffer[2];
     v3 = &TempVertexBuffer[3];
 
-    s32 vector[6];
-    vector[0] = v0->Position[0] - v2->Position[0];
-    vector[1] = v0->Position[1] - v2->Position[1];
-    vector[2] = v0->Position[3] - v2->Position[3];
-    vector[3] = v1->Position[0] - v2->Position[0];
-    vector[4] = v1->Position[1] - v2->Position[1];
-    vector[5] = v1->Position[3] - v2->Position[3];
+    normalX = ((s64)(v0->Position[1]-v1->Position[1]) * (v2->Position[3]-v1->Position[3]))
+        - ((s64)(v0->Position[3]-v1->Position[3]) * (v2->Position[1]-v1->Position[1]));
+    normalY = ((s64)(v0->Position[3]-v1->Position[3]) * (v2->Position[0]-v1->Position[0]))
+        - ((s64)(v0->Position[0]-v1->Position[0]) * (v2->Position[3]-v1->Position[3]));
+    normalZ = ((s64)(v0->Position[0]-v1->Position[0]) * (v2->Position[1]-v1->Position[1]))
+        - ((s64)(v0->Position[1]-v1->Position[1]) * (v2->Position[0]-v1->Position[0]));
 
-    bool facingview;
-    if ((vector[0] | vector[1] | vector[2]) == 0 || (vector[3] | vector[4] | vector[5]) == 0) [[unlikely]] // if either vector is 0 the polygon is accepted and treated as front facing.
+    while ((((normalX>>31) ^ (normalX>>63)) != 0) ||
+           (((normalY>>31) ^ (normalY>>63)) != 0) ||
+           (((normalZ>>31) ^ (normalZ>>63)) != 0))
     {
-        facingview = true;
+        normalX >>= 4;
+        normalY >>= 4;
+        normalZ >>= 4;
     }
-    else
+
+    dot = ((s64)v1->Position[0] * normalX) + ((s64)v1->Position[1] * normalY) + ((s64)v1->Position[3] * normalZ);
+
+    bool facingview = (dot <= 0);
+
+    if (dot < 0)
     {
-        // calculate z component of cross product
-        s64 crossx = ((s64)vector[1] * vector[5]) - ((s64)vector[2] * vector[4]);
-        s64 crossy = ((s64)vector[2] * vector[3]) - ((s64)vector[0] * vector[5]);
-        s64 crossz = ((s64)vector[0] * vector[4]) - ((s64)vector[1] * vector[3]);
-
-        crossx *= v2->Position[0];
-        crossy *= v2->Position[1];
-        crossz *= v2->Position[3];
-
-        s64 dot = crossx + crossy + crossz;
-
-        // set a flag for the rasterizer used for:
-        // unwinding vertices
-        // determining whether slopes are swapped
-        // the less than depth test's == special case
-        facingview = (dot >= 0);
-        
-        // cull polygon if corresponding render flag isn't set
-        if (((dot >= 0) && (CurPolygonAttr & (1<<7))) || // front facing
-            ((dot <= 0) && (CurPolygonAttr & (1<<6))));  // back facing
-        else
+        if (!(CurPolygonAttr & (1<<7)))
+        {
+            LastStripPolygon = NULL;
+            return;
+        }
+    }
+    else if (dot > 0)
+    {
+        if (!(CurPolygonAttr & (1<<6)))
         {
             LastStripPolygon = NULL;
             return;