rework polygon orientation calcs

but this time actually correct

src/GPU3D.cpp

@@ -965,26 +965,37 @@ void GPU3D::SubmitPolygon() noexcept
     VertexSlotCounter = 1;
     VertexSlotsFree = 0b11110;
 
-    // culling
+    // polygon orientation
     // TODO: work out how it works on the real thing
     // the normalization part is a wild guess
 
-    Vertex *v0, *v1, *v2, *v3;
+    Vertex *v0, *v1, *v2;
     s64 normalX, normalY, normalZ;
     s64 dot;
+    bool facingview;
 
     v0 = &TempVertexBuffer[0];
     v1 = &TempVertexBuffer[1];
     v2 = &TempVertexBuffer[2];
-    v3 = &TempVertexBuffer[3];
 
-    normalX = ((s64)(v0->Position[1]-v1->Position[1]) * (v2->Position[3]-v1->Position[3]))
-        - ((s64)(v0->Position[3]-v1->Position[3]) * (v2->Position[1]-v1->Position[1]));
-    normalY = ((s64)(v0->Position[3]-v1->Position[3]) * (v2->Position[0]-v1->Position[0]))
-        - ((s64)(v0->Position[0]-v1->Position[0]) * (v2->Position[3]-v1->Position[3]));
-    normalZ = ((s64)(v0->Position[0]-v1->Position[0]) * (v2->Position[1]-v1->Position[1]))
-        - ((s64)(v0->Position[1]-v1->Position[1]) * (v2->Position[0]-v1->Position[0]));
 
+    // setup vectors for v0->v2 and v1->v2, with w serving as the z axis
+    s32 vector0[3] = { v0->Position[0] - v2->Position[0], v0->Position[1] - v2->Position[1], v0->Position[3] - v2->Position[3] };
+    s32 vector1[3] = { v1->Position[0] - v2->Position[0], v1->Position[1] - v2->Position[1], v1->Position[3] - v2->Position[3] };
+
+    // if either vector is entirely 0, the polygon is accepted, and the front facing flag is set for the rasterizer
+    if (((vector0[0] | vector0[1] | vector0[2]) == 0) || ((vector1[0] | vector1[1] | vector1[2]) == 0))
+    {
+        facingview = true;
+    }
+    else
+    {
+        // calculate cross product of the two vectors
+        normalX = ((s64)vector0[1] * vector1[2]) - ((s64)vector0[2] * vector1[1]);
+        normalY = ((s64)vector0[2] * vector1[0]) - ((s64)vector0[0] * vector1[2]);
+        normalZ = ((s64)vector0[0] * vector1[1]) - ((s64)vector0[1] * vector1[0]);
+
+        // normalization (currently a guess)
         while ((((normalX>>31) ^ (normalX>>63)) != 0) ||
                (((normalY>>31) ^ (normalY>>63)) != 0) ||
                (((normalZ>>31) ^ (normalZ>>63)) != 0))
@@ -994,21 +1005,18 @@ void GPU3D::SubmitPolygon() noexcept
             normalZ >>= 4;
         }
         
-    dot = ((s64)v1->Position[0] * normalX) + ((s64)v1->Position[1] * normalY) + ((s64)v1->Position[3] * normalZ);
+        // calculate dot product against the "camera vector" (we can use any of the vertices to do that)
+        dot = (v1->Position[0] * normalX) + (v1->Position[1] * normalY) + (v1->Position[3] * normalZ);
 
-    bool facingview = (dot <= 0);
+        // front facing flag is set for the rasterizer to be used for:
+        // unwinding vertices
+        // determining whether slopes should be treated as "swapped"
+        // and whether to overwrite a pixel with equal depth when using the < depth test
+        facingview = (dot >= 0);
 
-    if (dot < 0)
+        // cull polygon faces
-    {
+        if (!(((dot >= 0) && (CurPolygonAttr & (1<<7))) || // front facing
-        if (!(CurPolygonAttr & (1<<7)))
+              ((dot <= 0) && (CurPolygonAttr & (1<<6)))))  // back facing
-        {
-            LastStripPolygon = NULL;
-            return;
-        }
-    }
-    else if (dot > 0)
-    {
-        if (!(CurPolygonAttr & (1<<6)))
         {
             LastStripPolygon = NULL;
             return;