VideoSW: rewrite lighting attenuation

- Fixes remaining lighting issues (Mario Tennis, etc)
- Apply same fixes to Software Renderer
- Corrected zero length light direction vector to resolve with normal direction (essentially becomes LIGHTDIF_NONE which was what I was after)

Source/Core/VideoBackends/Software/TransformUnit.cpp

@@ -212,38 +212,33 @@ static void LightColor(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 {
 	const LightPointer *light = (const LightPointer*)&xfmem.lights[lightNum];
 
-	if (!(chan.attnfunc & 1))
-	{
-		// atten disabled
-		switch (chan.diffusefunc)
-		{
-			case LIGHTDIF_NONE:
-				AddIntegerColor(light->color, lightCol);
-				break;
-			case LIGHTDIF_SIGN:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = ldir * normal;
-					AddScaledIntegerColor(light->color, diffuse, lightCol);
-				}
-				break;
-			case LIGHTDIF_CLAMP:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = std::max(0.0f, ldir * normal);
-					AddScaledIntegerColor(light->color, diffuse, lightCol);
-				}
-				break;
-			default: _assert_(0);
-		}
-	}
-	else // spec and spot
-	{
-		// not sure about divide by zero checks
 	Vec3 ldir = light->pos - pos;
-		float attn;
+	float attn = 1.0f;
 
-		if (chan.attnfunc == 3) // spot
+	switch (chan.attnfunc)
+	{
+		case LIGHTATTN_NONE:
+		case LIGHTATTN_DIR:
+		{
+			ldir = ldir.Normalized();
+			if (ldir == Vec3(0, 0, 0))
+				ldir = Vec3(1, 1, 1);
+			break;
+		}
+		case LIGHTATTN_SPEC:
+		{
+			ldir = ldir.Normalized();
+			attn = (ldir * normal) >= 0.0 ? std::max(0.0f, light->dir * normal) : 0;
+			Vec3 attLen = Vec3(1.0, attn, attn*attn);
+			Vec3 cosAttn = light->cosatt;
+			Vec3 distAttn = light->distatt;
+			if (chan.diffusefunc != LIGHTDIF_NONE)
+				distAttn = distAttn.Normalized();
+
+			attn = SafeDivide(std::max(0.0f, attLen * cosAttn), attLen * distAttn);
+			break;
+		}
+		case LIGHTATTN_SPOT:
 		{
 			float dist2 = ldir.Length2();
 			float dist = sqrtf(dist2);
@@ -253,81 +248,58 @@ static void LightColor(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 			float cosAtt = light->cosatt.x + (light->cosatt.y * attn) + (light->cosatt.z * attn * attn);
 			float distAtt = light->distatt.x + (light->distatt.y * dist) + (light->distatt.z * dist2);
 			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
+			break;
 		}
-		else if (chan.attnfunc == 1) // specular
+		default:
-		{
-			// donko - what is going on here?  655.36 is a guess but seems about right.
-			attn = (light->pos * normal) > -655.36 ? std::max(0.0f, (light->dir * normal)) : 0;
-			ldir.set(1.0f, attn, attn * attn);
-
-			float cosAtt = std::max(0.0f, light->cosatt * ldir);
-			float distAtt = light->distatt * ldir;
-			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
-		}
-		else
-		{
 			PanicAlert("LightColor");
-			return;
 	}
 
+	float difAttn = ldir * normal;
 	switch (chan.diffusefunc)
 	{
 		case LIGHTDIF_NONE:
 			AddScaledIntegerColor(light->color, attn, lightCol);
 			break;
 		case LIGHTDIF_SIGN:
-				{
-					float difAttn = ldir * normal;
 			AddScaledIntegerColor(light->color, attn * difAttn, lightCol);
-				}
 			break;
-
 		case LIGHTDIF_CLAMP:
-				{
+			difAttn = std::max(0.0f, difAttn);
-					float difAttn = std::max(0.0f, ldir * normal);
 			AddScaledIntegerColor(light->color, attn * difAttn, lightCol);
-				}
 			break;
 		default: _assert_(0);
 	}
-	}
 }
 
 static void LightAlpha(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const LitChannel &chan, float &lightCol)
 {
 	const LightPointer *light = (const LightPointer*)&xfmem.lights[lightNum];
 
-	if (!(chan.attnfunc & 1))
-	{
-		// atten disabled
-		switch (chan.diffusefunc)
-		{
-			case LIGHTDIF_NONE:
-				lightCol += light->color[0];
-				break;
-			case LIGHTDIF_SIGN:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = ldir * normal;
-					lightCol += light->color[0] * diffuse;
-				}
-				break;
-			case LIGHTDIF_CLAMP:
-				{
-					Vec3 ldir = (light->pos - pos).Normalized();
-					float diffuse = std::max(0.0f, ldir * normal);
-					lightCol += light->color[0] * diffuse;
-				}
-				break;
-			default: _assert_(0);
-		}
-	}
-	else // spec and spot
-	{
 	Vec3 ldir = light->pos - pos;
-		float attn;
+	float attn = 1.0f;
 
-		if (chan.attnfunc == 3) // spot
+	switch (chan.attnfunc)
+	{
+		case LIGHTATTN_NONE:
+		case LIGHTATTN_DIR:
+		{
+			ldir = ldir.Normalized();
+			break;
+		}
+		case LIGHTATTN_SPEC:
+		{
+			ldir = ldir.Normalized();
+			attn = (ldir * normal) >= 0.0 ? std::max(0.0f, light->dir * normal) : 0;
+			Vec3 attLen = Vec3(1.0, attn, attn*attn);
+			Vec3 cosAttn = light->cosatt;
+			Vec3 distAttn = light->distatt;
+			if (chan.diffusefunc != LIGHTDIF_NONE)
+				distAttn = distAttn.Normalized();
+
+			attn = SafeDivide(std::max(0.0f, attLen * cosAttn), attLen * distAttn);
+			break;
+		}
+		case LIGHTATTN_SPOT:
 		{
 			float dist2 = ldir.Length2();
 			float dist = sqrtf(dist2);
@@ -337,39 +309,27 @@ static void LightAlpha(const Vec3 &pos, const Vec3 &normal, u8 lightNum, const L
 			float cosAtt = light->cosatt.x + (light->cosatt.y * attn) + (light->cosatt.z * attn * attn);
 			float distAtt = light->distatt.x + (light->distatt.y * dist) + (light->distatt.z * dist2);
 			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
+			break;
 		}
-		else /* if (chan.attnfunc == 1) */ // specular
+		default:
-		{
+			PanicAlert("LightColor");
-			// donko - what is going on here?  655.36 is a guess but seems about right.
-			attn = (light->pos * normal) > -655.36 ? std::max(0.0f, (light->dir * normal)) : 0;
-			ldir.set(1.0f, attn, attn * attn);
-
-			float cosAtt = light->cosatt * ldir;
-			float distAtt = light->distatt * ldir;
-			attn = SafeDivide(std::max(0.0f, cosAtt), distAtt);
 	}
 
+	float difAttn = ldir * normal;
 	switch (chan.diffusefunc)
 	{
 		case LIGHTDIF_NONE:
 			lightCol += light->color[0] * attn;
 			break;
 		case LIGHTDIF_SIGN:
-				{
-					float difAttn = ldir * normal;
 			lightCol += light->color[0] * attn * difAttn;
-				}
 			break;
-
 		case LIGHTDIF_CLAMP:
-				{
+			difAttn = std::max(0.0f, difAttn);
-					float difAttn = std::max(0.0f, ldir * normal);
 			lightCol += light->color[0] * attn * difAttn;
-				}
 			break;
 		default: _assert_(0);
 	}
-	}
 }
 
 void TransformColor(const InputVertexData *src, OutputVertexData *dst)

Source/Core/VideoCommon/LightingShaderGen.h

@@ -63,7 +63,8 @@ static void GenerateLightShader(T& object, LightingUidData& uid_data, int index,
 		case LIGHTATTN_NONE:
 		case LIGHTATTN_DIR:
 			object.Write("ldir = normalize(" LIGHT_POS".xyz - pos.xyz);\n", LIGHT_POS_PARAMS(index));
-			object.Write("attn = 1.0f;\n");
+			object.Write("attn = 1.0;\n");
+			object.Write("if (length(ldir) == 0.0)\n\t ldir = _norm0;\n");
 			break;
 		case LIGHTATTN_SPEC:
 			object.Write("ldir = normalize(" LIGHT_POS".xyz - pos.xyz);\n", LIGHT_POS_PARAMS(index));
@@ -82,6 +83,7 @@ static void GenerateLightShader(T& object, LightingUidData& uid_data, int index,
 			object.Write("attn = max(0.0, " LIGHT_COSATT".x + " LIGHT_COSATT".y*attn + " LIGHT_COSATT".z*attn*attn) / dot(" LIGHT_DISTATT".xyz, float3(1.0,dist,dist2));\n",
 				LIGHT_COSATT_PARAMS(index), LIGHT_COSATT_PARAMS(index), LIGHT_COSATT_PARAMS(index), LIGHT_DISTATT_PARAMS(index));
 			break;
+		default: _assert_(0);
 	}
 
 	switch (chan.diffusefunc)