From 3e12966919e4f0826093f2e2f7b53aa081ca2df1 Mon Sep 17 00:00:00 2001 From: gabest11 Date: Wed, 27 May 2009 05:55:32 +0000 Subject: [PATCH] GSdx: Even more alpha test voodoo magic, it must be nailed now! And there is even an explanation. The tfx functions calculate At * Af >> 7, which means modulating by 0x80 should return At as the result. With the evil floating point pixel shader however 0x80 translates to 128/255 (0.502), not exactly 0.5, modulation as At' * Af' * 2 (' means 0 - 1.0 range) is not the same as with integers. At' = Af' = 0.502 At' * Af' * 2 = 0.504 If the alpha test happens to be "not equal to 0x80", then abs(0.504 - 128/255) < 0.5/255 will just miss. Solution is to re-scale those values to the integer range, do the calculations, and then back to float again, but in the end it just simplifies down to At' * Af' * 255/128, doh... At * Af >> 7 => ((At' * 255) * (Af' * 255) / 128) / 255 => At' * Af' * 255/128 At' = Af' = 0.502 0.502 * 0.502 * 255/128 = 0.502 (w00t!) git-svn-id: http://pcsx2.googlecode.com/svn/trunk@1272 96395faa-99c1-11dd-bbfe-3dabce05a288 --- plugins/GSdx/GSDevice10.cpp | 2 +- plugins/GSdx/GSDevice9.cpp | 2 +- plugins/GSdx/res/tfx10.fx | 8 ++++---- plugins/GSdx/res/tfx9.fx | 10 +++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/GSdx/GSDevice10.cpp b/plugins/GSdx/GSDevice10.cpp index bf22deb23d..a32bd1e7d4 100644 --- a/plugins/GSdx/GSDevice10.cpp +++ b/plugins/GSdx/GSDevice10.cpp @@ -511,7 +511,7 @@ void GSDevice10::IASetVertexBuffer(const void* vertices, size_t stride, size_t c { ASSERT(m_vertices.count == 0); - if(count > m_vertices.limit) + if(count * stride > m_vertices.limit * m_vertices.stride) { m_vertices.vb_old = m_vertices.vb; m_vertices.vb = NULL; diff --git a/plugins/GSdx/GSDevice9.cpp b/plugins/GSdx/GSDevice9.cpp index e84029c303..7d4f12a940 100644 --- a/plugins/GSdx/GSDevice9.cpp +++ b/plugins/GSdx/GSDevice9.cpp @@ -648,7 +648,7 @@ void GSDevice9::IASetVertexBuffer(const void* vertices, size_t stride, size_t co { ASSERT(m_vertices.count == 0); - if(count > m_vertices.limit) + if(count * stride > m_vertices.limit * m_vertices.stride) { m_vertices.vb_old = m_vertices.vb; m_vertices.vb = NULL; diff --git a/plugins/GSdx/res/tfx10.fx b/plugins/GSdx/res/tfx10.fx index 8f0fe77a9b..78d1c4403d 100644 --- a/plugins/GSdx/res/tfx10.fx +++ b/plugins/GSdx/res/tfx10.fx @@ -414,11 +414,11 @@ PS_OUTPUT ps_main(PS_INPUT input) { if(TCC == 0) { - c.rgb = c.rgb * t.rgb * 2; + c.rgb = c.rgb * t.rgb * 255.0f / 128; } else { - c = c * t * 2; + c = c * t * 255.0f / 128; } } else if(TFX == 1) @@ -434,7 +434,7 @@ PS_OUTPUT ps_main(PS_INPUT input) } else if(TFX == 2) { - c.rgb = c.rgb * t.rgb * 2 + c.a; + c.rgb = c.rgb * t.rgb * 255.0f / 128 + c.a; if(TCC == 1) { @@ -443,7 +443,7 @@ PS_OUTPUT ps_main(PS_INPUT input) } else if(TFX == 3) { - c.rgb = c.rgb * t.rgb * 2 + c.a; + c.rgb = c.rgb * t.rgb * 255.0f / 128 + c.a; if(TCC == 1) { diff --git a/plugins/GSdx/res/tfx9.fx b/plugins/GSdx/res/tfx9.fx index d825cf187b..384b85780d 100644 --- a/plugins/GSdx/res/tfx9.fx +++ b/plugins/GSdx/res/tfx9.fx @@ -241,11 +241,11 @@ float4 ps_main(PS_INPUT input) : COLOR { if(TCC == 0) { - c.rgb = c.rgb * t.rgb * 2; + c.rgb = c.rgb * t.rgb * 255.0f / 128; } else { - c = c * t * 2; + c = c * t * 255.0f / 128; } } else if(TFX == 1) @@ -261,7 +261,7 @@ float4 ps_main(PS_INPUT input) : COLOR } else if(TFX == 2) { - c.rgb = c.rgb * t.rgb * 2 + c.a; + c.rgb = c.rgb * t.rgb * 255.0f / 128 + c.a; if(TCC == 1) { @@ -270,7 +270,7 @@ float4 ps_main(PS_INPUT input) : COLOR } else if(TFX == 3) { - c.rgb = c.rgb * t.rgb * 2 + c.a; + c.rgb = c.rgb * t.rgb * 255.0f / 128 + c.a; if(TCC == 1) { @@ -292,7 +292,7 @@ float4 ps_main(PS_INPUT input) : COLOR } else if(ATST == 4) // e { - clip(0.6f / 256 - abs(c.a - AREF)); // FIXME: 0.5f is too small + clip(0.6f / 255 - abs(c.a - AREF)); // FIXME: 0.5f is too small } else if(ATST == 5 || ATST == 6) // ge, g {