Merge branch 'scissoring-fixed'

Fixes various cases in scissor rect emulation. This should fix glitches in various games, right now it's only known to (partially) fix Baten Kaitos: Eternal Wings.
Fix by delroth, port to D3D9/11 by godisgovernment, cleanup by me.

Source/Core/VideoCommon/Src/BPFunctions.cpp

@@ -44,9 +44,24 @@ void SetGenerationMode(const BPCmd &bp)
 	g_renderer->SetGenerationMode();
 }
 
-void SetScissor(const BPCmd &bp)
+void SetScissor()
 {
-	g_renderer->SetScissorRect();
+	const int xoff = bpmem.scissorOffset.x * 2 - 342;
+	const int yoff = bpmem.scissorOffset.y * 2 - 342;
+
+	EFBRectangle rc (bpmem.scissorTL.x - xoff - 342, bpmem.scissorTL.y - yoff - 342,
+					bpmem.scissorBR.x - xoff - 341, bpmem.scissorBR.y - yoff - 341);
+
+	if (rc.left < 0) rc.left = 0;
+	if (rc.top < 0) rc.top = 0;
+	if (rc.right > EFB_WIDTH) rc.right = EFB_WIDTH;
+	if (rc.bottom > EFB_HEIGHT) rc.bottom = EFB_HEIGHT;
+
+	if (rc.left > rc.right) rc.right = rc.left;
+	if (rc.top > rc.bottom) rc.bottom = rc.top;
+
+	TargetRectangle trc = g_renderer->ConvertEFBRectangle(rc);
+	g_renderer->SetScissorRect(trc);
 }
 
 void SetLineWidth(const BPCmd &bp)

Source/Core/VideoCommon/Src/BPFunctions.h

@@ -38,7 +38,7 @@ enum
 
 void FlushPipeline();
 void SetGenerationMode(const BPCmd &bp);
-void SetScissor(const BPCmd &bp);
+void SetScissor();
 void SetLineWidth(const BPCmd &bp);
 void SetDepthMode(const BPCmd &bp);
 void SetBlendMode(const BPCmd &bp);

Source/Core/VideoCommon/Src/BPStructs.cpp

@@ -162,7 +162,7 @@ void BPWritten(const BPCmd& bp)
 	case BPMEM_SCISSORTL: // Scissor Rectable Top, Left
 	case BPMEM_SCISSORBR: // Scissor Rectable Bottom, Right
 	case BPMEM_SCISSOROFFSET: // Scissor Offset
-		SetScissor(bp);
+		SetScissor();
 		break;
 	case BPMEM_LINEPTWIDTH: // Line Width
 		SetLineWidth(bp);

Source/Core/VideoCommon/Src/RenderBase.h

@@ -54,7 +54,7 @@ public:
 
 	virtual void SetColorMask() = 0;
 	virtual void SetBlendMode(bool forceUpdate) = 0;
-	virtual bool SetScissorRect() = 0;
+	virtual void SetScissorRect(const TargetRectangle& rc) = 0;
 	virtual void SetGenerationMode() = 0;
 	virtual void SetDepthMode() = 0;
 	virtual void SetLogicOpMode() = 0;
@@ -175,16 +175,4 @@ extern Renderer *g_renderer;
 
 void UpdateViewport(Matrix44& vpCorrection);
 
-template <typename R>
-void GetScissorRect(MathUtil::Rectangle<R> &rect)
-{
-	const int xoff = bpmem.scissorOffset.x * 2 - 342;
-	const int yoff = bpmem.scissorOffset.y * 2 - 342;
-
-	rect.left   = (R)(bpmem.scissorTL.x - xoff - 342);
-	rect.top    = (R)(bpmem.scissorTL.y - yoff - 342);
-	rect.right  = (R)(bpmem.scissorBR.x - xoff - 341);
-	rect.bottom = (R)(bpmem.scissorBR.y - yoff - 341);
-}
-
 #endif // _COMMON_RENDERBASE_H_

Source/Core/VideoCommon/Src/VideoCommon.h

@@ -70,8 +70,7 @@ void HandleGLError();
 // This structure should only be used to represent a rectangle in EFB
 // coordinates, where the origin is at the upper left and the frame dimensions
 // are 640 x 528.
-struct EFBRectangle : public MathUtil::Rectangle<int>
+typedef MathUtil::Rectangle<int> EFBRectangle;
-{};
 
 // This structure should only be used to represent a rectangle in standard target
 // coordinates, where the origin is at the lower left and the frame dimensions

Source/Plugins/Plugin_VideoDX11/Src/Render.cpp

@@ -42,6 +42,7 @@
 #include "Movie.h"
 #include "Television.h"
 #include "Host.h"
+#include "BPFunctions.h"
 
 namespace DX11
 {
@@ -444,51 +445,9 @@ bool Renderer::CheckForResize()
 	return false;
 }
 
-bool Renderer::SetScissorRect()
+void Renderer::SetScissorRect(const TargetRectangle& rc)
 {
-	TargetRectangle rc;
+	D3D::context->RSSetScissorRects(1, rc.AsRECT());
-	GetScissorRect(rc);
-
-	if (rc.left < 0) rc.left = 0;
-	if (rc.right < 0) rc.right = 0;
-	if (rc.top < 0) rc.top = 0;
-	if (rc.bottom < 0) rc.bottom = 0;
-
-	if (rc.left > EFB_WIDTH) rc.left = EFB_WIDTH;
-	if (rc.right > EFB_WIDTH) rc.right = EFB_WIDTH;
-	if (rc.top > EFB_HEIGHT) rc.top = EFB_HEIGHT;
-	if (rc.bottom > EFB_HEIGHT) rc.bottom = EFB_HEIGHT;
-
-	rc.left = EFBToScaledX(rc.left);
-	rc.right = EFBToScaledX(rc.right);
-	rc.top = EFBToScaledY(rc.top);
-	rc.bottom = EFBToScaledY(rc.bottom);
-
-	if (rc.left > rc.right)
-	{
-		int temp = rc.right;
-		rc.right = rc.left;
-		rc.left = temp;
-	}
-	if (rc.top > rc.bottom)
-	{
-		int temp = rc.bottom;
-		rc.bottom = rc.top;
-		rc.top = temp;
-	}
-
-	if (rc.right >= rc.left && rc.bottom >= rc.top)
-	{
-		D3D::context->RSSetScissorRects(1, rc.AsRECT());
-		return true;
-	}
-	else
-	{
-		//WARN_LOG(VIDEO, "Bad scissor rectangle: %i %i %i %i", rc.left, rc.top, rc.right, rc.bottom);
-		*rc.AsRECT() = CD3D11_RECT(0.f, 0.f, s_target_width, s_target_height);
-		D3D::context->RSSetScissorRects(1, rc.AsRECT());
-		return false;
-	}
 }
 
 void Renderer::SetColorMask()
@@ -1163,7 +1122,7 @@ void Renderer::RestoreAPIState()
 	D3D::stateman->PopDepthState();
 	D3D::stateman->PopRasterizerState();
 	VertexShaderManager::SetViewportChanged();
-	SetScissorRect();
+	BPFunctions::SetScissor();
 }
 
 void Renderer::ApplyState(bool bUseDstAlpha)

Source/Plugins/Plugin_VideoDX11/Src/Render.h

@@ -15,7 +15,7 @@ public:
 
 	void SetColorMask();
 	void SetBlendMode(bool forceUpdate);
-	bool SetScissorRect();
+	void SetScissorRect(const TargetRectangle& rc);
 	void SetGenerationMode();
 	void SetDepthMode();
 	void SetLogicOpMode();

Source/Plugins/Plugin_VideoDX9/Src/Render.cpp

@@ -53,6 +53,7 @@
 #include "Debugger.h"
 #include "Core.h"
 #include "Movie.h"
+#include "BPFunctions.h"
 
 namespace DX9
 {
@@ -426,55 +427,9 @@ bool Renderer::CheckForResize()
 	return false;
 }
 
-bool Renderer::SetScissorRect()
+void Renderer::SetScissorRect(const TargetRectangle& rc)
 {
-	TargetRectangle rc;
+	D3D::dev->SetScissorRect(rc.AsRECT());
-	GetScissorRect(rc);
-
-	if (rc.left < 0) rc.left = 0;
-	if (rc.right < 0) rc.right = 0;
-	if (rc.top < 0) rc.top = 0;
-	if (rc.bottom < 0) rc.bottom = 0;
-
-	if (rc.left > EFB_WIDTH) rc.left = EFB_WIDTH;
-	if (rc.right > EFB_WIDTH) rc.right = EFB_WIDTH;
-	if (rc.top > EFB_HEIGHT) rc.top = EFB_HEIGHT;
-	if (rc.bottom > EFB_HEIGHT) rc.bottom = EFB_HEIGHT;
-
-	if (rc.left > rc.right)
-	{
-		int temp = rc.right;
-		rc.right = rc.left;
-		rc.left = temp;
-	}
-	if (rc.top > rc.bottom)
-	{
-		int temp = rc.bottom;
-		rc.bottom = rc.top;
-		rc.top = temp;
-	}
-
-	rc.left   = EFBToScaledX(rc.left);
-	rc.top    = EFBToScaledY(rc.top);
-	rc.right  = EFBToScaledX(rc.right);
-	rc.bottom = EFBToScaledY(rc.bottom);
-
-	// Check that the coordinates are good
-	if (rc.right != rc.left && rc.bottom != rc.top)
-	{
-		D3D::dev->SetScissorRect(rc.AsRECT());
-		return true;
-	}
-	else
-	{
-		//WARN_LOG(VIDEO, "Bad scissor rectangle: %i %i %i %i", rc.left, rc.top, rc.right, rc.bottom);
-		rc.left   = 0;
-		rc.top    = 0;
-		rc.right  = s_target_width;
-		rc.bottom = s_target_height;
-		D3D::dev->SetScissorRect(rc.AsRECT());
-	}
-	return false;
 }
 
 void Renderer::SetColorMask()
@@ -1293,7 +1248,7 @@ void Renderer::RestoreAPIState()
 	D3D::SetRenderState(D3DRS_FILLMODE, g_ActiveConfig.bWireFrame ? D3DFILL_WIREFRAME : D3DFILL_SOLID);
 	D3D::SetRenderState(D3DRS_SCISSORTESTENABLE, TRUE);
 	VertexShaderManager::SetViewportChanged();
-	SetScissorRect();
+	BPFunctions::SetScissor();
 	if (bpmem.zmode.testenable) {
 		D3D::SetRenderState(D3DRS_ZENABLE, TRUE);
 		if (bpmem.zmode.updateenable)

Source/Plugins/Plugin_VideoDX9/Src/Render.h

@@ -15,7 +15,7 @@ public:
 
 	void SetColorMask();
 	void SetBlendMode(bool forceUpdate);
-	bool SetScissorRect();
+	void SetScissorRect(const TargetRectangle& rc);
 	void SetGenerationMode();
 	void SetDepthMode();
 	void SetLogicOpMode();

Source/Plugins/Plugin_VideoOGL/Src/Render.cpp

@@ -60,6 +60,7 @@
 #include "Core.h"
 #include "Movie.h"
 #include "Host.h"
+#include "BPFunctions.h"
 
 #include "main.h" // Local
 #ifdef _WIN32
@@ -642,49 +643,9 @@ TargetRectangle Renderer::ConvertEFBRectangle(const EFBRectangle& rc)
 // Renderer::GetTargetHeight() = the fixed ini file setting
 // donkopunchstania - it appears scissorBR is the bottom right pixel inside the scissor box
 // therefore the width and height are (scissorBR + 1) - scissorTL
-bool Renderer::SetScissorRect()
+void Renderer::SetScissorRect(const TargetRectangle& rc)
 {
-	MathUtil::Rectangle<float> rc;
+	glScissor(rc.left, rc.bottom, rc.GetWidth(), rc.GetHeight());
-	GetScissorRect(rc);
-
-	if (rc.left < 0) rc.left = 0;
-	if (rc.top < 0) rc.top = 0;
-	if (rc.right > EFB_WIDTH) rc.right = EFB_WIDTH;
-	if (rc.bottom > EFB_HEIGHT) rc.bottom = EFB_HEIGHT;
-
-	if (rc.left > rc.right)
-	{
-		int temp = rc.right;
-		rc.right = rc.left;
-		rc.left = temp;
-	}
-	if (rc.top > rc.bottom)
-	{
-		int temp = rc.bottom;
-		rc.bottom = rc.top;
-		rc.top = temp;
-	}
-
-	// Check that the coordinates are good
-	if (rc.right != rc.left && rc.bottom != rc.top)
-	{
-		glScissor(
-			EFBToScaledX(rc.left), // x = 0 for example
-			EFBToScaledY(EFB_HEIGHT - rc.bottom), // y = 0 for example
-			EFBToScaledX(rc.right - rc.left), // width = 640 for example
-			EFBToScaledY(rc.bottom - rc.top)); // height = 480 for example
-		return true;
-	}
-	else
-	{
-		glScissor(
-			0,
-			0,
-			Renderer::GetTargetWidth(),
-			Renderer::GetTargetHeight()
-			);
-	}
-	return false;
 }
 
 void Renderer::SetColorMask()
@@ -1432,7 +1393,7 @@ void Renderer::RestoreAPIState()
 	// Gets us back into a more game-like state.
 	glEnable(GL_SCISSOR_TEST);
 	SetGenerationMode();
-	SetScissorRect();
+	BPFunctions::SetScissor();
 	SetColorMask();
 	SetDepthMode();
 	SetBlendMode(true);

Source/Plugins/Plugin_VideoOGL/Src/Render.h

@@ -15,7 +15,7 @@ public:
 
 	void SetColorMask();
 	void SetBlendMode(bool forceUpdate);
-	bool SetScissorRect();
+	void SetScissorRect(const TargetRectangle& rc);
 	void SetGenerationMode();
 	void SetDepthMode();
 	void SetLogicOpMode();