GSDX-TextureCache: Add proper scaling for custom resolution

* Improve frame buffer height management on custom resolution. Width seems to be fine with the same size as scaled image output.
* Prevent offset issues on Persona 3 based on the data from merge circuit.

Note: Fixes custom resolution upscaling on ICO 50Hz/60Hz mode when large frame buffer is enabled. previously 60Hz mode only displayed half of the screen and 50Hz mode only worked due to the scissor hack.

plugins/GSdx/GSRenderer.h

@@ -69,6 +69,7 @@ public:
 	virtual void KeyEvent(GSKeyEventData* e);
 	virtual bool CanUpscale() {return false;}
 	virtual int GetUpscaleMultiplier() {return 1;}
+	virtual GSVector2i GetCustomResolution() {return GSVector2i(0,0);}
 	GSVector2i GetInternalResolution();
 	void SetAspectRatio(int aspect) {m_aspectratio = aspect;}
 	void SetVSync(bool enabled);

plugins/GSdx/GSRendererHW.cpp

@@ -44,8 +44,8 @@ GSRendererHW::GSRendererHW(GSTextureCache* tc)
 	}
 
 	if (!m_upscale_multiplier) { //Custom Resolution
-		m_width  = theApp.GetConfigI("resx");
-		m_height = theApp.GetConfigI("resy");
+		m_custom_width = m_width = theApp.GetConfigI("resx");
+		m_custom_height = m_height = theApp.GetConfigI("resy");
 	}
 
 	if (m_upscale_multiplier == 1) { // hacks are only needed for upscaling issues.
@@ -103,6 +103,20 @@ void GSRendererHW::SetScaling()
 	int upscaled_fb_w = fb_width * m_upscale_multiplier;
 	int upscaled_fb_h = fb_height * m_upscale_multiplier;
 	bool good_rt_size = m_width >= upscaled_fb_w && m_height >= upscaled_fb_h;
+	bool initialized_register_state = (m_context->FRAME.FBW > 1) && (crtc_size.y > 1);
+
+	if (!m_upscale_multiplier && initialized_register_state)
+	{
+		if (m_height == m_custom_height)
+		{
+			float ratio = ceil(static_cast<float>(m_height) / crtc_size.y);
+			float buffer_scale_offset = (m_large_framebuffer) ? ratio : 0.5f;
+			ratio = round(ratio + buffer_scale_offset);
+
+			m_tc->RemovePartial();
+			m_height = crtc_size.y * ratio;
+		}
+	}
 
 	// No need to resize for native/custom resolutions as default size will be enough for native and we manually get RT Buffer size for custom.
 	// don't resize until the display rectangle and register states are stabilized.
@@ -139,8 +153,12 @@ bool GSRendererHW::CanUpscale()
 
 int GSRendererHW::GetUpscaleMultiplier()
 {
-	// Custom resolution (currently 0) needs an upscale multiplier of 1.
-	return m_upscale_multiplier ? m_upscale_multiplier : 1;
+	return m_upscale_multiplier;
+}
+
+GSVector2i GSRendererHW::GetCustomResolution()
+{
+	return GSVector2i(m_custom_width, m_custom_height);
 }
 
 void GSRendererHW::Reset()

plugins/GSdx/GSRendererHW.h

@@ -32,6 +32,8 @@ class GSRendererHW : public GSRenderer
 private:
 	int m_width;
 	int m_height;
+	int m_custom_width;
+	int m_custom_height;
 	bool m_reset;
 	int m_upscale_multiplier;
 
@@ -162,6 +164,7 @@ public:
 	void SetGameCRC(uint32 crc, int options);
 	bool CanUpscale();
 	int GetUpscaleMultiplier();
+	GSVector2i GetCustomResolution();
 	void SetScaling();
 
 	void Reset();

plugins/GSdx/GSSettingsDlg.cpp

@@ -389,7 +389,7 @@ void GSSettingsDlg::UpdateControls()
 		ShowWindow(GetDlgItem(m_hWnd, IDC_TC_DEPTH), ogl ? SW_SHOW : SW_HIDE);
 
 		EnableWindow(GetDlgItem(m_hWnd, IDC_CRC_LEVEL), hw);
-		EnableWindow(GetDlgItem(m_hWnd, IDC_LARGE_FB), integer_scaling > 1 && hw);
+		EnableWindow(GetDlgItem(m_hWnd, IDC_LARGE_FB), integer_scaling != 1 && hw);
 		EnableWindow(GetDlgItem(m_hWnd, IDC_CRC_LEVEL_TEXT), hw);
 		EnableWindow(GetDlgItem(m_hWnd, IDC_OPENCL_DEVICE), ocl);
 		EnableWindow(GetDlgItem(m_hWnd, IDC_RESX), hw && !integer_scaling);

plugins/GSdx/GSTextureCache.cpp

@@ -464,22 +464,26 @@ GSTextureCache::Target* GSTextureCache::LookupTarget(const GIFRegTEX0& TEX0, int
 
 	if(m_renderer->CanUpscale())
 	{
-		int multiplier = m_renderer->GetUpscaleMultiplier();
+		float multiplier = static_cast<float>(m_renderer->GetUpscaleMultiplier());
+		GSVector2 scale_factor(multiplier, multiplier);
 
-		if(multiplier > 1)
+		if(!multiplier) //Custom Resolution
 		{
-			dst->m_texture->SetScale(GSVector2((float)multiplier, (float)multiplier));
-		}
-		else  // Custom resolution hack
-		{
-			int ww = m_renderer->GetDisplayRect().width();
-			int hh = m_renderer->GetDisplayRect().height();
+			int width = m_renderer->GetDisplayRect().width();
+			int height = m_renderer->GetDisplayRect().height();
+			int real_height = static_cast<int>(round(m_renderer->GetInternalResolution().y / dst->m_texture->GetScale().y));
 
-			if(ww && hh)
-			{
-				dst->m_texture->SetScale(GSVector2((float)w / ww, (float)h / hh));
-			}
+			// Fixes offset issues on Persona 3 (512x511) where real value of height is 512
+			if(real_height % height == 1)
+				height = real_height;
+
+			GSVector2i custom_resolution = m_renderer->GetCustomResolution();
+			scale_factor.x = static_cast<float>(custom_resolution.x) / width;
+			scale_factor.y = static_cast<float>(custom_resolution.y) / height;
 		}
+
+		if(scale_factor.x && scale_factor.y)
+			dst->m_texture->SetScale(scale_factor);
 	}
 
 	if(used)