From a0f8b7a894043783676ca93d0fc03314709485a8 Mon Sep 17 00:00:00 2001 From: rogerman Date: Fri, 2 Dec 2016 10:40:14 -0800 Subject: [PATCH] GPU: Fix crashing bug where target.lineLayerID can be overrun at custom framebuffer sizes. Fixes SourceForge issue #1600. (Regression from r5512.) --- desmume/src/GPU.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/desmume/src/GPU.cpp b/desmume/src/GPU.cpp index cf1e2d04c..bd82708f5 100644 --- a/desmume/src/GPU.cpp +++ b/desmume/src/GPU.cpp @@ -2600,9 +2600,9 @@ void GPUEngineBase::_RenderPixelsCustom(GPUEngineCompositorInfo &compInfo) } #endif - compInfo.target.lineColor16 = (u16 *)compInfo.target.lineColorHead; - compInfo.target.lineColor32 = (FragmentColor *)compInfo.target.lineColorHead; - compInfo.target.lineLayerID = compInfo.target.lineLayerIDHead; + compInfo.target.lineColor16 = (u16 *)compInfo.target.lineColorHeadCustom; + compInfo.target.lineColor32 = (FragmentColor *)compInfo.target.lineColorHeadCustom; + compInfo.target.lineLayerID = compInfo.target.lineLayerIDHeadCustom; #ifdef ENABLE_SSE2 const size_t ssePixCount = (compInfo.line.widthCustom - (compInfo.line.widthCustom % 16)); @@ -2707,9 +2707,9 @@ void GPUEngineBase::_RenderPixelsCustomVRAM(GPUEngineCompositorInfo &compInfo) compInfo.target.xNative = 0; compInfo.target.xCustom = 0; - compInfo.target.lineColor16 = (u16 *)compInfo.target.lineColorHead; - compInfo.target.lineColor32 = (FragmentColor *)compInfo.target.lineColorHead; - compInfo.target.lineLayerID = compInfo.target.lineLayerIDHead; + compInfo.target.lineColor16 = (u16 *)compInfo.target.lineColorHeadCustom; + compInfo.target.lineColor32 = (FragmentColor *)compInfo.target.lineColorHeadCustom; + compInfo.target.lineLayerID = compInfo.target.lineLayerIDHeadCustom; size_t i = 0; @@ -3881,10 +3881,14 @@ void GPUEngineBase::_RenderLine_Layers(const size_t l) if (this->isLineRenderNative[compInfo.line.indexNative]) { + compInfo.target.lineColorHead = compInfo.target.lineColorHeadNative; + compInfo.target.lineLayerIDHead = compInfo.target.lineLayerIDHeadNative; this->_RenderLine_LayerBG(compInfo); } else { + compInfo.target.lineColorHead = compInfo.target.lineColorHeadCustom; + compInfo.target.lineLayerIDHead = compInfo.target.lineLayerIDHeadCustom; this->_RenderLine_LayerBG(compInfo); } } //layer enabled @@ -4462,8 +4466,6 @@ FORCEINLINE void GPUEngineBase::_RenderLine_LayerBG_Final(GPUEngineCompositorInf // up into separate steps. If rendering at a custom size, do the pixel rendering step now. if ( !ISDEBUGRENDER && (ISCUSTOMRENDERINGNEEDED || !this->isLineRenderNative[compInfo.line.indexNative]) ) { - compInfo.target.lineLayerID = compInfo.target.lineLayerIDHeadCustom; - if (useCustomVRAM) { this->_RenderPixelsCustomVRAM(compInfo);