From be270259ee007eaff38a6fa5f2033b2b8b04a917 Mon Sep 17 00:00:00 2001 From: Flyinghead Date: Fri, 25 Mar 2022 15:41:16 +0100 Subject: [PATCH] crash with RTT and copy to VRAM if width > linestride RTT texture width must not exceed FB_W_LINESTRIDE fixes Flag to flag crash --- core/hw/pvr/Renderer_if.cpp | 1 + core/hw/pvr/ta_ctx.h | 16 ++++++++++++++-- core/rend/TexCache.cpp | 2 +- core/rend/dx11/dx11_renderer.cpp | 12 ++++++------ core/rend/dx9/d3d_renderer.cpp | 12 ++++++------ core/rend/gles/gltex.cpp | 14 +++++++------- core/rend/vulkan/drawer.cpp | 12 ++++++------ core/rend/vulkan/oit/oit_drawer.cpp | 13 +++++++------ 8 files changed, 48 insertions(+), 34 deletions(-) diff --git a/core/hw/pvr/Renderer_if.cpp b/core/hw/pvr/Renderer_if.cpp index 17e750b97..4193250a5 100644 --- a/core/hw/pvr/Renderer_if.cpp +++ b/core/hw/pvr/Renderer_if.cpp @@ -374,6 +374,7 @@ void rend_start_render() ctx->rend.fb_X_CLIP = FB_X_CLIP; ctx->rend.fb_Y_CLIP = FB_Y_CLIP; + ctx->rend.fb_W_LINESTRIDE = FB_W_LINESTRIDE.stride; ctx->rend.fog_clamp_min = FOG_CLAMP_MIN; ctx->rend.fog_clamp_max = FOG_CLAMP_MAX; diff --git a/core/hw/pvr/ta_ctx.h b/core/hw/pvr/ta_ctx.h index 777b205cf..4a761b6cb 100644 --- a/core/hw/pvr/ta_ctx.h +++ b/core/hw/pvr/ta_ctx.h @@ -126,8 +126,9 @@ struct rend_context bool isRTT; bool isRenderFramebuffer; - FB_X_CLIP_type fb_X_CLIP; - FB_Y_CLIP_type fb_Y_CLIP; + FB_X_CLIP_type fb_X_CLIP; + FB_Y_CLIP_type fb_Y_CLIP; + u32 fb_W_LINESTRIDE; RGBAColor fog_clamp_min; RGBAColor fog_clamp_max; @@ -160,6 +161,17 @@ struct rend_context fZ_max= 1.0f; isRenderFramebuffer = false; } + + u32 getFramebufferWidth() const { + u32 w = fb_X_CLIP.max + 1; + if (fb_W_LINESTRIDE != 0) + // Happens for Flag to Flag, Virtua Tennis? + w = std::min(fb_W_LINESTRIDE * 4, w); + return w; + } + u32 getFramebufferHeight() const { + return fb_Y_CLIP.max + 1; + } }; #define TA_DATA_SIZE (8 * 1024 * 1024) diff --git a/core/rend/TexCache.cpp b/core/rend/TexCache.cpp index 3ca209343..7d4ddf811 100644 --- a/core/rend/TexCache.cpp +++ b/core/rend/TexCache.cpp @@ -937,7 +937,7 @@ void WriteTextureToVRam(u32 width, u32 height, u8 *data, u16 *dst, u32 fb_w_ctrl else fb_w_ctrl = FB_W_CTRL; u32 padding = (linestride == ~0u ? FB_W_LINESTRIDE.stride * 8 : linestride); - if (padding != 0) + if (padding / 2 > width) padding = padding / 2 - width; const u16 kval_bit = (fb_w_ctrl.fb_kval & 0x80) << 8; diff --git a/core/rend/dx11/dx11_renderer.cpp b/core/rend/dx11/dx11_renderer.cpp index f0c4eaec8..fd4b85821 100644 --- a/core/rend/dx11/dx11_renderer.cpp +++ b/core/rend/dx11/dx11_renderer.cpp @@ -1188,10 +1188,10 @@ void DX11Renderer::setBaseScissor() void DX11Renderer::prepareRttRenderTarget(u32 texAddress) { - u32 fbw = pvrrc.fb_X_CLIP.max + 1; - u32 fbh = pvrrc.fb_Y_CLIP.max + 1; + u32 fbw = pvrrc.getFramebufferWidth(); + u32 fbh = pvrrc.getFramebufferHeight(); DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x", - FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, fbw, fbh, texAddress); + FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, fbw, fbh, texAddress); // Find the smallest power of two texture that fits the viewport u32 fbh2 = 2; while (fbh2 < fbh) @@ -1221,8 +1221,8 @@ void DX11Renderer::prepareRttRenderTarget(u32 texAddress) void DX11Renderer::readRttRenderTarget(u32 texAddress) { - u32 w = pvrrc.fb_X_CLIP.max + 1; - u32 h = pvrrc.fb_Y_CLIP.max + 1; + u32 w = pvrrc.getFramebufferWidth(); + u32 h = pvrrc.getFramebufferHeight(); const u8 fb_packmode = FB_W_CTRL.fb_packmode; if (config::RenderToTextureBuffer) { @@ -1268,7 +1268,7 @@ void DX11Renderer::readRttRenderTarget(u32 texAddress) deviceContext->Unmap(stagingTex, 0); u16 *dst = (u16 *)&vram[texAddress]; - WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst); + WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8); } else { diff --git a/core/rend/dx9/d3d_renderer.cpp b/core/rend/dx9/d3d_renderer.cpp index 00f40fa91..978171b0f 100644 --- a/core/rend/dx9/d3d_renderer.cpp +++ b/core/rend/dx9/d3d_renderer.cpp @@ -857,10 +857,10 @@ void D3DRenderer::setBaseScissor() void D3DRenderer::prepareRttRenderTarget(u32 texAddress) { - u32 fbw = pvrrc.fb_X_CLIP.max + 1; - u32 fbh = pvrrc.fb_Y_CLIP.max + 1; + u32 fbw = pvrrc.getFramebufferWidth(); + u32 fbh = pvrrc.getFramebufferHeight(); DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x", - FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, fbw, fbh, texAddress); + FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, fbw, fbh, texAddress); // Find the smallest power of two texture that fits the viewport u32 fbh2 = 2; while (fbh2 < fbh) @@ -893,8 +893,8 @@ void D3DRenderer::prepareRttRenderTarget(u32 texAddress) void D3DRenderer::readRttRenderTarget(u32 texAddress) { - u32 w = pvrrc.fb_X_CLIP.max + 1; - u32 h = pvrrc.fb_Y_CLIP.max + 1; + u32 w = pvrrc.getFramebufferWidth(); + u32 h = pvrrc.getFramebufferHeight(); const u8 fb_packmode = FB_W_CTRL.fb_packmode; if (config::RenderToTextureBuffer) { @@ -926,7 +926,7 @@ void D3DRenderer::readRttRenderTarget(u32 texAddress) verifyWin(offscreenSurface->UnlockRect()); u16 *dst = (u16 *)&vram[texAddress]; - WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst); + WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8); } else { diff --git a/core/rend/gles/gltex.cpp b/core/rend/gles/gltex.cpp index 2cb51f308..f895829f5 100644 --- a/core/rend/gles/gltex.cpp +++ b/core/rend/gles/gltex.cpp @@ -155,10 +155,10 @@ GLuint BindRTT(bool withDepthBuffer) WARN_LOG(RENDERER, "Invalid framebuffer format: 7"); return 0; } - u32 fbw = pvrrc.fb_X_CLIP.max + 1; - u32 fbh = pvrrc.fb_Y_CLIP.max + 1; + u32 fbw = pvrrc.getFramebufferWidth(); + u32 fbh = pvrrc.getFramebufferHeight(); u32 texAddress = FB_W_SOF1 & VRAM_MASK; - DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, + DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, fbw, fbh, texAddress); if (gl.rtt.texAddress != ~0u) @@ -266,8 +266,8 @@ GLuint BindRTT(bool withDepthBuffer) void ReadRTTBuffer() { - u32 w = pvrrc.fb_X_CLIP.max + 1; - u32 h = pvrrc.fb_Y_CLIP.max + 1; + u32 w = pvrrc.getFramebufferWidth(); + u32 h = pvrrc.getFramebufferHeight(); const u8 fb_packmode = FB_W_CTRL.fb_packmode; @@ -295,7 +295,7 @@ void ReadRTTBuffer() gl.rtt.height = h; u16 *dst = gl.gl_major >= 3 ? nullptr : (u16 *)&vram[tex_addr]; - gl.rtt.linestride = FB_W_LINESTRIDE.stride * 8; + gl.rtt.linestride = pvrrc.fb_W_LINESTRIDE * 8; if (gl.rtt.linestride == 0) gl.rtt.linestride = w * 2; @@ -324,7 +324,7 @@ void ReadRTTBuffer() u8 *p = (u8 *)tmp_buf.data(); glReadPixels(0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, p); - WriteTextureToVRam(w, h, p, dst); + WriteTextureToVRam(w, h, p, dst, -1, gl.rtt.linestride); gl.rtt.texAddress = ~0; } } diff --git a/core/rend/vulkan/drawer.cpp b/core/rend/vulkan/drawer.cpp index bd4187a44..20fb6799c 100644 --- a/core/rend/vulkan/drawer.cpp +++ b/core/rend/vulkan/drawer.cpp @@ -377,13 +377,13 @@ void TextureDrawer::Init(SamplerManager *samplerManager, ShaderManager *shaderMa vk::CommandBuffer TextureDrawer::BeginRenderPass() { - DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, + DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, pvrrc.fb_X_CLIP.max + 1, pvrrc.fb_Y_CLIP.max + 1, FB_W_SOF1 & VRAM_MASK); matrices.CalcMatrices(&pvrrc); textureAddr = FB_W_SOF1 & VRAM_MASK; - u32 origWidth = pvrrc.fb_X_CLIP.max + 1; - u32 origHeight = pvrrc.fb_Y_CLIP.max + 1; + u32 origWidth = pvrrc.getFramebufferWidth(); + u32 origHeight = pvrrc.getFramebufferHeight(); u32 heightPow2 = 8; while (heightPow2 < origHeight) heightPow2 *= 2; @@ -516,8 +516,8 @@ void TextureDrawer::EndRenderPass() { currentCommandBuffer.endRenderPass(); - u32 clippedWidth = pvrrc.fb_X_CLIP.max + 1; - u32 clippedHeight = pvrrc.fb_Y_CLIP.max + 1; + u32 clippedWidth = pvrrc.getFramebufferWidth(); + u32 clippedHeight = pvrrc.getFramebufferHeight(); if (config::RenderToTextureBuffer) { @@ -552,7 +552,7 @@ void TextureDrawer::EndRenderPass() PixelBuffer tmpBuf; tmpBuf.init(clippedWidth, clippedHeight); colorAttachment->GetBufferData()->download(clippedWidth * clippedHeight * 4, tmpBuf.data()); - WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst); + WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8); } else { diff --git a/core/rend/vulkan/oit/oit_drawer.cpp b/core/rend/vulkan/oit/oit_drawer.cpp index 5a5b83031..94f3fd40d 100644 --- a/core/rend/vulkan/oit/oit_drawer.cpp +++ b/core/rend/vulkan/oit/oit_drawer.cpp @@ -508,15 +508,16 @@ void OITScreenDrawer::MakeFramebuffers(const vk::Extent2D& viewport) vk::CommandBuffer OITTextureDrawer::NewFrame() { - DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, + DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, pvrrc.fb_X_CLIP.max + 1, pvrrc.fb_Y_CLIP.max + 1, FB_W_SOF1 & VRAM_MASK); NewImage(); matrices.CalcMatrices(&pvrrc); textureAddr = FB_W_SOF1 & VRAM_MASK; - u32 origWidth = pvrrc.fb_X_CLIP.max + 1; - u32 origHeight = pvrrc.fb_Y_CLIP.max + 1; + u32 origWidth = pvrrc.getFramebufferWidth(); + u32 origHeight = pvrrc.getFramebufferHeight(); + float upscale = 1.f; if (!config::RenderToTextureBuffer) upscale = config::RenderResolution / 480.f; @@ -641,8 +642,8 @@ void OITTextureDrawer::EndFrame() { currentCommandBuffer.endRenderPass(); - u32 clippedWidth = pvrrc.fb_X_CLIP.max + 1; - u32 clippedHeight = pvrrc.fb_Y_CLIP.max + 1; + u32 clippedWidth = pvrrc.getFramebufferWidth(); + u32 clippedHeight = pvrrc.getFramebufferHeight(); if (config::RenderToTextureBuffer) { @@ -679,7 +680,7 @@ void OITTextureDrawer::EndFrame() PixelBuffer tmpBuf; tmpBuf.init(clippedWidth, clippedHeight); colorAttachment->GetBufferData()->download(clippedWidth * clippedHeight * 4, tmpBuf.data()); - WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst); + WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8); } else {