ShuriZma
/
flycast
mirror of https://github.com/flyinghead/flycast.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

crash with RTT and copy to VRAM if width > linestride 

RTT texture width must not exceed FB_W_LINESTRIDE
fixes Flag to flag crash
This commit is contained in:
Flyinghead 2022-03-25 15:41:16 +01:00
parent 03f9955f8b
commit be270259ee
8 changed files with 48 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
core/hw/pvr/Renderer_if.cpp
View File

@ -374,6 +374,7 @@ void rend_start_render()
ctx->rend.fb_X_CLIP = FB_X_CLIP;
ctx->rend.fb_Y_CLIP = FB_Y_CLIP;
ctx->rend.fb_W_LINESTRIDE = FB_W_LINESTRIDE.stride;
ctx->rend.fog_clamp_min = FOG_CLAMP_MIN;
ctx->rend.fog_clamp_max = FOG_CLAMP_MAX;

16
core/hw/pvr/ta_ctx.h
View File

@ -126,8 +126,9 @@ struct rend_context
bool isRTT;
bool isRenderFramebuffer;
FB_X_CLIP_type fb_X_CLIP;
FB_Y_CLIP_type fb_Y_CLIP;
FB_X_CLIP_type fb_X_CLIP;
FB_Y_CLIP_type fb_Y_CLIP;
u32 fb_W_LINESTRIDE;
RGBAColor fog_clamp_min;
RGBAColor fog_clamp_max;
@ -160,6 +161,17 @@ struct rend_context
fZ_max= 1.0f;
isRenderFramebuffer = false;
}
u32 getFramebufferWidth() const {
u32 w = fb_X_CLIP.max + 1;
if (fb_W_LINESTRIDE != 0)
// Happens for Flag to Flag, Virtua Tennis?
w = std::min(fb_W_LINESTRIDE * 4, w);
return w;
}
u32 getFramebufferHeight() const {
return fb_Y_CLIP.max + 1;
}
};
#define TA_DATA_SIZE (8 * 1024 * 1024)

2
core/rend/TexCache.cpp
View File

@ -937,7 +937,7 @@ void WriteTextureToVRam(u32 width, u32 height, u8 *data, u16 *dst, u32 fb_w_ctrl
else
fb_w_ctrl = FB_W_CTRL;
u32 padding = (linestride == ~0u ? FB_W_LINESTRIDE.stride * 8 : linestride);
if (padding != 0)
if (padding / 2 > width)
padding = padding / 2 - width;
const u16 kval_bit = (fb_w_ctrl.fb_kval & 0x80) << 8;

12
core/rend/dx11/dx11_renderer.cpp
View File

@ -1188,10 +1188,10 @@ void DX11Renderer::setBaseScissor()
void DX11Renderer::prepareRttRenderTarget(u32 texAddress)
{
u32 fbw = pvrrc.fb_X_CLIP.max + 1;
u32 fbh = pvrrc.fb_Y_CLIP.max + 1;
u32 fbw = pvrrc.getFramebufferWidth();
u32 fbh = pvrrc.getFramebufferHeight();
DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x",
FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, fbw, fbh, texAddress);
FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, fbw, fbh, texAddress);
// Find the smallest power of two texture that fits the viewport
u32 fbh2 = 2;
while (fbh2 < fbh)
@ -1221,8 +1221,8 @@ void DX11Renderer::prepareRttRenderTarget(u32 texAddress)
void DX11Renderer::readRttRenderTarget(u32 texAddress)
{
u32 w = pvrrc.fb_X_CLIP.max + 1;
u32 h = pvrrc.fb_Y_CLIP.max + 1;
u32 w = pvrrc.getFramebufferWidth();
u32 h = pvrrc.getFramebufferHeight();
const u8 fb_packmode = FB_W_CTRL.fb_packmode;
if (config::RenderToTextureBuffer)
{
@ -1268,7 +1268,7 @@ void DX11Renderer::readRttRenderTarget(u32 texAddress)
deviceContext->Unmap(stagingTex, 0);
u16 *dst = (u16 *)&vram[texAddress];
WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst);
WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8);
}
else
{

12
core/rend/dx9/d3d_renderer.cpp
View File

@ -857,10 +857,10 @@ void D3DRenderer::setBaseScissor()
void D3DRenderer::prepareRttRenderTarget(u32 texAddress)
{
u32 fbw = pvrrc.fb_X_CLIP.max + 1;
u32 fbh = pvrrc.fb_Y_CLIP.max + 1;
u32 fbw = pvrrc.getFramebufferWidth();
u32 fbh = pvrrc.getFramebufferHeight();
DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x",
FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8, fbw, fbh, texAddress);
FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8, fbw, fbh, texAddress);
// Find the smallest power of two texture that fits the viewport
u32 fbh2 = 2;
while (fbh2 < fbh)
@ -893,8 +893,8 @@ void D3DRenderer::prepareRttRenderTarget(u32 texAddress)
void D3DRenderer::readRttRenderTarget(u32 texAddress)
{
u32 w = pvrrc.fb_X_CLIP.max + 1;
u32 h = pvrrc.fb_Y_CLIP.max + 1;
u32 w = pvrrc.getFramebufferWidth();
u32 h = pvrrc.getFramebufferHeight();
const u8 fb_packmode = FB_W_CTRL.fb_packmode;
if (config::RenderToTextureBuffer)
{
@ -926,7 +926,7 @@ void D3DRenderer::readRttRenderTarget(u32 texAddress)
verifyWin(offscreenSurface->UnlockRect());
u16 *dst = (u16 *)&vram[texAddress];
WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst);
WriteTextureToVRam<2, 1, 0, 3>(w, h, (u8 *)tmp_buf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8);
}
else
{

14
core/rend/gles/gltex.cpp
View File

@ -155,10 +155,10 @@ GLuint BindRTT(bool withDepthBuffer)
WARN_LOG(RENDERER, "Invalid framebuffer format: 7");
return 0;
}
u32 fbw = pvrrc.fb_X_CLIP.max + 1;
u32 fbh = pvrrc.fb_Y_CLIP.max + 1;
u32 fbw = pvrrc.getFramebufferWidth();
u32 fbh = pvrrc.getFramebufferHeight();
u32 texAddress = FB_W_SOF1 & VRAM_MASK;
DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8,
DEBUG_LOG(RENDERER, "RTT packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8,
fbw, fbh, texAddress);
if (gl.rtt.texAddress != ~0u)
@ -266,8 +266,8 @@ GLuint BindRTT(bool withDepthBuffer)
void ReadRTTBuffer()
{
u32 w = pvrrc.fb_X_CLIP.max + 1;
u32 h = pvrrc.fb_Y_CLIP.max + 1;
u32 w = pvrrc.getFramebufferWidth();
u32 h = pvrrc.getFramebufferHeight();
const u8 fb_packmode = FB_W_CTRL.fb_packmode;
@ -295,7 +295,7 @@ void ReadRTTBuffer()
gl.rtt.height = h;
u16 *dst = gl.gl_major >= 3 ? nullptr : (u16 *)&vram[tex_addr];
gl.rtt.linestride = FB_W_LINESTRIDE.stride * 8;
gl.rtt.linestride = pvrrc.fb_W_LINESTRIDE * 8;
if (gl.rtt.linestride == 0)
gl.rtt.linestride = w * 2;
@ -324,7 +324,7 @@ void ReadRTTBuffer()
u8 *p = (u8 *)tmp_buf.data();
glReadPixels(0, 0, w, h, GL_RGBA, GL_UNSIGNED_BYTE, p);
WriteTextureToVRam(w, h, p, dst);
WriteTextureToVRam(w, h, p, dst, -1, gl.rtt.linestride);
gl.rtt.texAddress = ~0;
}
}

12
core/rend/vulkan/drawer.cpp
View File

@ -377,13 +377,13 @@ void TextureDrawer::Init(SamplerManager *samplerManager, ShaderManager *shaderMa
vk::CommandBuffer TextureDrawer::BeginRenderPass()
{
DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8,
DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8,
pvrrc.fb_X_CLIP.max + 1, pvrrc.fb_Y_CLIP.max + 1, FB_W_SOF1 & VRAM_MASK);
matrices.CalcMatrices(&pvrrc);
textureAddr = FB_W_SOF1 & VRAM_MASK;
u32 origWidth = pvrrc.fb_X_CLIP.max + 1;
u32 origHeight = pvrrc.fb_Y_CLIP.max + 1;
u32 origWidth = pvrrc.getFramebufferWidth();
u32 origHeight = pvrrc.getFramebufferHeight();
u32 heightPow2 = 8;
while (heightPow2 < origHeight)
heightPow2 *= 2;
@ -516,8 +516,8 @@ void TextureDrawer::EndRenderPass()
{
currentCommandBuffer.endRenderPass();
u32 clippedWidth = pvrrc.fb_X_CLIP.max + 1;
u32 clippedHeight = pvrrc.fb_Y_CLIP.max + 1;
u32 clippedWidth = pvrrc.getFramebufferWidth();
u32 clippedHeight = pvrrc.getFramebufferHeight();
if (config::RenderToTextureBuffer)
{
@ -552,7 +552,7 @@ void TextureDrawer::EndRenderPass()
PixelBuffer<u32> tmpBuf;
tmpBuf.init(clippedWidth, clippedHeight);
colorAttachment->GetBufferData()->download(clippedWidth * clippedHeight * 4, tmpBuf.data());
WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst);
WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8);
}
else
{

13
core/rend/vulkan/oit/oit_drawer.cpp
View File

@ -508,15 +508,16 @@ void OITScreenDrawer::MakeFramebuffers(const vk::Extent2D& viewport)
vk::CommandBuffer OITTextureDrawer::NewFrame()
{
DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, FB_W_LINESTRIDE.stride * 8,
DEBUG_LOG(RENDERER, "RenderToTexture packmode=%d stride=%d - %d x %d @ %06x", FB_W_CTRL.fb_packmode, pvrrc.fb_W_LINESTRIDE * 8,
pvrrc.fb_X_CLIP.max + 1, pvrrc.fb_Y_CLIP.max + 1, FB_W_SOF1 & VRAM_MASK);
NewImage();
matrices.CalcMatrices(&pvrrc);
textureAddr = FB_W_SOF1 & VRAM_MASK;
u32 origWidth = pvrrc.fb_X_CLIP.max + 1;
u32 origHeight = pvrrc.fb_Y_CLIP.max + 1;
u32 origWidth = pvrrc.getFramebufferWidth();
u32 origHeight = pvrrc.getFramebufferHeight();
float upscale = 1.f;
if (!config::RenderToTextureBuffer)
upscale = config::RenderResolution / 480.f;
@ -641,8 +642,8 @@ void OITTextureDrawer::EndFrame()
{
currentCommandBuffer.endRenderPass();
u32 clippedWidth = pvrrc.fb_X_CLIP.max + 1;
u32 clippedHeight = pvrrc.fb_Y_CLIP.max + 1;
u32 clippedWidth = pvrrc.getFramebufferWidth();
u32 clippedHeight = pvrrc.getFramebufferHeight();
if (config::RenderToTextureBuffer)
{
@ -679,7 +680,7 @@ void OITTextureDrawer::EndFrame()
PixelBuffer<u32> tmpBuf;
tmpBuf.init(clippedWidth, clippedHeight);
colorAttachment->GetBufferData()->download(clippedWidth * clippedHeight * 4, tmpBuf.data());
WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst);
WriteTextureToVRam(clippedWidth, clippedHeight, (u8 *)tmpBuf.data(), dst, -1, pvrrc.fb_W_LINESTRIDE * 8);
}
else
{