From 04172f47dea8b7f1f6fadcb58dc8c9a43b63a5dd Mon Sep 17 00:00:00 2001 From: StapleButter Date: Fri, 14 Jul 2017 00:12:35 +0200 Subject: [PATCH] fix potential overflow in fog density calculation --- src/GPU3D_Soft.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/GPU3D_Soft.cpp b/src/GPU3D_Soft.cpp index 9e0ccdb8..56660c13 100644 --- a/src/GPU3D_Soft.cpp +++ b/src/GPU3D_Soft.cpp @@ -1545,15 +1545,19 @@ void ScanlineFinalPass(s32 y) } else { - z = (z - fogoffset) << fogshift; - densityid = z >> 19; + // technically: Z difference is shifted left by fog shift + // then bit 0-18 are the fractional part and bit 19-23 are the density index + // things are a little different here to avoid overflow in 32-bit range + + z -= fogoffset; + densityid = z >> (19-fogshift); if (densityid >= 32) { densityid = 32; densityfrac = 0; } else - densityfrac = z & 0x7FFFF; + densityfrac = (z << fogshift) & 0x7FFFF; } // checkme