From ec626d723fbb4716a697ad3cac0b0b29943dba00 Mon Sep 17 00:00:00 2001 From: Vicki Pfau Date: Wed, 29 Jul 2020 18:03:04 -0700 Subject: [PATCH] DS GX: Fix overflow causing a division crash --- CHANGES | 1 + src/ds/gx/software.c | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index f8f01aaec..d337c1993 100644 --- a/CHANGES +++ b/CHANGES @@ -22,6 +22,7 @@ Bugfixes: - DS: Properly close save file on close - DS Video: Fix size of VRAM zeroes buffer - DS GX: Fix depth test scaling + - DS GX: Fix overflow causing a division crash Misc: - DS GX: Clean up and unify texture mapping - DS Core: Add symbol loading diff --git a/src/ds/gx/software.c b/src/ds/gx/software.c index 87af8bec0..acb71fdd5 100644 --- a/src/ds/gx/software.c +++ b/src/ds/gx/software.c @@ -500,7 +500,7 @@ static void _preparePoly(struct DSGXRenderer* renderer, struct DSGXVertex* verts struct DSGXVertex* v1; int32_t v0w = v0->viewCoord[3]; - if (!v0w) { + if (!v0w || v0w == INT32_MIN) { v0w = 1; } @@ -518,7 +518,7 @@ static void _preparePoly(struct DSGXRenderer* renderer, struct DSGXVertex* verts v1 = &verts[poly->poly->vertIds[v]]; int32_t v1w = v1->viewCoord[3]; - if (!v1w) { + if (!v1w || v1w == INT32_MIN) { v1w = 1; } int32_t v1x = (v1->viewCoord[0] + v1w) * (int64_t) (v1->viewportWidth << 12) / (v1w * 2) + (v1->viewportX << 12); @@ -577,7 +577,7 @@ static void _preparePoly(struct DSGXRenderer* renderer, struct DSGXVertex* verts v1 = &verts[poly->poly->vertIds[0]]; int32_t v1w = v1->viewCoord[3]; - if (!v1w) { + if (!v1w || v1w == INT32_MIN) { v1w = 1; } int32_t v1x = (v1->viewCoord[0] + v1w) * (int64_t) (v1->viewportWidth << 12) / (v1w * 2) + (v1->viewportX << 12);