From 4c14a96888e7c4256e7b60d9dae1562f936b462f Mon Sep 17 00:00:00 2001 From: flyinghead Date: Sun, 12 Dec 2021 18:42:50 +0100 Subject: [PATCH] vk, gl4: out of bound array index when sorting pixels --- core/rend/gl4/abuffer.cpp | 11 ++++++++--- core/rend/vulkan/oit/oit_shaders.cpp | 9 +++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/core/rend/gl4/abuffer.cpp b/core/rend/gl4/abuffer.cpp index 8fafa4f6f..d8b15265c 100644 --- a/core/rend/gl4/abuffer.cpp +++ b/core/rend/gl4/abuffer.cpp @@ -42,9 +42,12 @@ uint pixel_list[MAX_PIXELS_PER_FRAGMENT]; int fillAndSortFragmentArray(ivec2 coords) { - // Load fragments into a local memory array for sorting uint idx = imageLoad(abufferPointerImg, coords).x; - int count = 0; + if (idx == EOL) + return 0; + int count = 1; + pixel_list[0] = idx; + idx = pixels[idx].next; for (; idx != EOL && count < MAX_PIXELS_PER_FRAGMENT; count++) { const Pixel p = pixels[idx]; @@ -56,11 +59,13 @@ int fillAndSortFragmentArray(ivec2 coords) { pixel_list[j + 1] = pixel_list[j]; j--; - jp = pixels[pixel_list[j]]; + if (j >= 0) + jp = pixels[pixel_list[j]]; } pixel_list[j + 1] = idx; idx = p.next; } + return count; } diff --git a/core/rend/vulkan/oit/oit_shaders.cpp b/core/rend/vulkan/oit/oit_shaders.cpp index 825b127d3..cd7b445a9 100644 --- a/core/rend/vulkan/oit/oit_shaders.cpp +++ b/core/rend/vulkan/oit/oit_shaders.cpp @@ -452,7 +452,11 @@ int fillAndSortFragmentArray(ivec2 coords) { // Load fragments into a local memory array for sorting uint idx = abufferPointer.pointers[coords.x + coords.y * uniformBuffer.viewportWidth]; - int count = 0; + if (idx == EOL) + return 0; + int count = 1; + pixel_list[0] = idx; + idx = PixelBuffer.pixels[idx].next; for (; idx != EOL && count < MAX_PIXELS_PER_FRAGMENT; count++) { const Pixel p = PixelBuffer.pixels[idx]; @@ -464,7 +468,8 @@ int fillAndSortFragmentArray(ivec2 coords) { pixel_list[j + 1] = pixel_list[j]; j--; - jp = PixelBuffer.pixels[pixel_list[j]]; + if (j >= 0) + jp = PixelBuffer.pixels[pixel_list[j]]; } pixel_list[j + 1] = idx; idx = p.next;