vk and pvr fixes. Renderer::Process returns void. N2 light 0 is no-light

pvr: Heap use after free due to PolyParam vector reallocation
vk: fix wrong buffer size calculation
Renderer::Process now returns void since it can't fail anymore
naomi2: light model at index 0 is "no light"
dx9, gles: throw if naomi 2 not supported

core/emulator.cpp

@@ -502,9 +502,6 @@ void Emulator::loadGame(const char *path, LoadProgress *progress)
 		dc_reset(true);
 		memset(&settings.network.md5, 0, sizeof(settings.network.md5));
 
-		if (settings.platform.isNaomi2() && config::RendererType == RenderType::DirectX9)
-			throw FlycastException("DirectX 9 doesn't support Naomi 2 games. Select a different graphics API");
-
 		if (settings.platform.isConsole())
 		{
 			if (settings.content.path.empty())

core/hw/pvr/Renderer_if.cpp

@@ -183,29 +183,25 @@ private:
 			retro_resize_renderer(_pvrrc->rend.framebufferWidth, _pvrrc->rend.framebufferHeight,
 					getOutputFramebufferAspectRatio());
 #endif
-		bool proc;
 		{
 			FC_PROFILE_SCOPE_NAMED("Renderer::Process");
-			proc = renderer->Process(_pvrrc);
+			renderer->Process(_pvrrc);
 		}
 
-		if (!proc || renderToScreen)
+		if (renderToScreen)
 			// If rendering to texture or in full framebuffer emulation, continue locking until the frame is rendered
 			renderEnd.Set();
 		rend_allow_rollback();
-		if (proc)
 		{
-			{
+			FC_PROFILE_SCOPE_NAMED("Renderer::Render");
-				FC_PROFILE_SCOPE_NAMED("Renderer::Render");
+			renderer->Render();
-				renderer->Render();
-			}
-
-			if (!renderToScreen)
-				renderEnd.Set();
-			else if (config::DelayFrameSwapping && fb_w_cur == FB_R_SOF1)
-				present();
 		}
 
+		if (!renderToScreen)
+			renderEnd.Set();
+		else if (config::DelayFrameSwapping && fb_w_cur == FB_R_SOF1)
+			present();
+
 		//clear up & free data ..
 		FinishRender(_pvrrc);
 		_pvrrc = nullptr;

core/hw/pvr/Renderer_if.h

@@ -56,7 +56,7 @@ struct Renderer
 	virtual bool Init() = 0;
 	virtual void Term() = 0;
 
-	virtual bool Process(TA_context *ctx) = 0;
+	virtual void Process(TA_context *ctx) = 0;
 	virtual bool Render() = 0;
 	virtual void RenderFramebuffer(const FramebufferInfo& info) = 0;
 	virtual bool RenderLastFrame() { return false; }

core/hw/pvr/ta.h

@@ -14,7 +14,7 @@ void ta_vtx_SoftReset();
 void DYNACALL ta_vtx_data32(const SQBuffer *data);
 void ta_vtx_data(const SQBuffer *data, u32 size);
 
-bool ta_parse(TA_context *ctx, bool primRestart);
+void ta_parse(TA_context *ctx, bool primRestart);
 
 class TaTypeLut
 {

core/hw/pvr/ta_vtx.cpp

@@ -154,10 +154,10 @@ protected:
 	static ModTriangle* lmr;
 
 	static u32 CurrentList;
-	static PolyParam* CurrentPP;
 	static TaListFP *VertexDataFP;
 public:
 	static std::vector<PolyParam> *CurrentPPlist;
+	static PolyParam* CurrentPP;
 	static TaListFP* TaCmd;
 	static bool fetchTextures;
 };
@@ -633,12 +633,10 @@ private:
 
 		if (CurrentPP->count > 0)
 		{
-			CurrentPPlist->emplace_back();
+			CurrentPPlist->push_back(*CurrentPP);
-			PolyParam* d_pp = &CurrentPPlist->back();
+			CurrentPP = &CurrentPPlist->back();
-			*d_pp = *CurrentPP;
+			CurrentPP->first = vd_rc.verts.size();
-			CurrentPP = d_pp;
+			CurrentPP->count = 0;
-			d_pp->first = vd_rc.verts.size();
-			d_pp->count = 0;
 		}
 	}
 	
@@ -954,7 +952,6 @@ private:
 	//Sprites
 	static void AppendSpriteParam(TA_SpriteParam* spr)
 	{
-		//printf("Sprite\n");
 		PolyParam* d_pp = CurrentPP;
 		if (CurrentPP == NULL || CurrentPP->count != 0)
 		{
@@ -1087,9 +1084,8 @@ private:
 
 		update_fz(cv[0].z);
 
-		CurrentPPlist->emplace_back();
+		CurrentPPlist->push_back(*CurrentPP);
 		PolyParam *d_pp = &CurrentPPlist->back();
-		*d_pp = *CurrentPP;
 		CurrentPP = d_pp;
 		d_pp->first = vd_rc.verts.size();
 		d_pp->count = 0;
@@ -1298,13 +1294,12 @@ static void ta_parse_naomi2(TA_context* ctx, bool primRestart)
 	ctx->rend.fb_Y_CLIP.max = std::min(ctx->rend.fb_Y_CLIP.max, ymax + 31);
 }
 
-bool ta_parse(TA_context *ctx, bool primRestart)
+void ta_parse(TA_context *ctx, bool primRestart)
 {
 	if (settings.platform.isNaomi2())
 		ta_parse_naomi2(ctx, primRestart);
 	else
 		ta_parse_vdrc(ctx, primRestart);
-	return true;
 }
 
 //
@@ -1329,6 +1324,7 @@ const float defaultProjMat[] {
 
 constexpr int IdentityMatIndex = 0;
 constexpr int DefaultProjMatIndex = 1;
+constexpr int NoLightIndex = 0;
 
 static void setDefaultMatrices()
 {
@@ -1339,6 +1335,12 @@ static void setDefaultMatrices()
 	}
 }
 
+static void setDefaultLight()
+{
+	if (ta_ctx->rend.lightModels.empty())
+		ta_ctx->rend.lightModels.emplace_back();
+}
+
 void ta_add_poly(const PolyParam& pp)
 {
 	verify(ta_ctx != nullptr);
@@ -1347,6 +1349,7 @@ void ta_add_poly(const PolyParam& pp)
 	BaseTAParser::startList(pp.pcw.ListType);
 
 	BaseTAParser::CurrentPPlist->push_back(pp);
+	BaseTAParser::CurrentPP = nullptr; // might be invalidated
 	n2CurrentPP = &BaseTAParser::CurrentPPlist->back();
 	n2CurrentPP->first = ta_ctx->rend.verts.size();
 	n2CurrentPP->count = 0;
@@ -1358,6 +1361,9 @@ void ta_add_poly(const PolyParam& pp)
 		n2CurrentPP->normalMatrix = IdentityMatIndex;
 	if (n2CurrentPP->projMatrix == -1)
 		n2CurrentPP->projMatrix = DefaultProjMatIndex;
+	setDefaultLight();
+	if (n2CurrentPP->lightModel == -1)
+		n2CurrentPP->lightModel = NoLightIndex;
 	vd_ctx = nullptr;
 }
 
@@ -1413,6 +1419,7 @@ int ta_add_matrix(const float *matrix)
 
 int ta_add_light(const N2LightModel& light)
 {
+	setDefaultLight();
 	ta_ctx->rend.lightModels.push_back(light);
 	return ta_ctx->rend.lightModels.size() - 1;
 }

core/rend/dx11/dx11_naomi2.cpp

@@ -434,15 +434,7 @@ void Naomi2Helper::setConstants(const PolyParam& pp, u32 polyNumber, const rend_
 	if (pp.lightModel != lastModel)
 	{
 		lastModel = pp.lightModel;
-		if (pp.lightModel != -1)
+		setConstBuffer(lightConstantsBuffer, ctx.lightModels[pp.lightModel]);
-		{
-			setConstBuffer(lightConstantsBuffer, ctx.lightModels[pp.lightModel]);
-		}
-		else
-		{
-			N2LightModel lightModel{};
-			setConstBuffer(lightConstantsBuffer, lightModel);
-		}
 		deviceContext->VSSetConstantBuffers(2, 1, &lightConstantsBuffer.get());
 	}
 }

core/rend/dx11/dx11_renderer.cpp

@@ -301,13 +301,13 @@ BaseTextureCacheData *DX11Renderer::GetTexture(TSP tsp, TCW tcw)
 	return tf;
 }
 
-bool DX11Renderer::Process(TA_context* ctx)
+void DX11Renderer::Process(TA_context* ctx)
 {
 	if (KillTex)
 		texCache.Clear();
 	texCache.Cleanup();
 
-	return ta_parse(ctx, true);
+	ta_parse(ctx, true);
 }
 
 void DX11Renderer::configVertexShader()

core/rend/dx11/dx11_renderer.h

@@ -36,7 +36,7 @@ struct DX11Renderer : public Renderer
 {
 	bool Init() override;
 	void Term() override;
-	bool Process(TA_context* ctx) override;
+	void Process(TA_context* ctx) override;
 	bool Render() override;
 	void RenderFramebuffer(const FramebufferInfo& info) override;
 

core/rend/dx9/d3d_renderer.cpp

@@ -308,19 +308,21 @@ void D3DRenderer::RenderFramebuffer(const FramebufferInfo& info)
 	theDXContext.setFrameRendered();
 }
 
-bool D3DRenderer::Process(TA_context* ctx)
+void D3DRenderer::Process(TA_context* ctx)
 {
 	if (!theDXContext.isReady()) {
 		// force a Present
 		frameRendered = true;
-		return false;
+		return;
 	}
+	if (settings.platform.isNaomi2())
+		throw FlycastException("DirectX 9 doesn't support Naomi 2 games. Select a different graphics API");
 
 	if (KillTex)
 		texCache.Clear();
 	texCache.Cleanup();
 
-	return ta_parse(ctx, false);
+	ta_parse(ctx, false);
 }
 
 inline void D3DRenderer::setTexMode(D3DSAMPLERSTATETYPE state, u32 clamp, u32 mirror)

core/rend/dx9/d3d_renderer.h

@@ -100,7 +100,7 @@ struct D3DRenderer : public Renderer
 {
 	bool Init() override;
 	void Term() override;
-	bool Process(TA_context* ctx) override;
+	void Process(TA_context* ctx) override;
 	bool Render() override;
 	bool RenderLastFrame() override;
 	bool Present() override

core/rend/gles/gles.cpp

@@ -1139,8 +1139,11 @@ void OpenGLRenderer::DrawOSD(bool clear_screen)
 	bindVertexArray(0);
 }
 
-bool OpenGLRenderer::Process(TA_context* ctx)
+void OpenGLRenderer::Process(TA_context* ctx)
 {
+	if (gl.gl_major < 3 && settings.platform.isNaomi2())
+		throw FlycastException("OpenGL ES 3.0+ required for Naomi 2");
+
 	if (KillTex)
 		TexCache.Clear();
 	TexCache.Cleanup();
@@ -1155,7 +1158,7 @@ bool OpenGLRenderer::Process(TA_context* ctx)
 		updatePaletteTexture(getPaletteTextureSlot());
 		palette_updated = false;
 	}
-	return ta_parse(ctx, gl.prim_restart_fixed_supported || gl.prim_restart_supported);
+	ta_parse(ctx, gl.prim_restart_fixed_supported || gl.prim_restart_supported);
 }
 
 static void upload_vertex_indices()

core/rend/gles/gles.h

@@ -408,7 +408,7 @@ struct OpenGLRenderer : Renderer
 	bool Init() override;
 	void Term() override;
 
-	bool Process(TA_context* ctx) override;
+	void Process(TA_context* ctx) override;
 
 	bool Render() override;
 

core/rend/gles/naomi2.h

@@ -127,62 +127,59 @@ void setN2Uniforms(const PolyParam *pp, ShaderType *shader, const rend_context&
 	if (pp->lightModel != shader->lastLightModel)
 	{
 		shader->lastLightModel = pp->lightModel;
-		if (pp->lightModel != -1)
+		const N2LightModel *const lightModel = &ctx.lightModels[pp->lightModel];
+		for (int vol = 0; vol < 2; vol++)
 		{
-			const N2LightModel *const lightModel = &ctx.lightModels[pp->lightModel];
+			glUniform1i(shader->ambientMaterialBase[vol], lightModel->ambientMaterialBase[vol]);
+			glUniform1i(shader->ambientMaterialOffset[vol], lightModel->ambientMaterialOffset[vol]);
+			glUniform4fv(shader->ambientBase[vol], 1, lightModel->ambientBase[vol]);
+			glUniform4fv(shader->ambientOffset[vol], 1, lightModel->ambientOffset[vol]);
+		}
+		glUniform1i(shader->useBaseOver, lightModel->useBaseOver);
+		glUniform1i(shader->bumpId0, lightModel->bumpId1);
+		glUniform1i(shader->bumpId1, lightModel->bumpId2);
+
+		glUniform1i(shader->lightCount, lightModel->lightCount);
+		for (int i = 0; i < lightModel->lightCount; i++)
+		{
+			const N2Light& light = lightModel->lights[i];
+			glUniform1i(shader->lights[i].parallel, light.parallel);
+
+			glUniform4fv(shader->lights[i].color, 1, light.color);
+			glUniform4fv(shader->lights[i].direction, 1, light.direction);
+			glUniform4fv(shader->lights[i].position, 1, light.position);
+
 			for (int vol = 0; vol < 2; vol++)
 			{
-				glUniform1i(shader->ambientMaterialBase[vol], lightModel->ambientMaterialBase[vol]);
+				glUniform1i(shader->lights[i].diffuse[vol], light.diffuse[vol]);
-				glUniform1i(shader->ambientMaterialOffset[vol], lightModel->ambientMaterialOffset[vol]);
+				glUniform1i(shader->lights[i].specular[vol], light.specular[vol]);
-				glUniform4fv(shader->ambientBase[vol], 1, lightModel->ambientBase[vol]);
-				glUniform4fv(shader->ambientOffset[vol], 1, lightModel->ambientOffset[vol]);
 			}
-			glUniform1i(shader->useBaseOver, lightModel->useBaseOver);
+			glUniform1i(shader->lights[i].routing, light.routing);
-			glUniform1i(shader->bumpId0, lightModel->bumpId1);
+			glUniform1i(shader->lights[i].dmode, light.dmode);
-			glUniform1i(shader->bumpId1, lightModel->bumpId2);
+			glUniform1i(shader->lights[i].smode, light.smode);
+			glUniform1i(shader->lights[i].distAttnMode, light.distAttnMode);
 
-			glUniform1i(shader->lightCount, lightModel->lightCount);
+			glUniform1f(shader->lights[i].attnDistA, light.attnDistA);
-			for (int i = 0; i < lightModel->lightCount; i++)
+			glUniform1f(shader->lights[i].attnDistB, light.attnDistB);
-			{
+			glUniform1f(shader->lights[i].attnAngleA, light.attnAngleA);
-				const N2Light& light = lightModel->lights[i];
+			glUniform1f(shader->lights[i].attnAngleB, light.attnAngleB);
-				glUniform1i(shader->lights[i].parallel, light.parallel);
-
-				glUniform4fv(shader->lights[i].color, 1, light.color);
-				glUniform4fv(shader->lights[i].direction, 1, light.direction);
-				glUniform4fv(shader->lights[i].position, 1, light.position);
-
-				for (int vol = 0; vol < 2; vol++)
-				{
-					glUniform1i(shader->lights[i].diffuse[vol], light.diffuse[vol]);
-					glUniform1i(shader->lights[i].specular[vol], light.specular[vol]);
-				}
-				glUniform1i(shader->lights[i].routing, light.routing);
-				glUniform1i(shader->lights[i].dmode, light.dmode);
-				glUniform1i(shader->lights[i].smode, light.smode);
-				glUniform1i(shader->lights[i].distAttnMode, light.distAttnMode);
-
-				glUniform1f(shader->lights[i].attnDistA, light.attnDistA);
-				glUniform1f(shader->lights[i].attnDistB, light.attnDistB);
-				glUniform1f(shader->lights[i].attnAngleA, light.attnAngleA);
-				glUniform1f(shader->lights[i].attnAngleB, light.attnAngleB);
-			}
 		}
-		else
+	}
+	else
+	{
+		float white[] { 1.f, 1.f, 1.f, 1.f };
+		float black[4]{};
+		for (int vol = 0; vol < 2; vol++)
 		{
-			float white[] { 1.f, 1.f, 1.f, 1.f };
+			glUniform1i(shader->ambientMaterialBase[vol], 0);
-			float black[4]{};
+			glUniform1i(shader->ambientMaterialOffset[vol], 0);
-			for (int vol = 0; vol < 2; vol++)
+			glUniform4fv(shader->ambientBase[vol], 1, white);
-			{
+			glUniform4fv(shader->ambientOffset[vol], 1, black);
-				glUniform1i(shader->ambientMaterialBase[vol], 0);
-				glUniform1i(shader->ambientMaterialOffset[vol], 0);
-				glUniform4fv(shader->ambientBase[vol], 1, white);
-				glUniform4fv(shader->ambientOffset[vol], 1, black);
-			}
-			glUniform1i(shader->useBaseOver, 0);
-			glUniform1i(shader->lightCount, 0);
-			glUniform1i(shader->bumpId0, -1);
-			glUniform1i(shader->bumpId1, -1);
 		}
+		glUniform1i(shader->useBaseOver, 0);
+		glUniform1i(shader->lightCount, 0);
+		glUniform1i(shader->bumpId0, -1);
+		glUniform1i(shader->bumpId1, -1);
 	}
 	glUniform1i(shader->bumpMapping, pp->pcw.Texture == 1 && pp->tcw.PixelFmt == PixelBumpMap);
 }

core/rend/norend/norend.cpp

@@ -8,8 +8,8 @@ struct norend : Renderer
 	}
 	void Term() override { }
 
-	bool Process(TA_context* ctx) override {
+	void Process(TA_context* ctx) override {
-		return ta_parse(ctx, true);
+		ta_parse(ctx, true);
 	}
 
 	bool Render() override {

core/rend/vulkan/drawer.cpp

@@ -322,11 +322,11 @@ void Drawer::UploadMainBuffer(const VertexShaderUniforms& vertexUniforms, const
 	BufferPacker packer;
 
 	// Vertex
-	packer.add(&pvrrc.verts[0], pvrrc.verts.size() + sizeof(decltype(pvrrc.verts[0])));
+	packer.add(&pvrrc.verts[0], pvrrc.verts.size() * sizeof(decltype(pvrrc.verts[0])));
 	// Modifier Volumes
-	offsets.modVolOffset = packer.add(&pvrrc.modtrig[0], pvrrc.modtrig.size() + sizeof(decltype(pvrrc.modtrig[0])));
+	offsets.modVolOffset = packer.add(&pvrrc.modtrig[0], pvrrc.modtrig.size() * sizeof(decltype(pvrrc.modtrig[0])));
 	// Index
-	offsets.indexOffset = packer.add(&pvrrc.idx[0], pvrrc.idx.size() + sizeof(decltype(pvrrc.idx[0])));
+	offsets.indexOffset = packer.add(&pvrrc.idx[0], pvrrc.idx.size() * sizeof(decltype(pvrrc.idx[0])));
 	// Uniform buffers
 	offsets.vertexUniformOffset = packer.addUniform(&vertexUniforms, sizeof(vertexUniforms));
 	offsets.fragmentUniformOffset = packer.addUniform(&fragmentUniforms, sizeof(fragmentUniforms));

core/rend/vulkan/drawer.h

@@ -144,18 +144,21 @@ protected:
 
 	vk::DeviceSize packNaomi2Lights(BufferPacker& packer)
 	{
-		constexpr static N2LightModel noLight{};
+		vk::DeviceSize offset = -1;
-		vk::DeviceSize offset = packer.addUniform(&noLight, sizeof(noLight));
 
 		size_t n2LightSize = sizeof(N2LightModel) + align(sizeof(N2LightModel), GetContext()->GetUniformBufferAlignment());
 		if (n2LightSize == sizeof(N2LightModel))
 		{
-			packer.addUniform(&pvrrc.lightModels[0], pvrrc.lightModels.size() * sizeof(decltype(pvrrc.lightModels[0])));
+			offset = packer.addUniform(&pvrrc.lightModels[0], pvrrc.lightModels.size() * sizeof(decltype(pvrrc.lightModels[0])));
 		}
 		else
 		{
 			for (const N2LightModel& model : pvrrc.lightModels)
-				packer.addUniform(&model, sizeof(N2LightModel));
+			{
+				vk::DeviceSize ioffset = packer.addUniform(&model, sizeof(N2LightModel));
+				if (offset == (vk::DeviceSize)-1)
+					offset = ioffset;
+			}
 		}
 
 		return offset;

core/rend/vulkan/oit/oit_pipeline.h

@@ -187,11 +187,7 @@ public:
 				writeDescriptorSets.emplace_back(perPolyDescSet, 2, 0, vk::DescriptorType::eUniformBuffer, nullptr, uniBufferInfo);
 
 				size = sizeof(N2LightModel) + align(sizeof(N2LightModel), uniformAlignment);
-				// light at index 0 is no light
+				lightBufferInfo = vk::DescriptorBufferInfo{ buffer, lightOffset + poly.lightModel * size, sizeof(N2LightModel) };
-				if (poly.lightModel != -1)
-					lightBufferInfo = vk::DescriptorBufferInfo{ buffer, lightOffset + (poly.lightModel + 1) * size, sizeof(N2LightModel) };
-				else
-					lightBufferInfo = vk::DescriptorBufferInfo{ buffer, lightOffset, sizeof(N2LightModel) };
 				writeDescriptorSets.emplace_back(perPolyDescSet, 3, 0, vk::DescriptorType::eUniformBuffer, nullptr, lightBufferInfo);
 			}
 

core/rend/vulkan/pipeline.h

@@ -104,11 +104,7 @@ public:
 				writeDescriptorSets.emplace_back(perPolyDescSet, 2, 0, vk::DescriptorType::eUniformBuffer, nullptr, uniBufferInfo);
 
 				size = sizeof(N2LightModel) + align(sizeof(N2LightModel), uniformAlignment);
-				// light at index 0 is no light
+				lightBufferInfo = vk::DescriptorBufferInfo{ buffer, lightOffset + poly.lightModel * size, sizeof(N2LightModel) };
-				if (poly.lightModel != -1)
-					lightBufferInfo = vk::DescriptorBufferInfo{ buffer, lightOffset + (poly.lightModel + 1) * size, sizeof(N2LightModel) };
-				else
-					lightBufferInfo = vk::DescriptorBufferInfo{ buffer, lightOffset, sizeof(N2LightModel) };
 				writeDescriptorSets.emplace_back(perPolyDescSet, 3, 0, vk::DescriptorType::eUniformBuffer, nullptr, lightBufferInfo);
 			}
 

core/rend/vulkan/vulkan_renderer.h

@@ -122,7 +122,7 @@ public:
 		return tf;
 	}
 
-	bool Process(TA_context* ctx) override
+	void Process(TA_context* ctx) override
 	{
 		if (KillTex)
 			textureCache.Clear();
@@ -134,27 +134,17 @@ public:
 		texCommandBuffer = texCommandPool.Allocate();
 		texCommandBuffer.begin(vk::CommandBufferBeginInfo(vk::CommandBufferUsageFlagBits::eOneTimeSubmit));
 
-		bool result = ta_parse(ctx, true);
+		ta_parse(ctx, true);
 
-		if (result)
-		{
 #ifdef LIBRETRO
-			if (!ctx->rend.isRTT)
+		if (!ctx->rend.isRTT)
-				overlay->Prepare(texCommandBuffer, true, true, textureCache);
+			overlay->Prepare(texCommandBuffer, true, true, textureCache);
 #endif
-			CheckFogTexture();
+		CheckFogTexture();
-			CheckPaletteTexture();
+		CheckPaletteTexture();
-			texCommandBuffer.end();
+		texCommandBuffer.end();
-			if (!ctx->rend.isRTT)
+		if (!ctx->rend.isRTT)
-				framebufferRendered = false;
+			framebufferRendered = false;
-		}
-		else
-		{
-			texCommandBuffer.end();
-			texCommandPool.EndFrame();
-		}
-
-		return result;
 	}
 
 	void ReInitOSD()