rsx: Improve surface options invalidation

rpcs3/Emu/RSX/RSXThread.cpp

@@ -1037,9 +1037,9 @@ namespace rsx
 			if ((!mask || !active_write_op) && rsx::method_registers.two_sided_stencil_test_enabled())
 			{
 				mask |= rsx::method_registers.back_stencil_mask();
-				active_write_op |= (rsx::method_registers.stencil_op_zpass() != rsx::stencil_op::keep ||
+				active_write_op |= (rsx::method_registers.back_stencil_op_zpass() != rsx::stencil_op::keep ||
-					rsx::method_registers.stencil_op_fail() != rsx::stencil_op::keep ||
+					rsx::method_registers.back_stencil_op_fail() != rsx::stencil_op::keep ||
-					rsx::method_registers.stencil_op_zfail() != rsx::stencil_op::keep);
+					rsx::method_registers.back_stencil_op_zfail() != rsx::stencil_op::keep);
 			}
 
 			layout.zeta_write_enabled = (mask && active_write_op);
@@ -1300,6 +1300,136 @@ namespace rsx
 		layout.ignore_change = false;
 	}
 
+	void thread::on_framebuffer_options_changed(u32 opt)
+	{
+		auto evaluate_depth_buffer_state = [&]()
+		{
+			m_framebuffer_layout.zeta_write_enabled =
+				(rsx::method_registers.depth_test_enabled() && rsx::method_registers.depth_write_enabled());
+		};
+
+		auto evaluate_stencil_buffer_state = [&]()
+		{
+			if (!m_framebuffer_layout.zeta_write_enabled &&
+				rsx::method_registers.stencil_test_enabled() &&
+				m_framebuffer_layout.depth_format == rsx::surface_depth_format::z24s8)
+			{
+				// Check if stencil data is modified
+				auto mask = rsx::method_registers.stencil_mask();
+				bool active_write_op = (rsx::method_registers.stencil_op_zpass() != rsx::stencil_op::keep ||
+					rsx::method_registers.stencil_op_fail() != rsx::stencil_op::keep ||
+					rsx::method_registers.stencil_op_zfail() != rsx::stencil_op::keep);
+
+				if ((!mask || !active_write_op) && rsx::method_registers.two_sided_stencil_test_enabled())
+				{
+					mask |= rsx::method_registers.back_stencil_mask();
+					active_write_op |= (rsx::method_registers.back_stencil_op_zpass() != rsx::stencil_op::keep ||
+						rsx::method_registers.back_stencil_op_fail() != rsx::stencil_op::keep ||
+						rsx::method_registers.back_stencil_op_zfail() != rsx::stencil_op::keep);
+				}
+
+				m_framebuffer_layout.zeta_write_enabled = (mask && active_write_op);
+			}
+		};
+
+		auto evaluate_color_buffer_state = [&]() -> bool
+		{
+			const auto mrt_buffers = rsx::utility::get_rtt_indexes(m_framebuffer_layout.target);
+			bool any_found = false;
+
+			for (uint i = 0; i < mrt_buffers.size(); ++i)
+			{
+				if (rsx::method_registers.color_write_enabled(i))
+				{
+					const auto real_index = mrt_buffers[i];
+					m_framebuffer_layout.color_write_enabled[real_index] = true;
+					any_found = true;
+				}
+			}
+
+			return any_found;
+		};
+
+		if (m_rtts_dirty)
+		{
+			// Nothing to do
+			return;
+		}
+
+		switch (opt)
+		{
+		case NV4097_SET_DEPTH_TEST_ENABLE:
+		case NV4097_SET_DEPTH_MASK:
+		{
+			auto old_state = m_framebuffer_layout.zeta_write_enabled;
+			evaluate_depth_buffer_state();
+
+			if (m_framebuffer_state_contested &&
+				!old_state && m_framebuffer_layout.zeta_write_enabled)
+			{
+				// Z buffer needs to be recreated
+				m_rtts_dirty = true;
+			}
+			break;
+		}
+		case NV4097_SET_TWO_SIDED_STENCIL_TEST_ENABLE:
+		case NV4097_SET_STENCIL_TEST_ENABLE:
+		case NV4097_SET_STENCIL_MASK:
+		case NV4097_SET_STENCIL_OP_ZPASS:
+		case NV4097_SET_STENCIL_OP_FAIL:
+		case NV4097_SET_STENCIL_OP_ZFAIL:
+		case NV4097_SET_BACK_STENCIL_MASK:
+		case NV4097_SET_BACK_STENCIL_OP_ZPASS:
+		case NV4097_SET_BACK_STENCIL_OP_FAIL:
+		case NV4097_SET_BACK_STENCIL_OP_ZFAIL:
+		{
+			// Stencil takes a back seat to depth buffer stuff
+			bool old_state = m_framebuffer_layout.zeta_write_enabled;
+			evaluate_depth_buffer_state();
+
+			if (!m_framebuffer_layout.zeta_write_enabled)
+			{
+				evaluate_stencil_buffer_state();
+			}
+
+			if (m_framebuffer_state_contested &&
+				!old_state && m_framebuffer_layout.zeta_write_enabled)
+			{
+				// Z|S buffer needs to be recreated
+				m_rtts_dirty = true;
+			}
+			break;
+		}
+		case NV4097_SET_COLOR_MASK:
+		case NV4097_SET_COLOR_MASK_MRT:
+		{
+			if (!m_framebuffer_state_contested) [[likely]]
+			{
+				// Update write masks and continue
+				evaluate_color_buffer_state();
+			}
+			else
+			{
+				bool old_state = false;
+				for (const auto& enabled : m_framebuffer_layout.color_write_enabled)
+				{
+					if (old_state = enabled) break;
+				}
+
+				const auto new_state = evaluate_color_buffer_state();
+				if (!old_state && new_state)
+				{
+					// Color buffers now in use
+					m_rtts_dirty = true;
+				}
+			}
+			break;
+		}
+		default:
+			rsx_log.fatal("Unhandled framebuffer option changed 0x%x", opt);
+		}
+	}
+
 	bool thread::get_scissor(areau& region, bool clip_viewport)
 	{
 		if (!(m_graphics_state & rsx::pipeline_state::scissor_config_state_dirty))

rpcs3/Emu/RSX/RSXThread.h

@@ -617,7 +617,7 @@ namespace rsx
 		// Framebuffer setup
 		rsx::gcm_framebuffer_info m_surface_info[rsx::limits::color_buffers_count];
 		rsx::gcm_framebuffer_info m_depth_surface_info;
-		framebuffer_layout m_framebuffer_layout;
+		framebuffer_layout m_framebuffer_layout{};
 		bool framebuffer_status_valid = false;
 
 		// Overlays
@@ -692,7 +692,7 @@ namespace rsx
 		u32 main_mem_size{0};
 		u32 local_mem_size{0};
 
-		bool m_rtts_dirty;
+		bool m_rtts_dirty = true;
 		std::array<bool, 16> m_textures_dirty;
 		std::array<bool, 4> m_vertex_textures_dirty;
 		bool m_framebuffer_state_contested = false;
@@ -727,8 +727,10 @@ namespace rsx
 		 * returns whether surface is a render target and surface pitch in native format
 		 */
 		void get_current_fragment_program(const std::array<std::unique_ptr<rsx::sampled_image_descriptor_base>, rsx::limits::fragment_textures_count>& sampler_descriptors);
+
 	public:
 		bool invalidate_fragment_program(u32 dst_dma, u32 dst_offset, u32 size);
+		void on_framebuffer_options_changed(u32 opt);
 
 	public:
 		u64 target_rsx_flip_time = 0;

rpcs3/Emu/RSX/rsx_methods.cpp

@@ -720,10 +720,12 @@ namespace rsx
 			set_surface_dirty_bit(rsx, reg, arg);
 		}
 
-		void set_surface_options_dirty_bit(thread* rsx, u32, u32)
+		void set_surface_options_dirty_bit(thread* rsx, u32 reg, u32)
 		{
-			if (rsx->m_framebuffer_state_contested)
+			if (reg != method_registers.register_previous_value)
-				rsx->m_rtts_dirty = true;
+			{
+				rsx->on_framebuffer_options_changed(reg);
+			}
 		}
 
 		template <u32 RsxFlags>
@@ -3092,6 +3094,16 @@ namespace rsx
 		bind<NV4097_SET_DEPTH_MASK, nv4097::set_surface_options_dirty_bit>();
 		bind<NV4097_SET_COLOR_MASK, nv4097::set_surface_options_dirty_bit>();
 		bind<NV4097_SET_COLOR_MASK_MRT, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_TWO_SIDED_STENCIL_TEST_ENABLE, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_STENCIL_TEST_ENABLE, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_STENCIL_MASK, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_STENCIL_OP_ZPASS, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_STENCIL_OP_FAIL, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_STENCIL_OP_ZFAIL, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_BACK_STENCIL_MASK, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_BACK_STENCIL_OP_ZPASS, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_BACK_STENCIL_OP_FAIL, nv4097::set_surface_options_dirty_bit>();
+		bind<NV4097_SET_BACK_STENCIL_OP_ZFAIL, nv4097::set_surface_options_dirty_bit>();
 		bind<NV4097_WAIT_FOR_IDLE, nv4097::sync>();
 		bind<NV4097_INVALIDATE_L2, nv4097::set_shader_program_dirty>();
 		bind<NV4097_SET_SHADER_PROGRAM, nv4097::set_shader_program_dirty>();