rsx: Rewrite async decompiler

rpcs3/Emu/RSX/Common/ProgramStateCache.h

@@ -111,7 +111,6 @@ class program_state_cache
 	using binary_to_vertex_program = std::unordered_map<RSXVertexProgram, vertex_program_type, program_hash_util::vertex_program_storage_hash, program_hash_util::vertex_program_compare> ;
 	using binary_to_fragment_program = std::unordered_map<RSXFragmentProgram, fragment_program_type, program_hash_util::fragment_program_storage_hash, program_hash_util::fragment_program_compare>;
 
-
 	struct pipeline_key
 	{
 		u32 vertex_program_id;
@@ -139,41 +138,26 @@ class program_state_cache
 		}
 	};
 
-public:
+	struct async_decompiler_job
-	struct async_link_task_entry
 	{
-		const vertex_program_type& vp;
+		RSXVertexProgram vertex_program;
-		const fragment_program_type& fp;
+		RSXFragmentProgram fragment_program;
-		pipeline_properties props;
+		pipeline_properties properties;
 
-		async_link_task_entry(const vertex_program_type& _V, const fragment_program_type& _F, pipeline_properties _P)
+		std::vector<u8> local_storage;
-			: vp(_V), fp(_F), props(std::move(_P))
-		{}
-	};
 
-	struct async_decompile_task_entry
+		async_decompiler_job(RSXVertexProgram v, const RSXFragmentProgram f, pipeline_properties p) :
+			vertex_program(std::move(v)), fragment_program(f), properties(std::move(p))
 		{
-		RSXVertexProgram vp;
+			local_storage.resize(fragment_program.ucode_length);
-		RSXFragmentProgram fp;
+			std::memcpy(local_storage.data(), fragment_program.addr, fragment_program.ucode_length);
-		bool is_fp;
+			fragment_program.addr = local_storage.data();
-
-		std::vector<u8> tmp_cache;
-
-		async_decompile_task_entry(RSXVertexProgram _V)
-			: vp(std::move(_V)), is_fp(false)
-		{
-		}
-
-		async_decompile_task_entry(const RSXFragmentProgram& _F)
-			: fp(_F), is_fp(true)
-		{
-			tmp_cache.resize(fp.ucode_length);
-			std::memcpy(tmp_cache.data(), fp.addr, fp.ucode_length);
-			fp.addr = tmp_cache.data();
 		}
 	};
 
 protected:
+	using decompiler_callback_t = std::function<void(const pipeline_properties&, const RSXVertexProgram&, const RSXFragmentProgram&)>;
+
 	shared_mutex m_vertex_mutex;
 	shared_mutex m_fragment_mutex;
 	shared_mutex m_pipeline_mutex;
@@ -181,14 +165,14 @@ protected:
 
 	atomic_t<size_t> m_next_id = 0;
 	bool m_cache_miss_flag; // Set if last lookup did not find any usable cached programs
-	bool m_program_compiled_flag; // Set if last lookup caused program to be linked
 
 	binary_to_vertex_program m_vertex_shader_cache;
 	binary_to_fragment_program m_fragment_shader_cache;
-	std::unordered_map <pipeline_key, pipeline_storage_type, pipeline_key_hash, pipeline_key_compare> m_storage;
+	std::unordered_map<pipeline_key, pipeline_storage_type, pipeline_key_hash, pipeline_key_compare> m_storage;
 
-	std::unordered_map <pipeline_key, std::unique_ptr<async_link_task_entry>, pipeline_key_hash, pipeline_key_compare> m_link_queue;
+	std::deque<async_decompiler_job> m_decompile_queue;
-	std::deque<async_decompile_task_entry> m_decompile_queue;
+	std::unordered_map<pipeline_key, bool, pipeline_key_hash, pipeline_key_compare> m_decompiler_map;
+	decompiler_callback_t notify_pipeline_compiled;
 
 	vertex_program_type __null_vertex_program;
 	fragment_program_type __null_fragment_program;
@@ -349,12 +333,7 @@ public:
 public:
 	program_state_cache() = default;
 	~program_state_cache()
-	{
+	{}
-		for (auto& pair : m_fragment_shader_cache)
-		{
-			free(pair.first.addr);
-		}
-	}
 
 	// Returns 2 booleans.
 	// First flag hints that there is more work to do (busy hint)
@@ -366,31 +345,58 @@ public:
 		// NOTE: Linking is much slower than decompilation step, so always decompile at least 1 unit
 		// TODO: Use try_lock instead
 		bool busy = false;
+		bool sync = false;
 		u32 count = 0;
-		std::unique_ptr<async_decompile_task_entry> decompile_task;
 
 		while (true)
 		{
 			{
-				std::lock_guard lock(m_decompiler_mutex);
+				reader_lock lock(m_decompiler_mutex);
 				if (m_decompile_queue.empty())
 				{
 					break;
 				}
-				else
+			}
+
+			// Decompile
+			const auto& vp_search = search_vertex_program(m_decompile_queue.front().vertex_program, true);
+			const auto& fp_search = search_fragment_program(m_decompile_queue.front().fragment_program, true);
+
+			const bool already_existing_fragment_program = std::get<1>(fp_search);
+			const bool already_existing_vertex_program = std::get<1>(vp_search);
+			const vertex_program_type& vertex_program = std::get<0>(vp_search);
+			const fragment_program_type& fragment_program = std::get<0>(fp_search);
+			const pipeline_key key = { vertex_program.id, fragment_program.id, m_decompile_queue.front().properties };
+
+			// Retest
+			bool found = false;
+			if (already_existing_vertex_program && already_existing_fragment_program)
 			{
-					decompile_task = std::make_unique<async_decompile_task_entry>(std::move(m_decompile_queue.front()));
+				if (auto I = m_storage.find(key); I != m_storage.end())
-					m_decompile_queue.pop_front();
+				{
+					found = true;
 				}
 			}
 
-			if (decompile_task->is_fp)
+			if (!found)
 			{
-				search_fragment_program(decompile_task->fp);
+				pipeline_storage_type pipeline = backend_traits::build_pipeline(vertex_program, fragment_program, m_decompile_queue.front().properties, std::forward<Args>(args)...);
+				rsx_log.success("New program compiled successfully");
+				sync = true;
+
+				if (notify_pipeline_compiled)
+				{
+					notify_pipeline_compiled(m_decompile_queue.front().properties, m_decompile_queue.front().vertex_program, m_decompile_queue.front().fragment_program);
 				}
-			else
+
+				std::scoped_lock lock(m_pipeline_mutex);
+				m_storage[key] = std::move(pipeline);
+			}
+
 			{
-				search_vertex_program(decompile_task->vp);
+				std::scoped_lock lock(m_decompiler_mutex);
+				m_decompile_queue.pop_front();
+				m_decompiler_map.erase(key);
 			}
 
 			if (++count >= max_decompile_count)
@@ -402,30 +408,7 @@ public:
 			}
 		}
 
-		async_link_task_entry* link_entry;
+		return { busy, sync };
-		pipeline_key key;
-		{
-			reader_lock lock(m_pipeline_mutex);
-			if (!m_link_queue.empty())
-			{
-				auto It = m_link_queue.begin();
-				link_entry = It->second.get();
-				key = It->first;
-			}
-			else
-			{
-				return { busy, false };
-			}
-		}
-
-		pipeline_storage_type pipeline = backend_traits::build_pipeline(link_entry->vp, link_entry->fp, link_entry->props, std::forward<Args>(args)...);
-		rsx_log.success("New program compiled successfully");
-
-		std::lock_guard lock(m_pipeline_mutex);
-		m_storage[key] = std::move(pipeline);
-		m_link_queue.erase(key);
-
-		return { (busy || !m_link_queue.empty()), true };
 	}
 
 	template<typename... Args>
@@ -434,6 +417,7 @@ public:
 		const RSXFragmentProgram& fragmentShader,
 		pipeline_properties& pipelineProperties,
 		bool allow_async,
+		bool allow_notification,
 		Args&& ...args
 		)
 	{
@@ -442,100 +426,66 @@ public:
 
 		const bool already_existing_fragment_program = std::get<1>(fp_search);
 		const bool already_existing_vertex_program = std::get<1>(vp_search);
+		const vertex_program_type& vertex_program = std::get<0>(vp_search);
+		const fragment_program_type& fragment_program = std::get<0>(fp_search);
+		const pipeline_key key = { vertex_program.id, fragment_program.id, pipelineProperties };
 
-		bool link_only = false;
 		m_cache_miss_flag = true;
-		m_program_compiled_flag = false;
 
 		if (!allow_async || (already_existing_vertex_program && already_existing_fragment_program))
 		{
-			const vertex_program_type &vertex_program = std::get<0>(vp_search);
-			const fragment_program_type &fragment_program = std::get<0>(fp_search);
-
 			backend_traits::validate_pipeline_properties(vertex_program, fragment_program, pipelineProperties);
-			pipeline_key key = { vertex_program.id, fragment_program.id, pipelineProperties };
 
 			{
 				reader_lock lock(m_pipeline_mutex);
-				const auto I = m_storage.find(key);
+				if (const auto I = m_storage.find(key); I != m_storage.end())
-				if (I != m_storage.end())
 				{
 					m_cache_miss_flag = false;
 					return I->second;
 				}
 			}
 
-			if (allow_async)
+			if (!allow_async)
-			{
-				// Programs already exist, only linking required
-				link_only = true;
-			}
-			else
 			{
 				rsx_log.notice("Add program (vp id = %d, fp id = %d)", vertex_program.id, fragment_program.id);
-				m_program_compiled_flag = true;
-
 				pipeline_storage_type pipeline = backend_traits::build_pipeline(vertex_program, fragment_program, pipelineProperties, std::forward<Args>(args)...);
+
+				if (allow_notification && notify_pipeline_compiled)
+				{
+					notify_pipeline_compiled(pipelineProperties, vertexShader, fragmentShader);
+					rsx_log.success("New program compiled successfully");
+				}
+
 				std::lock_guard lock(m_pipeline_mutex);
 				auto &rtn = m_storage[key] = std::move(pipeline);
-				rsx_log.success("New program compiled successfully");
 				return rtn;
 			}
 		}
 
 		verify(HERE), allow_async;
 
-		if (link_only)
+		std::scoped_lock lock(m_decompiler_mutex, m_pipeline_mutex);
+
+		// Rechecks
+		if (already_existing_vertex_program && already_existing_fragment_program)
 		{
-			const vertex_program_type &vertex_program = std::get<0>(vp_search);
+			if (const auto I = m_storage.find(key); I != m_storage.end())
-			const fragment_program_type &fragment_program = std::get<0>(fp_search);
+			{
-			pipeline_key key = { vertex_program.id, fragment_program.id, pipelineProperties };
+				m_cache_miss_flag = false;
+				return I->second;
+			}
 
-			reader_lock lock(m_pipeline_mutex);
+			if (const auto I = m_decompiler_map.find(key); I != m_decompiler_map.end())
-
-			if (m_link_queue.find(key) != m_link_queue.end())
 			{
 				// Already in queue
 				return __null_pipeline_handle;
 			}
 
-			rsx_log.notice("Add program (vp id = %d, fp id = %d)", vertex_program.id, fragment_program.id);
+			m_decompiler_map[key] = true;
-			m_program_compiled_flag = true;
-
-			lock.upgrade();
-			m_link_queue[key] = std::make_unique<async_link_task_entry>(vertex_program, fragment_program, pipelineProperties);
-		}
-		else
-		{
-			reader_lock lock(m_decompiler_mutex);
-
-			auto vertex_program_found = std::find_if(m_decompile_queue.begin(), m_decompile_queue.end(), [&](const auto& V)
-			{
-				if (V.is_fp) return false;
-				return program_hash_util::vertex_program_compare()(V.vp, vertexShader);
-			});
-
-			auto fragment_program_found = std::find_if(m_decompile_queue.begin(), m_decompile_queue.end(), [&](const auto& F)
-			{
-				if (!F.is_fp) return false;
-				return program_hash_util::fragment_program_compare()(F.fp, fragmentShader);
-			});
-
-			const bool add_vertex_program = (vertex_program_found == m_decompile_queue.end());
-			const bool add_fragment_program = (fragment_program_found == m_decompile_queue.end());
-
-			if (add_vertex_program)
-			{
-				lock.upgrade();
-				m_decompile_queue.emplace_back(vertexShader);
 		}
 
-			if (add_fragment_program)
+		// Enqueue if not already queued
-			{
+		m_decompile_queue.emplace_back(vertexShader, fragmentShader, pipelineProperties);
-				lock.upgrade();
-				m_decompile_queue.emplace_back(fragmentShader);
-			}
-		}
 
 		return __null_pipeline_handle;
 	}
@@ -599,6 +549,16 @@ public:
 
 	void clear()
 	{
+		std::scoped_lock lock(m_vertex_mutex, m_fragment_mutex, m_decompiler_mutex, m_pipeline_mutex);
+
+		for (auto& pair : m_fragment_shader_cache)
+		{
+			free(pair.first.addr);
+		}
+
+		notify_pipeline_compiled = {};
+		m_fragment_shader_cache.clear();
+		m_vertex_shader_cache.clear();
 		m_storage.clear();
 	}
 };

rpcs3/Emu/RSX/GL/GLFragmentProgram.cpp

@@ -412,15 +412,8 @@ void GLFragmentProgram::Delete()
 	shader.clear();
 
 	if (id)
-	{
-		if (Emu.IsStopped())
-		{
-			rsx_log.warning("GLFragmentProgram::Delete(): glDeleteShader(%d) avoided", id);
-		}
-		else
 	{
 		glDeleteShader(id);
-		}
 		id = 0;
 	}
 }

rpcs3/Emu/RSX/GL/GLGSRender.cpp

@@ -903,6 +903,15 @@ void GLGSRender::on_init_thread()
 
 	m_gl_texture_cache.initialize();
 
+	m_prog_buffer.initialize
+	(
+		[this](void* const& props, const RSXVertexProgram& vp, const RSXFragmentProgram& fp)
+		{
+			// Program was linked or queued for linking
+			m_shaders_cache->store(props, vp, fp);
+		}
+	);
+
 	if (!m_overlay_manager)
 	{
 		m_frame->hide();
@@ -1196,16 +1205,10 @@ bool GLGSRender::load_program()
 
 	void* pipeline_properties = nullptr;
 	m_program = m_prog_buffer.get_graphics_pipeline(current_vertex_program, current_fragment_program, pipeline_properties,
-			!g_cfg.video.disable_asynchronous_shader_compiler).get();
+			!g_cfg.video.disable_asynchronous_shader_compiler, true).get();
 
 	if (m_prog_buffer.check_cache_missed())
 	{
-		if (m_prog_buffer.check_program_linked_flag())
-		{
-			// Program was linked or queued for linking
-			m_shaders_cache->store(pipeline_properties, current_vertex_program, current_fragment_program);
-		}
-
 		// Notify the user with HUD notification
 		if (g_cfg.misc.show_shader_compilation_hint)
 		{

rpcs3/Emu/RSX/GL/GLProgramBuffer.h

@@ -86,21 +86,26 @@ struct GLTraits
 	}
 };
 
-class GLProgramBuffer : public program_state_cache<GLTraits>
+struct GLProgramBuffer : public program_state_cache<GLTraits>
 {
-public:
+	GLProgramBuffer() = default;
 
-	u64 get_hash(void*&)
+	void initialize(decompiler_callback_t callback)
+	{
+		notify_pipeline_compiled = callback;
+	}
+
+	u64 get_hash(void* const&)
 	{
 		return 0;
 	}
 
-	u64 get_hash(RSXVertexProgram &prog)
+	u64 get_hash(const RSXVertexProgram &prog)
 	{
 		return program_hash_util::vertex_program_utils::get_vertex_program_ucode_hash(prog);
 	}
 
-	u64 get_hash(RSXFragmentProgram &prog)
+	u64 get_hash(const RSXFragmentProgram &prog)
 	{
 		return program_hash_util::fragment_program_utils::get_fragment_program_ucode_hash(prog);
 	}
@@ -109,7 +114,7 @@ public:
 	void add_pipeline_entry(RSXVertexProgram &vp, RSXFragmentProgram &fp, void* &props, Args&& ...args)
 	{
 		vp.skip_vertex_input_check = true;
-		get_graphics_pipeline(vp, fp, props, false, std::forward<Args>(args)...);
+		get_graphics_pipeline(vp, fp, props, false, false, std::forward<Args>(args)...);
 	}
 
     void preload_programs(RSXVertexProgram &vp, RSXFragmentProgram &fp)
@@ -122,9 +127,4 @@ public:
 	{
 		return m_cache_miss_flag;
 	}
-
-	bool check_program_linked_flag() const
-	{
-		return m_program_compiled_flag;
-	}
 };

rpcs3/Emu/RSX/GL/GLVertexProgram.cpp

@@ -314,15 +314,8 @@ void GLVertexProgram::Delete()
 	shader.clear();
 
 	if (id)
-	{
-		if (Emu.IsStopped())
-		{
-			rsx_log.warning("GLVertexProgram::Delete(): glDeleteShader(%d) avoided", id);
-		}
-		else
 	{
 		glDeleteShader(id);
-		}
 		id = 0;
 	}
 }

rpcs3/Emu/RSX/VK/VKGSRender.cpp

@@ -537,7 +537,14 @@ VKGSRender::VKGSRender() : GSRender()
 	m_video_output_pass = std::make_unique<vk::video_out_calibration_pass>();
 	m_video_output_pass->create(*m_device);
 
-	m_prog_buffer = std::make_unique<VKProgramBuffer>();
+	m_prog_buffer = std::make_unique<VKProgramBuffer>
+	(
+		[this](const vk::pipeline_props& props, const RSXVertexProgram& vp, const RSXFragmentProgram& fp)
+		{
+			// Program was linked or queued for linking
+			m_shaders_cache->store(props, vp, fp);
+		}
+	);
 
 	if (g_cfg.video.disable_vertex_cache || g_cfg.video.multithreaded_rsx)
 		m_vertex_cache = std::make_unique<vk::null_vertex_cache>();
@@ -2473,18 +2480,12 @@ bool VKGSRender::load_program()
 	vertex_program.skip_vertex_input_check = true;
 	fragment_program.unnormalized_coords = 0;
 	m_program = m_prog_buffer->get_graphics_pipeline(vertex_program, fragment_program, properties,
-			!g_cfg.video.disable_asynchronous_shader_compiler, *m_device, pipeline_layout).get();
+			!g_cfg.video.disable_asynchronous_shader_compiler, true, *m_device, pipeline_layout).get();
 
 	vk::leave_uninterruptible();
 
 	if (m_prog_buffer->check_cache_missed())
 	{
-		if (m_prog_buffer->check_program_linked_flag())
-		{
-			// Program was linked or queued for linking
-			m_shaders_cache->store(properties, vertex_program, fragment_program);
-		}
-
 		// Notify the user with HUD notification
 		if (g_cfg.misc.show_shader_compilation_hint)
 		{

rpcs3/Emu/RSX/VK/VKProgramBuffer.h

@@ -186,26 +186,22 @@ struct VKTraits
 
 struct VKProgramBuffer : public program_state_cache<VKTraits>
 {
-	VKProgramBuffer() = default;
+	VKProgramBuffer(decompiler_callback_t callback)
-
-	void clear()
 	{
-		program_state_cache<VKTraits>::clear();
+		notify_pipeline_compiled = callback;
-		m_vertex_shader_cache.clear();
-		m_fragment_shader_cache.clear();
 	}
 
-	u64 get_hash(vk::pipeline_props &props)
+	u64 get_hash(const vk::pipeline_props &props)
 	{
 		return rpcs3::hash_struct<vk::pipeline_props>(props);
 	}
 
-	u64 get_hash(RSXVertexProgram &prog)
+	u64 get_hash(const RSXVertexProgram &prog)
 	{
 		return program_hash_util::vertex_program_utils::get_vertex_program_ucode_hash(prog);
 	}
 
-	u64 get_hash(RSXFragmentProgram &prog)
+	u64 get_hash(const RSXFragmentProgram &prog)
 	{
 		return program_hash_util::fragment_program_utils::get_fragment_program_ucode_hash(prog);
 	}
@@ -214,7 +210,7 @@ struct VKProgramBuffer : public program_state_cache<VKTraits>
 	void add_pipeline_entry(RSXVertexProgram &vp, RSXFragmentProgram &fp, vk::pipeline_props &props, Args&& ...args)
 	{
 		vp.skip_vertex_input_check = true;
-		get_graphics_pipeline(vp, fp, props, false, std::forward<Args>(args)...);
+		get_graphics_pipeline(vp, fp, props, false, false, std::forward<Args>(args)...);
 	}
 
     void preload_programs(RSXVertexProgram &vp, RSXFragmentProgram &fp)
@@ -228,9 +224,4 @@ struct VKProgramBuffer : public program_state_cache<VKTraits>
 	{
 		return m_cache_miss_flag;
 	}
-
-	bool check_program_linked_flag() const
-	{
-		return m_program_compiled_flag;
-	}
 };

rpcs3/Emu/RSX/rsx_cache.h

@@ -616,7 +616,7 @@ namespace rsx
 			dlg->close();
 		}
 
-		void store(pipeline_storage_type &pipeline, RSXVertexProgram &vp, RSXFragmentProgram &fp)
+		void store(const pipeline_storage_type &pipeline, const RSXVertexProgram &vp, const RSXFragmentProgram &fp)
 		{
 			if (g_cfg.video.disable_on_disk_shader_cache)
 			{
@@ -739,7 +739,7 @@ namespace rsx
 			return std::make_tuple(pipeline, vp, fp);
 		}
 
-		pipeline_data pack(pipeline_storage_type &pipeline, RSXVertexProgram &vp, RSXFragmentProgram &fp)
+		pipeline_data pack(const pipeline_storage_type &pipeline, const RSXVertexProgram &vp, const RSXFragmentProgram &fp)
 		{
 			pipeline_data data_block = {};
 			data_block.pipeline_properties = pipeline;

rpcs3/Emu/RSX/rsx_utils.h

@@ -772,7 +772,7 @@ namespace rsx
 	}
 
 	template <int N>
-	void unpack_bitset(std::bitset<N>& block, u64* values)
+	void unpack_bitset(const std::bitset<N>& block, u64* values)
 	{
 		constexpr int count = N / 64;
 		for (int n = 0; n < count; ++n)
@@ -893,7 +893,7 @@ namespace rsx
 
 		simple_array(const std::initializer_list<Ty>& args)
 		{
-			reserve(args.size());
+			reserve(::size32(args));
 
 			for (const auto& arg : args)
 			{